
    'iM              	       F   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
Z
d dlZddlmZ ddlmZmZmZmZ ddlmZ dd	lmZ dOd
eej,                     dededdfdZe j2                  dej4                  ej6                     fd       ZdeddfdZdPdZdeddfdZe
j@                  jC                  dd      de"deddfd       Z#deddfdZ$e j2                  dQdejJ                  e&   de"fd       Z'd
edejP                  fdZ)deddfdZ*deddfdZ+deddfdZ,dPdZ-dPd Z.dPd!Z/deddfd"Z0deddfd#Z1dPd$Z2e
j@                  jC                  d%g d&      d'ed(eddfd)       Z3e
j@                  jC                  d*d+d,gfd-d.gfd/d0gfd1g d2fd3d4gfg      d5ed6ejh                  ejj                  ejl                  e&ef   d7f      ddfd8       Z7e
j@                  jC                  d9d:d;g      d<ed6eddfd=       Z8dPd>Z9dPd?Z:dPd@Z;dPdAZ<e j2                  dBedej4                  ejz                     fdC       Z>dPdDZ?dPdEZ@dPdFZAdPdGZBdHeddfdIZCe
j@                  jC                  dJi dKdig      dJej                  ee&f   deddfdL       ZEdeddfdMZFdeddfdNZGy)R    N)Path)patch   )helpers)job_environment
submission	test_coreutils)Job   )slurmjobprintserrorsreturnc           	         | j                   j                  | j                   j                  fD cg c]0  }t        |      j	                  dt        | j
                              2 }}t        |||f      D ]9  \  }}t        |      j                  d      5 }|j                  |       ddd       ; yc c}w # 1 sw Y   KxY w)zWrite fake log files%jwN)
pathsstdoutstderrstrreplacejob_idzipr   openwrite)r   r   r   x	filepathsfilepathmsgfs           S/var/www/stems/demucs_env/lib/python3.12/site-packages/submitit/slurm/test_slurm.py_mock_log_filesr$      s    ADAQAQSVS\S\ScSc@de1Qc#**o6eIeY(89 #(^  % 	GGCL	 	 f	 	s   5B1B66B?	c               #   8  K   t        j                  dg      } 	 | j                         5  |  d d d        t        j                  j
                  j                          y # 1 sw Y   2xY w# t        j                  j
                  j                          w xY ww)Nsrun)
known_cmds)r	   MockedSubprocesscontextr   SlurmJobwatcherclearmocks    r#   mocked_slurmr/       sp     %%&:D'\\^ 	J	 	$$&		 	 	$$&s1   BA- A!A- )B!A*&A- -*BBtmp_pathc                     t               5 }|j                  dd       t        j                  | d      }|j                  dk(  sJ |j                  d       d d d        y # 1 sw Y   y xY w)N12z       UNKNOWNFtimeout)r/   set_job_stater   r*   state
_interrupt)r0   r.   r   s      r#   test_mocked_missing_stater9   +   s^    	 &44+$)NN8T$ByyI%%%u%	& & &s   AA  A)c                      t               5 } | j                  dd       | j                  d      5  t        j                         j
                  dk(  sJ 	 d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr2   RUNNINGr   )r/   r6   job_contextr   JobEnvironmentclusterr-   s    r#   test_job_environmentr?   3   s|    	 G44+d# 	G"113;;wFFF	GG G	G 	GG Gs"   $A1$A%A1%A.	*A11A:c                 V   t               5 }t        j                  |       }|j                  t        j
                  ddd      }|j                  dk(  sJ |j                  dk(  sJ |j                         J t        |dd	
       d|_
        t        j                  t        j                        5  |j                          d d d        t        |dd	
       |j!                  |j                        5  t#        j$                  |j&                  j(                         d d d        |j+                         dk(  sJ |j                         d	k(  sJ |j-                         dk(  sJ dt.        j0                  vsJ d       	 d d d        y # 1 sw Y   xY w# 1 sw Y   wxY w# 1 sw Y   y xY w)Nfolderr   r      blublur2   r;   This is the error log
hop)r   r   r      _USELESS_TEST_ENV_VAR_z(Test context manager seems to be failing)r/   r   SlurmExecutorsubmitr	   
do_nothingr   r7   r   r$   _results_timeout_spytestraisesr
   UncompletedJobError_get_outcome_and_resultr<   r   process_jobr   rB   resultr   osenviron)r0   r.   executorr   s       r#   test_slurm_job_mockedrW   :   ss   	 f4&&h7ooi22AqoCzzT!!!yyI%%%zz|###$=eL!"]]5445 	*'')	*$=eLcjj) 	5""399#3#34	5zz|r!!!zz|u$$$zz|8888'rzz9e;ee9'f f	* 	*	5 	5f fs=   B&F1F1F3*FA FF	FF	FF(use_batch_api)FTc           
         d}t               5 }t        j                  |      }|j                  d       t	        |      t	        dd|z         cdt
        dt
        dt
        ffd	}g }| rQ|j                         5  t              D ]'  \  }}|j                  |j                  |||             ) 	 d d d        n|j                  |      }|d
   j                  j                  d      d
   }	t	        |      D 
cg c]	  }
|	 d|
  c}
|D cg c]  }|j                   c}k(  sJ |D ]`  }|j                  dk(  sJ |j                  |j                        5  t        j                   |j"                  j$                         d d d        b |d
   j'                  d       t)        t+        |            |D cg c]  }|j-                          c}k(  sJ t/        ||	      j"                  j0                  j3                         }|j5                         D cg c]  }d|v s|j7                          }}|dgk(  sJ 	 d d d        y # 1 sw Y   xY wc c}
w c c}w # 1 sw Y   AxY wc c}w c c}w # 1 sw Y   y xY w)N   rA   rC   )array_parallelism
   r   yr   c                 &    | v sJ |v sJ | |z   S N )r   r]   data1data2s     r#   addz(test_slurm_job_array_mocked.<locals>.addY   s$    ::::q5L    r   _r;   Tr4   r   z--arrayz#SBATCH --array=0-4%3)r/   r   rJ   update_parametersrangeintbatchr   appendrK   	map_arrayr   splitr7   r<   r   rR   r   rB   r8   listmaprS   r   submission_file	read_text
splitlinesstrip)rX   r0   nr.   rV   rc   jobsd1d2array_idajr   sbatchl
array_linera   rb   s                   @@r#   test_slurm_job_array_mockedr~   Q   s_   	A	 74&&h7""Q"7Qxr26!2u	3 	3 	3 	
 #%! >!%/ >FBKKR <=>> > %%c5%8D7>>'',Q/+084a8*AaS!448Pa8PPPP 	9C99	)))!!#**- 9&&syy'7'789 9	9
 	Q4(CUE*+D/Iq
/IIIIXh/55EEOOQ)/):):)<OA	QaggiO
O56666=7 7> > 58P9 9 0J P;7 7st   A1J  7I8A
J IJ I$)8J !*I)8J I6AJ "	I;,I;>
J I	J )I3.J  J	c                    t               5 }t        j                  |       }|j                  dd       |j	                  t
        j                  ddd      }|j                  |j                        5  t        j                  t              5  t        j                  |j                  j                         d d d        d d d        t!        |d	
       t        j                  t"        j$                        5  |j'                          d d d        |j)                         }t+        |t"        j$                        sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   WxY w# 1 sw Y   y xY w)NrA      r   )timegpus_per_noder   r   rH   )errorrF   )r   )r/   r   rJ   rg   rK   r	   rL   r<   r   rN   rO   
ValueErrorr   rR   r   rB   r$   r
   FailedJobErrorrS   	exception
isinstance)r0   r.   rV   r   r   s        r#   test_slurm_error_mockedr   u   s    	 ;4&&h7""!"<ooi22AqoCcjj) 	9z* 9&&syy'7'789	9 	$=>]]5//0 	JJL	MMO	)U%9%9:::; ;
9 9	9 	9	 	; ;sT   A(E*3E*E7E?8E*7E4E*EEE	E*E'	#E**E3called_with
not_calledc              #     K   |s| J t        dd       }|5 }	 d  |r|j                          n|j                  |        	 d d d        y # |r|j                          w |j                  |        w xY w# 1 sw Y   y xY ww)Nz1submitit.slurm.slurm.SlurmJobEnvironment._requeue)return_value)r   assert_not_calledassert_called_with)r   r   requeue_patchs       r#   mock_requeuer      s     000GVZ[G	 7F	7((*))+67 7 ((*))+67 7s2   BA5A%A5	B&A22A55A>:Bc                     t        j                         }t        j                  j	                  | j
                  j                        }t        j                  || j
                  |      }|S r_   )	r   SlurmJobEnvironmentr
   DelayedSubmissionloadr   submitted_pickler   SignalHandler)r   envdelayedsigs       r#   get_signal_handlerr      sM    

#
#
%C%%**399+E+EFG

'
'SYY
@CJrd   c                    t         j                  j                         }t        j                         }|j                  t        j                  d       t        |t        j                        sJ t               5  t        j                  | d      }|j                  d       |j                  |      }d d d        t              } |d       t!        j"                  t$              5  t'        d      5  |j)                  |       d d d        d d d        t        |      } |d	       t!        j"                  t$              5  t'        d
      5  |j)                  |       d d d        d d d        t        |      } |d       t        j*                  j                         }t'        d      5  t!        j"                  t,        j.                  d      5  |j)                  |       d d d        d d d        y # 1 sw Y   IxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   QxY w# 1 sw Y   y xY w)Nr\   r   rB   max_num_timeout<   r      minutesr   2   r   7   Tr   ztimed-out too many times.match)submititr=   _usr_sigr   FunctionSequencerc   r	   _three_timer   Checkpointabler/   r   rJ   rg   rK   r   rN   rO   
SystemExitr   checkpoint_and_try_requeuer   r
   rP   )r0   fast_forward_clockusr_sigfs0rV   r   r   s          r#   test_requeuing_checkpointabler      s   %%..0G

"
"
$CGGI!!2&c711222 
 #&&hJ"""+ooc"#
 S
!Cr"	z	" 0LQ$? 0&&w/0 0 S
!Cr" 
z	" 0LQ$? 0&&w/0 0 S
!Cr" ''002G		& 0!!)D) 0 	&&w/0 0 0;# #0 0 0 00 0 0 00 0 0 0sx   9;G;)H6HHH.H"$H.2&IH:*I;HH	HH"H+	'H..H7:I	?IIc                    t         j                  j                         }t               5  t	        j
                  | d      }|j                  d       |j                  t        j                  d      }d d d        t              } |d       t        d	      5  |j                  t        j                  j                         d d d        t!        j"                  t$              5  t        d
      5  |j'                  |       d d d        d d d        t        |      } |d       t        d	      5  t!        j"                  t(        j*                  d      5  |j'                  |       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   CxY w# 1 sw Y   y xY w)Nr   r   r   r   r\   r   r   Tr   r   r   z timed-out and not checkpointabler   )r   r=   r   r/   r   rJ   rg   rK   r	   r   r   r   bypasssignalSignalsSIGTERMrN   rO   r   r   r
   rP   )r0   r   r   rV   r   r   s         r#   !test_requeuing_not_checkpointabler      sq   %%..0G	 9&&hJ"""+ooi33R89 S
!Cr"		& +

6>>))*+ 
z	" 0LQ$? 0&&w/0 0 S
!Cr" 
	& 0!!)K) 0 	&&w/0 0 0+9 9+ +0 0 0 00 0 0 0s`   A
E>*F&F#3FF#5&F;F/-F;>FFF 	F##F,/F8	4F;;Gc                 n   t         j                  j                         }t               5  t	        j
                  | d      }|j                  d       |j                  t        j                  d      }d d d        t              }t        j                  t              5  t        d      5  |j                  |       d d d        d d d        t         j"                  j%                  |j&                  j(                        }|j*                  dk(  sJ y # 1 sw Y   xY w# 1 sw Y   bxY w# 1 sw Y   fxY w)Nr   r   r   r   r\   Tr   )r   r=   r   r/   r   rJ   rg   rK   r	   r   r   rN   rO   r   r   checkpoint_and_exitr
   r   r   r   r   _timeout_countdown)r0   r   rV   r   r   r   s         r#   test_checkpoint_and_exitr      s    %%..0G	 9&&hJ"""+ooi33R89
 S
!C	z	" )LD$A )() ) %%**399+E+EFG%%***9 9) ) ) )s0   A
DD+,D>D+DD(	$D++D4c            
      `   t        j                  ddddddig d      } d	| v sJ d
| vsJ d| vsJ t        t              j                  dz  }|j                         s|j                  |        |j                         }g }t        t        | j                         |j                                     D ],  \  }\  }}||k7  s|j                  d|dz    d| d| d       . |rLt        |        t        d       t        |       dg|z   }|ddd| gz  }t        dj                  |            y )Nz
blublu bar/tmp	learnfairTrE   rH   )z-vvz
--cpu-bindnone)commandrB   	partition	exclusiveadditional_parameters	srun_argsr   z	--command
constraintz_sbatch_test_record.txtzline #r   z: "z" -> ""z	# # # # #zDifference with reference file: z)Delete the record file if this is normal:zrm 
)r   _make_sbatch_stringr   __file__parentexists
write_textrq   	enumerater   rr   rk   printAssertionErrorjoin)stringrecord_filerecordedchangeskline1line2messages           r#   test_make_sbatch_stringr      sT   &&'n/F &   f$$$v%%%x.''*CCKv&$$&HG&s6+<+<+>@S@S@U'VW E>E5E>NNVAE7#eWF5'CDE fkh45?BCs;-EXYYTYYw/00 rd   c                  @    t        j                  ddd      } d| v sJ y )NrE   r   r   )r   rB   r   z--gpus-per-node=2r   r   r   s    r#   test_make_sbatch_string_gpur     s%    &&xVWXF&(((rd   c                  @    t        j                  ddd      } d| vsJ y )NrE   r   T)r   rB   stderr_to_stdout--errorr   r   s    r#   test_make_sbatch_stderrr     s%    &&xY]^FF"""rd   c                     t               5  t        j                  |       }d d d        j                  d       |j                  j
                  d   dk(  sJ y # 1 sw Y   :xY w)NrA   g      @)mem_gbmem3584MB)r/   r   AutoExecutorrg   	_executor
parametersr0   rV   s     r#   test_update_parametersr     s\    	 :((9:c*((/8;;;: :s   AA$c                     t               5  t        j                  |       }d d d        t        j                  t
              5  j                  d       d d d        y # 1 sw Y   >xY w# 1 sw Y   y xY w)NrA   rH   rD   )r/   r   rJ   rN   rO   r   rg   r   s     r#   test_update_parameters_errorr     sc    	 8&&h78	z	" .""""-. .8 8. .s   AA+A(+A4c                      d} t        j                         j                  |       }|d   dddk(  sJ |d   dddk(  sJ t        |      h dk(  sJ y )	NzoJobID|State
5610980|RUNNING
5610980.ext+|RUNNING
5610980.0|RUNING
20956421_0|RUNNING
20956421_[2-4%25]|PENDING
5610980r;   )JobIDState
20956421_2z20956421_[2-4%25]PENDING>   r   
20956421_0r   
20956421_3
20956421_4)r   SlurmInfoWatcher	read_infoset)exampleoutputs     r#   test_read_infor   &  sh    G ##%//8F))i HHHH,-@9#UUUUv;]]]]rd   z
name,state))12_0R)12_1U)12_2X)12_3r  )12_4r  namer7   c                     d}t        j                         }dD ]  }|j                  |        |j                  |      }|j	                  | i       j	                  dd      |k(  sJ y )Nz JobID|State
12_0|R
12_[2,4-12]|X)r  r  r   r  )r   r   register_jobr   get)r  r7   r   r+   jobidr   s         r#   test_read_info_arrayr  4  sj     3G$$&G! $U#$w'F::dB##GS1U:::rd   zjob_id,expectedr2   )rH   r   )rH   r   z20_[2-7%56]   r      z20_[2-7,12-17,22%56])r  )r  rH      )r     z20_[0%1])r  r   r   expected.c                     t        j                  |       }||D cg c]  }t        d |D               c}k(  sJ y c c}w )Nc              3   2   K   | ]  }t        |        y wr_   )r   ).0r   s     r#   	<genexpr>z#test_read_job_id.<locals>.<genexpr>L  s     2qCF2s   )r   read_job_idtuple)r   r  r   groups       r#   test_read_job_idr  @  s;     v&FIue2E22IIIIIs   <zstring,expected)s   Submitted batch job 5610208
5610208)zSubmitted batch job 5610208
r  r   c                 P    t         j                  j                  |       }||k(  sJ y r_   )r   rJ   #_get_job_id_from_submission_command)r   r  r   s      r#   #test_get_id_from_submission_commandr  O  s(    
   DDVLFXrd   c                      t        j                  t        j                        5  t        j
                  j                  d       d d d        y # 1 sw Y   y xY w)Ns   blublur   )rN   rO   r
   FailedSubmissionErrorr   rJ   r  r`   rd   r#   )test_get_id_from_submission_command_raiser"  X  sD    	u22	3 R??y?QR R Rs   !AAc                     t               5 } t        j                         }| j                  dd       |j                  dk(  sJ |j                  d      }t        |j                  j                               dhk(  sJ |j                  dhk(  sJ |dk(  sJ | j                  dd       |j                  dd	      }|dk(  sJ |j                  ddhk(  sJ |j                  dhk(  sJ 	 d d d        y # 1 sw Y   y xY w)
Nr2   r;   r   11rf   r3   FAILEDforce)r   mode)r/   r   r   r6   	num_calls	get_stater   
_info_dictkeys_registered	_finished)r.   r+   r7   s      r#   test_watcherr.  ]  s   	 +4((*4+  A%%%!!!.7%%**,-$777""tf,,,	!!!4*!!G!<   ""tTl222  TF***+ + +s   CC22C;c                  @    t        j                         } | d   dk(  sJ y )Nnodesr   )r   _get_default_parameters)defaultss    r#   test_get_default_parametersr3  o  s#    ,,.HG!!!rd   c                  J    t         j                  j                         dk(  sJ y )Nr   )r   rJ   r  r`   rd   r#   	test_namer5  t  s    ##%000rd   	node_listc              #      K   dt         j                  d<   | t         j                  d<   t        j                          t         j                  d= t         j                  d= y w)N1SLURM_JOB_IDSLURM_JOB_NODELIST)rT   rU   r   r   )r6  s    r#   with_slurm_job_nodelistr;  x  sK     !$BJJ~'0BJJ#$

#
#
%%


'(


>"s   A A"c                     t        d      5 } dg| j                  k(  sJ 	 d d d        t        d      5 } ddg| j                  k(  sJ 	 d d d        t        d      5 } ddg| j                  k(  sJ 	 d d d        t        d      5 } d	d
g| j                  k(  sJ 	 d d d        t        d      5 } g d| j                  k(  sJ 	 d d d        t        d      5 } g d| j                  k(  sJ 	 d d d        t        d      5 } g d| j                  k(  sJ 	 d d d        t        d      5 } g d| j                  k(  sJ 	 d d d        t        d      5 } ddg| j                  k(  sJ 	 d d d        t        d      5 } g d| j                  k(  sJ 	 d d d        y # 1 sw Y   sxY w# 1 sw Y   XxY w# 1 sw Y   =xY w# 1 sw Y   "xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nzcompute-b24zcompute-a1,compute-b2
compute-a1z
compute-b2zcompute-b2[1,2]compute-b21compute-b22zcompute-b2[011,022]zcompute-b2011zcompute-b2022zcompute-b2[1-3])r>  r?  compute-b23zcompute-b2[1-3,5,6,8])r>  r?  r@  compute-b25compute-b26compute-b28zcompute-b2[1-3,5-6,8]z compute-b2[1-3,5-6,8],compute-a1)r>  r?  r@  rA  rB  rC  r=  zcompute[042,044]
compute042
compute044zcompute[042-043,045,048-049])rD  
compute043
compute045
compute048
compute049r;  	hostnamesr   s    r#   test_slurm_node_listrM    sH   	 	/ 03#--///0	 !8	9 =Sl+s}}<<<=	 !2	3 ?s}->>>?	 !6	7 C31S]]BBBC	 !2	3 Ns<MMMN	 !8	9 S
 ]] 	  
!!8	9 S
 ]] 	  
!!C	D 	
 ]] 	 	 
!!3	4 =l+s}}<<<=	 !?	@ gCUY\YfYffffg gQ0 0= =? ?C CN N  	 	= =g gsv   FFF+F8+GG;G#G*G63HFF(+F58GGGG'*G36G?Hc                  l    t        d      5 } g d| j                  k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nz)compute-b24-[1-3,5-9],compute-b25-[1,4,8])zcompute-b24-1zcompute-b24-2zcompute-b24-3zcompute-b24-5zcompute-b24-6zcompute-b24-7zcompute-b24-8zcompute-b24-9zcompute-b25-1zcompute-b25-4zcompute-b25-8rJ  rL  s    r#   )test_slurm_node_list_online_documentationrO    s@    	 !L	M QT
 ]] 	   s   *3c                     t        j                  t        j                        5  t	        d      5 } t        | j                         d d d        d d d        t        j                  t        j                        5  t	        d      5 } t        | j                         d d d        d d d        y # 1 sw Y   gxY w# 1 sw Y   kxY w# 1 sw Y   *xY w# 1 sw Y   y xY w)Nzcompute-b2[1-,4]zcompute-b2[1,2,compute-b3])rN   rO   r   SlurmParseExceptionr;  r   rK  rL  s    r#   test_slurm_invalid_parserR    s    	u00	1 !$%78 	!C#-- 	!! 
u00	1 !$%AB 	!c#-- 	!! !	! 	!! !	! 	!! !sF   B8B,B89CCC,B5	1B88CC		CCc                  ~    t        d      5 } | j                  g| j                  k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr   )r;  hostnamerK  rL  s    r#   test_slurm_missing_node_listrU    s9    	 	$ /~.../ / /s   3<weird_tmp_pathc                 `   d| j                   v rt        j                  d       t               5  t	        j
                  |       }|j                  t        j                  ddd      }d d d        j                  j                  j                  d       |j                  j                  j                  d       i }|j                  j                  j                         j                         D ]w  }|j!                  d	      sd
|vr|t#        d	      d  j%                         j'                  d
d      \  }}|j)                  d|j*                        j)                  dd      ||<   y t-        j.                  d|d   z   d       t-        j.                  d|d   z   d       y # 1 sw Y   =xY w)Nr   z0test doesn't support newline in 'weird_tmp_path'rA   r   r   rC   rD   r   z#SBATCH=r   z%t0zls z--outputT)shellr   )r  rN   skipr/   r   rJ   rK   r	   rL   r   r   r   r   rp   rq   rr   
startswithlenrs   rm   r   r   
subprocess
check_call)rV  rV   r   sbatch_argsr|   keyvals          r#   test_slurm_weird_dirrc    sm   ~"""FG	 D&&n=ooi22AqoCD
 II#II# KYY&&002==? L||I&a<S^%&,,.44S!<S;;tSZZ8@@sKCL %+j"99F%+i"88E+D Ds   :F##F-paramsr   c                    t               5  t        j                  |      } |j                  di | dddii |j	                  t
        j                  ddd      }d d d        j                  j                  j                         }|j                         D cg c]	  }d	|v s| }}t        |      dk(  s
J d
|        y # 1 sw Y   hxY wc c}w )NrA   slurm_additional_parametersmem_per_gpurH   r   r   rC   rD   z#SBATCH --memzUnexpected lines: r`   )r/   r   r   rg   rK   r	   rL   r   rp   rq   rr   r]  )rd  r0   rV   r   textr   	mem_liness          r#   test_slurm_through_autorj    s    	 D((9"""]V]-Y[I\]ooi22AqoCD 99$$..0D OO-FqA1EFIFy>Q@"4YK @@D D
 Gs   AC	C%CCc                 :   d }t               5 }t        j                  |       }|j                  |      }t	        |d       d|_        t        j                  t        j                  d      5  |j                          d d d        |j                  dd	       |j                  |      }d|_        t        j                  t        j                  d
      5  |j                          d d d        d d d        y # 1 sw Y   zxY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nc                      t        d      )NzToo bad)r   r`   rd   r#   fail_silentlyz/test_slurm_job_no_stderr.<locals>.fail_silently  s    ##rd   rA   zjob is running ...
)r   r   zjob is running ...r   13r;   z!No output/error stream produced !)r/   r   rJ   rK   r$   rM   rN   rO   r
   rP   rQ   r6   )r0   rm  r.   rV   r   s        r#   test_slurm_job_no_stderrro    s    $ 
 *4&&h7oom,$:;!"]]544<PQ 	*'')	* 	4+oom,!" ]]544<_` 	*'')	** *	* 	*	* 	** *s=   A!D/C9 ADD(D9D	>DD	
DDc                 D   t               5  t        j                  |       }|j                  ddidgdg       |j	                  t
        j                  ddd	
      }d d d        j                  j                  j                         }d|v sJ d|v sJ y # 1 sw Y   :xY w)NrA   rg  rH   zThis is a setup commandzThis is a teardown command)rf  slurm_setupslurm_teardownr   r   rC   rD   )
r/   r   r   rg   rK   r	   rL   r   rp   rq   )r0   rV   r   rh  s       r#   /test_slurm_through_auto_with_setup_and_teardownrs  
  s    	 D((9"")6(;2389 	# 	

 ooi22AqoCD 99$$..0D$,,,'4///D Ds   ABB)r   r   )r   N)NF)H
contextlibrT   r   r^  typingtppathlibr   unittest.mockr   rN   r   r   r   corer   r   r	   r
   	core.corer   r   Anyr   r$   contextmanagerIteratorr(   r/   r9   r?   rW   markparametrizeboolr~   r   Optionalri   r   r   r   r   r   r   r   r   r   r   r   r   r  ListTupleUnionr  r  r"  r.  r3  r5  r   r;  rM  rO  rR  rU  rc  Dictrj  ro  rs  r`   rd   r#   <module>r     s    	         @ @  RVV c  T  'bkk)"<"<= ' '& & &GfD fT f. -8 7t  7t  7  7 9 7F;d ;t ; 
7bkk#. 
74 
7 
7C O$A$A '0D '0 '0T0 0T 0<+t + + 1:)
#
<T <d <.4 .D .^ ];s ;3 ;4 ;; 	w	'	%	!EF	gY	JS JBGGBHHRXXc3h=OQT=T4U,V J[_ J	J
 24`a s t 	
R
+$"
1 #s #r{{5;T;T/U # #*gZ"!/
F F$ F6 B4(8#9:ABGGCH$5 A A$ A ;A*t * *.0d 0t 0rd   