miAdZddlZddlmZddlmZddlmZmZddl Z ddl m cm Z ddlmZmZmZmZddlmZdd lmZdd lmZmZdd lmZdd lmZmZej<eZ d Z!Gdde"Z#y)zMain training loop.N)get_xp)write_and_rename) LogProgressbold)augmentdistribstates pretrained) apply_model)ModelEMA)evaluatenew_sdr) svd_penalty) pull_metricEMAcNdjd|jDS)N | c3LK|]\}}|jd|yw)=N) capitalize).0keyvals G/var/www/stems/demucs_env/lib/python3.12/site-packages/demucs/solver.py z_summary..s'Rhc3)*!C51Rs"$)joinitems)metricss r_summaryr s ::R'--/R RRcLeZdZdZdZdZdedefdZdedefdZdZ d d Z y ) Solverc .||_||_||_||_t j |j|j |j|_tj||_ tt|jjj|_ggd|_|jj!D]i}t#|j$|}|dk(r |jnd}|s/|D]6}|j|j't)|j||8kt+j,t/|j0j2|j0j4z|j*j6g} |j*j8r+| t+j:t+j<gz } dD]X} t#|j*| } | j>s&| j't#t*| jAd i| ZtCjDjF| |_tI} | jJ|_%| jJdz |_&| jJdz |_'tPjSd |jLjUd|_+d |_,| jZ|_-|jZj\|_.|j_y) N)batchepochr%cpu)device)shiftsame)scaleremix checkpoint.thzbest.thzCheckpoint will be saved to %sF)0argsloadersmodel optimizerr get_quantizerquant quantizerr wrapdmodelnextiter parametersr(emaskeysgetattremaappendr rShiftintdset samplerater) shift_sameflip FlipChannelsFlipSignprobartorchnn Sequentialrfoldercheckpoint_file best_fileloggerdebugresolve best_state best_changedlinkhistory_reset) selfr0r1r2r/kinddecaysr(decayaugmentsaugkwxps r__init__zSolver.__init__sP   "--djj$**dnnUll5) 4 5 5 789@@ !2. IINN$ WDTXXt,F$(GOT[[F#WEIIdO**8DJJf+UVW  WMMDII,@,@499??,R(S'+||'>'>@A <<   --/1A1A1CD DH% JCs+Bxx B1A B HR HI Jxx**H5 Xii !yy?:Y. 5t7K7K7S7S7UV!GG yy((  r!c i}|jj|d<|jj|d<|j|d<|j|d<|j |d<|j jD]1\}}t|D]\}}|j|d|d|< 3t|j5}tj||ddd|j j}|re|dz|zd k(rZ|dz|j jk7r>t|jd |dzd z 5}tj||ddd|j rut|j"5}t%j&|j|j }|j|d<tj||dddd |_yy#1swY xYw#1swYxYw#1swY+xYw) Nstater2rUrRr/ema__rr checkpoint_z.thF)r1 state_dictr2rUrRr/r;r enumeraterrMrIsave save_everyepochsrLrSrNr serialize_model) rWr&packagerXr;kr>tmprhs r _serializezSolver._serializeMs::002#~~88: !\\  $ ))))//+ >JD$#D/ >3-0^^-=$tfAaS)* > >d22 3 %s JJw $ %YY)) 519 2a7EAIIYIYr1rprootcfs rrVz Solver._resetgs    & & ( KK4T5I5I4JK Ljj!5!5u=G JJ & &ww'7 8 NN * *7;+? @%i0DLLO%l3DO"iioo/ D d'oDFAs''$tfAaS0A(BCD DYY * *((YY22YY..0E JJ & &u'7'7'9 : YY $ $"D;;%%DDII3344t;B KK)2 .jjU+G%l3DOyy&& **7<+@*O **77+;E*Jyy%%..w{/CD&%r!rreturnctt|ddt|ddd}d|vrt|dd|d<|jt|dd|d<d |vrt|d d|d <d |vrt|d d|d <d |vr|d |d <d |vrt|d d|d <d |vrt|d d|d <|S)z#Formatting for train/valid metrics.lossz.4freco)rrnsdr.3fmsz.2fgradbestbnamepenaltyhloss)formatr5)rWrlossess r _format_trainzSolver._format_trains76?E276?E2  W #GFOU;F6N >> %!'$-7F4L W #GFOU;F6N W #GFOU;F6N g %g.F7O   &wy'95 AF9  g $WW%5u=F7O r!ci}d|vrt|dd|d<d|vrt|dd|d<|jjD]9}d|}||vrt||d||<d|}||vs(t||d||<;|S)zFormatting for test metrics.sdrrrsdr_nsdr_)rr1sources)rWrrsourcers r _format_testzSolver._format_tests G "75>59F5M W #GFOU;F6Njj(( :F/Cg~$WS\59s &"Cg~$WS\59s  : r!c |jrtjdt|jD]\}}|j |d}tjt d|dzdt ||j |d}tjt d|dzdt |d|vs|j|d}|stjt d |dzdt |d }tt|j|jjD]}|jji}tjd tjd |j||d<|j |d}tjt d|dzdt |tjd tjd |jjt!j"5|j|d}|}d}t%j&|jj)}i|d<||dd<|jj*j,}|j.j1D]\} } t| D]x\} } | j35|j|d}dddd| d| } ||d| <||}||}|j5dr| }| }||ksi|}| j6}| }z|dj9|||dd< ddd|d}t;|j<jd||gz}|j5dr t?|}n tA|}||dd<|jjBjDd kDrmtG|jjB}|jIdt!j"5tK|jfddi|}ddd|dd<|j |d}tjt d|dzdt |||k(s |jjLjNr@tjt d|t%j&|_(d|_)|dz|jj*jTzd k(}||jjdz k(}|s|rBtjd tjd|jj*jVr |jP}n-t%j&|jj)}|jj*jXxr|}t%jZ|j|5t!j"5t]|||d<dddddd|j|d}tjt d |dzdt ||j<j_|t`jbd k(r?|je|tjgd|jhjk|syy#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYxYw)Nz#Replaying metrics from previous runtrainzTrain Summary | Epoch rrvalidzValid Summary | Epoch testzTest Summary | Epoch rzF----------------------------------------------------------------------z Training...zCross validation...F)rmainrbrcrrzvalid.rrexactTzNew best valid loss %.4fzEvaluating on the test set...) compute_sdrzCheckpoint saved to %s)6rUrOrtrfrrr rrangelenr/rir1r_run_one_epochevalrIno_gradr copy_staterermetricr;rswap startswithraupdaterrTmaxminsvdrdictpoprrB train_validrRrSeveryrr swap_stater push_metricsr rankrnrPrMrQ)rWr&r formattedrbvalidrrarrXr;rlr>rpab valid_lossmets best_lossr]r should_evalis_lastrs rrz Solver.trains` << KK= >' 5 cNE7**77+; KK'.>?d E JJ   G KK ! KK &#2259GG **77+;>&1!"A!"Aq5%*F$'IIE$(E )G$++F305GG$W-!6 64!)#.Jtyy00F3%.AZLPD~~f%I I '0GG V $yy}}$$q($))--(y!]]_H)$**GDGBGGH.5 +**77+;>&& OOE"--djj.C.C.EFE"iinn00  D#8 3xPYGZF[!\]^ II " "7 +||q & 5t7K7K7S7S7UVId 2LL 6 6HHHLRRRRs[B3\'5\ >\'0\'\4+] ]] \$\''\1 4\> ] ]  ] c|j}|r|jdn|jd}tjdkDr|r|jj |ddg|}|d|dzz}t |}|jrt||j}tt|||jjj|}t} t|D]E\} } | j|j } |r$|j#| } | j%d} n| ddd f} | ddddf} |sN|jj&r8t)|j*| |jj,j.d } n|j1| } |r2t3|j*d r|j*j5| | } | j6| j6k(sJ| j6| j6ft9t;d | j=}|j>j@d k(r;tCjD| | d}|jG|jGd }|}n{|j>j@dk(r@tCjH| | d}|jG|}|dz}|jGd }n"tKd|jj@tMjN|jPj| }||zj%|j%z }d }|jR|jRjU}|jVjXr||jVjX|zz }i}||zj%|j%z |d<||d<|s{t[| | j]jGd }d }t_|j*j`||D]\}}}||d|<|||zz }||j%z |d<|rv|jbjdd kDr]tg|jb}|jidtk|j*fi|}||d<||jbjd|zz }||d<t|j*j`D]\}}|||d|<|r|jmd }g}|j*joD]^}|jp||jpjrjud zz }|jw|jpjr`|dz|d<|j>jxrVtLjzj|j|j*jo|j>jx|jjdk(r;|j*jD]\}}|jptd| |jj|jj|jdD]}|j| |}|j|} |jd i| ~~ ~~|j| k(rn8|jjr|rn|jjdk(sFn|r$|jdD]}|jtj dzS)!NrrrValidTrainz | Epoch )totalupdatesrp)dimr)splitoverlaptransform_targetl1none) reductionmseg?z Invalid loss rrrrrrreco_runszno gradr%rPr&r.)Kr/r0r world_sizesampler set_epochr max_batchesrrrOmisc num_printsrrftor(rsum valid_applyr r1rrr7hasattrrshapetuplerroptimrFl1_lossmeanmse_loss ValueErrorrItensorweightsr5 model_sizer4diffqrdetachziprrrrrrbackwardr:rdatanormr? clip_gradrJutilsclip_grad_norm_flagnamed_parametersprintr2step zero_gradr;rrrPaverage)!rWr&rr/ data_loaderlabelrprlogprogaverageridxrmixestimatedimsrrrrrnsdrsrrwr]rrl grad_normgradspnr>logss! rrzSolver._run_one_epoch#s1yy/4dll7+$,,w:O    !e    ) )% 0'"5)519+..K    t//0Efk&*iinn&?&?dL5%g._ LCjj-G,,w/kkak(adm!!QR%.TYY22&tzz3diinn>R>R\]^;;s+-?@**55c7C>>W]]2 SX^^W]]4S S2q'++-01Dzz$&yy7fEyy++A.E)zz(GvFyySyyy| =0@!ABBll4<<033G>U* $ ; ; =0166>!)Q/0##%((*99W-!CJJL!f%F%%f-D GNN "T "hb3&yy5yy~~(_ @ yy)   vsQw//r!N)T) __name__ __module__ __qualname__r_rnrVrrrrrr.r!rr#r#sF,\&4E>Td,DT unr0r!r#)$__doc__loggingdorar dora.utilsrdora.logrrrItorch.nn.functionalrJ functionalrrr r r applyr r>r rrrrrrr getLoggerrrOr objectr#r.r!rr sZ '& 22'#   8 $Sw0Vw0r!