mie1~dZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZddlmZddlZd d lmZd d lmZmZd d lmZmZd dej6eej:fdefdZGdde Z Gdde Z!Gdde Z"GddZ#d d d ddZ$GddZ%dZ&Gdde Z'y)!z Support for PyTorch lightning. You should just replace the call to `Trainer(...)` with `get_trainer(...)`. For using `dora.log.LogProgress` as a progress bar with PL, see `PLLogProgress`. N)LightningModule)Callback)ProgressBarBase)ClusterEnvironment)Trainer)from_argparse_args)distrib)get_xpis_xp)bold LogProgressTmetricsepochc|i}|jD]\}}|r|jdr|s|jdr.|jds|jdr|jddd}t|tj r#|j dk(r|j}|||<|S)aFilters metrics before formatting, in particular to remove the `_step` or `_epoch` suffix. This will also convert torch tensors to float. Args: metrics: dict given by PL. epoch: if True, keep only epoch level metrics, otherwise, keep only step level metrics. _step_epoch_r r)itemsendswithrsplit isinstancetorchTensornumelitem)rroutkeyvalues H/var/www/stems/demucs_env/lib/python3.12/site-packages/dora/lightning.py_filter_metricsr!s Cmmo  U S\\'* h/  << CLL$:**S!$Q'C eU\\ *u{{}/AJJLEC  JceZdZfdZdefdZedefdZdefdZ deddfdZ defd Z d eddfd Z defd Z defd ZedefdZedefdZedefdZxZS)DoraEnvironmentc|t|tj|_tj yN)super__init__r get_distrib_specspecset_distrib_envself __class__s r r(zDoraEnvironment.__init__4s* ,,. !r"returncyNTr-s r creates_childrenz DoraEnvironment.creates_children9sr"cyr1r2r3s r creates_processes_externallyz,DoraEnvironment.creates_processes_externally<sr"c.|jjSr&)r* world_sizer3s r r8zDoraEnvironment.world_size@yy###r"sizeNcyr&r2)r-r:s r set_world_sizezDoraEnvironment.set_world_sizeC r"c.|jjSr&)r*rankr3s r global_rankzDoraEnvironment.global_rankFsyy~~r"r?cyr&r2)r-r?s r set_global_rankzDoraEnvironment.set_global_rankIr=r"c.|jjSr&)r* local_rankr3s r rDzDoraEnvironment.local_rankLr9r"c.|jjSr&)r* node_rankr3s r rFzDoraEnvironment.node_rankOsyy"""r"cyNFr2r2r"r detectzDoraEnvironment.detectRsr"c(tjdS)N MAIN_ADDR)osenvironr3s r main_addresszDoraEnvironment.main_addressVszz+&&r"c:ttjdS)N MAIN_PORT)intrLrMr3s r main_portzDoraEnvironment.main_portZs2::k*++r")__name__ __module__ __qualname__r(boolr4propertyr6rQr8r<r@rBrDrF staticmethodrIstrrNrR __classcell__r.s@r r$r$3s" $d$C$ 3 4 S C D $C$#3#D'c'',3,,r"r$c"eZdZdZdZdZdZy)DoraCheckpointSyncz>Make sure Dora history, and checkpoint state are in sync. c"t|_yr&)r xpr3s r r(zDoraCheckpointSync.__init__bs (r"cX|d}|jjj|y)Ndora_link_history)r_linkupdate_history)r-trainer pl_module checkpointhistorys r on_load_checkpointz%DoraCheckpointSync.on_load_checkpointes#01  ##G,r"c|jjj|d<|jj|d<|jj|d<|S)Nradora_sigdora_cfg)r_rbrgsigcfgr-rdrerfs r on_save_checkpointz%DoraCheckpointSync.on_save_checkpointisE*.'',,*>*> &'!% :!% :r"N)rSrTrU__doc__r(rhror2r"r r]r]_s-r"r]c4eZdZdZfdZdZdZdZxZS)DoraHistoryLoggerz,Save metrics to Dora using the XP link. cTt|tj|_yr&)r'r(r rbr,s r r(zDoraHistoryLogger.__init__ss HMM r"cd|_yr1 _first_validr-rdres r on_fit_startzDoraHistoryLogger.on_fit_startws  r"cd|_yrHrurws r on_train_epoch_startz&DoraHistoryLogger.on_train_epoch_startzs !r"c|jry|j}t|d}|jj |y)NTr)rvlogged_metricsr!rb push_metrics)r-rdrers r on_epoch_endzDoraHistoryLogger.on_epoch_end}s7    ((!'6 w'r") rSrTrUrpr(rxrzrrZr[s@r rrrrps"!"(r"rrceZdZdZy)_DummySLURMConnectorcyr&r2r3s r register_slurm_signal_handlersz3_DummySLURMConnector.register_slurm_signal_handlersr=r"N)rSrTrUrr2r"r rrs r"r) auto_resumeadd_dora_loggerno_unfinished_epochscts tdtj}t |dr|j }t |drt j|dgt|zfi|}|d=|jdxsg}t}ttjj|j}|jdkDr||dgz }||d<|jddxsg} | j!t#| |d<|d  td |d dk7r td ||d <|j$j&|d <t)j*|d <|r|dj!t-|j/d} |r<| :t)j*dz } | j1r t3| } nd} | |d<tdi|} |rt5| _| S)aReturn a PL trainer, adding the necessary glue code to make everything works. The arguments are exactly the same as for `pytorch_lightning.trainer.Trainer`, with a few extras documented after. ..note:: You should not pass `gpus=` or `num_nodes=` arguments as those will be filled by Dora. Args: auto_resume (bool): if True, automatically resume previous checkpoints. You are still responsible for creating the `ModelCheckpoint` callback, this only handles the `resume_from_checkpoint` part. add_dora_logger (bool): if True, adds a Dora Logger to automatically forward the metrics (those logged with per_epoch=True), otherwise pushing metrics will be up to you. no_unfinished_epochs (bool): if True, deactivates SLURM signal handling by PL, which can result in half finished epoch with each interruption. It is recommended to instead dump a checkpoint every epoch and resume from that one so that training is reliable. z.This can only be called from inside a Dora XP. __wrapped__Nr-pluginsr ddp callbacksgpuszCYou cannot specify the number of GPUs, as this is provided by Dora. num_nodeszDYou cannot specify the number of nodes, as this is provided by Dora.default_root_dirresume_from_checkpointz last.ckptr2)r RuntimeErrorrr(hasattrrinspect getcallargslistpopr$minrcuda device_countr8appendr]r*rr folderrrgetis_filerYrslurm_connector)rrrargskwargsinitrenvrrrlastresumerds r get_trainerrs* 7KLL   D $ & $ &  vT ': Ef EFvjj#)rG  C uzz&&(#..*: ;D ~~!C<F9 ;-3I ')*#F; f~!`aa kaabbF6N((,,F;!'F {""#4#67#ZZ(@A-5x, <<>YFF+1'(G"6"8 Nr"cPeZdZejej dZy) _Interceptc ||_||_yr&)rr)r-rrs r r(z_Intercept.__init__s  r"N)rSrTrU functoolswrapsrr(r2r"r rrs%Y__W%%&'r"rc btt|fi|}t|ji|jSr&)rrrrr)rr intercepts r trainer_from_argparse_argsrs-":t>v>I   ;)*:*: ;;r"c"eZdZdZdfd Zdeddffd ZfdZede fdZ dd e jee jfded efd Zed Zd ZfdZfdZdZfdZfdZdZdZfdZfdZfdZdZdZdZxZS) PLLogProgressz<`dora.log.LogProgress` support for Pytorch-Lightning. r/Nc Lt|||_||_d|_yr&)r'r(loggerr _pl_module)r-rrr.s r r(zPLLogProgress.__init__s#   8<r"stagecDt||||||_g|_yr&)r'setupr_replay_history)r-rdrerr.s r rzPLLogProgress.setups"  gy%0#02r"cBt|||d|_d|_y)NFT)r'rx _in_trainrvr-rdrer.s r rxzPLLogProgress.on_fit_starts! Wi0 r"c6|jJ|jSr&)rr3s r rezPLLogProgress.pl_modules***r"rrczi}|jD]%\}}t|tst|d||<'|S)agDefault method to format metrics for displaying in the progress bar. To customize, you can define a `format_metrics()` method on your Lightning module. Args: metrics: dict of metrics given by PL. stage: "train" or "valid". epoch: if True, provided metrics are for the end of epoch summary. z.5f)rrfloatformat)r-rrrrrrs r format_metricszPLLogProgress.format_metricssC!--/ 0JC%'!%/C 0 r"cDt|jd|jS)Nr)getattrrerr3s r _format_metricszPLLogProgress._format_metrics st~~'79L9LMMr"c|jjd|jj|dk(rdnd|jd|jjdzz}|dk(rt |j }n)|dk(rt |j}ntd|t|}t|j|f||d |j|_ t|jy) NzF----------------------------------------------------------------------trainz Training...z Validating...z | Epoch r validzInvalid stage )totalname)rinfo capitalizerd current_epochrQtotal_train_batchestotal_val_batchesrrangerrlogprogiter)r-rrrloaders r _on_epoch_startzPLLogProgress._on_epoch_starts " %7*:P!i 0J0JQ0N/O$PP G 001E g ../Ew78 8u"4;;^e$^RVR]R]^  T\\r"cb|jdd|_d|_t|||S)NrTF)rrrvr'rzrs r rzz"PLLogProgress.on_train_epoch_starts2 W%!w+GY??r"cF|jdt| ||SNr)rr'on_validation_epoch_startrs r rz'PLLogProgress.on_validation_epoch_start$s# W%w0)DDr"c|j|j|j}t|d}|j ||d}|j j di|t|j y)NFr|r2) get_metricsrdrer!rrupdatenext)r-rr formatteds r _on_batch_endzPLLogProgress._on_batch_end(sb""4<<@!'7((%u(E  (i( T\\r"cFt||i||jdy)Nr)r'on_train_batch_endrr-rrr.s r rz PLLogProgress.on_train_batch_end/s# "D3F3 7#r"cFt||i||jdyr)r'on_validation_batch_endrrs r rz%PLLogProgress.on_validation_batch_end3s# '88 7#r"c||dk(r=|jjjjj dd}nF|jjjj jj dd}t |d}|j||jj|y)NrFlogr|) rdfit_loop epoch_loop_resultsrval_loopr!_show_epoch_summaryr)r-rrs r _on_stage_endzPLLogProgress._on_stage_end7s G ll++66??GGNuUGll++66??HHPPQVWX]^G!'7    (B(BGLr"c 2|jj|||f|j||d}|j}dj d|j D}|j jt|d|dzd|y)NTr|z | c3LK|]\}}|jd|yw)=N)r).0rvals r z4PLLogProgress._show_epoch_summary..Ds, ,4Cs~~ # ' s"$z Summary | End of Epoch r ) rrrrjoinrrrr )r-rrrrrsummarys r rz!PLLogProgress._show_epoch_summary@s ##UE7$;<((%t(D !** 8A8I   &>uqykWIVWXr"ct||||js|jsJ|js|j dd|_yyNrF)r' on_train_endrrvrrs r on_validation_startz!PLLogProgress.on_validation_startIsJ Wi0~~!2!222    w '"DN!r"cnt||||jr|jdd|_yr)r'rrrrs r rzPLLogProgress.on_epoch_endPs. Wi0 >>   w 'r"cHt||||jdyr)r'on_validation_endrrs r rzPLLogProgress.on_validation_endVs  !'95 7#r"cyr&r2r3s r disablezPLLogProgress.disableZs r"c|jdg}|r|jjd|D]}|j|y)Ndora_replay_historyzReplaying past metrics...)rrrr)r-rdrerfreplay_historysteps r rhz PLLogProgress.on_load_checkpoint_sH#(=rB  KK  8 9" ,D $D $ $d + ,r"c$|j|d<|S)Nr)rrns r roz PLLogProgress.on_save_checkpointfs,0,@,@ ()r")r/N)F) rSrTrUrpr(rYrrxrWrretpDictAnyrVrrrrzrrrrrrrrrrrhrorZr[s@r rrs = 3s3t3 ! ? 27bggc266k&:!*."NN @ E$$MY# $ ,r"r)T)(rprrrLtypingrpytorch_lightningrpytorch_lightning.callbacksr$pytorch_lightning.callbacks.progressr&pytorch_lightning.plugins.environmentsrpytorch_lightning.trainerr$pytorch_lightning.utilities.argparserrr r_r r rr rrrYrrVr!r$r]rrrrrrrr2r"r rs  -0@E-C "RWWS"&&[1$*),(),X"((,  $(TXCL< IOIr"