o Me#@s<ddlmZmZddlZddlZddlZGdddeZdS)) Controller ControleModeNc@s0eZdZeddZddZddZddZd S) PSControllercCsd|jjtjks|jjst|jjdks|jjst|jjdkr0|j d |j tj|j_dSdS)Nrz {} enabledTF) argsZrun_moderZPS server_numlenservers trainer_numtrainersloggerdebugformat__name__)clsctxrXD:\Projects\ConvertPro\env\Lib\site-packages\paddle/distributed/launch/controllers/ps.pyenables  zPSController.enablecCs,|jjjr|jjjr|dS|dS)N)rrr r _build_pod_with_args_build_pod_with_masterselfrrr build_pod"s  zPSController.build_podc s&d|jjjvr dn|jjjdd|jjjdD}dd|jjjdD}fdd|jjjdD}fdd|jjjdD}t|}t|}|||j_ | ||gddl }| }t j|rot||jjj} d |dd d| } d || |jjjd } t|D]2} |jjj|jjj|| d d dddt||jjjd} | | d| }|j| |dqd}|D] }|rn|d 7}qt|D]8} d|d||| d d ddd| |dt||jjjd} | | d| }|j| |dqdS)Nz 127.0.0.1cSg|]}|qSrr.0srrr .z5PSController._build_pod_with_args..,cSrrrrrrrr/rcg|] }|r|qSr startswithrhostrrr0  cr!rr"rr$rrr3r&r{}:{}:3ZPADDLE_GLOO_RENDEZVOUSZPADDLE_GLOO_FS_PATHZPADDLE_GLOO_HTTP_ENDPOINTZPADDLE_WITH_GLOOrPSERVER{})PADDLE_PSERVERS_IP_PORT_LISTPADDLE_TRAINER_ENDPOINTS PADDLE_PORT PADDLE_ROLE TRAINING_ROLEPADDLE_TRAINERS_NUMPOD_IP serverlog.{}Zenvslog_fileTRAINER)r-r.r/r0r1PADDLE_TRAINER_IDr2r3 workerlog.{})rrr nodeipsplitr rpodreplicas save_pod_logtempfilemkdtempospathexistsshutilrmtree gloo_portr with_gloorangeupdate add_containerr#join)rserver_endpointstrainer_endpointsr r rr r@gloo_rendezvous_dirrG gloo_http _gloo_envsier6trainer_rank_offsetrrr$rr(sr                z!PSController._build_pod_with_argsc stjjjj_jjjpd}fddjj|D}jjjp$d}fddjj|D}t jj jj||jjj j jjd}jdjjjj |jjjj\}}jjd|dd|D}|d d|D}d d|D} td d|d|D} td d|d|D} |j_||j_d dl} | } tj| rt| |d d}d|d  dd |}d| |jjj!d}t"|D]:}djjd#|d#| |||  dddddt$| jjj%d}|&|d|}j'||dqt"|D]A}djjd#|d#| | ||  ddddd|| dt$| jjj%d }|&|d|}j'||dqdS)Nrcg|] }djjj|qSr'rrr:r;rprrrruz7PSController._build_pod_with_master..crUrVrWrXrrrrzrZ)namerankr r dtyperGz/{}/infozsync peers done {}cSsg|]}t|qSr)jsonloadsrrRrrrrscSg|] }|dD]}|qqSr rrrRjrrrrcSrar rrcrrrrrecSg|]}t|dqSrbrr`rrrrcSrgrfrhr`rrrrrirrGr'r(r)r*r,r r+) PADDLE_NNODESr-r.r/r0r1r2r3r4r5r7) rjr-r.r/r0r1r8r2r3r9)(intrrr\r=rr:Zget_free_portsr r^dumpsr[Zdevicer]Z get_free_portZmasterZ sync_peersrZjobidr>r r r?sumr@rArBrCrDrErFr<rHrIrLrr;rJrK)rrr r r dataZ peer_listr\rMrNZserver_rank_offsetrTr@rOrGrPrQrRrSr6rrrrps                       z#PSController._build_pod_with_masterN)r __module__ __qualname__ classmethodrrrrrrrrrs    Hr) controllerrrr^rBrErrrrrs