o
    Me                     @   sT   d dl Z d dlmZ d dlmZ d dlmZ d dlT d dlmZ G dd deZdS )	    N)launch_utils)cloud_utils)ascend_utils)*)LauncherInterfacec                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
CollectiveLauncherc                 C   s   || _ g | _d S )N)argsprocs)selfr    r   [D:\Projects\ConvertPro\env\Lib\site-packages\paddle/distributed/fleet/elastic/collective.py__init__   s   
zCollectiveLauncher.__init__c                 C   s   t d | j}t | _tjjj	
|\}}tjjj	|| j}t|||j|j|j|d| _t| jD ]\}}t d|jj| q4d S )Nzcollective lauchner launch ...)training_scripttraining_script_argslog_dirZenvszlaunch proc_id:{} idx:{})loggerinfor   tempfilemkdtemptmp_dirZpaddledistributedZfleetlaunchZget_cluster_infoZget_global_envsZstart_local_trainersr   r   r   r	   	enumerateformatprocpid)r
   r   ZclusterZpodZglobal_envsidxr   r   r   r   r      s$   


zCollectiveLauncher.launchc                 C   s>   t d |  st d tj| jrt	| j d S d S )Nzcollective lauchner stop ...zkill process failed)
r   r   Z_terminate_procserrorospathexistsr   shutilrmtree)r
   r   r   r   stop2   s   

zCollectiveLauncher.stopc                 C   s:   t d | jD ]}|jr|jdkrt| q|  }|S )Nzcollective lauchner watch ...r   )r   debugr	   Zlog_fnZ
local_rankZpull_worker_logZ_check_procs)r
   pretr   r   r   watch9   s   

zCollectiveLauncher.watchN)__name__
__module____qualname__r   r   r#   r'   r   r   r   r   r      s
    r   )	r   Zpaddle.distributed.fleetr   r   r   Z%paddle.distributed.fleet.launch_utilsZ(paddle.distributed.fleet.elastic.managerr   r   r   r   r   r   <module>   s   