o
    Með  ã                   @   sP   d dl Z d dlZd dlmZmZmZ d dlmZ g Zdd„ Zdd„ Z	dd	„ Z
dS )
é    N)Úget_clusterÚget_gpusÚget_cluster_from_args)Úloggerc              
      s,  t  d¡}|dusJ dƒ‚t  d¡}|dusJ dƒ‚t  d¡}|dus'J dƒ‚tt  d¡ƒ}|dus6J d	ƒ‚| d
¡}t|ƒ}t|ƒ}|dkrT||krTt d ||¡¡ | dkrh| d
 |¡krht d | |¡¡ t  d¡}	|	du rÔ|}
|dkr¬ztt  dd¡ƒ}|t|ƒkr”||kr”t d |¡¡ |}
W n t	y« } z
t
|ƒ W Y d}~nd}~ww |
du r²d}
dd„ t|
|
t|ƒ ƒD ƒ}g }	|D ]‰ |	 ‡ fdd„|D ƒ¡ qÄn'|	 d
¡}g }	|| t|ƒksåJ ‚t|ƒD ]}|	 ||| |d | … ¡ qét d ||||	¡¡ t|||	|ƒ\}}||j| fS )zZ
    args_node_ips:string, args_node_ip:string, args_port: int, selected_devices:list
    ZPADDLE_TRAINERSNz"PADDLE_TRAINERS should not be NoneZPOD_IPzPOD_IP should not be NoneZPADDLE_TRAINER_IDz$PADDLE_TRAINER_ID should not be NoneZTRAINER_PORTS_NUMz$TRAINER_PORTS_NUM should not be Noneú,z	127.0.0.1z¡Please NOTE: When using paddlecloud, node_ip is automatically got from POD_IP. Your input node_ip: {} doesn't equals to node_ip: {} from paddlecloud environment.zÛPlease NOTE: When using paddlecloud, cluster_node_ips is automatically got from PADDLE_TRAINERS(multi nodes) or POD_IP(single node).Your input cluster_node_ips: {} doesn't equals to IPs: {} from paddlecloud environment.ZDISTRIBUTED_TRAINER_ENDPOINTSé   ZPADDLE_PORTÚ zUse Cloud specified port:{}.i  c                 S   s   g | ]}|‘qS © r	   )Ú.0Úxr	   r	   úND:\Projects\ConvertPro\env\Lib\site-packages\paddle/distributed/cloud_utils.pyÚ
<listcomp>M   s    ÿz%get_cloud_cluster.<locals>.<listcomp>c                    s   g | ]}d ˆ |f ‘qS )z%s:%dr	   )r
   Úport©Úipr	   r   r   R   s    zRparsed from args: node_ips:{}         node_ip:{} node_rank:{} trainer_endpoints:{})ÚosÚgetenvÚintÚsplitÚlenr   ÚwarningÚformatÚjoinÚ	ExceptionÚprintÚrangeÚappendÚdebugr   Zpods)Zargs_node_ipsZargs_node_ipZ	args_portÚselected_devicesZnode_ipsÚnode_ipZ	node_rankZpaddle_ports_numZ	num_nodesZtrainer_endpointsÚstarted_portZpaddle_portÚeZportsZtrainer_endpoints_oriÚiÚclusterÚpodr	   r   r   Úget_cloud_cluster   s†   



þü
ÿÿ€€þÿÿ
ÿÿÿÿÿr%   c                   C   s   t t dd¡ƒS )NZPADDLE_TRAINERS_NUMÚ1)r   r   r   r	   r	   r	   r   Ú_get_trainers_nume   s   r'   c                 C   sŽ   t | jƒ}tƒ }t d ||¡¡ d }d }| jr4|dkr4t| j| j	| j
|ƒ\}}t d |¡¡ ||fS t| |ƒ\}}t d |¡¡ ||fS )Nz5parsed from args trainerss_num:{} selected_devices:{}r   zget cluster from cloud:{}zget cluster from args:{})r   r   r'   r   r   r   Zuse_paddlecloudr%   Zcluster_node_ipsr   r    Úinfor   )Úargsr   Ztrainers_numr#   r$   r	   r	   r   Úget_cluster_and_podi   s    
ÿ
ÿýr*   )r   ZpaddleZ%paddle.distributed.utils.launch_utilsr   r   r   r   Ú__all__r%   r'   r*   r	   r	   r	   r   Ú<module>   s   N