o
    Ne"                     @   s   d dl mZ ddlmZmZ ddlmZmZmZ d dl	Z	d dl	m
Z
mZ dd	d
ZdddZ			 	dddZdddZdddZdddZdd Zdd ZdS )    )print_function   )LayerHelperunique_name)Variablein_dygraph_mode_in_legacy_dygraphN)_C_ops_legacy_C_opssumFc                 C   s   t di t }d}|dkrd}n|dkrd}n|dkrd}n|dkr&d	}ntd
|d u rF|jtd| jdg| j| j	| j
| jdd}|jdd| gid|gi||dd |S )N	allreducer   r   prod   maxr   min   z*reduce type can only be [sum|prod|max|min].tmpT)nameshapedtypetypepersistableZstop_gradientXOut)reduce_type	sync_moder   inputsoutputsattrs)r   r   locals	TypeErrorcreate_variabler   generate_with_ignorable_keyjoinr   r   r   r   r   	append_op)xoutr   r   helperZred_typ_int r+   ND:\Projects\ConvertPro\env\Lib\site-packages\paddle/fluid/layers/collective.py
_allreduce   s<   r-   c                 C   s8   t di t }|jdd| gid| gi||dd | S )N	broadcastr   r   )r   rootr   )r.   r   r"   r'   )r(   r/   r   r*   r+   r+   r,   
_broadcast8   s   r1   c                 C   s   t di t }|dvrtdd| }|d u r/|jtd| j|g| j| j	| j
| jd}|j|d| gid|gi||d	d
 |S )Nc_allreduce)r   Zprobr   r   z+reduce type can only be "sum|prod|max|min]"Zc_allreduce_r   r   r   r   r   r   r   r   )ring_iduse_calc_streamr   )r2   r!   )r(   r)   r   r4   r5   r*   op_typer+   r+   r,   _c_allreduceD   s,   r7   c                 C   s@   d}t |fi t }|j|d| gid| gi|||dd | S )NZc_broadcastr   r   )r/   r4   r5   r   r0   )r(   r/   r4   r5   r6   r*   r+   r+   r,   _c_broadcastb   s   r8   c                 C   s  d}t  r,tjj }t| j}|d  |9  < t|| j}|j	
| |}|  |S t r@d|d|d|f}	tj| g|	R  S t|fi t }
t| jd d  }|d dkr`|d  |9  < |
jtd| j|g|| j| j| jd}|
j|d| gid	|gi|||d
d |S )Nc_allgatherr   nranksr4   r5   r   r3   r   r   r:   r4   r5   r   )r   paddledistributedZ
collectiveZ_get_default_grouplistr   emptyr   Zprocess_groupZ
all_gatherwaitr   r
   r9   r   r"   r$   r   r%   r&   r   r   r   r'   )r(   r:   r4   r5   r6   groupZtensor_shaper)   taskr    r*   	out_shaper+   r+   r,   _c_allgatherp   sF   

rD   c                 C   s   t | ts	td| jd dkr$| jd | dkr$td| jd |f d}t|fi t }t| jd d  }|d dkrF|d  |  < |jt	
d| j|g|| j| j| jd}|j|d| gid|gi|||d	d
 |S )Nzx must be a Variabler   z5x.shape[0](%d) cannot be evenly divided by nranks(%d)Zc_reducescatterr   r3   r   r   r;   r   )
isinstancer   r#   r   
ValueErrorr   r"   r>   r$   r   r%   r&   r   r   r   r   r'   )r(   r:   r4   r5   r6   r*   rC   r)   r+   r+   r,   _c_reducescatter   s<   
 rG   c                 C   s6   d}t |fi t }|j|d| gid| gid | S )NZc_sync_calc_streamr   r   )r   r   r   r0   )r(   r6   r*   r+   r+   r,   _c_sync_calc_stream   s   rH   c                 C   s<   d}t |fi t }|j|d| gid| gid|id | S )NZc_sync_comm_streamr   r   r4   r   r0   )r(   r4   r6   r*   r+   r+   r,   _c_sync_comm_stream   s   rI   )Nr   F)F)Nr   r   F)r   r   F)r   F)
__future__r   Zlayer_helperr   r   Z	frameworkr   r   r   r<   r	   r
   r-   r1   r7   r8   rD   rG   rH   rI   r+   r+   r+   r,   <module>   s"   

"



&