o
    Me#                     @   st   d dl Z d dlmZ ddlmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlZd d	l mZmZ d
d ZdS )    N)dygraph_only   )ParallelMode)fleet)
MethodType)core)to_variable)_C_ops_legacy_C_opsc                 C   s.   dd }t j }|j tjurt|| | _| S )Nc           
      S   s|  | j sd S t|dd rMt|jd trMg }g }g }|jD ]/}|d D ](}| d urJ||  | jtj	j
jkrC||  q"||  q"qndd |jD }dd |jD }dd |jD }ttdgtj}ttdgtj}t|rt|| j|| t|rt|| j|| |s|rdnd| _tj| jgd	d
}	tjj|	tjjjd d |	 d | _d S )N_param_groupsr   paramsc                 S   s    g | ]}|  d ur|  qS N)
_grad_ivar.0param r   OD:\Projects\ConvertPro\env\Lib\site-packages\paddle/distributed/fleet/scaler.py
<listcomp>.   s
    z>distributed_scaler.<locals>.unscale_method.<locals>.<listcomp>c                 S   4   g | ]}|  d ur|  jtjjjkr|  qS r   )r   dtyper   VarDescVarTypeFP16r   r   r   r   r   2       c                 S   r   r   )r   r   r   r   r   ZFP32r   r   r   r   r   7   r   r   Zint32)r   )opgroup)Z_enablegetattr
isinstancer   dictr   appendr   r   r   r   r   Z_parameter_listr   nparrayZastypeZbool_lenr
   Zcheck_finite_and_unscaleZ_scaleZ
_found_infpaddleZ	to_tensordistributedZ
all_reduceZReduceOpMAXnumpy)
selfZ	optimizerZparam_gradsZparam_grads_fp16Zparam_grads_fp32r   r   Ztemp_found_inf_fp16Ztemp_found_inf_fp32Zis_found_infr   r   r   unscale_method   sd   



z*distributed_scaler.<locals>.unscale_method)r   Z_hcgZget_parallel_moder   ZDATA_PARALLELr   Z_unscale)Zscalerr)   Z	fleet_envr   r   r   distributed_scaler   s
   9r*   )r$   Zpaddle.fluid.frameworkr   Zbase.topologyr   Zpaddle.distributedr   typesr   Zpaddle.fluidr   Zpaddle.fluid.dygraphr   r'   r!   r	   r
   r*   r   r   r   r   <module>   s   