o
    Ne                     @   sp   d dl mZ d dl mZ ddlmZ ddlZddlmZ ddlm	Z	 dd	lm
Z
 dd
lmZ G dd deZdS )   )DownpourServer)DownpourWorker   )append_backward    N)find_distributed_lookup_table)$find_distributed_lookup_table_inputs)%find_distributed_lookup_table_outputs)text_formatc                   @   s*   e Zd ZdZd	ddZ			d
ddZdS )DownpourSGDa  
    Distributed optimizer of downpour stochastic gradient descent
    Standard implementation of Google's Downpour SGD
    in Large Scale Distributed Deep Networks

    Args:
        learning_rate (float): the learning rate used to update parameters. \
        Can be a float value
    Examples:
        .. code-block:: python
    
             opt = fluid.DistributedOptimizer(sgd_opt)
             opt.minimize()

             downpour_sgd = fluid.distributed.DownpourSGD(learning_rate=0.2)
             downpour_sgd.minimize(cost)
    MbP?r   c                 C   s    || _ || _d| _g d| _d S )NZdownpour)z.batch_sizez.batch_square_sumz
.batch_sumz.batch_size@GRADz.batch_square_sum@GRADz.batch_sum@GRAD)learning_rate_window_typedata_norm_name)selfZlearning_rateZwindow r   QD:\Projects\ConvertPro\env\Lib\site-packages\paddle/fluid/distributed/downpour.py__init__+   s   zDownpourSGD.__init__Nc                 C   s  t |ts	tdt|d jj}t|d jj|}t|d jj|}t	 }t
 }	t| j}
d}|	|| j|| |
|| j|| d}g }g }tt|D ]}|jj }tt|| jj|_|j|g |j|g tt|| ||dd d}|| g }g }g }g }|D ]$}d}| jD ]}|d j|rd}||d  q|s||d  q|D ]$}d}| jD ]}|d j|rd}||d  q|s||d  q|	 || j|| |
 || j|| |j!|g |j"|g t|dkr+t|dkr+|d7 }|	#|| j|| |
 || j|| |j!|g |j"|g |d7 }|| qO|j$%|	&  |j%|
&  |D ]}|jj|g qGd	d
g}|jj'| i }d|d< d|d< d|d< ||d< ||d< |D ]}||jj_(qvd|fS )a  
        DownpounSGD is a distributed optimizer so
        that user can call minimize to generate backward
        operators and optimization operators within minimize function
        Args:
            loss(Variable): loss variable defined by user
            startup_program(Program): startup program that defined by user
            parameter_list(str list): parameter names defined by users
            no_grad_set(set): a set of variables that is defined by users
            so that these variables do not need gradient computation
        Returns:
            [ps_param, worker_skipped_ops]
            ps_param: parameter server protobuf desc
            worker_skipped_ops: operator names that need
            to be skipped during execution
        z(losses is a list, just lick [model.cost]r   r   c                 S   s
   | d j S )Nr   )name)xr   r   r   <lambda>f   s   
 z&DownpourSGD.minimize.<locals>.<lambda>)keyFTZlookup_tableZlookup_table_gradZDistMultiTrainerZtrainerr   Zdevice_workerZ	optimizerZ
fleet_descworker_skipped_opsN))
isinstancelist
ValueErrorr   blockprogramr   r	   pslibZPSParameterr   r   r   Zadd_sparse_tabler   rangelenZtrainer_paramprogram_configaddstridZ
program_idZpull_sparse_table_idextendZpush_sparse_table_idsortedr   appendr   r   endswithZadd_dense_tableZpull_dense_table_idZpush_dense_table_idZadd_data_norm_tableZserver_paramZCopyFromZget_descZskip_opZ
_fleet_opt)r   ZlossesZstartup_programZparameter_listZno_grad_setZ
table_nameZprefetch_slotsZprefetch_slots_embZps_paramserverZworkerZsparse_table_indexZdense_table_indexZprogram_configsZparam_grads_listZ
loss_indexr"   Zparams_gradsparamsZgradsZdata_norm_paramsZdata_norm_gradsiZis_data_norm_datar   Zdata_norm_gradr   Zopt_infoZlossr   r   r   minimize6   s   











	zDownpourSGD.minimize)r   r   )NNN)__name__
__module____qualname____doc__r   r-   r   r   r   r   r      s    
r   )noder   r   Zbackwardr   Zps_pb2r   Z$paddle.fluid.distribute_lookup_tabler   r   r	   Zgoogle.protobufr
   objectr   r   r   r   r   <module>   s   