o
    Ne&                     @   sl   d Z ddlm  mZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ edddgd	G d
d dejZdS )zNadam optimizer implementation.    N)backend_config)optimizer_v2)learning_rate_schedule)keras_exportzkeras.optimizers.legacy.Nadamzkeras.optimizers.Nadam)v1c                       sl   e Zd ZdZdZ					 d fdd	Zd	d
 Zdd Z fddZdddZ	dddZ
 fddZ  ZS )Nadama
  Optimizer that implements the NAdam algorithm.
    Much like Adam is essentially RMSprop with momentum, Nadam is Adam with
    Nesterov momentum.

    Args:
      learning_rate: A Tensor or a floating point value.  The learning rate.
      beta_1: A float value or a constant float tensor. The exponential decay
        rate for the 1st moment estimates.
      beta_2: A float value or a constant float tensor. The exponential decay
        rate for the exponentially weighted infinity norm.
      epsilon: A small constant for numerical stability.
      name: Optional name for the operations created when applying gradients.
        Defaults to `"Nadam"`.
      **kwargs: keyword arguments. Allowed arguments are `clipvalue`,
        `clipnorm`, `global_clipnorm`.
        If `clipvalue` (float) is set, the gradient of each weight
        is clipped to be no higher than this value.
        If `clipnorm` (float) is set, the gradient of each weight
        is individually clipped so that its norm is no higher than this value.
        If `global_clipnorm` (float) is set the gradient of all weights is
        clipped so that their global norm is no higher than this value.

    Usage Example:
      >>> opt = tf.keras.optimizers.legacy.Nadam(learning_rate=0.2)
      >>> var1 = tf.Variable(10.0)
      >>> loss = lambda: (var1 ** 2) / 2.0
      >>> step_count = opt.minimize(loss, [var1]).numpy()
      >>> "{:.1f}".format(var1.numpy())
      9.8

    Reference:
      - [Dozat, 2015](http://cs229.stanford.edu/proj2015/054_report.pdf).
    TMbP??+?Hz>c                    s   | dd|d< |d|}t|tjrtdt j|fi | | d|d| | d| j	 | d| | d| |pDt
 | _d | _d S )	NZschedule_decaygMbp?decaylrzdThe Nadam optimizer does not support tf.keras.optimizers.LearningRateSchedules as the learning rate.learning_ratebeta_1beta_2)popget
isinstancer   ZLearningRateSchedule
ValueErrorsuper__init__Z
_set_hyper_initial_decayr   epsilon_m_cache)selfr   r   r   r   namekwargs	__class__ SD:\Projects\ConvertPro\env\Lib\site-packages\keras/optimizers/optimizer_v2/nadam.pyr   D   s   

zNadam.__init__c                 C   sp   |d j j}| jd u r | jdg |ddtjjd| _| j| j |D ]}| 	|d q"|D ]}| 	|d q-d S )Nr   Zmomentum_cacheZonesF)shapedtypeZinitializerZ	trainableZaggregationmv)
r"   
base_dtyper   Z
add_weighttfZVariableAggregationZONLY_FIRST_REPLICAZ_weightsappendZadd_slot)r   var_list	var_dtypevarr   r   r    _create_slotsa   s    
zNadam._create_slotsc                 C   s@  t | d|}t | d|}t | d|}t | jd |}t | jd |}t d|}	|ddt |	| j|    }
|ddt |	| j|    }t | j||
 }|| jj	u rot t j
jj| j|| jd	}|| }t|| t | j||||
|d| d| d|
 d| d| dt || d
|||f< d S )Nr   r   r         gQ?g      ?g      ?Zuse_locking)lr_tneg_lr_tr   beta_1_tbeta_2_tm_tm_t_1one_minus_beta_1_tone_minus_beta_2_tone_minus_m_tone_minus_m_schedule_newone_minus_m_schedule_nextv_t_prime_denominator)r&   identityZ
_get_hypercastZ
iterationspowr   _m_cache_readr   r"   compatr   assign_use_lockingdictZconvert_to_tensorr   )r   
var_devicer)   apply_stater/   r1   r2   Z
local_stepZ	next_stepZ
decay_baser3   r4   Zm_schedule_newZm_schedule_nextr   r   r    _prepare_localu   sF   
zNadam._prepare_localc                    s   t | j| _t |S N)r&   r;   r   r>   r   _prepare)r   r(   r   r   r    rG      s   zNadam._prepareNc                 C   s  |j |jj}}|pi ||fp| ||}| |d}| |d}||d  }	|d | |d |  }
tjjj	||
| j
d}
|
|d  }|d | |d	 t|  }tjjj	||| j
d}||d
  }|d |	 |d |  }||d | t||d    }tjjj	||| j
djS )Nr#   r$   r8   r1   r5   r.   r9   r2   r6   r:   r7   r4   r/   r   )devicer"   r%   r   _fallback_apply_stateget_slotr&   r?   r   r@   rA   Zsquaresqrtop)r   gradr*   rD   rC   r)   coefficientsr#   r$   g_primer3   	m_t_primev_t	v_t_primem_t_barZvar_tr   r   r    _resource_apply_dense   s<   




zNadam._resource_apply_densec                 C   s  |j |jj}}|pi ||fp| ||}| |d}| |d}	||d  }
||d  }tjjj	|||d  | j
d}t|g | |||}t||}W d    n1 s\w   Y  ||d  }|d |
 |d	 |  }|| |d
  }tjjj	|	|	|d  | j
d}t|g | |	||}t||}W d    n1 sw   Y  ||d  }t||d  }| |||d | | }tj|||g S )Nr#   r$   r8   r5   r1   r.   r9   r7   r4   r6   r2   r:   r   r0   )rH   r"   r%   r   rI   rJ   r&   r?   r   r@   rA   Zcontrol_dependenciesZ_resource_scatter_addgatherrK   group)r   rM   r*   indicesrD   rC   r)   rN   r#   r$   rO   Zm_scaled_g_valuesr3   Z	m_t_slicerP   rS   Zv_scaled_g_valuesrQ   Z	v_t_slicerR   Zv_prime_sqrt_plus_epsZ
var_updater   r   r    _resource_apply_sparse   sL   


zNadam._resource_apply_sparsec                    s:   t   }|| d| j| d| d| jd |S )Nr   r   r   )r   r   r   r   r   )r   
get_configupdateZ_serialize_hyperparameterr   r   )r   configr   r   r    rY      s   
zNadam.get_config)r   r	   r
   r   r   rF   )__name__
__module____qualname____doc__Z_HAS_AGGREGATE_GRADr   r+   rE   rG   rT   rX   rY   __classcell__r   r   r   r    r      s    ")

/r   )r_   Ztensorflow.compat.v2r?   v2r&   Zkerasr   Zkeras.optimizers.optimizer_v2r   Zkeras.optimizers.schedulesr   Z tensorflow.python.util.tf_exportr   ZOptimizerV2r   r   r   r   r    <module>   s   