o
    Ne6!                     @   s   d Z ddlm  mZ ddlmZ ddlmZ dddZ	dd Z
G d	d
 d
ejZG dd deZG dd deZG dd dejZG dd dejZdS )z6Contains testing utilities related to mixed precision.    N)regularizers)
base_layerc                    s$   t jfdd  fdd}|S )a  Returns a function that asserts it's gradient has a certain value.

    This serves as a hook to assert intermediate gradients have a certain value.
    This returns an identity function. The identity's gradient function is also
    the identity function, except it asserts that the gradient equals
    `expected_gradient` and has dtype `expected_dtype`.

    Args:
      expected_gradient: The gradient function asserts that the gradient is this
        value.
      expected_dtype: The gradient function asserts the gradient has this dtype.

    Returns:
      An identity function whose gradient function asserts the gradient has a
      certain value.
    c                    s"   t    fdd} |fS )z8Function that asserts it's gradient has a certain value.c                    s    r| j  ksJ d  d| j  tj| j dd}tg tjj| |}W d   n1 s4w   Y  t|g t| } W d   | S 1 sPw   Y  | S )zLGradient function that asserts the gradient has a certain
            value.zdx.dtype should be z	 but is: expected_gradient)dtypenameN)r   tfZconvert_to_tensorZcontrol_dependenciescompatv1Zassert_equalidentity)dxZexpected_tensorZ	assert_op)expected_dtyper   x OD:\Projects\ConvertPro\env\Lib\site-packages\keras/mixed_precision/test_util.pygrad.   s    
zScreate_identity_with_grad_check_fn.<locals>._identity_with_grad_check.<locals>.gradr   r
   r   r   )r   r   r   r   _identity_with_grad_check)   s   
zEcreate_identity_with_grad_check_fn.<locals>._identity_with_grad_checkc                        | S Nr   r   )r   r   r   identity_with_grad_checkE      zDcreate_identity_with_grad_check_fn.<locals>.identity_with_grad_checkr   Zcustom_gradient)r   r   r   r   )r   r   r   r   "create_identity_with_grad_check_fn   s   r   c                    s"   t jfdd  fdd}|S )a{  Returns a function that optionally has NaN gradients.

    This serves as a hook to introduce NaN gradients to a model. This returns an
    identity function. The identity's gradient function will check if the
    boolean tensor `have_nan_gradients` is True. If so, the gradient will be
    NaN.  Otherwise, the gradient will also be the identity.

    Args:
      have_nan_gradients: A scalar boolean tensor. If True, gradients will be
        NaN. Otherwise, the gradient function is the identity function.

    Returns:
      An identity function whose gradient function will return NaNs, if
      `have_nan_gradients` is True.
    c                    s   t | }  fdd}| |fS )z@Function whose gradient is NaN iff `have_nan_gradients` is True.c                    s   t  fdd fddS )Nc                      s    t d S )NNaN)floatr   r   r   r   <lambda>c   s    zkcreate_identity_with_nan_gradients_fn.<locals>._identity_with_nan_gradients.<locals>.grad.<locals>.<lambda>c                      s    S r   r   r   r   r   r   r   c   s    )r   Zcondr   have_nan_gradientsr   r   r   a   s   zYcreate_identity_with_nan_gradients_fn.<locals>._identity_with_nan_gradients.<locals>.gradr   r   r   r   r   _identity_with_nan_gradients\   s   
zKcreate_identity_with_nan_gradients_fn.<locals>._identity_with_nan_gradientsc                    r   r   r   r   )r!   r   r   identity_with_nan_gradientsj   r   zJcreate_identity_with_nan_gradients_fn.<locals>.identity_with_nan_gradientsr   )r    r"   r   )r!   r    r   %create_identity_with_nan_gradients_fnK   s   r#   c                       s*   e Zd ZdZd fdd	Zdd Z  ZS )AssertTypeLayerz5A layer which asserts it's inputs are a certain type.Nc                    s,   |rt |jnd | _t jdi | d S )Nr   )r   Zas_dtyper   _assert_typesuper__init__)selfassert_typekwargs	__class__r   r   r'   s   s   zAssertTypeLayer.__init__c                 C   sH   | j r tj|}|D ]}|jj| j ksJ d|jj| j f qdS dS )zMAsserts `inputs` are of the correct type. Should be called in
        call().z<Input tensor has type %s which does not match assert type %sN)r%   r   nestflattenr   Z
base_dtyper   )r(   inputsZinputs_flattenedinpr   r   r   assert_input_typesy   s   z"AssertTypeLayer.assert_input_typesr   )__name__
__module____qualname____doc__r'   r1   __classcell__r   r   r+   r   r$   p   s    r$   c                       sN   e Zd ZdZ				d fdd	Zdd Zd	d
 Zdd Z fddZ  Z	S )MultiplyLayerz8A layer which multiplies its input by a scalar variable.NFvc                    sl   || _ t|trtj|t d| _ || _t|tr"tj|t d| _|| _|| _t	 j
dd| ji| dS )a"  Initializes the MultiplyLayer.

        Args:
          regularizer: The weight regularizer on the scalar variable.
          activity_regularizer: The activity regularizer.
          use_operator: If True, add using the * operator. If False, add using
            tf.multiply.
          var_name: The name of the variable. It can be useful to pass a name
            other than 'v', to test having the attribute name (self.v) being
            different from the variable name.
          **kwargs: Passed to AssertTypeLayer constructor.
        )Zcustom_objectsactivity_regularizerNr   )_regularizer
isinstancedictr   Zdeserializeglobals_activity_regularizer_use_operator	_var_namer&   r'   )r(   regularizerr9   use_operatorvar_namer*   r+   r   r   r'      s"   



zMultiplyLayer.__init__c                 C   s"   | j | jdd| jd| _d| _d S )Nr   ones)initializerrA   T)
add_weightr@   r:   r8   built)r(   _r   r   r   build   s   
zMultiplyLayer.buildc                 C   s   |  | | || jS r   )r1   	_multiplyr8   r(   r/   r   r   r   call   s   
zMultiplyLayer.callc                 C   s   | j r|| S t||S r   )r?   r   multiply)r(   r   yr   r   r   rJ      s   zMultiplyLayer._multiplyc                    sL   t   }t| j|d< t| j|d< | j|d< | j|d< | j|d< |S )NrA   r9   rB   rC   r)   )	r&   
get_configr   	serializer:   r>   r?   r@   r%   )r(   configr+   r   r   rO      s   



zMultiplyLayer.get_config)NNFr8   )
r2   r3   r4   r5   r'   rI   rL   rJ   rO   r6   r   r   r+   r   r7      s    %	r7   c                   @   s    e Zd ZdZdd Zdd ZdS )MultiplyLayerWithoutAutoCastz:Same as MultiplyLayer, but does not use AutoCastVariables.c                 C   s6   | j }|dv r	d}| jddd|d| jd| _d| _d S )	N)Zfloat16Zbfloat16float32r8   r   rD   F)rE   r   Zexperimental_autocastrA   T)r   rF   r:   r8   rG   )r(   rH   r   r   r   r   rI      s   
z"MultiplyLayerWithoutAutoCast.buildc                 C   s:   |  | | jjtjtjfv sJ | |t| j|jS r   )r1   r8   r   r   rS   Zfloat64rJ   castrK   r   r   r   rL      s   
z!MultiplyLayerWithoutAutoCast.callN)r2   r3   r4   r5   rI   rL   r   r   r   r   rR      s    rR   c                   @      e Zd Zdd Zdd ZdS )IdentityRegularizerc                 C   s   |j tjksJ t|S r   )r   r   rS   r
   r(   r   r   r   r   __call__   s   
zIdentityRegularizer.__call__c                 C      i S r   r   r(   r   r   r   rO         zIdentityRegularizer.get_configNr2   r3   r4   rX   rO   r   r   r   r   rV      s    rV   c                   @   rU   )ReduceSumRegularizerc                 C   s
   t |S r   )r   Z
reduce_sumrW   r   r   r   rX      s   
zReduceSumRegularizer.__call__c                 C   rY   r   r   rZ   r   r   r   rO      r[   zReduceSumRegularizer.get_configNr\   r   r   r   r   r]      s    r]   r   )r5   Ztensorflow.compat.v2r   v2r   Zkerasr   Zkeras.enginer   r   r#   ZLayerr$   r7   rR   ZRegularizerrV   r]   r   r   r   r   <module>   s   
4%G	