o
    Meh                     @   sL  d Z ddlZddlZddlZddlm  mZ ddl	m
Z
 ddlZddlmZ ddlmZ ddlmZ ddlmZ G dd	 d	ejjZG d
d dejje
jZG dd dejje
jZG dd dejje
jZG dd dejje
jZG dd dejje
jZG dd dejje
jZG dd dejje
jZedkrejj j!"  dS dS )zKTests for tf.keras models with callbacks, checkpointing with dist
strategy.    N)parameterized)losses)distribute_strategy_test)distributed_training_utils_v1)optimizer_combinationsc                   @   s    e Zd ZdZdd Zdd ZdS )CounterzCounts the number of times each callback method was run.

    Attributes:
      method_counts: dict. Contains the counts of time  each callback method was
        run.
    c              
   C   s<   t t| _g d}|D ]}t| || |t| | qd S )N)on_batch_beginon_batch_endon_epoch_beginon_epoch_endon_predict_batch_beginon_predict_batch_endon_predict_beginon_predict_endon_test_batch_beginon_test_batch_endon_test_beginon_test_endon_train_batch_beginon_train_batch_endon_train_beginon_train_end)collectionsdefaultdictintmethod_countssetattrwrap_with_countsgetattr)selfZmethods_to_countmethod_name r!   QD:\Projects\ConvertPro\env\Lib\site-packages\keras/distribute/keras_utils_test.py__init__(   s   zCounter.__init__c                    s    fdd}|S )Nc                     s    j   d7  <  | i |S )N   )r   )argskwargsmethodr    r   r!   r"   _call_and_countD   s   z1Counter.wrap_with_counts.<locals>._call_and_countr!   )r   r    r(   r)   r!   r'   r"   r   C   s   zCounter.wrap_with_countsN)__name__
__module____qualname____doc__r#   r   r!   r!   r!   r"   r       s    r   c                   @   s   e Zd Zejjjejjj	e
 dd Zejjjejjj	e
 dd Zejjjejjj	e
 dd ZdS )%TestDistributionStrategyWithCallbacksc           
      C   s   |   t }|jdddgd W d    n1 sw   Y  t|}t }d}d}d}|j|||d|||gd	 t|tj	j
jjjr[t s[|jj}|| }	|| rZ|	d
7 }	n|}	| |j||	 ||	 |||| || ||||	 ||	 d
d
d d S )Nsgdmsemae	optimizerlossmetrics         r   )epochssteps_per_epochverboseZvalidation_datavalidation_steps	callbacksr$   )r   r	   r
   r   r   r   r   r   r   r   r   r   )scopekeras_test_lib	get_modelcompileget_datasetr   fit
isinstancetfcompatv1
distributeexperimentalTPUStrategyexecuting_eagerlyextendedsteps_per_runassertDictEqualr   )
r   distributionmodeldatasetcounterr9   r:   r<   rM   Znum_batch_call_per_epochr!   r!   r"   test_callbacks_in_fitN   s\   

	z;TestDistributionStrategyWithCallbacks.test_callbacks_in_fitc                 C   s   |   t }|jdddgd W d    n1 sw   Y  t|}t }|j|d|gd | |jddddd d S )	Nr/   r0   r1   r2   r7   stepsr=   r$   )r   r   r   r   )	r>   r?   r@   rA   rB   r   evaluaterN   r   r   rO   rP   rQ   rR   r!   r!   r"   test_callbacks_in_eval   s   

z<TestDistributionStrategyWithCallbacks.test_callbacks_in_evalc                 C   s   |   t }|jdddgd W d    n1 sw   Y  t|}t }|jt|d|gd | |j	ddddd d S )	Nr/   r0   r1   r2   r7   rT   r$   )r   r   r   r   )
r>   r?   r@   rA   rB   r   predictget_predict_datasetrN   r   rW   r!   r!   r"   test_callbacks_in_predict   s&   

z?TestDistributionStrategyWithCallbacks.test_callbacks_in_predictN)r*   r+   r,   rE   __internal__rH   combinationsgeneratetesttimesr?   all_strategy_combinationsrS   rX   r[   r!   r!   r!   r"   r.   K   s&    



8



r.   c                   @   sH  e Zd Zejjjejjjj	ejjjj
gdgddd Zejjjejjjj	ejjjj
gddgddd Zejjjejjjj	ejjjj
gddgddd	 Zejjjejjjj	ejjjj
ejjjjgddgdd
d Zejjjejjjj	ejjjj
ejjjjgddgddd Zejjje dd ZdS )"TestDistributionStrategyErrorCasesgraphrO   modec              
   C      |   N tjdd }||}| td# |  t||d  W d    n1 s.w   Y  W d    n1 s=w   Y  W d    d S W d    d S 1 sUw   Y  d S )Nc                  S   s>   t j } | jjdrt ddggS t ddgddggS )NGPU:0r$   r6   )rE   rH   get_replica_contextreplica_id_in_sync_groupdeviceendswithconstantctxr!   r!   r"   run   s   
ziTestDistributionStrategyErrorCases.test_validating_dataset_input_tensors_with_shape_mismatch.<locals>.runzLInput tensor shapes do not match for distributed tensor inputs PerReplica:.+	cached_sessionrE   functionro   assertRaisesRegex
ValueErrorr>   r   Z#validate_distributed_dataset_inputsr   rO   ro   xr!   r!   r"   9test_validating_dataset_input_tensors_with_shape_mismatch   $   



"z\TestDistributionStrategyErrorCases.test_validating_dataset_input_tensors_with_shape_mismatcheagerc              
   C   rf   )Nc                  S   sD   t j } | jjdrt jddggt jdS t jddggt jdS )Nrg   r$   r6   Zdtype)	rE   rH   rh   ri   rj   rk   rl   Zint32Zfloat64rm   r!   r!   r"   ro      s   
ziTestDistributionStrategyErrorCases.test_validating_dataset_input_tensors_with_dtype_mismatch.<locals>.runzLInput tensor dtypes do not match for distributed tensor inputs PerReplica:.+rp   ru   r!   r!   r"   9test_validating_dataset_input_tensors_with_dtype_mismatch   rx   z\TestDistributionStrategyErrorCases.test_validating_dataset_input_tensors_with_dtype_mismatchc           	   
   C   s  |    | ! t }tjjjd}d}dg}|j	|||d W d    n1 s-w   Y  t
|}| t |j|dddddd	 W d    n1 sRw   Y  tjd
}| t |j|ddd|d W d    n1 sww   Y  | }| t |j|ddd W d    n1 sw   Y  | t |j|dd W d    n1 sw   Y  | t |j|dd W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )NMbP?r0   r1   r5   r$   r6   r   g      ?)r9   r:   r;   Zvalidation_splitr<   )
   )r9   r:   r;   sample_weight)r9   r;   )r;   )rq   r>   r?   r@   rE   rF   rG   trainGradientDescentOptimizerrA   rB   assertRaisesrt   rC   nprandomrepeatrV   rY   )	r   rO   re   rP   r3   r4   r5   rQ   r   r!   r!   r"   test_unsupported_features  sT   
	

'"z<TestDistributionStrategyErrorCases.test_unsupported_featuresc              	   C   s   |  C G dd dtj}|d}t s2| td |d W d    n1 s,w   Y  n|d W d    d S W d    d S 1 sJw   Y  d S )Nc                       s$   e Zd Z fddZdd Z  ZS )zeTestDistributionStrategyErrorCases.test_distribution_strategy_on_subclassed_model.<locals>._SimpleMLPc                    s   t    tj|| _d S N)superr#   keraslayersDensedense)r   Z
num_labels	__class__r!   r"   r#   O  s   
znTestDistributionStrategyErrorCases.test_distribution_strategy_on_subclassed_model.<locals>._SimpleMLP.__init__c                 S   s
   |  |S r   )r   )r   inputsr!   r!   r"   callS  s   
zjTestDistributionStrategyErrorCases.test_distribution_strategy_on_subclassed_model.<locals>._SimpleMLP.call)r*   r+   r,   r#   r   __classcell__r!   r!   r   r"   
_SimpleMLPN  s    r   r8   We currently do not support distribution strategy with a `Sequential` model that is created without `input_shape`/`input_dim` set in its first layer or a subclassed model.r/   )r>   r   ModelrE   rK   rs   rt   rA   )r   rO   r   rP   r!   r!   r"   .test_distribution_strategy_on_subclassed_modelB  s   

" zQTestDistributionStrategyErrorCases.test_distribution_strategy_on_subclassed_modelc              	   C   s   |  Y tj }|tjjddd |tjjddd t r*|	d n#| 
td |	d W d    n1 s@w   Y  W d    d S W d    d S W d    d S 1 s`w   Y  d S )N   Zrelu)Z
activationr8   Zsoftmaxr/   r   )r>   r   models
Sequentialaddr   r   rE   rK   rA   rs   rt   )r   rO   rP   r!   r!   r"   7test_distribution_strategy_on_deferred_sequential_modelc  s"   

"zZTestDistributionStrategyErrorCases.test_distribution_strategy_on_deferred_sequential_modelc              	   C   s   |  7 t }| td tddg}||| W d    n1 s&w   Y  W d    d S W d    d S 1 s>w   Y  d S )NzNPlease use `tf.keras.losses.Reduction.SUM` or `tf.keras.losses.Reduction.NONE`r$   r   )r>   r   ZMeanSquaredErrorrs   rt   r   Zasarray)r   rO   Zloss_objectyr!   r!   r"   +test_standalone_loss_without_loss_reduction  s   
"zNTestDistributionStrategyErrorCases.test_standalone_loss_without_loss_reductionN)r*   r+   r,   rE   r\   rH   r]   r^   r_   combine"mirrored_strategy_with_gpu_and_cpurw   r{   r   Zone_device_strategyr   r   r?   'all_strategy_combinations_minus_defaultr   r!   r!   r!   r"   rb      s^    












+



	




	
rb   c                   @   sF   e Zd Zejjjejjjj	ejjjj
gddgejddd ZdS )'TestDistributionStrategyWithLossMaskingrc   ry   )rO   re   r3   c              
   C   s8  |    tjd tdgdggdgdggg}| / tj }|	tj
jddd |	tj
tj
jddd |jd| d	 W d    n1 sPw   Y  tdgdggdgdggg}tjj||f}|d
}|d}|j|ddd}| |jd d d W d    d S 1 sw   Y  d S )Ni9  r$   r   )r6   r$   )Z
mask_valueinput_shapeone)Zkernel_initializerr0   r4   r3   d   r~   r6   )rv   r9   r:   r4   )rq   r   r   seedarrayr>   r   r   r   r   r   ZMaskingZTimeDistributedr   rA   rE   dataDatasetfrom_tensor_slicesr   batchrC   assertEqualhistory)r   rO   r3   rv   rP   r   rQ   histr!   r!   r"   test_masking  s,   





"z4TestDistributionStrategyWithLossMasking.test_maskingN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r   r   r   &gradient_descent_optimizer_keras_v2_fnr   r!   r!   r!   r"   r     s    


	r   c                	   @   s   e Zd Zejjjejjj	e
 ejjjjddgejddd Zejjjejjj	e
 ejjjjejddd Zd	S )
.TestDistributionStrategyWithNormalizationLayerTF)fusedr3   c           
   	   C   sh  |    | # tj }tjjdd|d}|| |jd| d W d    n1 s/w   Y  t	j
jdddd	}|d
}tjj||f}|d}t|d|}tjj|}|d}t|d|}|j|dddd |j|dd}	|	tj|j8 }	|	tj|j }	t	jj|	 ddd t	jj|	 ddd W d    d S 1 sw   Y  d S )Nr~         皙?)r   momentumr   r0   r         @      $@i  r~   r   r   locscalesizefloat32r          r   r~   r9   r;   r:   r6   rU           皙?Zatol      ?rq   r>   r   r   r   r   ZBatchNormalizationr   rA   r   r   normalZastyperE   r   r   r   r   r?   Zbatch_wrapperrC   rY   backendevalbetagammatestingZassert_allclosemeanZstd)
r   rO   r   r3   rP   normrv   rQ   predict_datasetoutr!   r!   r"   test_batchnorm_correctness  s8   




	


"zITestDistributionStrategyWithNormalizationLayer.test_batchnorm_correctnessr3   c           	   	   C   sj  |    | $ tj }tjjddddd}|| |jd| d W d    n1 s0w   Y  t	j
jdd	d
d}|d}tjj||f}|d}t|d|}tjj|}|d}t|d|}|j|dddd |j|dd}|tj|j8 }|tj|j }t	jj| ddd t	jj| ddd W d    d S 1 sw   Y  d S )Nr   r   FT)r   r   r   Zrenormr0   r   r   r   r   r   r   r   r   r   r   r~   r   r6   r   r   r   r   r   r   )	r   rO   r3   rP   r   rv   rQ   r   r   r!   r!   r"   &test_batchnorm_correctness_with_renorm  s:   
	






"zUTestDistributionStrategyWithNormalizationLayer.test_batchnorm_correctness_with_renormN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r`   r?   ra   r   r   r   r   Ztpu_strategy_combinationsr   r!   r!   r!   r"   r     s(    



	
&

r   c                   @   s   e Zd Zejjjejjj	e
 ejjjjejddd Zejjjejjj	e
 ejjjjejddd ZdS )'TestDistributionStrategySaveLoadWeightsr   c              	   C   s   |   i t|}| G t }|| d |j|ddd td}|	| t }|| d |
| |jt|dd |j|ddd W d    n1 sXw   Y  W d    d S W d    d S 1 spw   Y  d S )Nr0   r$   r9   r:   z.h5r6   r   )rq   r?   rB   r>   r@   rA   rC   tempfilemktempsave_weightsload_weightsrY   rZ   r   rO   r3   rQ   rP   Zweights_fileZmodel_2r!   r!   r"   test_save_load_h5  s&   
	





"z9TestDistributionStrategySaveLoadWeights.test_save_load_h5c              	   C   s   t |tjjjtjjjjjfr|jjdkr| 	d | 
 h t|}| F t }|| d |j|ddd t }|| t }|| d || |jt|dd |j|ddd W d    n1 sqw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr$   z8MultiStep TPU Strategy deadlocks with optimizer restore.r0   r   r6   r   )rD   rE   rH   rI   rJ   rF   rG   rL   rM   ZskipTestrq   r?   rB   r>   r@   rA   rC   r   r   r   r   rY   rZ   r   r!   r!   r"   test_save_load_trackable4  s<   





"z@TestDistributionStrategySaveLoadWeights.test_save_load_trackableN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r`   r?   r   r   r   Zrmsprop_optimizer_keras_v2_fnr   r   r!   r!   r!   r"   r     s&    






r   c                   @   sP   e Zd Zejjjejjj	e
 dd Zejjje
 dd ZdS )"TestDistributionStrategyValidationc              
   C   s   |   l | tdM tjjddd}tjjddd|}| ' t||}t	j
jjd}d	}d
tj g}|j|||d W d    n1 sLw   Y  W d    n1 s[w   Y  W d    d S W d    d S 1 ssw   Y  d S Nz,was not created in the distribution strategyr8   input)shapenamer   r   r   r|   r0   r1   r}   )rq   rs   rt   r   r   Inputr   r>   r   rE   rF   rG   r   r   r5   CategoricalAccuracyrA   r   rO   rv   r   rP   r3   r4   r5   r!   r!   r"   test_layer_outside_scopea  s(   


"z;TestDistributionStrategyValidation.test_layer_outside_scopec              
   C   s   |   l | tdM tjjddd}tjjddd|}t||}| ! t	j
jjd}d	}d
tj g}|j|||d W d    n1 sLw   Y  W d    n1 s[w   Y  W d    d S W d    d S 1 ssw   Y  d S r   )rq   rs   rt   r   r   r   r   r   r>   rE   rF   rG   r   r   r5   r   rA   r   r!   r!   r"   test_model_outside_scopev  s(   


"z;TestDistributionStrategyValidation.test_model_outside_scopeN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r`   r?   r   r   r   r!   r!   r!   r"   r   ^  s    



r   c                   @   sx   e Zd Zejjjejjjj	ejjjj
gddgddd Zejjjejjjj	ejjjj
gddgddd ZdS )	(TestDistributionStrategyWithStaticShapesrc   ry   rd   c              	   C   s|   |  0 | td tjjdddd W d    n1 sw   Y  W d    d S W d    d S 1 s7w   Y  d S )NzQThe `batch_size` argument \(5\) must be divisible by the number of replicas \(2\)r   r7   r   r   Z
batch_sizer   )r>   rs   rt   r   r   r   )r   rO   r!   r!   r"   3test_input_batch_size_not_divisible_by_num_replicas  s   
	"z\TestDistributionStrategyWithStaticShapes.test_input_batch_size_not_divisible_by_num_replicasc                 C   s   t jdt jd}t jdt jd}tjj||f}|d}|jddd}|	 * t
jjddd	d
}t
jjddd|}t
||}|jdddgd W d    n1 sVw   Y  |j|ddd |j|dd || d S )N)r~   r8   rz   )r~   r   r   r~   T)Zdrop_remainderr   r   r   r   r   r   r/   r0   r1   r2   r$   r7   r   r   )r   Zzerosr   rE   r   r   r   r   r   r>   r   r   r   r   r   rA   rC   rV   rY   )r   rO   r   targetsrQ   rv   r   rP   r!   r!   r"   test_static_input_batch_size  s   	

zETestDistributionStrategyWithStaticShapes.test_static_input_batch_sizeN)r*   r+   r,   rE   r\   rH   r]   r^   r_   r   r   r   r   r!   r!   r!   r"   r     s"    




	

r   __main__)#r-   r   r   numpyr   Ztensorflow.compat.v2rF   v2rE   Zabsl.testingr   r   r   Zkeras.distributer   r?   r   r   r=   ZCallbackr   r_   ZTestCaser.   rb   r   r   r   r   r   r*   r\   rH   Zmulti_process_runnerZ	test_mainr!   r!   r!   r"   <module>   sJ   
+

w
 
O

&

`

H

,
.