o
    Men                    @   s  d Z ddlZddlZddlm  mZ ddlm	Z	 ddl
Z
ddl
mZ ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  ddl!m"Z" ddl#m$Z$ ddl#m%Z% ddl&m'Z' dZ(dZ)dZ*dZ+dd Z,e+fddZ-dd Z.d d! Z/dZd"d#Z0d$d% Z1d&d' Z2d(d) Z3d*d+ Z4dZd,d-Z5d.d/ Z6d0d1 Z7d2d3 Z8d4d5 Z9d6d7 Z:d8d9 Z;d:d; Z<d<d= Z=d>d? Z>G d@dA dAe
j?j@ZAG dBdC dCejBjCe	jCZDG dDdE dEejBjCe	jCZEG dFdG dGejBjCe	jCZFG dHdI dIejBjCe	jCZGe"HdJG dKdL dLejBjCe	jCZIdMdN ZJdOdP ZKdQdR ZLG dSdT dTejBjCe	jCZMG dUdV dVe
jNZOG dWdX dXejBjCe	jCZPeQdYkrkeR  ejSjTjUV  dS dS )[z7Tests for tf.keras models using tf.distribute.Strategy.    N)parameterized)backend)distributed_training_utils)distributed_training_utils_v1)multi_worker_testing_utils)optimizer_combinations)all_strategies) multi_worker_mirrored_strategies)"strategies_minus_default_minus_tpu)strategies_minus_tpu)tpu_strategies)base_layer_utils)policy	optimizer)gradient_descent)
test_utils)losses_utils)np_utils)SimpleClusterResolveri9     
      c                  C   sN   t j } | t jjddtd | t jd | t jjtdd | S )N   relu
activationinput_shape皙?softmaxr   )	kerasmodels
SequentialaddlayersDense_INPUT_SIZEDropout
_NUM_CLASSmodel r-   YD:\Projects\ConvertPro\env\Lib\site-packages\keras/distribute/distribute_strategy_test.pysimple_sequential_modelC   s   
r/   c                 C   s   G dd dt j}|| S )Nc                       s$   e Zd Z fddZdd Z  ZS )z+simple_subclassed_model.<locals>._SimpleMLPc                    s   t    tj|| _d S N)super__init__r"   r&   r'   dense)self
num_labels	__class__r-   r.   r2   O   s   
z4simple_subclassed_model.<locals>._SimpleMLP.__init__c                 S   s
   |  |S r0   )r3   r4   inputsr-   r-   r.   callS      
z0simple_subclassed_model.<locals>._SimpleMLP.call)__name__
__module____qualname__r2   r:   __classcell__r-   r-   r6   r.   
_SimpleMLPN   s    r@   )r"   Model)r5   r@   r-   r-   r.   simple_subclassed_modelM   s   rB   c                  C   s|   t jjddd} t jjddd}t jj| |gdd}t jjddd	d
|}t jjdddd
|}t jj| |g||gd}|S )Nr   input_ashapenameinput_bmergerG      r    dense_2r   rG   r   Zdense_3r9   outputs)r"   r&   InputZconcatenater'   r#   rA   )rD   rH   mergedoutput_coutput_dr,   r-   r-   r.   'simple_multi_inputs_multi_outputs_modelY   s   rT   c                  C   s   t jtdddtd\\} }\}}t jtdddtd\\}}\}}t jtdddtd\\}}	\}
}	t|}t|}t|}t|}| ||||d}|||
||d}||fS )N2   rC   rK   )Ztrain_samplesZtest_samplesr   num_classesZrandom_seedr   )   )rD   rH   Zinput_mrR   rS   )r   Zget_test_data_TRAIN_SIZE_RANDOM_SEEDr   Zto_categorical)Za_trainZc_trainZa_testZc_testZb_trainZd_trainZb_testZd_testZm_train_Zm_testZ
train_dataZ	test_datar-   r-   r.   #get_multi_inputs_multi_outputs_dataj   sL   



r[   c                 C   s0   |r|  |} t|r| j|ddS | |S )NTZdrop_remainder)repeatr   is_tpu_strategybatch)dataset
batch_sizedistributionr]   r-   r-   r.   batch_wrapper   s
   


rc   c                  C   s4   t jjddd} t jjddd| }t | |}|S )NrK   inputrE      r3   rJ   r"   r&   rP   r'   rA   xyr,   r-   r-   r.   	get_model   s   rk   c                  C   s8   t jjddd} t jjddddd| }t | |}|S )	N   re   rE   rm   oneszerosr3   )kernel_initializerZbias_initializerrG   rg   rh   r-   r-   r.   get_sample_weights_model   s   rq   c                 C   sL   t jdt jd}t jdt jd}tjj||f}|d}t|d| }|S )Nr   rK   Zdtyper   rf   d   r   	npro   float32tfdataDatasetfrom_tensor_slicesr]   rc   )rb   r9   targetsr`   r-   r-   r.   get_dataset   s   
r~   c                 C   s8   t jdt jd}tjj|}|d}t|d| }|S )Nrr   rs   ru   r   rv   )rb   r9   r`   r-   r-   r.   get_predict_dataset   s
   
r   c                 C   sL   |d ur| |f}dd }n| }dd }t jj|}||jddd}|S )Nc                 S      dS NTr-   )inptargetr-   r-   r.   <lambda>       zCconvert_numpy_to_dataset_with_unknown_cardinality.<locals>.<lambda>c                 S   r   r   r-   r   r-   r-   r.   r      r   r   Tr\   )ry   rz   r{   r|   filterr_   )r9   r}   Zinput_slicesdummy_opZoriginal_datasetZds_with_unknown_cardinalityr-   r-   r.   1convert_numpy_to_dataset_with_unknown_cardinality   s   

r   c                  C   s~   t jjddd} t jjddd}t jjddd}t jjdd	d}|| }||}t jjd
dd|}t j| |g||g}|S )Nrd   rD   rE   )   rH      dense_1rJ   rL         ?Zdropout)r"   r&   rP   r'   r)   r#   rA   )abr   rL   cder,   r-   r-   r.   multi_input_output_model   s   r   c                   C      t jjjjtddgdS Ngrapheagerrb   mode)ry   __internal__testcombinationscombiner   r-   r-   r-   r.   strategy_minus_tpu_combinations      
r   c                   C   r   r   ry   r   r   r   r   r   r-   r-   r-   r.   tpu_strategy_combinations   r   r   c                   C      t jjjjtdgdS )Nr   r   r   r-   r-   r-   r.   $tpu_strategy_combinations_graph_only      
r   c                   C   r   )Nr   r   )ry   r   r   r   r   r	   r-   r-   r-   r.   -multi_worker_strategy_combinations_eager_only   r   r   c                   C      t  t  t  S r0   )r   r   r   r-   r-   r-   r.   all_strategy_combinations      r   c                   C   s@   t jjjjt jjjjt jjjjt jjjjt jjjj	gddgdS r   )
ry   r   r   r   r   
distributeone_device_strategyone_device_strategy_gpu"mirrored_strategy_with_gpu_and_cpumirrored_strategy_with_two_gpusr-   r-   r-   r.   /all_strategy_minus_default_and_tpu_combinations  s   




r   c                   C   r   r0   )r   r   r   r-   r-   r-   r.   'all_strategy_combinations_minus_default  r   r   c                  C   s   t jjjt t jjjjtjtj	tj
tjtjtjtjtjtjtjtjtjgd} t jjjjtdgtjtj	tj
tjtjtjtjtjgd}t jjjjtdgtjtjtjtjgd}t jjjjtdgtjtjtjtjtjtjtjtjgd}| | | | S )Nr   r   )rb   r   r   r   )ry   r   r   r   timesr   r   r   Zadagrad_optimizer_v1_fnZadam_optimizer_v1_fnZ gradient_descent_optimizer_v1_fnZrmsprop_optimizer_v1_fnZadadelta_optimizer_keras_v2_fnZadagrad_optimizer_keras_v2_fnZadam_optimizer_keras_v2_fnZadamax_optimizer_keras_v2_fnZ&gradient_descent_optimizer_keras_v2_fnZnadam_optimizer_keras_v2_fnZrmsprop_optimizer_keras_v2_fnZftrl_optimizer_keras_v2_fnr   r	   )Znon_tpu_strategiesZtpu_strategies_graphZtpu_strategies_eagerZmulti_worker_eagerr-   r-   r.   #strategy_and_optimizer_combinations  sz   





r   c                       sX   e Zd Z fddZdddZdddZddd	Zdd
dZdddZdddZ	  Z
S )BatchCountingCBc                    s2   t    g | _g | _g | _g | _g | _g | _d S r0   )r1   r2   train_begin_batchestrain_end_batchestest_begin_batchestest_end_batchespredict_begin_batchespredict_end_batchesr4   r6   r-   r.   r2   X  s   

zBatchCountingCB.__init__Nc                 C      | j | d S r0   )r   appendr4   r_   Zlogsr-   r-   r.   on_train_batch_begina     z$BatchCountingCB.on_train_batch_beginc                 C   r   r0   )r   r   r   r-   r-   r.   on_train_batch_endd  r   z"BatchCountingCB.on_train_batch_endc                 C   r   r0   )r   r   r   r-   r-   r.   on_test_batch_beging  r   z#BatchCountingCB.on_test_batch_beginc                 C   r   r0   )r   r   r   r-   r-   r.   on_test_batch_endj  r   z!BatchCountingCB.on_test_batch_endc                 C   r   r0   )r   r   r   r-   r-   r.   on_predict_batch_beginm  r   z&BatchCountingCB.on_predict_batch_beginc                 C   r   r0   )r   r   r   r-   r-   r.   on_predict_batch_endp  r   z$BatchCountingCB.on_predict_batch_endr0   )r<   r=   r>   r2   r   r   r   r   r   r   r?   r-   r-   r6   r.   r   W  s    
	



r   c                   @   s(  e Zd Zejjje dd Z	ejjje dd Z
ejjje dd Zejjje dd Zejjje d	d
 Zejjje dd Zejjje dd Zejjjejjjjejjjjgddgddd Zejjje dd Zejjjejjjjeddgdejjjjedgd dd Zejjje dd Zejjjejjje ejjjjddgddd Zejjjejjje dd  Zejjje d!d" Zejjjejjje d#d$ Zejjje d%d& Zd'S )('TestDistributionStrategyWithNumpyArraysc                 C   s   d}t |s
|j}|  ; tj|dd d d\}}| |d|  | |d tj|dd d d\}}| |d|  | |d W d    d S 1 sLw   Y  d S )N      ?@   stepsra       r      rm   r   global_batch_size_supportednum_replicas_in_synccached_sessionr   get_input_paramsassertEqualr4   rb   Zreplica_scale_factorr   ra   r-   r-   r.   4test_calculating_input_params_no_steps_no_batch_sizew  s"   


"z\TestDistributionStrategyWithNumpyArrays.test_calculating_input_params_no_steps_no_batch_sizec              	   C   sz  d}t |s
|j}|   tj|ddd d\}}| |d|  | |d tj|ddd d\}}| |d|  | |d | td tj|ddd d W d    n1 s[w   Y  |dkr|tj|dd	d d\}}| |d
 | |d	 n'| td tj|ddd d W d    n1 sw   Y  W d    d S W d    d S W d    d S 1 sw   Y  d S )Nr   r   rm   r   r   r   znot divisible by steps?   rK      z4could not be sharded evenly across the sync replicas)	r   r   r   r   r   r   r   assertRaisesRegex
ValueErrorr   r-   r-   r.   6test_calculating_input_params_with_steps_no_batch_size  sV   


"%"z^TestDistributionStrategyWithNumpyArrays.test_calculating_input_params_with_steps_no_batch_sizec                 C   s   d}t |s
|j}|  ; tj|dd dd\}}| |d | |d|  tj|dd dd\}}| |d | |d|  W d    d S 1 sLw   Y  d S )Nr   r   r   r   rf   r   r   r   r   r-   r-   r.   6test_calculating_input_params_no_steps_with_batch_size  s"   


"z^TestDistributionStrategyWithNumpyArrays.test_calculating_input_params_no_steps_with_batch_sizec              	   C   s   |   G tj|dddd\}}| |d | |d | td tj|dddd W d    n1 s6w   Y  W d    d S W d    d S 1 sNw   Y  d S )Nr   r   rK   r   zless than samples requiredr      )r   r   r   r   r   r   )r4   rb   r   ra   r-   r-   r.   8test_calculating_input_params_with_steps_with_batch_size  s    

	"z`TestDistributionStrategyWithNumpyArrays.test_calculating_input_params_with_steps_with_batch_sizec           	   
   C   s  |   s | V tj}|d}t }d}dg}|j|||d tjdtjd}tjdtjd}|j	||dd	d
||fd |
|| |j
||dd || |j|dd W d    n1 sbw   Y  W d    d S W d    d S 1 szw   Y  d S )NMbP?msemaemetricsr   rK   rs   r   rf   rm   r   r   epochsra   verbosevalidation_datarW   ra   )r   scopegradient_descent_kerasSGDrk   compilerw   ro   rx   fitevaluatepredict)	r4   rb   optimizer_fnr   r,   lossr   r9   r}   r-   r-   r.   $test_calling_model_with_numpy_arrays  s4   


"zLTestDistributionStrategyWithNumpyArrays.test_calling_model_with_numpy_arraysc                 C   s  t |tjjjjjtjjjtjjjtjjjjjfr| d t	
|r&d}nd}|   |  t|s tj}|d}tjjddd}tjjdd	d
|}tjjddd|}t||}d}dg}	|j|||	d tjddd}
tjddd}|j|
|ddd|
|fd ||
| |j|
|dd ||
 |j|
dd W d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nb/152097775mixed_bfloat16mixed_float16r   rd   re   rE   rf   r3   rJ   r    rx   rs   r   r   r   r   r   rm   r   r   r   rW   r   )
isinstancery   compatv1r   experimentalParameterServerStrategyCentralStorageStrategyskipTestr   r^   r   r   r   Zpolicy_scoper   r   r"   r&   rP   r'   Z
ActivationrA   r   rw   ro   r   r   r   )r4   rb   policy_namer   r   ri   rj   r,   r   r   r9   r}   r-   r-   r.   'test_calling_model_with_mixed_precision'  sR   
	
	
P zOTestDistributionStrategyWithNumpyArrays.test_calling_model_with_mixed_precisionc              	      s  t |tjjjjjtjjjtjjjtjjjjjfr| d t	
|r&d}nd}G dd dtjj}|  J | - ||d  fdd}t rPt|}||\}}| | | | W d    n1 skw   Y  W d    d S W d    d S 1 sw   Y  d S )	Nr   r   r   c                   @      e Zd Zdd Zdd ZdS )z_TestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precision.<locals>.MyLayerc                 S   s    |  dd| _|  dd| _d S )Nvr-   )
add_weightr   v2r4   rZ   r-   r-   r.   buildt  s   zeTestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precision.<locals>.MyLayer.buildc                 S   s   || j 7 }| j| S r0   r   r   )r4   r   r-   r-   r.   r:   x  s   

zdTestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precision.<locals>.MyLayer.callNr<   r=   r>   r  r:   r-   r-   r-   r.   MyLayers      r  rs   c                     s\   t dg} t } | }W d    n1 sw   Y  || j jg\}}||fS Nr   )rw   arrayry   GradientTapegradientr   r   )ri   taperj   grad_v1grad_v2layerr-   r.   run_fn  s   

z^TestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precision.<locals>.run_fn)r   ry   r   r   r   r   r   r   r   r   r^   r"   r&   Layerr   r   executing_eagerlyfunctionrunZassertIsNotNone)r4   rb   r   r  r  r  r  r-   r  r.   &test_operator_overload_mixed_precision[  s,   
	
	


PzNTestDistributionStrategyWithNumpyArrays.test_operator_overload_mixed_precisionr   r   r   c              
   C   s  |   y | \ tjtjdg}tdgg}t	
 }||}W d    n1 s/w   Y  |||j}t }| td |t||j W d    n1 sYw   Y  W d    n1 shw   Y  W d    d S W d    d S 1 sw   Y  d S )Nrm   r   z)cannot be called in cross-replica context)r   r   r"   r#   r$   r&   r'   rw   r  ry   r	  r
  trainable_variablesr   r   r   RuntimeErrorapply_gradientszip)r4   rb   r,   ri   r  rj   Z	gradientsr   r-   r-   r.   4test_optimizer_in_cross_replica_context_raises_error  s"   


Pz\TestDistributionStrategyWithNumpyArrays.test_optimizer_in_cross_replica_context_raises_errorc              	   C   s4  |    |  tj}|dd}t }d}||| W d    n1 s'w   Y  tjtjdtj	d}tjtjdtj	d}||g}tjtjdtj	d}	tjtjdtj	d}
|	|
g}|j
||dd	d
d ||| |j||d	d || |j|d	d W d    d S 1 sw   Y  d S )Nr   Zlearning_rater   r   rs   )r   r   )r   r   rm   rW   r   )r   ra   r   r   )r   r   r   r   r   r   rw   asarrayrandomrx   r   r   r   )r4   rb   r   r   r,   r   
input_a_np
input_b_npr9   output_d_npoutput_e_npr}   r-   r-   r.   +test_calling_model_with_nested_numpy_arrays  s0   



"zSTestDistributionStrategyWithNumpyArrays.test_calling_model_with_nested_numpy_arraysc           	   	   C   s  |    | b t }tjjjjdd}d}||| t	
dgdgdgdggt	j}t	
dgdgd	gd
ggt	j}t	
g dt	j}|j||d|dd}| |d |j||ddd}| |d W d    n1 snw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   r  r   r   rm   r   rK   rf      rW   g      ?r   g      ?rm   )ra   Zsample_weightr        @%@)ra   r         +@)r   r   rq   ry   r   r   trainRMSPropOptimizerr   rw   r  rx   r   assertAllClose)	r4   rb   r,   r   r   r9   r}   sample_weightsresultr-   r-   r.   test_numpy_with_sample_weights  s&   	PzFTestDistributionStrategyWithNumpyArrays.test_numpy_with_sample_weightsc           
   	   C   s   |   m |  t }tj}|dd}d}||| W d    n1 s'w   Y  tjtjdtj	d}tjtjdtj	d}||g}|
|}	| |	d | dd	g|	d
 j | dd	g|	d j W d    d S 1 stw   Y  d S )Nr   r  r   )r#  rK   rs   )r#  r   r   r#  r   r   rm   )r   r   r   r   r   r   rw   r  r  rx   r   	assertLenassertAllEqualrF   )
r4   rb   r,   r   r   r   r  r  r9   Zoutsr-   r-   r.   test_flatten_predict_outputs  s    


	
"zDTestDistributionStrategyWithNumpyArrays.test_flatten_predict_outputsrf   r#  r   c              	   C   sN  |    tjjjd}d}dtj g}|	  t
 }|j|||d W d    n1 s0w   Y  t
 }|j|||d tjdd}tjdd}	||  |||	}
td| }| j|j||	||d	d
d  |
d
d  ddd | j|j||	|dd
d  |
d
d  ddd W d    d S 1 sw   Y  d S )Nr   r   r   r   rr   rx   rt         $@ra   r   rm   h㈵>ZatolZrtolr   )r   ry   r   r   r'  GradientDescentOptimizerr"   r   CategoricalAccuracyr   rk   r   rw   r  astypeset_weightsget_weightsr   ceilr)  )r4   rb   ra   r   r   r   model_with_ds_strategy	cpu_modelri   rj   Zevaluate_ground_truthr   r-   r-   r.    test_evaluate_with_partial_batch  sD   




"zHTestDistributionStrategyWithNumpyArrays.test_evaluate_with_partial_batchc              	   C   s   |   l tjjjd}d}|  t }||| W d    n1 s'w   Y  t }||| t	j

dt	j}||  ||}| j|j|ddd|ddd | j|j|dd	|ddd W d    d S 1 ssw   Y  d S )
Nr   r   rr   rf   rK   r1  r2  r3  r   )r   ry   r   r   r'  r4  r   rk   r   rw   r  r6  rx   r7  r8  r   r)  )r4   rb   r   r   r:  r;  r9   Zpredict_ground_truthr-   r-   r.   test_predict_with_partial_batchL  s2   


"zGTestDistributionStrategyWithNumpyArrays.test_predict_with_partial_batchc              	   C   s  |   { tjjjd}G dd dtjj}|	 L tj
 }|tjjddtd ||  |tjjtdd || td	tj}|j|d
dd |j|d
d |j|d
d W d    n1 sjw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   c                   @   s   e Zd ZdddZdS )zMTestDistributionStrategyWithNumpyArrays.test_no_target_model.<locals>.MyLayerNc                 S   s   | j t|dd |S )NT)r9   )add_lossry   
reduce_sum)r4   r9   trainingr-   r-   r.   r:   |  s   zRTestDistributionStrategyWithNumpyArrays.test_no_target_model.<locals>.MyLayer.callr0   )r<   r=   r>   r:   r-   r-   r-   r.   r  {  s    r  r   r   r   r    r!   r   r   rm   r   r   steps_per_epochr   )r   ry   r   r   r'  r4  r"   r&   r  r   r#   r$   r%   r'   r(   r*   r   rw   ro   rx   r   r   r   )r4   rb   r   r  r,   r9   r-   r-   r.   test_no_target_modelt  s*   



"z<TestDistributionStrategyWithNumpyArrays.test_no_target_modelc           	   	   C   s   |   ` tjjjd}d}|  t }||| W d    n1 s'w   Y  t }||| t	 \}}|d |d d}|
|  | j|j|ddd||d	d	d
 W d    d S 1 sgw   Y  d S )Nr   r   rD   rH   rD   rH         r1  -C6?r3  )r   ry   r   r   r'  r4  r   rT   r   r[   r7  r8  r)  r   )	r4   rb   r   r   r:  r;  
input_datarZ   
input_dictr-   r-   r.   2test_predict_multi_output_model_with_partial_batch  s0   


	"zZTestDistributionStrategyWithNumpyArrays.test_predict_multi_output_model_with_partial_batchc                 C   s   t  s	| d G dd dtjj}|  t|dddg}|dd W d    n1 s2w   Y  t	j
jd	d
}t	j
jd	d
}| d }||| | d }| || d S )Nz.None gradients are not supported in graph modec                       s   e Zd Z fddZ  ZS )z]TestDistributionStrategyWithNumpyArrays.test_gradients_are_none.<locals>.DenseWithExtraWeightc                    s4   | j dddd| _t | | j dddd| _d S )Nextra_weight_1r-   rn   )rF   initializerextra_weight_2)r   rM  r1   r  rO  r4   r   r6   r-   r.   r    s   zcTestDistributionStrategyWithNumpyArrays.test_gradients_are_none.<locals>.DenseWithExtraWeight.build)r<   r=   r>   r  r?   r-   r-   r6   r.   DenseWithExtraWeight  s    rQ  rf   )rf   )r   adamr   r   sizerm   )ry   r  r   r"   r&   r'   r   r$   r   rw   r  normalr8  r   ZassertNotAllEqual)r4   rb   rQ  r,   r9   r}   Z
old_kernelZ
new_kernelr-   r-   r.   test_gradients_are_none  s   

z?TestDistributionStrategyWithNumpyArrays.test_gradients_are_noneN) r<   r=   r>   ry   r   r   r   generater   r   r   r   r   r   r   r  r   r   r   r  r"  r   r	   r,  r/  r   r   r<  r=  rE  rL  rV  r-   r-   r-   r.   r   t  s    




8





 

1

/





 



(




.


#




"r   c                   @   s  e Zd Zejjje dd Z	ejjje dd Z
ejjje dd Zejjje dd Zejjje d	d
 Zejjje dd Zejjje dd Zejjje dd Zejjje dd Zejjje dd Zejjjejjjjejjjjejjjjgddgddd Zejjjejjjjejjjjgddgddd Zejjjejjjjejjjjejjjjgddgddd Zejjje dd Zejjjejjje ejjjjd d!gd"d#d$ Zejjjejjje d%d& Z ejjjejjje d'd( Z!ejjje" d)d* Z#ejjjejjjje$ddgdejjjje%dgd d+d, Z&d-S ).$TestDistributionStrategyWithDatasetsc              	   C   s   |   [ | # tj}|d}t }d}dtj g}|j|||d W d    n1 s/w   Y  t	|}|j
|ddd|dd |j
|ddd|dd |jt|dd	 W d    d S 1 sbw   Y  d S )
Nr   r   r   r   rm   r   r   )r   rC  r   r   validation_stepsrD  )r   r   r   r   rk   r"   r   r5  r   r~   r   r   r   r4   rb   r   r   r,   r   r   r`   r-   r-   r.   "test_calling_model_on_same_dataset  s8   

"zGTestDistributionStrategyWithDatasets.test_calling_model_on_same_datasetc           
   
   C   st  |    | 6 tj}t }|j|dddtj gd t }|	|
  |j|dddtj gd W d    n1 sBw   Y  t|}|j|ddd|ddd}g }tdD ]}|j|ddddd	 ||j|dd
 q]| |jd dd |D  |jd}	|	s|jd }	| |	dd |D  | |jd dd |D  W d    d S 1 sw   Y  d S )Nr   r   r   )r   r   r   rm   F)r   rC  r   r   rY  shuffle)r   rC  r   r\  rD  Zval_lossc                 S      g | ]}|d  qS )r   r-   .0ri   r-   r-   r.   
<listcomp>3      zhTestDistributionStrategyWithDatasets.test_model_interleaved_eval_same_as_direct_eval.<locals>.<listcomp>val_mean_absolute_errorZval_maec                 S   r]  rl   r-   r^  r-   r-   r.   r`  <  ra  Zval_categorical_accuracyc                 S   r]  )r   r-   r^  r-   r-   r.   r`  @  ra  )r   r   r   r   rk   r   r"   r   r5  r7  r8  r~   r   ranger   r   r   historyget)
r4   rb   r   Zuser_controlled_modelZinterleaved_modelr`   Zinterleaved_outputZuser_controlled_outputrZ   rb  r-   r-   r.   /test_model_interleaved_eval_same_as_direct_eval  st   


"zTTestDistributionStrategyWithDatasets.test_model_interleaved_eval_same_as_direct_evalc              	   C   sX  |    | $ tj}|dd}t }d}dtj g}|j|||d W d    n1 s0w   Y  t	j

dd}t	j

dd}t	j

d	d}	t	j

d	d}
tjj||f|	|
ff}|d
}|d}|j|dddd tjj||d|	|
ff}|d
}|d}|j|dddd W d    d S 1 sw   Y  d S )Nr   r  r   r   r   rr   rx   r   r   )r   r   ru   r   rm   r   r   rC  r   rF  )r   r   r   r   r   r"   r   r5  r   rw   r  r6  ry   rz   r{   r|   r]   r_   r   )r4   rb   r   r   r,   r   r   r  r  r   r!  Zdataset_tupleZdataset_dictr-   r-   r.   +test_fit_with_tuple_and_dict_dataset_inputsC  s8   






"zPTestDistributionStrategyWithDatasets.test_fit_with_tuple_and_dict_dataset_inputsc              	      s  t |r
| d dd  |   | b tjjg ddd}tjjg ddd}tjjd	d	gd
d}tjjdddgdd|}tjj	 fdddd}||||g}tj
j|||g||gd}||dj |jdd W d    n1 s|w   Y  t rdd }	ndd }	tjg dtjd}
tjg dtjd}tjg dtjd}tjj|
||f|	d}|| W d    d S 1 sw   Y  d S )Nzb/142805125c                 S   s   t j|| }t|| S r0   )r"   lossesZbinary_crossentropyry   reduce_mean)r   labelweightZbcer-   r-   r.   custom_lossr  s   zlTestDistributionStrategyWithDatasets.test_fit_with_dictionary_in_the_dataset_b135161171.<locals>.custom_loss)r   r   rK   imgrJ   )r   r   rm   lblr   rm  r   rm   Zsamepaddingc                    s    |  S r0   r-   ri   rn  r-   r.   r         ziTestDistributionStrategyWithDatasets.test_fit_with_dictionary_in_the_dataset_b135161171.<locals>.<lambda>my_lossrN   rR  r   c                 S   s   | ||d}|fS N)ro  rp  rm  r-   ro  rp  rm  r9   r-   r-   r.   map_fn  s   zgTestDistributionStrategyWithDatasets.test_fit_with_dictionary_in_the_dataset_b135161171.<locals>.map_fnc                 S   s   | ||d}|i fS rw  r-   rx  r-   r-   r.   ry    s   )rU   r   r   rK   rs   )rU   r   r   rm   )rU   r   r   r   )r   r^   r   r   r   r"   r&   rP   Conv2DLambdar#   rA   r>  	get_layeroutputr   ry   r  rw   rn   rx   rz   r{   r|   mapr_   r   )r4   rb   Z	input_imgZ	input_lblZinput_weightr   Zloss_lambdarv  r,   ry  Z	fake_imgsZ	fake_lblsZfake_weightsrz   r-   rt  r.   2test_fit_with_dictionary_in_the_dataset_b135161171j  sH   




"zWTestDistributionStrategyWithDatasets.test_fit_with_dictionary_in_the_dataset_b135161171c              	   C   st  |    | # tj}|d}t }d}dtj g}|j|||d W d    n1 s/w   Y  t	j
dt	jd}t	j
dt	jd}|j||dd	d
j}	|j||d	d}
|j|d	d}tjj||f}|jd	dd}|j|ddj}||}tjj|}|jd	dd}||}| j|	|ddd | j|
|ddd | j||ddd W d    d S 1 sw   Y  d S )Nr   r   r   r     rK   rs   r  rf   rm   r   r   ra   r   Tr\   r   rI  r3  )r   r   r   r   rk   r"   r   r5  r   rw   ro   rx   r   rd  r   r   ry   rz   r{   r|   r_   r)  )r4   rb   r   r   r,   r   r   r9   r}   fit_with_numpyeval_with_numpypredict_with_numpyr`   fit_with_dseval_with_dspredict_datasetpredict_with_dsr-   r-   r.   :test_fit_eval_and_predict_methods_on_dataset_without_steps  sF   



"z_TestDistributionStrategyWithDatasets.test_fit_eval_and_predict_methods_on_dataset_without_stepsc              	   C   s  |dkrt |r| d |  k | # tj}|d}t }d}dtj	
 g}|j|||d W d    n1 s=w   Y  tjdtjd}|j|d	d
}	t|}
| tj tjj|
tjjj ||
}| j|	|ddd W d    d S 1 sw   Y  d S )Nr   3partial batch not supported with TPU in graph mode.r   r   r   r   r   rK   rs   r   r   rI  r3  )r   r^   r   r   r   r   r   rk   r"   r   r5  r   rw   ro   rx   r   r   r   	get_valuery   rz   r   cardinalityUNKNOWN_CARDINALITYr)  )r4   rb   r   r   r   r,   r   r   r9   r  r  r  r-   r-   r.   >test_predict_on_dataset_with_unknown_cardinality_without_steps  s6   



"zcTestDistributionStrategyWithDatasets.test_predict_on_dataset_with_unknown_cardinality_without_stepsc              	   C   s   |dkrt |r| d |dkrt |r| d |   | # tj}|d}t }d}dtj	
 g}|j|||d W d    n1 sKw   Y  tjd	tjd
}tjdtjd
}	|j||	dddj}
|j||	dddj}|j||	dd}|j|dd}t||	}t|}| tj tjj|tjjj | tj tjj|tjjj ||}||}| j||ddd | j||ddd |j|ddj}|j|ddj}| j|
|ddd | j||ddd W d    d S 1 sw   Y  d S )Nr   z'caused segfault with TPU in eager mode.r   r  r   r   r   r   )ru   rK   rs   )ru   rf   rm   r   r  r   r   rI  r3  r  )r   r^   r   r   r   r   r   rk   r"   r   r5  r   rw   ro   rx   r   rd  r   r   r   r   r  ry   rz   r   r  r  r)  )r4   rb   r   r   r   r,   r   r   r9   r}   r  Zfit_with_numpy_multiple_epochsr  r  r`   r  r  r  r  Zfit_with_ds_multiple_epochsr-   r-   r.   6test_on_dataset_with_unknown_cardinality_without_steps  s   





"z[TestDistributionStrategyWithDatasets.test_on_dataset_with_unknown_cardinality_without_stepsc              	   C   s  |    | " t }d}dtj g}|jtjj	j
d||d W d    n1 s.w   Y  tjdtjd}tjdtjd}|j||dd	}|j|dd	}t||}	t|}
| tjtjj|	tjjj | tjtjj|
tjjj |j|	d
d}|j|
d
d}| j||ddd | j||ddd | td |j|	dd W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   r   r   r   r  rs   r  r   r   ru   rD  rI  r3  z%Number of steps could not be inferredrm   r  )r   r   rk   r"   r   r5  r   ry   r   r   r'  r4  rw   ro   rx   r   r   r   r   r   r  rz   r   r  r  r)  r   r   r   )r4   rb   r,   r   r   r9   r}   r  r  r`   r  r  r  r-   r-   r.   (test_on_dataset_with_unknown_cardinalityB  sb   


/"zMTestDistributionStrategyWithDatasets.test_on_dataset_with_unknown_cardinalityc              	   C   s   |   V | # tj}|d}t }d}dtj g}|j|||d W d    n1 s/w   Y  t	|}|j
|dddd |j|ddd |jt|dd	 W d    d S 1 s]w   Y  d S )
Nr   r   r   r   rm   r   rh  r   r   rD  )r   r   r   r   rk   r"   r   r5  r   r~   r   r   r   r   rZ  r-   r-   r.   ,test_fit_eval_and_predict_methods_on_datasetz  s   

"zQTestDistributionStrategyWithDatasets.test_fit_eval_and_predict_methods_on_datasetc              	   C   s   |   G |  t }d}|| | W d    n1 s w   Y  t|}|j|dddd |j|ddd |jt|dd W d    d S 1 sNw   Y  d S )Nr   rm   r   rh  r  rD  )	r   r   rk   r   r~   r   r   r   r   )r4   rb   r   r,   r   r`   r-   r-   r.   (test_fit_eval_and_predict_with_optimizer  s   

"zMTestDistributionStrategyWithDatasets.test_fit_eval_and_predict_with_optimizerr   r   r   c           
   	   C   s"  |dkr	|  d |  z |  tj}|dd}t }d}||| W d    n1 s0w   Y  tjdtj	d}tjdtj	d}t
jj||f}	|	d	}	|	d
}	| td |j|	dddd W d    n1 srw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   zYTODO(b/120943676, b/120957836): Re-enable for graph once the validation code is restored.r   r  r   rg  rs   rt   ru   r   zis incompatible withrm   r   r   rh  )r   r   r   r   r   rk   r   rw   ro   rx   ry   rz   r{   r|   r]   r_   r   r   r   )
r4   rb   r   r   r   r,   r   r9   r}   r`   r-   r-   r.   test_dataset_wrong_input_shape  s.   





"zCTestDistributionStrategyWithDatasets.test_dataset_wrong_input_shapec           	   	   C   s   |   W |  tj}|dd}t }d}||| W d    n1 s'w   Y  tjdtjd}tjdtjd}t	j
j||f}|d}|j|dd	dd
 W d    d S 1 s^w   Y  d S )Nr   r  r   )ru   r   rK   rs   )ru   r   rf   ru   rm   r   rh  )r   r   r   r   rk   r   rw   ro   rx   ry   rz   r{   r|   r]   r   )	r4   rb   r   r   r,   r   r9   r}   r`   r-   r-   r.   ,test_dataset_external_batch_input_validation  s   
	


"zQTestDistributionStrategyWithDatasets.test_dataset_external_batch_input_validationc              	   C   s  |    | @ tjjddd}tjjddd|}tjd|}t||}| }t	j
}|d}d	}	d
g}
|j||	|
d W d    n1 sLw   Y  d}t|tjjtjjjjfred|j }tjdtjd}tjdtjd}tjj||f}| |}|j|dddd}| |jd
 d dd |  || W d    n1 sw   Y  tjdtjd}tjj|}| |}|j|dd}tjdtjd}|  ||d W d    d S 1 sw   Y  d S )Nrl   re   rE   rm   rn   rp   gH.?g{Gzt?r   accr   rW   r   rm   rs   r   rh  r   r   rD  )   rm   r   )!r   r   r"   r&   rP   r'   r)   rA   r8  r   r   r   r   ry   r   ZMirroredStrategyr   r   r   rw   rn   rx   rz   r{   r|   r]   r_   r   ZassertAlmostEqualrd  r7  r   assertArrayNear)r4   rb   ri   rj   zr,   Zinitial_weightsr   r   r   r   ra   r9   r}   r`   histr  r}  Z
ref_outputr-   r-   r.   test_learning_phase_value  sJ   




"z>TestDistributionStrategyWithDatasets.test_learning_phase_valuec              	   C   s   |   P |  t }td}d}||| W d    n1 s$w   Y  t|}dd }|j|dddtj	
|gd | d	tj|jj W d    d S 1 sWw   Y  d S )
N{Gz?r   c                 S   r   )Nr   r-   )rZ   r-   r-   r.   schedule"  s   zQTestDistributionStrategyWithDatasets.testOptimizerWithCallbacks.<locals>.schedulerm   r   r   )r   rC  r   	callbacksr   )r   r   rk   r   r   r   r~   r   r"   r  ZLearningRateSchedulerr)  r   r  r   lr)r4   rb   r,   r   r   r`   r  r-   r-   r.   testOptimizerWithCallbacks  s(   


"z?TestDistributionStrategyWithDatasets.testOptimizerWithCallbacksrf   r#  r   c              	   C   sd  |    tjjjd}d}dtj g}|	  t
 }|j|||d W d    n1 s0w   Y  t
 }|j|||d tjdd}tjdd}	tjj||	f}
||  |
|}td| }| j|j||d	d
d  |j||d	d
d  ddd | j||d
d  ||d
d  ddd W d    d S 1 sw   Y  d S )Nr   r   r   r   rr   rx   rt   r0  rD  rm   r2  r3  )r   ry   r   r   r'  r4  r"   r   r5  r   rk   r   rw   r  r6  rz   r{   r|   r7  r8  r_   r9  r)  r   )r4   rb   ra   r   r   r   r:  r;  ri   rj   r`   dataset_with_partial_batchr   r-   r-   r.   -test_evaluate_with_dataset_with_partial_batch0  sB   
	

"zRTestDistributionStrategyWithDatasets.test_evaluate_with_dataset_with_partial_batchc           	   	   C   s   |   i tjjjd}d}|  t }||| W d    n1 s'w   Y  t }||| t	j

dt	j}tjj|}|d}||  | j|j|dd|j|ddddd W d    d S 1 spw   Y  d S )	Nr   r   rr   rf   rK   rD  r2  r3  )r   ry   r   r   r'  r4  r   rk   r   rw   r  r6  rx   rz   r{   r|   r_   r7  r8  r)  r   )	r4   rb   r   r   r:  r;  r9   r`   r  r-   r-   r.   ,test_predict_with_dataset_with_partial_batch`  s,   


"zQTestDistributionStrategyWithDatasets.test_predict_with_dataset_with_partial_batchc              	   C   s   |   m tjjjd}d}|  t }||| W d    n1 s'w   Y  t }||| t	 \}}|d |d d}tj
j|}	|	d}
||  | j|j|
dd|j|
ddd	d	d
 W d    d S 1 stw   Y  d S )Nr   r   rD   rH   rF  rG  rH  rD  rI  r3  )r   ry   r   r   r'  r4  r   rT   r   r[   rz   r{   r|   r_   r7  r8  r)  r   )r4   rb   r   r   r:  r;  rJ  rZ   rK  r`   r  r-   r-   r.   ?test_predict_multi_output_model_with_dataset_with_partial_batch  s4   



"zdTestDistributionStrategyWithDatasets.test_predict_multi_output_model_with_dataset_with_partial_batchc              	   C   sn  dd }t jddt jt jddt jd}t jdt jd}tjj	||f}|j
dd	d
}|  v | Y | \}}}tjj||g|d}	|	dd |	 }
|	j|dddj}| \}}}tjj||d|d}|dd ||
 |j|dddj}| j||ddd W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nc                  S   sl   t jjddd} t jjddd}t jd| }t jd|}t j ||g}t jd|}| ||fS )NrC   z_input_sorted_lastrE   )r   a_input_sorted_firstr   r   )r"   r&   rP   r'   Add)rD   rH   Zintermediate_aZintermediate_brQ   r}  r-   r-   r.   "_create_model_input_output_tensors  s   
zTestDistributionStrategyWithDatasets.test_match_model_input_matches_with_dataset_tensors.<locals>._create_model_input_output_tensorsr   r   )r  r  )r   r   rs   rf   Tr\   rN   sgdr   rm   rC  r   rI  r3  )rw   r  randr6  rx   rn   ry   rz   r{   r|   r_   r   r   r"   r#   rA   r   r8  r   rd  r7  r)  )r4   rb   r  rK  r   r`   rD   rH   r}  Zmodel_with_array_inputZmodel_weightsZmodel_with_array_input_fitZmodel_with_dict_inputZmodel_with_dict_input_fitr-   r-   r.   3test_match_model_input_matches_with_dataset_tensors  sX   


"zXTestDistributionStrategyWithDatasets.test_match_model_input_matches_with_dataset_tensorsc           
   	   C   sB  |    | v t }tjjjjdd}d}||| t	
dgdgdgdggt	j}t	
dgdgd	gd
ggt	j}t	
g dt	j}tjj|||fd}|j|dd}	| |	d tjj||fd}|j|dd}	| |	d W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   r  r   r   rm   r   rK   rf   r#  rW   r$  )r   r%  r&  )r   r   rq   ry   r   r   r'  r(  r   rw   r  rx   rz   r{   r|   r_   r   r)  )
r4   rb   r,   r   r   r9   r}   r*  dsr+  r-   r-   r.    test_dataset_with_sample_weights  s&   	PzETestDistributionStrategyWithDatasets.test_dataset_with_sample_weightsN)'r<   r=   r>   ry   r   r   r   rW  r   r[  rf  ri  r  r  r  r  r   r  r  r   r  r   r   r   r   r  r  r   r  r  r   r  r  r  r   r  r   r	   r  r-   r-   r-   r.   rX    s    




B

$

6

'

%

J

5







	








	
3




*





&

<

rX  c                       s6   e Zd Z fddZejjje	 dd Z
  ZS )(TestDistributionStrategyWithDatasetsFilec                    sh   t    tj|  d| _tjdtj	d}t
jj|}|t
jj}t
jj| j}|| d S )Nzinput.tfrecordr  rs   )r1   setUpospathjoinget_temp_dirinput_file_namerw   ro   rx   ry   rz   r{   r|   r~  ioZserialize_tensorr   ZTFRecordWriterwrite)r4   r9   Zinput_datasetwriterr6   r-   r.   r    s   
z.TestDistributionStrategyWithDatasetsFile.setUpc           
      C   s   ~|   tj}|d}t }d}||| W d    n1 s"w   Y  tj| j}|	dd }dd }|
|jddd}tj }	tjjjj|	j_||	}|j|d	d
 d S )Nr   r   c                 S   s   t j| t jS r0   )ry   r  Zparse_tensorrx   rs  r-   r-   r.   r   =      z{TestDistributionStrategyWithDatasetsFile.test_predict_on_dataset_shard_options_file_multi_worker_mirrored.<locals>.<lambda>c                 S   r   r   r-   r   r-   r-   r.   r   ?  r   rW   Tr\   rm   rD  )r   r   r   rk   r   ry   rz   ZTFRecordDatasetr  r~  r   r_   Optionsr   ZAutoShardPolicyFILEZexperimental_distributeZauto_shard_policyZwith_optionsr   )
r4   rb   r   r   r   r,   r   r`   r   optionsr-   r-   r.   @test_predict_on_dataset_shard_options_file_multi_worker_mirrored(  s"   



ziTestDistributionStrategyWithDatasetsFile.test_predict_on_dataset_shard_options_file_multi_worker_mirrored)r<   r=   r>   r  ry   r   r   r   rW  r   r  r?   r-   r-   r6   r.   r    s    
r  c                   @   sf   e Zd ZG dd dejjZG dd dejjZ	e
dd Zejjjejjje dd Zd	S )
TestRegularizerLossc                   @   s   e Zd Zdd ZdS )z'TestRegularizerLoss.IdentityRegularizerc                 C   
   t |S r0   )ry   identityr4   ri   r-   r-   r.   __call__N  r;   z0TestRegularizerLoss.IdentityRegularizer.__call__N)r<   r=   r>   r  r-   r-   r-   r.   IdentityRegularizerM  s    r  c                   @   r   )zTestRegularizerLoss.AddLayerc                 C   s   | j dddt d| _d S )Nr   r-   rn   )rN  Zregularizer)r   r  r  r   r  r-   r-   r.   r  R  s   z"TestRegularizerLoss.AddLayer.buildc                 C   s
   || j  S r0   )r   r8   r-   r-   r.   r:   Z  r;   z!TestRegularizerLoss.AddLayer.callNr  r-   r-   r-   r.   AddLayerQ  s    r  c                 C   r  r0   )ry   rk  )rZ   Zy_predr-   r-   r.   loss_fn]  s   
zTestRegularizerLoss.loss_fnc                 C   s   d}t |s||j }| T tjjd|d}t |}tj	j
||d}td}|j|tjd |jtjdgdggtjdtjdgdggtjd|d | d	 }| d
| W d    d S 1 sgw   Y  d S )Nr   rl   )rF   ra   rN   r   r   rs   ri   rj   ra   r         )r   r   r   r   r"   r&   rP   r  r  r#   rA   r   r   r   r  r   rw   r  rx   r8  r   )r4   rb   ra   ri   rj   r,   optr   r-   r-   r.   test_regularizer_lossa  s&   


"z)TestRegularizerLoss.test_regularizer_lossN)r<   r=   r>   r"   ZregularizersZRegularizerr  r&   r  r  staticmethodr  ry   r   r   r   rW  r   r   r   r  r-   r-   r-   r.   r  L  s    


r  z$Uses Dense layers, which call matmulc                   @   s  e Zd Zejjje dd Z	ejjje dd Z
ejjjejjjjedgddd Zejjjejjjjedgdd	d
 Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjjjedgddd Zejjjejjje dd Zejjjejjjjejjjjejjjjgddgejjejjejjgddd Z ejjjejjje dd Z!ejjje" dd  Z#ejjjejjje" d!d" Z$ejjjejjjjejjjj%ejjjj&ejjjjejjjjgdgdd#d$ Z'ejjjejjje" d%d& Z(ejjjejjjje)e* dgdd'd( Z+ejjjejjjje)e* dgdd)d* Z,ejjjejjjje-e. e* dgdd+d, Z/ejjjejjjjddgd-d.d/ Z0d0S )1'TestDistributionStrategyWithKerasModelsc                 C   s   |  ) tj}|dd}t }d}||| tdtj}tdtj}W d    n1 s0w   Y  |j||ddd |j	|dd	 |j
||dd	 d S )
Nr   r  r   rA  )r   r   rm   r   r  r   )r   r   r   r/   r   rw   ro   rx   r   r   r   r4   rb   r   r   r,   r   r9   r}   r-   r-   r.   .test_distribution_strategy_on_sequential_model  s   


zVTestDistributionStrategyWithKerasModels.test_distribution_strategy_on_sequential_modelc                 C   s   |  + tj}|dd}t }d}||| tjdtjd}tjdtjd}W d    n1 s2w   Y  |j||dd |	| |
|| d S )	Nr   r  r   r   rs   r   rm   r  )r   r   r   rk   r   rw   ro   rx   r   r   r   r  r-   r-   r.   .test_distribution_strategy_on_functional_model  s   



zVTestDistributionStrategyWithKerasModels.test_distribution_strategy_on_functional_modelr   r   c           	   	   C   sv  |   G dd dtjj}ttjdg}|dd | }t	dt	d}}t
jj||f}|dd	}||}t
jj||f}|d
}||}|j|d|dd	|gd | |jd
 | |jd | |jd	 |jdkr| td |j|d	d W d    n1 sw   Y  W d    d S W d    d S W d    d S 1 sw   Y  d S )Nc                   @   2   e Zd Zdd Zd
ddZd
ddZd
dd	ZdS )zSTestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounterc                 S      d| _ d| _d| _d S Nr   r   train_batchestest_batchesr   r-   r-   r.   r2        
z\TestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounter.__init__Nc                 S      |  j d7  _ d S Nrm   r  r   r-   r-   r.   on_epoch_end     z`TestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounter.on_epoch_endc                 S   r  r  r  r   r-   r-   r.   r     r  zfTestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounter.on_train_batch_endc                 S   r  r  r  r   r-   r-   r.   r     r  zeTestDistributionStrategyWithKerasModels.test_distributed_dataset.<locals>.CBCounter.on_test_batch_endr0   r<   r=   r>   r2   r  r   r   r-   r-   r-   r.   	CBCounter  
    

r  rm   r  r   ru   r   ru   rm   r   r   r   r   rC  r   rY  r   r  %distributed dataset, you must specifyr  )r   r"   r  Callbackr$   r&   r'   r   rw   rn   ry   rz   r{   r|   r_   r]   experimental_distribute_datasetr   r   r  r  r   r   r   r   )	r4   rb   r  r,   
cb_counterri   rj   r  val_dsr-   r-   r.   test_distributed_dataset  sD   



	
,-"z@TestDistributionStrategyWithKerasModels.test_distributed_datasetc              	   C   s*  |   G dd dtjj}ttjdg}|dd | }dd }||}||}|j	|d|d	d
|gd | 
|jd | 
|jd | 
|jd
 |jdkr{| td |j	|d
d W d    n1 snw   Y  W d    d S W d    d S W d    d S 1 sw   Y  d S )Nc                   @   r  )zbTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounterc                 S   r  r  r  r   r-   r-   r.   r2     r  zkTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounter.__init__Nc                 S   r  r  r  r   r-   r-   r.   r    r  zoTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounter.on_epoch_endc                 S   r  r  r  r   r-   r-   r.   r     r  zuTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounter.on_train_batch_endc                 S   r  r  r  r   r-   r-   r.   r     r  ztTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.CBCounter.on_test_batch_endr0   r  r-   r-   r-   r.   r    r  r  rm   r  r   c                 S   s:   t dt d}}tjj||f}|d }|S )Nr  r  r   )rw   rn   ry   rz   r{   r|   r_   r]   )rZ   ri   rj   r  r-   r-   r.   make_dataset  s   zeTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_function.<locals>.make_datasetr   r   r   r  r   r  r  )r   r"   r  r  r$   r&   r'   r   Z!distribute_datasets_from_functionr   r   r  r  r   r   r   r   )r4   rb   r  r,   r  r  r  r  r-   r-   r.   'test_distributed_datasets_from_function  s@   

	
./"zOTestDistributionStrategyWithKerasModels.test_distributed_datasets_from_functionc                 C   s`  t |tjjr| d | , td}tjj	ddd|}tj
 |}tjd|}t||}W d    n1 s?w   Y  |jdddd	 t }td
td}}|j||dd|gd | |jg d | |jg d |j||d|gd | |jg d | |jg d |j|d|gd | |jg d | |jg d d S )Nb/172032817)r   r   rK   rK   )kernel_sizerm   r  r   r   steps_per_execution)ru   r   r   rK   r  r   ra   r   r  )r   r   r      (   )	         '   1   ra   r  )r   ry   r   MultiWorkerMirroredStrategyr   r   r"   rP   r&   rz  Flattenr'   rA   r   r   rw   rn   r   r   r   r   r   r   r   r   r   r   )r4   rb   r9   ri   rO   r,   bcrj   r-   r-   r.   test_host_training_loop"  s*   


z?TestDistributionStrategyWithKerasModels.test_host_training_loopc                 C   s>  t |tjjr| d |  td}tj	d|}t
||}W d    n1 s.w   Y  |jdddd t }tdtd	}}|j||d
d|gd | |jg d | |jg d |j||d
|gd | |jg d | |jg d |j|d
|gd | |jg d | |jg d d S )Nr  r   rm   r  r   r   r  r  r  r   r  r   r   r  r  r  r  r  )r   ry   r   r  r   r   r"   rP   r&   r'   rA   r   r   rw   rn   r   r   r   r   r   r   r   r   r   r   )r4   rb   r9   rO   r,   r   ri   rj   r-   r-   r.   .test_host_training_loop_last_partial_executionA  s&   


zVTestDistributionStrategyWithKerasModels.test_host_training_loop_last_partial_executionc                 C   s  t |tjjr| d |  td}tj	d|}t
||}W d    n1 s.w   Y  |jdddd tdtd	}}tjj||fd
}|dd }t }| td |j|d
|gd W d    n1 svw   Y  |d
}	|j|	dd
|gd | |jg d | |jg d | td |j||gd W d    n1 sw   Y  |d
}
|j|
d|gd | |jg d | |jg d |d
}|j|d|gd | |jg d | |j g d d S )Nr  r   rm   r  r   r   r  r  r  r   c                  _   r   r   r-   )argskwargsr-   r-   r.   r   o  r   zfTestDistributionStrategyWithKerasModels.test_host_training_loop_dataset_unknown_size.<locals>.<lambda>r  )r   r  rU   )rC  r   r  )r   r   r  r   r   r  )r  r  r  r  r  r  )r  )r   r  r  r  )!r   ry   r   r  r   r   r"   rP   r&   r'   rA   r   rw   rn   rz   r{   r|   r_   r   r   r   r   r   r]   r   r   r   r   r   r   r   r   r   )r4   rb   r9   rO   r,   ri   rj   r  r   Ztrain_dsZtest_dsZ
predict_dsr-   r-   r.   ,test_host_training_loop_dataset_unknown_size^  s<   





zTTestDistributionStrategyWithKerasModels.test_host_training_loop_dataset_unknown_sizec                 C   sV  t |tjjr| d |  td}tj	d|}t
||}W d    n1 s.w   Y  |jdddd tdtd	}}t }|j||d
d
|gd | |jddg | |jddg tdtd}}|j||d
|gd | |jdg | |jdg td}|j|d
|gd | |jdg | |jdg d S )Nr  r   rm   r  r   i  r  r  r  r   r  r   r  )rU   r   )rU   rm   r     )r   ry   r   r  r   r   r"   rP   r&   r'   rA   r   rw   rn   r   r   r   r   r   r   r   r   r   r   r   )r4   rb   r9   rO   r,   ri   rj   r   r-   r-   r.   )test_host_training_loop_truncate_to_epoch  s*   



zQTestDistributionStrategyWithKerasModels.test_host_training_loop_truncate_to_epochc                 C   s   G dd dt jj}tdtd}}|  | }t |g}tjdddd}W d    n1 s5w   Y  |	|d t
|tjjjtjjjjjfrq| td |j||d	d
d W d    d S 1 sjw   Y  d S |j||d	d
d | | |jd | | |jd d S )Nc                   @   r   )zOTestDistributionStrategyWithKerasModels.test_gradient_clipping.<locals>.MyLayerc                 S   s   t d| _t d| _d S r  ry   Variabler   r   r  r-   r-   r.   r       zUTestDistributionStrategyWithKerasModels.test_gradient_clipping.<locals>.MyLayer.buildc                 S   s   d| j  d| j  S )NrK   r  r  r-   r-   r.   r:     s   zTTestDistributionStrategyWithKerasModels.test_gradient_clipping.<locals>.MyLayer.callNr  r-   r-   r-   r.   r    r  r  r  r   g       @)ZclipnormZ	clipvaluer   znot supportedr   rm   ra   r   g      @r  )r"   r&   r  rw   rn   r   r$   r   r   r   r   ry   r   r   r   r   r   r   r   r   r)  r   r   )r4   rb   r  ri   rj   r  r,   r   r-   r-   r.   test_gradient_clipping  s,   
"z>TestDistributionStrategyWithKerasModels.test_gradient_clippingc           	      C   s   t |tjjjtjjjjjfr| d G dd dtj	j
}dd }tdtd}}|  | }t|g}tjd|gd}W d    n1 sMw   Y  ||d	 |j||d
dd | | |jd | | |jd d S )Nz+Not supported with `CentralStorageStrategy`c                   @   r   )z\TestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.MyLayerc                 S   s   t d| _t d| _d S )Nr   r  r
  r  r-   r-   r.   r    r  zbTestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.MyLayer.buildc                 S   s   || j  | j S r0   r  r  r-   r-   r.   r:     r   zaTestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.MyLayer.callNr  r-   r-   r-   r.   r    r  r  c                 S   s   dd | D S )Nc                 S   s   g | ]\}}t ||fqS r-   )ry   	ones_like)r_  gr   r-   r-   r.   r`    s    zyTestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.custom_transform.<locals>.<listcomp>r-   )Zgrads_and_varsr-   r-   r.   custom_transform  s   zeTestDistributionStrategyWithKerasModels.test_custom_gradient_transformation.<locals>.custom_transformr  r   )Zgradient_transformersr   r   rm   r          g       )r   ry   r   r   r   r   r   r   r"   r&   r  rw   rn   r   r$   r   r   r   r   r)  r   r   )	r4   rb   r  r  ri   rj   r  r,   r   r-   r-   r.   #test_custom_gradient_transformation  s*   

zKTestDistributionStrategyWithKerasModels.test_custom_gradient_transformationc                 C   s   |  F tjjdd}tjjddd|}tj|g|gd}|jddd	gd
 tjd	d}tjj
ddd}|j||ddd W d    d S 1 sMw   Y  d S )Nr   rF   rK   r    r!   rN   Zrmspropsparse_categorical_crossentropysparse_categorical_accuracyr   r   r   r   r   rx   r   rS  rm   r   rB  )r   r"   r&   rP   r'   rA   r   rw   r  r6  randintr   )r4   rb   r   outr,   ri   rj   r-   r-   r.   *test_distribution_strategy_one_dimensional  s   
"zRTestDistributionStrategyWithKerasModels.test_distribution_strategy_one_dimensionalr   )rb   r   	reductionc                 C   s   t jt dd }t jd}t jd}tjj||f}|d}| }|j	dt
jj|dd |j|d	d
dd}|   | }	|	j	dt
jj|dd |	j|d	d
dd}
W d    n1 scw   Y  | |jd |
jd d d S )Nc                  S   sV   t d} t jjddd| }t jjddd|}t jjddd|}t | |}|S Nr   r   ro   r  rm   r"   rP   r&   r'   rA   r9   x1Zx2rO   r,   r-   r-   r.   
_get_model	  s   
zpTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_loss_reduction_types.<locals>._get_modelr  r   rm   r   r  )r  r  r   rm   F)rC  r   r\  r   r2  )rw   r  seedrY   ry   rz   r{   r|   r_   r   r"   rj  MeanSquaredErrorr   r   r  rd  )r4   rb   r  r!  ri   rj   r`   r,   rd  ds_model
ds_historyr-   r-   r.   4test_distribution_strategy_with_loss_reduction_types
	  s.   

z\TestDistributionStrategyWithKerasModels.test_distribution_strategy_with_loss_reduction_typesc           	      C   s   dd }t dd}| }|d |j|dd}|  | }|d |j|dd}W d    n1 s9w   Y  | |j|j d S )Nc                  S   sv   t d} t jjddd| }t jjddd|}t jjddd|}t | |}|t| |t| |S r  )r"   rP   r&   r'   rA   r>  ry   rk  r  r-   r-   r.   _make_model_with_add_lossD	  s   
z|TestDistributionStrategyWithKerasModels.test_distribution_strategy_with_symbolic_add_loss.<locals>._make_model_with_add_lossr  rx   r  rm   r  )rw   rn   r6  r   r   r   r)  rd  )	r4   r   rb   r(  ri   r,   rd  r%  r&  r-   r-   r.   1test_distribution_strategy_with_symbolic_add_loss<	  s   



zYTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_symbolic_add_lossc           	      C   s   dd }t dd}t dd}| }| |jd |dd |j||d	dd
}| ! | }| |jd |dd |j||d	dd
}W d    n1 sUw   Y  | |j	|j	 d S )Nc                     sl   t d} t jjddd| }t jjddd|}t jjddd  |}t | |}| fdd |S )Nr   r   ro   r  rm   c                      s   dt  j S )Ng      Y@)ry   rk  Zkernelr-   r   r-   r.   r   i	  r  zTestDistributionStrategyWithKerasModels.DISABLED_test_distribution_strategy_with_callable_add_loss.<locals>._make_model.<locals>.<lambda>)r"   rP   r&   r'   rA   r>  r  r-   r*  r.   _make_modelb	  s   
zwTestDistributionStrategyWithKerasModels.DISABLED_test_distribution_strategy_with_callable_add_loss.<locals>._make_modelr  rx   r"  rm   r  r   r   r  )
rw   rn   r6  r-  rj  r   r   r   r)  rd  )	r4   rb   r+  ri   rj   r,   rd  r%  r&  r-   r-   r.   :DISABLED_test_distribution_strategy_with_callable_add_loss\	  s   

zbTestDistributionStrategyWithKerasModels.DISABLED_test_distribution_strategy_with_callable_add_lossc           
         
  G dd dt jj  fdd}tdd}tdd}| }| |jd |d	d
 |j	||||fddd}|
 3 | }| |jd |d	d
 |j	||||fddd}t rcdnd}	| |j|	 W d    n1 svw   Y  | |j|j d S )Nc                   @   r   )zhTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_call.<locals>.Biasc                 S   s   | j dddd| _d S )Nbiasro   r-   rG   rN  rF   )r   r.  rP  r-   r-   r.   r  	  s   znTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_call.<locals>.Bias.buildc                 S   s    | j t|ddd || j S )Nr.  meanrG   aggregation)
add_metricry   rk  r.  r8   r-   r-   r.   r:   	  s   
zmTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_call.<locals>.Bias.callNr  r-   r-   r-   r.   Bias	  s    r4  c                     L   t d} t jjddd| }  |}t jjddd|}t | |}|S r  r  r  r4  r-   r.   _make_model_with_add_metric	     

zTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_call.<locals>._make_model_with_add_metricr  rx   r"  rm   r  r   r   r   rY  r   r"   r&   r  rw   rn   r6  r-  r   r   r   r   ry   r  r)  rd  
r4   rb   r7  ri   rj   r,   rd  r%  r&  metrics_lenr-   r6  r.   2test_distribution_strategy_with_add_metric_in_call}	  s*   
zZTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_in_callc           
         r-  )Nc                   @   r   )zgTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_object.<locals>.Biasc                 S   s&   | j dddd| _tjjdd| _d S )Nr.  ro   r-   r/  r0  rJ   )r   r.  r"   r   ZMeanr0  rP  r-   r-   r.   r  	  s   zmTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_object.<locals>.Bias.buildc                 S   s   |  | | || j S r0   )r3  r0  r.  r8   r-   r-   r.   r:   	  s   
zlTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_object.<locals>.Bias.callNr  r-   r-   r-   r.   r4  	  s    r4  c                     r5  r  r  r  r6  r-   r.   "_make_model_with_add_metric_object	  r8  zTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_object.<locals>._make_model_with_add_metric_objectr  rx   r"  rm   r  r   r   r9  r:  )
r4   rb   r>  ri   rj   r,   rd  r%  r&  r<  r-   r6  r.   1test_distribution_strategy_with_add_metric_object	  s*   
zYTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_objectc           
      C   s   dd }t dd}t dd}| }| |jd |dd |j||||fd	d	d
}| 3 | }| |jd |dd |j||||fd	d	d
}t	 rWd	nd}	| |j|	 W d    n1 sjw   Y  | 
|j|j d S )Nc                  S   sX   t d} t jjddd| }t jjddd|}t | |}|jt|ddd |S )	Nr   r   ro   r  rm   Zmid_meanr0  r1  )r"   rP   r&   r'   rA   r3  ry   rk  )r9   r   rO   r,   r-   r-   r.   r7  	  s   
zTestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_outside_call.<locals>._make_model_with_add_metricr  rx   r"  rm   r  r   r   r9  )rw   rn   r6  r-  r   r   r   r   ry   r  r)  rd  r;  r-   r-   r.   7test_distribution_strategy_with_add_metric_outside_call	  s(   	

z_TestDistributionStrategyWithKerasModels.test_distribution_strategy_with_add_metric_outside_callc           	      C   s   G dd dt jj}t | g}tg dg dg}|j|dd}tddgddgddgg}tg d	}tdd
g}| |j| | |j	| | |j
| d S )Nc                   @   s   e Zd ZdZdd ZdS )zTTestDistributionStrategyWithKerasModels.test_sparse_tensor_outputs.<locals>.ToSparsez5Create a sparse tensor based on a given dense tensor.c                 S   s<   t t |d}t ||}t j|dd}t j|||dS )Nr   Zint64)Zout_type)dense_shape)ry   where	not_equalZ	gather_ndrF   ZSparseTensor)r4   r9   indicesvaluesrF   r-   r-   r.   r:   
  s   zYTestDistributionStrategyWithKerasModels.test_sparse_tensor_outputs.<locals>.ToSparse.callN)r<   r=   r>   __doc__r:   r-   r-   r-   r.   ToSparse
  s    rG  rm   r   r   r   rK   r   r   r   r   rm   )rm   r   rK   rK   )r"   r&   r  r$   rw   r  r   r.  rD  rE  rA  )	r4   rb   rG  r,   rJ  r}  Zexpected_indicesexpected_valuesZexpected_dense_shaper-   r-   r.   test_sparse_tensor_outputs
  s   	zBTestDistributionStrategyWithKerasModels.test_sparse_tensor_outputsc                 C   sh   G dd dt jj}t |ddg}tg dg dg}|j|dd}d	gdd
gg}| || d S )Nc                       s*   e Zd ZdZd fdd	Zdd Z  ZS )zTTestDistributionStrategyWithKerasModels.test_ragged_tensor_outputs.<locals>.ToRaggedz5Create a ragged tensor based on a given dense tensor.rm   c                    s"   t  jdi | || _|| _d S )Nr-   )r1   r2   _padding_ragged_rank)r4   rr  ragged_rankr  r6   r-   r.   r2   ;
  s   
z]TestDistributionStrategyWithKerasModels.test_ragged_tensor_outputs.<locals>.ToRagged.__init__c                 S   s   t jj|| j| jdS )N)rr  rN  )ry   ZRaggedTensorZfrom_tensorrL  rM  r8   r-   r-   r.   r:   @
  s   
zYTestDistributionStrategyWithKerasModels.test_ragged_tensor_outputs.<locals>.ToRagged.callrl   )r<   r=   r>   rF  r2   r:   r?   r-   r-   r6   r.   ToRagged8
  s    rO  r   rq  rH  rI  r   r   rm   rK   )r"   r&   r  r$   rw   r  r   r.  )r4   rb   rO  r,   rJ  r}  rJ  r-   r-   r.   test_ragged_tensor_outputs0
  s   zBTestDistributionStrategyWithKerasModels.test_ragged_tensor_outputsc                    s   d dd } fdd}  3 | tdtj fdd}| }td	D ]}|D ]}|| q3q/W d    d S 1 sFw   Y  d S )
Nr   c                  S   sj   t jjdd} t jjdd}t jjddd| }t jd|}t j| |g|}|t j|| |S )Nrl   r  r   r   r!   rm   )	r"   r&   rP   r'   r#   rA   r>  rj  mean_squared_error)r9   labelsri   rj   r,   r-   r-   r.   r!  Y
  s   zhTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>._get_modelc                     sJ   t jdd} d|  }| d} |d}tjj| |f}| }|S )Nr   rm   rK   rx   )	rw   r  r  r6  ry   rz   r{   r|   r_   )Zx_trainZy_trainr`   r   r-   r.   	_get_datab
  s   


zgTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>._get_datag?c                    s6    fdd}j || fd}jtjjj|d dS )Nc                    s   t  B}| }t j dd  tjdksJ t jd   }tj	| d |}t j
j| d}t jj|| W d    n1 sIw   Y  ||j}t|j |S )Nc                 S   s   d S r0   r-   r*  r-   r-   r.   r   w
  r   zTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>.train_step.<locals>.step_fn.<locals>.<lambda>rm   r   )Zglobal_batch_size)ry   r	  r   Zget_replica_contextZ
merge_calllenrj  r?  r"   rQ  nnZcompute_average_lossr   r   Zassert_equalr
  r  r  r  )r9   r  logitsZloss_from_modelr   Zgrads)ra   r,   r   r-   r.   step_fnq
  s*   


zyTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>.train_step.<locals>.step_fn)r  )Zaxis)r  reducery   r   ZReduceOpSUM)Zdist_inputsrW  Zper_replica_lossesra   rb   r,   r   r-   r.   
train_stepo
  s   "zhTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call.<locals>.train_stepr   )r   r   r   ry   r  r  rc  )r4   rb   r!  rS  r[  r`   rZ   ri   r-   rZ  r.   ,test_correctness_of_add_loss_with_merge_callN
  s   		
	
*
"zTTestDistributionStrategyWithKerasModels.test_correctness_of_add_loss_with_merge_call)r   c              	   C   s   t jddd}ttj|ddddid}tjjjj	
|}| |tjjjj	j
 | td	9 |  t }tjjjjd
d}d}||| W d    n1 sSw   Y  W d    d S W d    d S 1 skw   Y  d S )NrK   r   )Znum_workersZnum_psZworkerrm   ZGPUr   )cluster_specZ	task_typetask_idZnum_acceleratorszParameterServerStrategy*r   r  r   )r   Zcreate_in_process_clusterr   ry   r'  ZClusterSpecr   r   r   r   r   ZassertIsInstancer   NotImplementedErrorr   r/   r(  r   )r4   r]  Zcluster_resolverrb   r,   r   r   r-   r-   r.   ,test_unimplemented_parameter_server_strategy
  s>   


"zTTestDistributionStrategyWithKerasModels.test_unimplemented_parameter_server_strategyN)1r<   r=   r>   ry   r   r   r   rW  r   r  r  r   r   r   r  r  r  r  r  r	  r  r  r   r   r  r   r   r   ReductionV2ZAUTOSUM_OVER_BATCH_SIZErY  r'  r)  r   r,  r=  r   r   r?  r@  r   r	   rK  rP  r
   r   r\  r`  r-   r-   r-   r.   r    s   







3


5








$





!


$







$







-






,


%








Ir  c           
         s(  t j| dd}t jjdddd|}t jjdd|}t jjd	ddd|}t jjdd|}t jjd
d|}ttt	|d}t jj
dddd|}t jjdd dd|}t jj
|dd|}t j||d}	|	t j fdd| |	j|ddd |	||  |	j|ddd |	S )NimagesrJ   r   r   r   r  r   r   Z	pool_sizer   	embedding   sparse_embeddingrM   c                 S   s   t t | dS Nrg  ry   rk  r?  rs  r-   r-   r.   r   
      z6_functional_with_add_loss_and_metric.<locals>.<lambda>l1_lossrV  rN   c                    s   |   S r0   r-   rs  l2r-   r.   r   
  ru  r0  l2_lossr2  rG   )r"   rP   r&   rz  MaxPooling2Dr  ry   rk  r?  squarer'   r{  rA   r>  r3  )
r   rV   l1ro  r9   ri   rp  rm  rO   r,   r-   rn  r.   $_functional_with_add_loss_and_metric
  s(   ru  c                    s  t t jjddd| dt jjddt jjddddt jjddt jjd	d
t jjddddt jj|dd
g}|dd} t	
t	|d }|| |jt j fdd|ddd |d	d}t jjfdddd
|}|| |j| ddd |S )Nr   r   r   r  r   r   r   re  r   rd  rf  rJ   rh  ri  rM   rV  rg  c                    s   t |  S r0   )ry   dividers  )rt  r-   r.   r   
  s    z6_sequential_with_add_loss_and_metric.<locals>.<lambda>r0  rm  rq  c                    s    t t | |  d S rj  rk  rs  rn  r-   r.   r   
  s    rp  )r"   r$   r&   rz  rr  r  r'   r|  Zget_output_atry   rk  r?  r>  r3  r{  )r   rV   rt  ro  r,   ri   rm  rp  r-   )rt  ro  r.   $_sequential_with_add_loss_and_metric
  s>   

rx  c                 C   s   t t jjddd| dt jjddt jjddddt jjddt j t jjd	dd
t jj|ddg}t j| dd}||}t j||d}|t	
|}|t	|}	t	t	t	|| d}
||
|  |j|
ddd t	t	t	||	 d}|||  |j|ddd |S )Nr   r   r   rv  r   re  r   rd  rh  r!   rV  rJ   rc  rN   rg  r0  rp  rq  rm  )r"   r$   r&   rz  rr  r  r'   rP   rA   ry   Z
zeros_liker  rk  r?  rs  r>  r3  abs)r   rV   rt  ro  Z
base_modelr9   rV  r,   Zzero_logitsZ
one_logitsrp  rm  r-   r-   r.   _functional_with_layer_reuse   s0   rz  c                
   @   sT   e Zd ZdZejjjejj	j
e ejj	jjeeegdgdgddd ZdS )9TestDistributionStrategyWithMultipleAddLossAndMetricCallsz=Tests complex models with multiple add loss and metric calls.r  r   )model_fnrt  ro  c           
      C   sz  t jt tjjt jjdd	t j
t jjddddf}|djd|j dd	}tjj|d jd
d  }| . ||d||}|jtjjdtjjdtjjdtj tjjddgd W d    n1 sow   Y  tdD ]}| | qxt!t"|j#|$|}	| %|	d dd | &|	d d | &|	d d | '|	d | |	d |  |	d  |	d d d S )N)r      r}  rm   rS  r   r   r   r   rW   Tr\   rm   rI  )from_logitsr  )r  r  r   r  g{Gz?r   rp  r  rm  r  r   gư>)(rw   r  r#  rY   ry   rz   r{   r|   uniformr6  rx   r  r\  r_   r   r   Zget_structurerF   r   r   r"   
optimizersadam_v2Adamrj  ZSparseCategoricalCrossentropyr   ra  rb  r   ZSparseCategoricalAccuracyrc  r   dictr  metrics_namesr   ZassertBetweenZassertGreaterZ
assertNear)
r4   rb   r|  rt  ro  r`   r   r,   unused_epochresultsr-   r-   r.   test_fit_and_evaluate$  sP   




zOTestDistributionStrategyWithMultipleAddLossAndMetricCalls.test_fit_and_evaluateN)r<   r=   r>   rF  ry   r   r   r   rW  r   r   r   r   ru  rx  rz  r  r-   r-   r-   r.   r{    s     


r{  c                       s2   e Zd ZdZ fddZdd Zd	ddZ  ZS )
DeterministicModelzDeterministic Model that always outputs the same initial result.

    It verifies the `call` method is run inside the same distribution
    strategy that the model was initially passed.
    c                    s   t    d | _|| _d S r0   )r1   r2   ri   strategy)r4   r  r6   r-   r.   r2   i  s   

zDeterministicModel.__init__c                 C   s   t t jdd| _d S )Nr-   r  )ry   r  rn   ri   rP  r-   r-   r.   r  n  s   zDeterministicModel.buildNc                 C   s&   t j }|| jurtd| j| S )Nz0Model must execute call w/ the original strategy)ry   r   Zget_strategyr  r   ri   )r4   r9   r@  maskZactive_strategyr-   r-   r.   r:   q  s   


zDeterministicModel.call)NN)r<   r=   r>   rF  r2   r  r:   r?   r-   r-   r6   r.   r  b  s
    r  c                   @   sl   e Zd ZdZejjjejj	jj
edgddd Zejjjejj	jj
ejjjjdgddd ZdS )	TestModelCapturesStrategyz0Tests that model creation captures the strategy.r   r   c           
      C   s  t jjt jddt jddf}|d|j }|  t|}t	j
jd}W d    n1 s3w   Y  |j|t	j dgd | |jj d tdD ]}|| qR||}tt|j|}| |d d	 | |d
 d t jdd |jD }|D ]}| |j | q|jj!D ]}| |j | q|j"D ]}| |j | q|  t	j# }	W d    n1 sw   Y  |j|t	j |	gd t j$% r	| &t'd |jt	j
jdt	j t	j# gd W d    d S 1 sw   Y  d S d S )Nr~  r  rW   rI  Zbinary_accuracyr  r   r   r   r   r  c                 S   s   g | ]}|j qS r-   )	variables)r_  metricr-   r-   r.   r`    rl  zCTestModelCapturesStrategy.test_fit_and_evaluate.<locals>.<listcomp>zAll metrics must be created in)(ry   rz   r{   r|   rn   r_   r   r   r  r"   r  r  r  r   rj  r$  r   r   Z
iterationsnumpyrc  r   r   r  r  r  r)  nestflattenr   
assertTrueextendedZvariable_created_in_scopeZ_weightsr  ZBinaryAccuracyr   Zhas_strategyr   r   )
r4   rb   r`   r,   r   r  r  Zmetric_varsvarr  r-   r-   r.   r  {  sl   







$z/TestModelCapturesStrategy.test_fit_and_evaluatec                 C   s@  t j|  d}dd }| }tjdd }}|j||dd || | 3 | }|	| t
|jtjr@|j|j | |j  | t|j d  W d    n1 s_w   Y  |  | }W d    n1 svw   Y  |	| t
|jtjrd S | |j  | t|j d  d S )	NZckptc                  S   s6   t jt jdg} | jddd | d dg | S )Nrm   rR  r   )r   r   )r"   r#   r$   r&   r'   r   r  r+   r-   r-   r.   create_model  s   
z>TestModelCapturesStrategy.test_optimizer.<locals>.create_model)rm   rm   r  rm   r  r   )r  r  r  r  ry   rn   r   Zsave_weightsr   Zload_weightsr   r   optimizer_experimentalZ	Optimizerr  r  ZassertNotEmptyr  r  r   Zis_distributed_variable)r4   rb   temp_dirr  r,   ri   rj   r-   r-   r.   test_optimizer  s<   





z(TestModelCapturesStrategy.test_optimizerN)r<   r=   r>   rF  ry   r   r   r   rW  r   r   r   r  Zmirrored_strategy_with_one_cpur  r-   r-   r-   r.   r  x  s    



I

r  __main__r0   )WrF  r  r  rw   Ztensorflow.compat.v2r   r   ry   Zabsl.testingr   r"   r   Zkeras.distributer   r   r   r   Z&keras.distribute.strategy_combinationsr   r	   r
   r   r   Zkeras.enginer   Zkeras.mixed_precisionr   Z'keras.optimizers.optimizer_experimentalr   r  Zkeras.optimizers.optimizer_v2r   r   Zkeras.testing_infrar   Zkeras.utilsr   r   Z-tensorflow.python.distribute.cluster_resolverr   rY   rX   r(   r*   r/   rB   rT   r[   rc   rk   rq   r~   r   r   r   r   r   r   r   r   r   r   r   r  r  r   r   ZTestCaser   rX  r  r  Zrun_all_without_tensor_float_32r  ru  rx  rz  r{  rA   r  r  r<   Zenable_v2_dtype_behaviorr   r   Zmulti_process_runnerZ	test_mainr-   r-   r-   r.   <module>   s   	

.		
B

    
i
      
F
3:

      A#

C 
