o
    Me}                     @   s  d Z ddlZddlm  mZ ddlmZ ddlm	Z
 ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ dd Zd"ddZdd Z 								d#ddZ!	d$ddZ"	d$ddZ#G dd dej$Z%dd Z&G d d! d!ej$Z'dS )%zBPart of the Keras training engine related to distributed training.    N)backend)	callbacks)distribute_coordinator_utils)distributed_training_utils_v1)partial_batch_padding_handler)training_arrays_v1)training_utils_v1)Progbar)ModeKeys)	input_lib)
tf_loggingc                 C   s   |  |}|j|j|j|jfS N)Z_make_execution_functioninputsoutputs
updates_opZsession_kwargs)modelmode	exec_func r   TD:\Projects\ConvertPro\env\Lib\site-packages\keras/engine/training_distributed_v1.py_per_replica_execution_function#   s   
r   c                 C   s4   |j rtj|| |||d d S t|| ||| d S )N)r   targets)_compile_distribution
dist_utilsZclone_model_on_replicasZ_build_distributed_network)strategyr   r   r   r   r   r   r   _build_model-   s   


r   c                    s    fdd}|S )aA  Create step fn.

    Args:
      model: a Keras Model instance.
      mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
      strategy: a `tf.distribute.Strategy` instance.
      output_labels: the output labels for the step function.

    Returns:
      A step function to run by `tf.distribute.Strategy`.
    c                    s  t  ttfrt dkr \ }nd}t  tr# fddjD  t | jjt	t
fd\}}}}t
||||\}}}	}
tj||f|	dt d d|
}t|jD ]\}}|d	krqtjjj}ntjjj}| ||| qc|jS )
z"A step fn that returns update ops.   Nc                    s   g | ]} | qS r   r   ).0Z
input_namer   r   r   
<listcomp>S   s    z9_make_train_step_fn.<locals>._step_fn.<locals>.<listcomp>argsZdistributed_Z	_function)updatesnameloss)
isinstancetuplelistlendictZ_feed_input_namesr   extendedZcall_for_each_replicar   r   get_distributed_modelZunwrap_valuesr   functionstrzipr   tf
distributeReduceOpSUMMEANZset_last_step_outputr   )ctxr   r   Zgrouped_inputsZgrouped_outputsZgrouped_updatesZgrouped_session_argsZ
all_inputsZall_outputsZall_updatesZall_session_argsZcombined_fnlabeloutput	reduce_opr   r   output_labelsr   r   r   _step_fnE   sX   




z%_make_train_step_fn.<locals>._step_fnr   )r   r   r   r9   r:   r   r8   r   _make_train_step_fn8   s   ?r;   d      c
           '   
   C   s$  t j}
| j}t||jj}tj|ddd}t	||}tj
|dd}|  | jp*g }t| t j||}i }td|d< |  D ]}| }t|j|j||j< q@|jj||||d}|j}|j}t|}| jrot| |
 tj|| ||||d	|
d
}|jjg||jj  }||jj r|||jj  t|}| |
 | !||
}t"||D ]}t#|  |$| i }d}d}d}||k r)|| } |d| d}!|%|
d||! |du s| |krt& '|(|  | }zt)||g\}"}#W n tj*j+y	   t,-d| |  Y n w |!.|# |%|
d||! ||  }|d7 }|j/j0r%n||k s|rgt12|	|rgt,3d| | jrDt4| t j t5| ||||d}$t6|$t7sV|$g}$t8||$D ]\}%}&|&|d|% < q[|9|| |j/j0rt nqd| _:|;|
 | jrt4| t j |<ddd | j=S )a  Fit loop for training with TPU tf.distribute.Strategy.

    Args:
        model: Keras Model instance.
        dataset: Dataset that returns inputs and targets
        epochs: Number of times to iterate over the data
        verbose: Integer, Verbosity mode, 0, 1 or 2
        callbacks: List of callbacks to be called during training
        initial_epoch: Epoch at which to start training
            (useful for resuming a previous training run)
        steps_per_epoch: Total number of steps (batches of samples)
            before declaring one epoch finished and starting the
            next epoch. Ignored with the default value of `None`.
        val_dataset: Dataset for validation data.
        validation_steps: Number of steps to run validation for
            (only if doing validation from data tensors).
            Ignored with the default value of `None`.
        validation_freq: Only relevant if validation data is provided. Integer
            or `collections.abc.Container` instance (e.g. list, tuple, etc.). If
            an integer, specifies how many training epochs to run before a new
            validation run is performed, e.g. `validation_freq=2` runs
            validation every 2 epochs. If a Container, specifies the epochs on
            which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
            validation at the end of the 1st, 2nd, and 10th epochs.

    Returns:
        Returns `None`.

    Raises:
        ValueError: in case of invalid arguments.
    Zint32steps_per_run)valuedtyper#   r=   r   Zlearning_phaseg    cAr$   )Z
iterationsinitial_loop_valuesstepsdo_validationepochssteps_per_epochverboseZ
count_moder   r   N)batchsizeZ	num_stepsbeginzYour dataset iterator ran out of data; interrupting training. Make sure that your dataset can generate at least `steps_per_epoch * epochs` batches (in this case, %d batches).endz#Running validation at fit epoch: %s)rC   rH   r   Zval_T)>r
   TRAIN_distribution_strategyminr*   r>   r   variabler   get_iteratordistributed_scope	__enter__metrics_namesr;   r/   ZconstantZ_get_training_eval_metricsresultZzerosshaper@   r#   Z"experimental_run_steps_on_iteratorZrun_opZlast_step_outputsboolr   "_copy_weights_to_distributed_modelcbksconfigure_callbacksappendr(   _call_begin_hookZ#_maybe_load_initial_epoch_from_ckptrange_reset_metricsZon_epoch_begin_call_batch_hookZget_sessionrunZassignbatch_get_valueerrorsOutOfRangeErrorloggingwarningupdater   Zstop_trainingr   Zshould_run_validationinfoZ_copy_weights_to_original_modelexperimental_tpu_test_loopr%   r'   r.   Zon_epoch_endZ_successful_loop_finish_call_end_hook__exit__history)'r   datasetrF   rH   r   initial_epochrG   val_datasetvalidation_stepsvalidation_freqr   current_strategyZiteration_valuer>   iteratorscope
out_labelsZstep_fnrB   mZtensorr4   Ztrain_opoutput_tensorsrE   Zsteps_to_runtarget_stepsepochZ
epoch_logsZ
step_indexZprev_step_countcurrent_stepZ
step_count
batch_logs_r   Zval_outsr5   Zval_outr   r   r   experimental_tpu_fit_loop   s   +










 

r|   c              
      s  t j j}t||}tj|dd}|  j} fdd}	| }
|j	|	|
fd}i }t
||D ]\}}|dkrBtjjj}ntjjj}|j||dd||< q4tt| }|d	krdt|d
}jrmt  t tj|dd	||dt jd}|  dgtj }|dur|}ntdd}||k r|d	d}| d|| zt||g\}}W n tj j!y   d"|}t#$d|  |}Y nDw t%jD ]\}}|dkr||  || 7  < q|| ||< q|&|| }| d|| |d	kr|'|d	  |d	7 }||k s|d	kr|'| |(  |)ddd t|dkr9|d  |  < t|d	krD|d S |S )a  Test loop for evaluating with TPU tf.distribute.Strategy.

    Args:
        model: Keras Model instance.
        dataset: Dataset for input data.
        verbose: Integer, Verbosity mode 0 or 1.
        steps: Total number of steps (batches of samples)
            before declaring predictions finished.
            Ignored with the default value of `None`.
        callbacks: List of callbacks to be called during training

    Returns:
        Scalar loss (if the model has a single output and no metrics)
        or list of scalars (if the model has multiple outputs
        and/or metrics). The attribute `model.metrics_names` will give you
        the display labels for the outputs.
    r   rA   c                    s   t | ttfrt| dkr| \} }nd}tj jt | |fd t	t
  \}}}}t|g dd |D W  d   S 1 sGw   Y  dS )z-A fn that returns output of single test step.r   Nr    c                 S      g | ]}t |qS r   r/   identityr   outr   r   r   r   x      zEexperimental_tpu_test_loop.<locals>._test_step_fn.<locals>.<listcomp>)r%   r&   r'   r(   r/   r0   get_replica_context
merge_callr   r   r   r+   control_dependencies)r   r   r{   r   r"   r   r   r   r   _test_step_fng  s   

$z1experimental_tpu_test_loop.<locals>._test_step_fnr    r$   NZaxisr=   targetFrC   rD           TNumber of steps could not be inferred from the data, please pass the steps argument.rI   rJ   rK   ]Make sure that your dataset can generate at least `steps` batches (in this case, {} batches).@Your dataset iterator ran out of data; interrupting evaluation. rL   )*r
   TESTrN   r   rQ   rR   rS   rT   get_nextr`   r.   r/   r0   r1   r2   r3   reducegroupr'   valuesr	   r   rX   r^   rY   rZ   r\   r(   
ValueErrorr_   r   ra   rb   rc   formatrd   re   	enumerate	make_logsrf   ri   rj   )r   rl   rH   rC   r   rq   rr   rs   rt   r   Ztest_input_dataper_replica_outputsrv   r5   r6   r7   Ztest_opprogbarZoutsrw   ry   rz   r{   
batch_outswarning_msgir   r   r   rh   H  s   








 

rh   c              
      s  t j t|}d}|s@tj}t|\}}}	||_	|
|j|j|_||j}| }|j|dd}|	dur@||	}j}
t||
}tj|
dd}|   fdd}| }|
j||fd}t|
|}|d	krvt|d
}jrt  t tj|dd	||d d}|  t j!}dd t"|D }|dur|}nt#dd}||k r-|d	d}|$ d|| zt%&|}t'(||g\}}W n t%j)j*y   d+|}t,-d|  Y nCw t"|D ]}||
j. }||
j. }||| }|| /| q|0|| }|$ d|| |d	kr%|1|d	  |d	7 }||k s|d	kr7|1| |2  |3ddd t |d	krTt4j5|d dd}ndd |D }|rc|6|}|S )aX  Predict loop for predicting with TPU tf.distribute.Strategy.

    Args:
        model: Keras Model instance.
        dataset: Dataset for input data.
        verbose: Integer, Verbosity mode 0 or 1.
        steps: Total number of steps (batches of samples)
            before declaring `_predict_loop` finished.
            Ignored with the default value of `None`.
        callbacks: List of callbacks to be called during training

    Returns:
        Array of predictions (if the model has a single output)
        or list of arrays of predictions
        (if the model has multiple outputs).
    NT)Zdrop_remainderr   rA   c                    sr   t j jt | fd tt  \}}}}t |g dd |D W  d   S 1 s2w   Y  dS )z3A fn that returns output of single prediction step.r    c                 S   r}   r   r~   r   r   r   r   r     r   zKexperimental_tpu_predict_loop.<locals>._predict_step_fn.<locals>.<listcomp>N)	r/   r0   r   r   r   r   r   r+   r   )r   r{   r   r"   r   r   r   _predict_step_fn  s   

$z7experimental_tpu_predict_loop.<locals>._predict_step_fnr    r=   r   FrC   rD   c                 S   s   g | ]}g qS r   r   )r   r{   r   r   r   r   A  s    z1experimental_tpu_predict_loop.<locals>.<listcomp>r   r   rK   r   r   rL   r   c                 S   s   g | ]	}t j|d dqS )r   r   )npconcatenater   r   r   r   r   }  s    )7r
   PREDICTr   Zis_dataset_shape_fully_definedpadding_utilZPartialBatchPaddingHandlerZ_feed_output_shapesr   Z_get_dataset_attributesZpadded_batch_sizer   Zpadding_maskZupdate_maskmapZ	pad_batchZunbatchrI   ZprefetchrN   rQ   rR   rS   r   r`   Zflatten_per_replica_valuesr	   r   rX   r^   rY   rZ   r\   r(   Zoutput_namesr]   r   r_   r/   r   r   ra   rb   rc   r   rd   re   Znum_replicas_in_syncextendr   rf   ri   rj   r   r   Z
apply_mask)r   rl   rH   rC   r   Zdataset_fully_shapedZpadding_handler
batch_sizer{   Zprefetch_bufferrq   rr   rs   r   Zpredict_input_datar   rv   r   Znum_model_outputsZunconcatenated_outsrw   ry   rz   Zpredict_opsr   r   r   Zoutput_start_indexZoutput_end_indexZsingle_model_outputZprediction_resultr   r   r   experimental_tpu_predict_loop  s   













'


r   c                   @   sb   e Zd ZdZ															dddZ							dd	d
Z				dddZdS )$DistributionSingleWorkerTrainingLoopz;Training loop for distribution strategy with single worker.Nr=   r   Tr   c                 K   s  t j||jd t || t j|j|||tj|d\}}||||}|j	|||||||
|d}t 
|sY|j  |j||||||
d\}}}W d   n1 sTw   Y  d}|	rt|	\}}}t || t |j|||tj\}}|j	|||d|||
dd}n|rtdt|jrtj||||d	d
}|du rtdt st||||||||||d
S tj||||||||
||||d	dS )z%Fit loop for Distribution Strategies.)Zinput_callbacks	optimizer)validation_split)sample_weightclass_weightr   r   shufflerF   )r   r   r   r   r   NT)r   r   r   r   r   allow_partial_batchzHvalidation_split argument is not supported with distribution strategies.rG   
steps_namez^Number of steps could not be inferred from the data, please pass the steps_per_epoch argument.)rF   rH   r   rn   rm   rG   ro   rp   )r   rF   rH   r   Z
val_inputsr   rm   rG   ro   rp   r   )r   Zvalidate_callbacksr   validate_inputsprocess_batch_and_step_sizerN   r
   rM   _validate_or_infer_batch_size#_distribution_standardize_user_dataZis_distributing_by_cloningrs   Z_standardize_user_datar   Zunpack_validation_datar   r   r   is_tpu_strategyinfer_steps_for_datasetr/   executing_eagerlyr|   r   Zfit_loop)selfr   xyr   rF   rH   r   r   Zvalidation_datar   r   r   rm   rG   ro   rp   kwargsrl   r{   rn   Zval_xZval_yZval_sample_weightsr   r   r   fit  s   




z(DistributionSingleWorkerTrainingLoop.fitc	                 K   s   t || t |j|||tj\}}||||}|j||||dd}
t	|jrHt
j||
|dd}|du r;tdt sHt||
|||dS tj||
||||dS )	z*Evaluate loop for Distribution Strategies.T)r   r   r   rC   r   Nr   rH   rC   r   )r   r   rH   rC   r   )r   r   r   rN   r
   r   r   r   r   r   r   r   r   r/   r   rh   r   Z	test_loop)r   r   r   r   r   rH   r   rC   r   r   rl   r   r   r   evaluate  sH   z-DistributionSingleWorkerTrainingLoop.evaluatec           	      K   s   t j|dd t |j|||tj\}}||||}|j||dd}t	|jrGt
j|||dd}|du r:tdt sGt|||||dS tj||||||d	S )
z)Predict loop for Distribution Strategies.N)r   r   T)r   r   rC   r   r   r   )r   rH   rC   r   )r   r   r   rN   r
   r   r   r   r   r   r   r   r   r/   r   r   r   Zpredict_loop)	r   r   r   r   rH   rC   r   r   rl   r   r   r   predictH  s@   z,DistributionSingleWorkerTrainingLoop.predict)NNNr=   r=   Nr   NTNNr   NNr=   )NNNr=   NNN)Nr   NN)__name__
__module____qualname____doc__r   r   r   r   r   r   r   r     s@    
 
;r   c                    s    fdd}|S )zCDecorator handles multi worker training with distribution strategy.c                    s    fdd}t |jS )Nc                    s0     dd }t|}| d< fi  S )Nr   )popr   Zfilter_distributed_callbacks)r{   r   filtered_callbacks)r   methodr   r   r   
_worker_fnz  s   z=_train_with_multi_worker.<locals>.wrapper.<locals>._worker_fn)dcZrun_distribute_coordinatorrN   )r   r   r   r   )r   r   r   wrappery  s   z)_train_with_multi_worker.<locals>.wrapperr   )r   r   r   r   r   _train_with_multi_workerv  s   r   c                   @   s0   e Zd ZdZdd Zdd Zdd Zdd	 Zd
S )#DistributionMultiWorkerTrainingLoopz=Training loop for distribution strategy with multiple worker.c                 C   s
   || _ d S r   )_single_worker_loop)r   Zsingle_worker_loopr   r   r   __init__  s   
z,DistributionMultiWorkerTrainingLoop.__init__c                 O      t | jj|i |S r   )r   r   r   r   r!   r   r   r   r   r     
   
z'DistributionMultiWorkerTrainingLoop.fitc                 O   r   r   )r   r   r   r   r   r   r   r     r   z,DistributionMultiWorkerTrainingLoop.evaluatec                 O   s   | j j|i |S r   )r   r   r   r   r   r   r     s   z+DistributionMultiWorkerTrainingLoop.predictN)r   r   r   r   r   r   r   r   r   r   r   r   r     s    r   r   )r<   r=   Nr   NNNr=   )r   NN)(r   numpyr   Ztensorflow.compat.v2compatv2r/   Zkerasr   r   rY   Zkeras.distributer   r   r   r   Zkeras.enginer   r   r   r   Zkeras.utils.generic_utilsr	   Zkeras.utils.mode_keysr
   Ztensorflow.python.distributer   Ztensorflow.python.platformr   rd   r   r   r;   r|   rh   r   ZTrainingLoopr   r   r   r   r   r   r   <module>   sJ   

R
 C
 
 1 p