o
    Nem%  ã                   @   sÈ   d Z ddlZddlZddlZddlZddlmZ eeej	ddZ
ddlmZmZmZmZmZmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZmZ ddgZG dd„ deƒZ G dd„ deƒZ!dS )zDefination of TrainerFactory.é    N)Ú
get_loggerz&%(asctime)s-%(levelname)s: %(message)s)Úfmté   )ÚMultiTrainerÚDistMultiTrainerÚPipelineTrainerÚHeterXpuTrainerÚPSGPUTrainerÚHeterPipelineTrainer)ÚHogwildÚDownpourSGDÚDownpourLiteÚSectionÚDownpourSGDOPTÚHeterSection)ÚVariable)ÚProcessÚManagerÚTrainerFactoryÚFetchHandlerMonitorc                   @   s"   e Zd ZdZdd„ Zddd„ZdS )r   zš
    Create trainer and device worker.
    If opt_info is not None, it will get configs from opt_info,
    otherwise create MultiTrainer and Hogwild.
    c                 C   s   d S ©N© ©Úselfr   r   úLD:\Projects\ConvertPro\env\Lib\site-packages\paddle/fluid/trainer_factory.pyÚ__init__)   s   zTrainerFactory.__init__Nc                 C   s‚  d }d }|st ƒ }tƒ }| |¡ |S | dd¡}| dd¡}tƒ | ƒ }tƒ | ƒ }|d ur/| d¡d ur>| |d ¡ | d¡d urL| |d ¡ | d¡d urZ| |d ¡ | d¡d urh| |d ¡ | d	¡d urv| 	|d	 ¡ | d
¡d urt
| d
¡ƒdkr| |d
 ¡ | d¡d ur¤t
| d¡ƒdkr¤| |d ¡ | d¡d ur²| |d ¡ | d¡d urÀ| |d ¡ | d¡d ur×t
| d¡ƒdkr×| |d ¡ | d¡d urå| |d ¡ | d¡d uró| |d ¡ | d¡d ur| |d ¡ | d¡d ur| |d ¡ | d¡d ur | |d ¡ | d¡d ur/| |d ¡ d|v rº| |d ¡ | |d ¡ | d¡d urQ| |d ¡ | d¡d ur`| |d ¡ | d¡d uro| |d ¡ | d¡d ur~| |d ¡ | d¡d ur| |d ¡ | d¡d urœ| |d ¡ | d¡d ur«| |d ¡ | d¡d urº| |d ¡ | |¡ |S )NÚtrainerr   Údevice_workerr   ZtrainersZ
trainer_idZ	dump_slotZmpi_rankZmpi_sizeZdump_fieldsr   Zdump_fields_pathZdump_file_numZdump_converterZ
dump_paramZworker_placesZ
use_ps_gpuZis_dump_in_simple_modeZenable_random_dumpZdump_intervalZrandom_with_lineidZ
fleet_descZuse_cvmZno_cvmZ%scale_sparse_gradient_with_batch_sizeZscale_datanormZadjust_ins_weightZ
copy_tableZcheck_nan_var_namesZ
loss_names)r   r   Z_set_device_workerÚgetÚglobalsZ_set_trainersZ_set_trainer_idZ_set_dump_slotZ_set_mpi_rankZ_set_mpi_sizeÚlenZ_set_dump_fieldsZ_set_dump_fields_pathZ_set_dump_file_numZ_set_dump_converterZ_set_dump_paramZ_set_worker_placesZ_set_use_ps_gpuZ_set_is_dump_in_simple_modeZ_set_enable_random_dumpZ_set_dump_intervalZ_set_random_with_lineidZ_set_fleet_descZ_set_use_cvmZ_set_no_cvmZ&_set_scale_sparse_grad_with_batch_sizeZ_set_scale_datanormZ_set_adjust_ins_weightZ_set_copy_table_configZ_set_check_nan_var_namesZ_set_loss_names)r   Zopt_infor   r   Ztrainer_classZdevice_worker_classr   r   r   Ú_create_trainer,   sº   
Hº
ÿÿÿÿÿÿÿÿÿ
ÿÿÿÿÿ
zTrainerFactory._create_trainerr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r!   r   r   r   r   r   "   s    c                   @   s0   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
S )r   zN
    Defination of FetchHandlerMonitor class,
    it's for fetch handler.
    c                 C   s2   || _ tj| j|| j fd| _t ¡ | _d| _d S )N)ÚtargetÚargsF)Úfetch_instanceÚ	threadingÚThreadÚhandler_launch_funcÚfetch_threadÚLockÚrunning_lockÚrunning)r   ÚscopeÚhandlerr   r   r   r   „   s   ÿ

zFetchHandlerMonitor.__init__c                 C   sh  |}|j }i }|jD ]}t|j| tƒr|||j| j< q
t d |¡¡ ||d< q
d}	 | j 	¡  | j
dkr9d S ||k rGt d¡ |d7 }ngd}i }|D ]}| |¡}	|	||< |	d krft d || ¡¡ qMi }
|D ]=}|| }|| d kr|d |
|< qk||  ¡ |
|< |
|  ¡ }t|ƒdkr”tdƒ‚|
|  ¡ r¤t |
| ¡|
|< qkd |
|< qk| |
¡ | j ¡  q-)	Nz!the value of {} is not a VariablezNone.varr   TFr   z {} value currently not availableaŒ  Some of your fetched tensors                                             hold LoD information.                                             They can not be completely cast                                             to Python ndarray. We can                                             not return LoDTensor itself directly,                                             please choose another targets)Úperiod_secsZvar_dictÚ
isinstancer   ÚnameÚlocal_loggerÚwarningÚformatr.   Úacquirer/   ÚtimeÚsleepZfind_varZ
get_tensorÚlodr    ÚRuntimeErrorZ_is_initializedÚnpÚarrayr1   Úrelease)r   r0   r1   r(   r2   Zvar_name_to_keyÚkeyZelapsed_secsZ
fetch_dictÚvarZres_dictZ	user_namer;   r   r   r   r+   ‹   s\   
ÿ





ÿÿ€


Øz'FetchHandlerMonitor.handler_launch_funcc                 C   s4   | j  ¡  d| _| j  ¡  | j d¡ | j ¡  dS )zH
        start monitor,
        it will start a monitor thread.
        TN)r.   r8   r/   r?   r,   Ú	setDaemonÚstartr   r   r   r   rC   Á   s
   

zFetchHandlerMonitor.startc                 C   s   | j  ¡  d| _| j  ¡  d S )NF)r.   r8   r/   r?   r   r   r   r   ÚstopÌ   s   
zFetchHandlerMonitor.stopN)r"   r#   r$   r%   r   r+   rC   rD   r   r   r   r   r   ~   s    6)"r%   r)   r9   ÚloggingÚnumpyr=   Zpaddle.fluid.log_helperr   r"   ÚINFOr5   Ztrainer_descr   r   r   r   r	   r
   r   r   r   r   r   r   r   Z	frameworkr   Úmultiprocessingr   r   Ú__all__Úobjectr   r   r   r   r   r   Ú<module>   s"   þ  \