o
    Me’  ã                   @   sD   d dl mZmZmZ ddlZddlZddlmZ G dd„ deƒZ	dS )é   )Úget_gpu_processÚget_gpu_utilÚget_gpu_infoé    N)ÚThreadc                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )ÚWatcherc                 C   s¤   || _ d| _g | _| j jjp| j jjj| _t	| jƒdkr>t
j | j jjd | j jj¡¡}t
jt
j |¡dd t|dƒ| _nd S t| jd| _d| j_| j ¡  d S )Né   r   z
{}.gpu.logT)Úexist_okÚw)Útarget)ÚctxÚintervalZgpu_utilÚargsZdevicesÚnodeZdeviceÚlabelsÚgpusÚlenÚosÚpathÚjoinZlog_dirÚformatZjob_idÚmakedirsÚdirnameÚopenÚgpu_fdr   ÚwatchÚprocÚdaemonÚstart)Úselfr   Úfn© r!   ú]D:\Projects\ConvertPro\env\Lib\site-packages\paddle/distributed/launch/controllers/watcher.pyÚ__init__   s   ÿzWatcher.__init__c                 C   s€   t | jƒdks	d S |  ¡  d}| j |¡ | j d¡ | jj ¡ s2|  |¡ t	 
| j¡ | jj ¡ r!t| dƒr>| j ¡  d S d S )Nr   zDindex,utilization_gpu,memory_total,memory_used,memory_free,timestampÚ
r   )r   r   Ú_print_gpu_infor   Úwriter   ÚstatusZis_doneÚ_save_gpu_logÚtimeÚsleepr   ÚhasattrÚclose)r   Úutil_keyr!   r!   r"   r   .   s   
þ
ÿzWatcher.watchc                 C   sØ   z^d}| j  |¡ | j  d¡ t| jƒD ]}| j  | |¡¡ | j  d¡ q| j  d¡ d}| j  |¡ | j  d¡ t| jƒD ]}| j  | |¡¡ | j  d¡ q?| j  d¡ | j  ¡  W d S    | jj 	d¡ Y d S )NzEindex,uuid,driver_version,name,gpu_serial,display_active,display_moder$   z.pid,process_name,gpu_uuid,gpu_name,used_memoryzsave gpu info failed)
r   r&   r   r   Ústrr   Úflushr   ÚloggerÚwarning)r   Zinfo_keyÚlineZprocess_keyr!   r!   r"   r%   ?   s$   zWatcher._print_gpu_infoc                 C   sZ   zt | jƒD ]}| j | |¡¡ | j d¡ q| j ¡  W d S    | jj d¡ Y d S )Nr$   zsave gpu log failed)	r   r   r   r&   r.   r/   r   r0   r1   )r   r-   r2   r!   r!   r"   r(   U   s   zWatcher._save_gpu_logc                 C   s   t | dƒr	 d S d S )Nr   )r+   )r   r!   r!   r"   Ústop^   s   
ýzWatcher.stopN)Ú__name__Ú
__module__Ú__qualname__r#   r   r%   r(   r3   r!   r!   r!   r"   r      s    	r   )
Zutils.nvsmir   r   r   r)   r   Ú	threadingr   Úobjectr   r!   r!   r!   r"   Ú<module>   s
   