o
    eJ&                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 ddl
T ddlmZ dd	lmZ dd
lmZ ddlmZ d dlmZ edZe Zdd ZG dd deZdS )    N)Lock)Thread)Process)Queue   )*)load_profiler_json)ProfilerResult)RedirectStdStreams)
RunManager)bfilez+(.+)_time_(.+)\.paddle_trace\.((pb)|(json))c                 C   s   d| vrdS dS )a-  Determine whether it is a paddle profile file that can be read by vdl according to the file name.

    File name of a paddle profile file must contain `paddle_trace`.

    Args:
        path: File name to determine.
    Returns:
        True if the file is a paddle profile file, otherwise false.
    Zpaddle_traceFT )pathr   r   [D:\Projects\ConvertPro\env\Lib\site-packages\visualdl/component/profiler/profiler_reader.pyis_VDLProfiler_file#   s   
r   c                   @   s   e Zd ZdZd ddZedd Zdd Zd	d
 Zd!ddZ	d!ddZ
d"ddZdd Zdd Zdd Zdd Zdd Zdd ZdS )#ProfilerReaderzUProfile reader to read paddle profile files, support for frontend api in lib.py.
     c                 C   sf   t |tr
|g| _n|| _i | _i | _i | _i | _t | _d| _	|r1| 
  t| jdd  dS dS )zInstance of ProfileReader

        Args:
            logdir: The dir include paddle profile files, multiple subfolders allowed.
        Nr   targetargs)
isinstancestrdirwalksdisplayname2runsruns2displaynamerun_managersr   profile_result_queuetempfilerunsr   _get_data_from_queuestart)selflogdirr   r   r   __init__6   s   

zProfilerReader.__init__c                 C   s   | j S N)r   r"   r   r   r   r#   K   s   zProfilerReader.logdirc                 C   s8   i }| j D ]}t|D ]\}}}|||i qq|S r%   )r   r   walkupdate)r"   flush_walksr   rootdirsfilesr   r   r   get_all_walkO   s   
zProfilerReader.get_all_walkc                 C   s&   || j v r| j |   | j | S d S r%   )r   join)r"   runr   r   r   get_run_managerV   s   

zProfilerReader.get_run_managerFc                 C   s8   t  }| js|S |du r| j|d | jr|d |S )z1Get component tabs used by vdl frontend.
        Tr(   Zprofiler)setr#   r   r   add)r"   r(   component_tabsr   r   r   r4   ]   s   
zProfilerReader.component_tabsc                 C   s`   | j r|du r-|  }i }| D ]\}}dd |D }t|dkr)|||i q|| _ | j S )a6  Get profile run files.

        Every dir(means `run` in vdl) has may have more than one profiler file.

        Returns:
            walks: A dict like {"exp1": ["1587375595_paddle_trace.json", "1587375685_paddle_trace.json"],
                                "exp2": ["1587375686_paddle_trace.json"]}
        Tc                 S   s   g | ]}t |r|qS r   )r   ).0filenamer   r   r   
<listcomp>w   s    z/ProfilerReader.profile_runs.<locals>.<listcomp>r   )r   r-   itemslenr(   )r"   r(   r)   Z
walks_tempr/   	filenamesZ	tags_tempr   r   r   profile_runsi   s   	zProfilerReader.profile_runsTc              
   C   s   | j |d | j D ]N\}}|| jvrt|| j|< | j| | |D ]3}t! | j| |r:	 W d    q%| j| j	| W d    n1 sMw   Y  | 
|| q%qt| j S )Nr1   )r;   r   r8   r   r   Zset_all_filenames_lockZhas_handledZhandled_filenamesr3   
_read_datalistkeys)r"   r(   r/   r:   r6   r   r   r   r      s   
zProfilerReader.runsc                 C   s,   |dkrt tttttdS tttt	t
tdS )Nzh)Zoverview_environmentZoverview_model_perspectiveZ"overview_model_perspective_perstepZoverview_event_type_perspectiveZ%overview_event_type_model_perspectiveZdistributed_histogram)ZTOOLTIP_DEVICE_INFO_CNZTOOLTIP_MODEL_PERSPECTIVE_CNZ$TOOLTIP_MODEL_PERSPECTIVE_PERSTEP_CNZ!TOOLTIP_EVENT_TYPE_PERSPECTIVE_CNZ'TOOLTIP_EVENT_TYPE_MODEL_PERSPECTIVE_CNZ&TOOLTIP_EVENT_DISTRIBUTED_HISTOGRAM_CNZTOOLTIP_DEVICE_INFO_ENZTOOLTIP_MODEL_PERSPECTIVE_ENZ$TOOLTIP_MODEL_PERSPECTIVE_PERSTEP_ENZ!TOOLTIP_EVENT_TYPE_PERSPECTIVE_ENZ'TOOLTIP_EVENT_TYPE_MODEL_PERSPECTIVE_ENZ&TOOLTIP_EVENT_DISTRIBUTED_HISTOGRAM_EN)r"   langr   r   r   get_descriptions   s   zProfilerReader.get_descriptionsc                 C   s   |j | _|j| _d S r%   )Z	name2tagsr   Z	tags2namer   )r"   Z
log_readerr   r   r   set_displayname   s   zProfilerReader.set_displaynamec                 C   s   | S r%   r   r&   r   r   r   	__enter__      zProfilerReader.__enter__c                 C   s   d S r%   r   )r"   exc_typeexc_valexc_tbr   r   r   __exit__   rE   zProfilerReader.__exit__c              
   C   sb   	 z| j  \}}}}| j| ||| W n ty/ } ztd| W Y d }~nd }~ww q)NTz3Read profiler data error in multiprocess, error: {})r   getr   Zadd_profile_result	Exceptionprintformat)r"   r/   r6   worker_nameprofile_resulter   r   r   r       s   
z#ProfilerReader._get_data_from_queuec                    sp   t |}|r6|d}d|v r# fdd}t||||fd  d S  fdd}t||||fd  d S d S )Nr   z.pbc           	         s  t tjd}t||dr zdd l}W n ty) } z
td tt|d }~ww t	j
dt	j
|jkr;tdddlm} z|tj| |}|d u rStdt|}W n tyk } ztd	|d }~ww  j| |||f W d    d S 1 sw   Y  d S )
Nzw+)stdoutstderrr   a  Paddle is required to read protobuf file. "                                "Please install [paddlepaddle](https://www.paddlepaddle.org.cn/install/quick?"                                    "docurl=/documentation/docs/zh/develop/install/pip/linux-pip.html) first.z2.4.0z.Please make sure paddlepaddle version >= 2.4.0)load_profiler_resultzMissing required fields.zAn error occurred while loading the protobuf file, which may be caused by the outdated version of paddle that generated the profile file. Please make sure protobuf file is exported by paddlepaddle version >= 2.4.0. Error message: {})openosdevnullr
   paddlerK   rL   RuntimeErrorr   	packagingversionparse__version__Zpaddle.profilerrS   r   r.   r	   rM   r   put)	r/   r6   rN   rV   rW   rP   rS   contentrO   r&   r   r   _load_profiler_protobuf   sL   
"z:ProfilerReader._read_data.<locals>._load_profiler_protobufr   c                    s*   t tj| |} j| |||f d S r%   )r   rU   r   r.   r   r]   )r/   r6   rN   rO   r&   r   r   _load_profiler_json   s   
z6ProfilerReader._read_data.<locals>._load_profiler_json)_name_patternmatchgroupr   r!   )r"   r/   r6   rb   rN   r_   r`   r   r&   r   r=      s"   

"

zProfilerReader._read_dataN)r   )F)T)__name__
__module____qualname____doc__r$   propertyr#   r-   r0   r4   r;   r   rB   rC   rD   rI   r    r=   r   r   r   r   r   2   s     




r   )rU   re	threadingr   r   Zpackaging.versionrY   Zmultiprocessr   r   Zparser.const_descriptionZparser.event_noder   r	   Zparser.utilsr
   Zrun_managerr   Zvisualdl.ior   compilera   r<   r   objectr   r   r   r   r   <module>   s"   
