o
    eX                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlZ	d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ i ag ai Zg Zd
d Zdd Zdd Zdd Zdd Zdd Z dd Z!e" D ]
Z#e$de#e#f  qtdRddZ%dd Z&dd Z'd d! Z(d"d# Z)d$d% Z*dRd&d'Z+d(d) Z,d*d+ Z-d,d- Z.d.d/ Z/d0d1 Z0d2d3 Z1d4d5 Z2d6d7 Z3dSd8d9Z4dSd:d;Z5d<d= Z6d>d? Z7d@dA Z8dBdC Z9dDdE Z:		F	F	F	GdTdHdIZ;	F	GdUdJdKZ<dLdM Z=dNdO Z>dPdQ Z?dS )V    N)partial)
components)bfile)logger)calc_all_hyper_param_importance)duplicate_removal)
decode_tag)
encode_tagc                 C   s   | dk r| d S | S )Ni 5wi   	timestampr
   r
   CD:\Projects\ConvertPro\env\Lib\site-packages\visualdl/server/lib.pys2ms'   s   r   c                 C   s    t | s
t | rt| } | S N)mathisnanisinfstr)Zscalar_valuer
   r
   r   transfer_abnomal_scalar_value+   s   r   c                 C   s   | j dd}t|S )NT)update)r   list)
log_readerr   r
   r
   r   get_components1   s   r   c                 C   <   g }|   D ]}|| jv r|| j|  q|| q|S r   )runs	tags2nameappend)r   r   itemr
   r
   r   get_runs6      
r   c                 C   r   r   )r   Zruns2displaynamer   )graph_readerr   r   r
   r
   r   get_graph_runs@   r   r!   c                 C   s   |   S r   )tags)r   r
   r
   r   get_tagsJ   s   r#   c                 C   s  | j |j}i }|D ];}|d}|d| }t||d d  }|| v r0|| | n|g||< || j vrF|| j|< || j|< qi }| D ]\}	}
|	| jv r^|
|| j|	 < qM|
||	< qMg g d}| D ]\}}|d | |d | qlt	
d}|tvrt|di |rt| sd	t|< | j }| D ]K\}	}
|	tv rq|	|}|d
kr|	|t| d  }| j|	 | j||
i | j|
 | j|
|i ||d |d |	< n|	}t| q|S )N/r      )r   r"   r   r"   ZVISUALDL_RUN_PREFIXFT)data_managerget_reservoirkeysrfindr	   r   r   	name2tagsitemsosgetenvMODIFY_PREFIXr   copyMODIFIED_RUNSfindlenpopindex)r   	componentZall_tagr"   r   r5   runtagZ	fake_tagskeyvaluerun2tagZ
run_prefixZtemp_name2tagsZtemp_keyr
   r
   r   get_logsN   sT   









r<   z-get_%s_tags=partial(get_logs, component='%s')tsvc              	   C   s  t | }d|kr
dnd}dg}g }g }|D ]}||d  7 }||d  7 }qt }g }	|D ]}
|
|v r6q/||
 |	|
 q/t }g }|D ]}||v rOqH|| || qHg }|D ]!}d|ddi}||di  ||di  || q^||	 | }t 4}t	j
||d	}|| |D ]}g }|D ]}|||d q|| q| }|W  d    S 1 sw   Y  d S )
Nr=   	,zTrial IDhparamsmetricsname 	delimiter)get_hparam_listr)   setaddr   getr   ioStringIOcsvwriterwriterowgetvalue)r   typeresult	delimeterheaderZhparams_headerZmetrics_headerr   Zname_setZh_headerhparamZm_headerZmetricZtrans_resulttempfp
csv_writerrowZcol_namer
   r
   r   get_hparam_data   sP   



$rY   c                 C   s:   t | }dd |d D }dd |d D }t||}|S )Nc                 S      g | ]
}|d  dkr|qS rP   stringr
   .0r   r
   r
   r   
<listcomp>       z)get_hparam_importance.<locals>.<listcomp>r@   c                 S   rZ   r[   r
   r]   r
   r
   r   r_      r`   rA   )get_hparam_indicatorr   )r   Z	indicatorr@   rA   rQ   r
   r
   r   get_hparam_importance   s   
rb   c              	   C   s  t | d}|d }i }i }g }|D ]%}|| jv r| j| n|}|   | jd|td}|||g q|jdd d dd |D }|D ]\}}|d	 j	j
D ]}|d
}	d|	kr|j| vrq|jd|jgd||j< qR|j||j d vr||j d |j qRd|	kr|j| vr|jd|jgd||j< qR|j||j d vr||j d |j qRd|	kr|j| vr|jd|jgd||j< qR|j||j d vr||j d |j qRtd|	 |d	 j	jD ]N}
|
jdg d||
j< |D ]$}zt| ||
j}||
j d |d d  W  n
   td Y qt||
j d d	kr5||
j q||
j d qqHdd | D dd | D d}|S )Nhyper_parametersr   rT   c                 S      | d d j S Nr   r   xr
   r
   r   <lambda>       z&get_hparam_indicator.<locals>.<lambda>r9   c                 S      g | ]\}}|qS r
   r
   )r^   rr7   r
   r
   r   r_          z(get_hparam_indicator.<locals>.<listcomp>r   rP   float_valueZ
continuous)rB   rP   valuesro   string_valuer\   	int_valuenumeric&Invalid hparams param value type `%s`.r&   MMissing data of metrics! Please make sure use add_scalar to log metrics data.c                 S   rk   r
   r
   r^   r9   r:   r
   r
   r   r_     rm   c                 S   rk   r
   r
   ru   r
   r
   r   r_     rm   )r@   rA   )r<   r+   load_new_datar'   r(   	get_itemsr   r   sortrT   hparamInfos
WhichOneofrB   r)   rn   rp   rq   	TypeErrormetricInfosget_hparam_metricr   errorr3   r4   r,   )r   r;   r   r@   rA   records_listr7   records
hparamInforP   
metricInfometrics_dataresultsr
   r
   r   ra      s   


ra   c                 C   J   || j v r
| j | n|}|   | jd|t|}dd |D }|S )Nscalarc                 S   s$   g | ]}t |j|jt|jgqS r
   )r   r   idr   r:   r]   r
   r
   r   r_     s
    z%get_hparam_metric.<locals>.<listcomp>r+   rv   r'   r(   rw   r   r   r7   r8   r   r   r
   r
   r   r}        r}   c                 C   sb  t | d}|d }g }g }|D ]%}|| jv r| j| n|}|   | jd|td}|||g q|jdd d |D ]o\}}i }|d j	j
D ]-}|d}	d	|	kr]|j||j< qKd
|	krh|j||j< qKd|	krs|j||j< qKtd|	 i }
|d j	jD ]#}zt| ||j}|d d |
|j< W q   td d |
|j< Y q||||
d q?|S )Nrc   r   rT   c                 S   rd   re   r   rf   r
   r
   r   rh   .  ri   z!get_hparam_list.<locals>.<lambda>rj   r   rP   rn   rp   rq   rs   r&   rt   )rB   r@   rA   )r<   r+   rv   r'   r(   rw   r   r   rx   rT   ry   rz   rn   rB   rp   rq   r{   r|   r}   r   r~   )r   r;   r   r   r   r7   r   r@   r   Zhparam_typerA   r   r   r
   r
   r   rF   "  sR   

rF   c                 C   r   )Nr   c                 S   sL   g | ]"}| d dkrt|j|jt|jgnt|j|jt|jjgqS )	one_valuer:   )rz   r   r   r   r   r:   Z	tag_valuer]   r
   r
   r   r_   S  s    
zget_scalar.<locals>.<listcomp>r   r   r
   r
   r   
get_scalarM  s   r   c           	      C   s<  d}t j|t|ddr-|  t	t j
|t|d dkr-t j
|}d}|| jv r7| j| n|}|   |rI| d|t|}n| d|t|}td|d|dt| d	|krcd
nd}t +}tj||d}|r{|g d n|g d || | }|W  d    S 1 sw   Y  d S )NF%_ZscalarsTr   r7   r8   r=   r>   r?   rD   )r   r8   Zsub_tagr   r:   )r   r8   r   r:   )r-   pathbasename
startswithr   replacer"   rI   r   joindirnamer+   rv   Zget_log_dataprintrJ   rK   rL   rM   rN   	writerowsrO   )	r   r7   r8   rP   Z
is_scalarsrQ   rR   rV   rW   r
   r
   r   get_scalar_data^  s*   (

$r   c                 C   r   )Nimagec                 S      g | ]}|j t|jd qS )stepZwallTimer   r   r   r]   r
   r
   r   r_   |  
    
z'get_image_tag_steps.<locals>.<listcomp>r   r   r7   r8   r   rQ   r
   r
   r   get_image_tag_stepsw  r   r   c                 C   D   || j v r
| j | n|}|   | jd|t|}|| jjS )Nr   )r+   rv   r'   r(   rw   r   r   Zencoded_image_stringr   r7   r8   
step_indexr   r
   r
   r   get_individual_image     r   c                 C   r   )Ntextc                 S   r   r   r   r]   r
   r
   r   r_     r   z&get_text_tag_steps.<locals>.<listcomp>r   r   r
   r
   r   get_text_tag_steps  r   r   c                 C   r   )Nr   )r+   rv   r'   r(   rw   r   r   Zencoded_text_stringr   r
   r
   r   get_individual_text  r   r   c                 C   r   )Naudioc                 S   r   r   r   r]   r
   r
   r   r_     r   z'get_audio_tag_steps.<locals>.<listcomp>r   r   r
   r
   r   get_audio_tag_steps  r   r   c                 C   sH   || j v r
| j | n|}|   | jd|t|}|| jj}|S )Nr   )r+   rv   r'   r(   rw   r   r   Zencoded_audio_string)r   r7   r8   r   r   rQ   r
   r
   r   get_individual_audio  s   r   c                       || j v r
| j | n|}|   | jd|t|}g }|D ]<}|j}t|j  fddt	d d D }|
t|j|jt|jt|jt|jt|jt|jt|j|g	 q |S )Npr_curvec                       g | ]}t |  qS r
   floatr^   vlengthr
   r   r_         z get_pr_curve.<locals>.<listcomp>r%   )r+   rv   r'   r(   rw   r   r   r3   	precisionranger   r   r   r   r   ZrecallTPFPTNFN)r   r7   r8   r   r   r   r   num_thresholdsr
   r   r   get_pr_curve  (   

	r   c                    r   )N	roc_curvec                    r   r
   r   r   r   r
   r   r_     r   z!get_roc_curve.<locals>.<listcomp>r%   )r+   rv   r'   r(   rw   r   r   r3   Ztprr   r   r   r   r   r   Zfprr   r   r   r   )r   r7   r8   r   r   r   r   r   r
   r   r   get_roc_curve  r   r   c                 C   p   |}|| j v r| j | n|}t| }|d |d | d }|   | jd|t|}dd |D }|S )Nr"   r   r   r   c                 S      g | ]
}t |j|jgqS r
   r   r   r   r]   r
   r
   r   r_         z%get_pr_curve_step.<locals>.<listcomp>)r+   Zget_pr_curve_tagsr5   rv   r'   r(   rw   r   r   r7   r8   Zfake_runr;   r   r   r
   r
   r   get_pr_curve_step     r   c                 C   r   )Nr"   r   r   r   c                 S   r   r
   r   r]   r
   r
   r   r_     r   z&get_roc_curve_step.<locals>.<listcomp>)r+   Zget_roc_curve_tagsr5   rv   r'   r(   rw   r   r   r
   r
   r   get_roc_curve_step  r   r   c                 C   s   t | d}t|d |d D ]^\}}|| jv r| j| n|}|D ]K}tj|| }}|tv r5t    S t|||di | j	
d|t|}t|d jj}t|d jjd j}	||	g}
t||
|d q qtS )N
embeddingsr   r"   )r7   r8   r   )rB   shaper   )r<   zipr+   r-   r   r   EMBEDDING_NAMEembedding_namesr   r'   r(   rw   r   r3   r   vectorsr   )r   r;   r7   Z_tagsr8   rB   r   r   Zrow_lenZcol_lenr   r
   r
   r   get_embeddings_list  s.   

r   c           
      C   s   t | d }t | d }|   | jd|t|}g }|d jjD ]}||j q$|d jj	}|r:|g| }t
 }tj|dd}	|	| | }W d    |S 1 sZw   Y  |S )Nr7   r8   r   r   r>   rD   )r   rv   r'   r(   rw   r   r   r   label
label_metarJ   rK   rL   rM   r   rO   )
r   rB   r7   r8   r   labelsr   r   rV   rW   r
   r
   r   get_embedding_labels  s(   




r   c                 C   sx   t | d }t | d }|   | jd|t|}g }|d jjD ]}||j q$t	
| t	j }|S )Nr7   r8   r   r   )r   rv   r'   r(   rw   r   r   r   r   nparrayflattenZastypeZfloat32tobytes)r   rB   r7   r8   r   r   r   r
   r
   r   get_embedding_tensors&  s   r   c                 C   s   || j v r
| j | n|}|   | jd|t|}g }|D ]2}|j}|j}|j}g }	t	t
|D ]}
|	||
 ||
d  ||
 g q3|t|j|j|	g q |S )N	histogramr%   )r+   rv   r'   r(   rw   r   r   hist	bin_edgesr   r3   r   r   r   r   )r   r7   r8   r   r   r   r   r   r   Zhistogram_datar5   r
   r
   r   get_histogram3  s"   r   c                 C   sJ   d}| j r#t| j d}|| j }W d    |S 1 sw   Y  |S )N    rb)modelr   ZBFile	read_file)r   rQ   Zbfpr
   r
   r   get_static_graphG  s   
r   FTc                 C   s@   d}|| j v r| j | n|}|d urd}| ||||||}|S )NrC   F)displayname2runs	get_graph)r    r7   nodeidexpand
keep_state
expand_allrefreshrQ   r
   r
   r   r   O  s   
r   c                 C   s2   d}|| j v r| j | n|}| j||||d}|S )NrC   )r   is_node)r   Zsearch_graph_node)r    r7   r   r   r   rQ   r
   r
   r   get_graph_search`  s   
r   c                 C   s*   d}|| j v r| j | n|}| |}|S )NrC   )r   Zget_all_nodes)r    r7   rQ   r
   r
   r   get_graph_all_nodesj  s   

r   c              
   O   s   t | D ]:}z
||i |W   S  ty>   || d k r4dttt }td|  t	
| nddl}|  Y qw dS )zs
    try to execute `function` `ntimes`, if exception catched, the thread will
    sleep `time2sleep` seconds.
    r%   
zUnexpected error: %sr   N)r   	Exceptionr   mapr   sysexc_infor   r~   timesleep	traceback	print_exc)ZntimesfunctionZ
time2sleepargskwargsi
error_infor   r
   r
   r   retryr  s   r   c                    s    fdd}|S )Nc                    sB     | }|d u rtd|   ||i |} | | |S |S )Nzupdate cache %s)rI   r   warningrG   )r9   funcr   r   datacacher
   r   _handler  s   
zcache_get.<locals>._handlerr
   )r   r   r
   r   r   	cache_get  s   	r   )r=   r   )NFFFT)FT)@rL   rJ   r   r-   r   r   	functoolsr   numpyr   Zvisualdl.componentr   Zvisualdl.ior   Zvisualdl.server.logr   Zvisualdl.utils.importancer   Zvisualdl.utils.list_utilr   Zvisualdl.utils.string_utilr   r	   r/   r1   r   r   r   r   r   r   r!   r#   r<   r)   rB   execrY   rb   ra   r}   rF   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r
   r
   r
   r   <module>   s|   

7
*T+
	





