o
    Qe¥0  ã                   @   sd  d dl mZ d dl mZ d dl mZ d dlZd dlZd dlmZ d dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlmZ zd dlmZ W n   G dd„ deƒZY d dlZe e¡Zd	gZe d
¡ZdZdd„ Zd+dd	„Zdd„ Zdd„ Z				d,dd„Z dd„ Z!dd„ Z"e!e"dœZ#d-dd„Z$d+dd„Z%d d!„ Z&d"d#„ Z'd.d%d&„Z(d'd(„ Z)d)d*„ Z*dS )/é    )Úabsolute_import)Údivision)Úprint_functionN)ÚOrderedDict)Útqdmc                   @   s.   e Zd Zd
dd„Zdd„ Zdd„ Zdd	„ ZdS )r   Nc                 C   s   || _ d| _d S )Nr   )ÚtotalÚn)Úselfr   © r
   úED:\Projects\ConvertPro\env\Lib\site-packages\paddle/utils/download.pyÚ__init__%   s   
ztqdm.__init__c                 C   s^   |  j |7  _ | jd u rtj d | j ¡¡ ntj d d| j  t| jƒ ¡¡ tj ¡  d S )Nz{0:.1f} bytesz	{0:.1f}%éd   )r   r   ÚsysÚstderrÚwriteÚformatÚfloatÚflush)r	   r   r
   r
   r   Úupdate)   s   
ÿztqdm.updatec                 C   s   | S ©Nr
   )r	   r
   r
   r   Ú	__enter__2   s   ztqdm.__enter__c                 C   s   t j d¡ d S )NÚ
)r   r   r   )r	   Úexc_typeÚexc_valÚexc_tbr
   r
   r   Ú__exit__5   s   ztqdm.__exit__r   )Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r
   r
   r
   r   r   #   s
    
	r   Úget_weights_path_from_urlz~/.cache/paddle/hapi/weightsé   c                 C   s   |   d¡p	|   d¡S )zR
    Whether path is URL.
    Args:
        path (string): URL string or not.
    zhttp://zhttps://)Ú
startswith)Úpathr
   r
   r   Úis_urlD   s   r#   c                 C   s   t | t|ƒ}|S )a.  Get weights path from WEIGHT_HOME, if not exists,
    download it from url.

    Args:
        url (str): download url
        md5sum (str): md5 sum of download package

    Returns:
        str: a local path to save downloaded weights.

    Examples:
        .. code-block:: python

            from paddle.utils.download import get_weights_path_from_url

            resnet18_pretrained_weight_url = 'https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams'
            local_weight_path = get_weights_path_from_url(resnet18_pretrained_weight_url)

    )Úget_path_from_urlÚWEIGHTS_HOME)ÚurlÚmd5sumr"   r
   r
   r   r   M   s   c                 C   s   t  | ¡d }|}t  ||¡S )Néÿÿÿÿ)ÚospÚsplitÚjoin)r&   Úroot_dirÚfnameZfpathr
   r
   r   Ú	_map_pathe   s   r.   c                 C   s^   |   ¡  tƒ }tƒ }| D ]}| d¡d }||v rq| |¡ | |¡ qt d |¡¡ |S )Nú:r   zunique_endpoints {})ÚsortÚsetr*   ÚaddÚloggerÚinfor   )Útrainer_endpointsÚipsÚunique_endpointsZendpointÚipr
   r
   r   Ú_get_unique_endpointsl   s   
r9   TÚgetc           	      C   sÚ   ddl m} t| ƒsJ d | ¡ƒ‚t| |ƒ}t|ƒ jdd… ƒ}t |¡r5|r5t	||ƒr5t
 d |¡¡ n |ƒ j|v rDt| |||d}ntj |¡sUt d¡ tj |¡rJ|ƒ j|v rk|rkt |¡sgt |¡rkt|ƒ}|S )a¢   Download from given url to root_dir.
    if file or directory specified by url is exists under
    root_dir, return the path directly, otherwise download
    from url and decompress it, return the path.

    Args:
        url (str): download url
        root_dir (str): root dir for downloading, it should be
                        WEIGHTS_HOME or DATASET_HOME
        md5sum (str): md5 sum of download package
        decompress (bool): decompress zip or tar file. Default is `True`
        method (str): which download method to use. Support `wget` and `get`. Default is `get`.

    Returns:
        str: a local path to save downloaded models & weights & datasets.
    r   )ÚParallelEnvzdownloading from {} not a urlNzFound {})Úmethodé   )Zpaddle.fluid.dygraph.parallelr;   r#   r   r.   r9   r5   r)   ÚexistsÚ	_md5checkr3   r4   Zcurrent_endpointÚ	_downloadÚosr"   ÚtimeÚsleepÚtarfileÚ
is_tarfileÚzipfileÚ
is_zipfileÚ_decompress)	r&   r,   r'   Zcheck_existÚ
decompressr<   r;   Úfullpathr7   r
   r
   r   r$   {   s"   

ÿÿr$   c           
      C   sH  t  |¡}z	tj| dd}W n ty- } zt d || t|ƒ¡¡ W Y d }~dS d }~ww |j	dkr<t
d | |j	¡ƒ‚|d }|j d¡}t|d	ƒI}|r}tt|ƒd
 d d}|jddD ]}	| |	¡ | d¡ q`W d   ƒ n1 sww   Y  n|jddD ]	}	|	rŒ| |	¡ qƒW d   ƒ n1 s—w   Y  t ||¡ |S )NT)Ústreamz/Downloading {} from {} failed with exception {}FéÈ   z(Downloading from {} failed with code {}!Ú_tmpzcontent-lengthÚwbiÿ  i   )r   )Ú
chunk_sizer=   )r)   ÚbasenameÚrequestsr:   Ú	Exceptionr3   r4   r   ÚstrÚstatus_codeÚRuntimeErrorÚheadersÚopenr   ÚintÚiter_contentr   r   ÚshutilÚmove)
r&   Úfullnamer-   ÚreqÚeÚtmp_fullnameÚ
total_sizeÚfZpbarÚchunkr
   r
   r   Ú_get_download¬   s@   

ÿ€ý

ÿ
þÿ€
€÷
rc   c                 C   s^   |d }d  |t| ¡}tj|dtjtjd}| ¡ }|jdkr'td  || ¡ƒ‚t 	||¡ |S )NrM   zwget -O {} -t {} {}T)ÚshellÚstdoutr   r   z<{} failed. Please make sure `wget` is installed or {} exists)
r   ÚDOWNLOAD_RETRY_LIMITÚ
subprocessÚPopenÚPIPEÚcommunicateÚ
returncoderU   rZ   r[   )r&   r\   r_   ÚcommandZsubprcÚ_r
   r
   r   Ú_wget_downloadÎ   s"   ÿý
þrn   )r:   Zwgetc                 C   sÀ   |t v sJ d |¡ƒ‚t |¡st |¡ t | ¡d }t ||¡}d}t 	d || ¡¡ t |¡r7t
||ƒs^|tk r@|d7 }ntd | ¡ƒ‚t | | |ƒsTt d¡ q-t |¡r7t
||ƒr7|S )zù
    Download from url, save to path.

    url (str): download url
    path (str): download to given path
    md5sum (str): md5 sum of download package
    method (str): which download method to use. Support `wget` and `get`. Default is `get`.

    zmake sure `{}` implementedr(   r   zDownloading {} from {}r=   z,Download from {} failed. Retry limit reached)Ú_download_methodsr   r)   r>   rA   Úmakedirsr*   r+   r3   r4   r?   rf   rU   rB   rC   )r&   r"   r'   r<   r-   r\   Z	retry_cntr
   r
   r   r@   ê   s(   
ÿ


ÿ
÷r@   c                    sœ   |d u rdS t  d | ¡¡ t ¡ }t| dƒ‰ t‡ fdd„dƒD ]}| |¡ q!W d   ƒ n1 s3w   Y  | ¡ }||krLt  d | ||¡¡ dS dS )	NTzFile {} md5 checking...Úrbc                      s
   ˆ   d¡S )Ni   )Úreadr
   ©ra   r
   r   Ú<lambda>  s   
 z_md5check.<locals>.<lambda>ó    z.File {} md5 check failed, {}(calc) != {}(base)F)	r3   r4   r   ÚhashlibÚmd5rW   Úiterr   Ú	hexdigest)r\   r'   rw   rb   Zcalc_md5sumr
   rs   r   r?     s    ÿÿ
ÿr?   c                 C   sJ   t  d | ¡¡ t | ¡rt| ƒ}|S t | ¡rt| ƒ}|S t	d | ¡ƒ‚)z)
    Decompress for zip and tar file
    zDecompressing {}...zUnsupport compress file type {})
r3   r4   r   rD   rE   Ú_uncompress_file_tarrF   rG   Ú_uncompress_file_zipÚ	TypeError)r-   Úuncompressed_pathr
   r
   r   rH     s   

ûþrH   c                 C   s  t  | d¡x}| ¡ }tj | ¡}t|ƒr&|d }tj ||¡}| |¡ nOt	|ƒrKtj 
|d  tj¡¡d  tj¡d }tj ||¡}| |¡ n*tj 
| ¡d  tj¡d }tj ||¡}tj |¡skt |¡ | tj ||¡¡ |W  d   ƒ S 1 sw   Y  d S )NÚrr   r(   )rF   ÚZipFileÚnamelistrA   r"   ÚdirnameÚ_is_a_single_filer+   Ú
extractallÚ_is_a_single_dirÚsplitextÚstripÚsepr*   r>   rp   )ÚfilepathÚfilesÚ	file_listÚfile_dirÚrootpathr}   r
   r
   r   r{   4  s,   ÿÿ
$èr{   úr:*c                 C   s  t  | |¡x}| ¡ }tj | ¡}t|ƒr&|d }tj ||¡}| |¡ nOt	|ƒrKtj 
|d  tj¡¡d  tj¡d }tj ||¡}| |¡ n*tj 
| ¡d  tj¡d }tj ||¡}tj |¡skt |¡ | tj ||¡¡ |W  d   ƒ S 1 sw   Y  d S )Nr   r(   )rD   rW   ÚgetnamesrA   r"   r   r‚   r+   rƒ   r„   r…   r†   r‡   r*   r>   rp   )rˆ   Úmoder‰   rŠ   r‹   rŒ   r}   r
   r
   r   rz   P  s,   ÿÿ
$êrz   c                 C   s(   t | ƒdkr| d  tj¡dk rdS dS )Nr=   r   TF)ÚlenÚfindrA   r‡   )rŠ   r
   r
   r   r‚   j  s    r‚   c                 C   s   g }| D ]}d|v r|  dtj¡}nd|v r|  dtj¡}| |¡ q|d  tj¡d }tdt|ƒƒD ]}|||  tj¡d krE dS q4dS )Nú/ú\r   r=   FT)ÚreplacerA   r‡   Úappendr*   Úranger   )rŠ   Znew_file_listÚ	file_pathÚ	file_nameÚir
   r
   r   r„   p  s   ÿr„   r   )NTTr:   )Nr:   )r   )+Ú
__future__r   r   r   rA   r   Zos.pathr"   r)   rZ   rQ   rg   rv   rD   rF   rB   Úcollectionsr   r   ÚobjectÚloggingÚ	getLoggerr   r3   Ú__all__Ú
expanduserr%   rf   r#   r   r.   r9   r$   rc   rn   ro   r@   r?   rH   r{   rz   r‚   r„   r
   r
   r
   r   Ú<module>   sX   


	
û1"þ

#
