o
    eO                     @   s"  d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZmZ d dlmZmZ d dlZddlmZmZ dd	lmZmZmZmZmZ dd
lmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& eG dd dZ'G dd dej(Z)G dd dej*j+Z,de-de-fddZ.dS )    N)	dataclass)datetime)	has_magic)AnyDictListOptionalTupleUnion)quoteunquote   )CommitOperationCopyCommitOperationDelete)DEFAULT_REVISIONENDPOINTREPO_TYPE_MODELREPO_TYPES_MAPPINGREPO_TYPES_URL_PREFIXES)HfApi)EntryNotFoundErrorHFValidationErrorRepositoryNotFoundErrorRevisionNotFoundErrorhf_raise_for_statushttp_backoffpaginateparse_datetimec                   @   s@   e Zd ZU dZeed< eed< eed< eed< defddZd	S )
HfFileSystemResolvedPathzUData structure containing information about a resolved Hugging Face file system path.	repo_typerepo_idrevisionpath_in_reporeturnc                 C   s2   t | jd| j  dt| j d| j dS )N @/)r   getr   r    
safe_quoter!   r"   rstripself r,   ND:\Projects\ConvertPro\env\Lib\site-packages\huggingface_hub/hf_file_system.py	unresolve$   s   *z"HfFileSystemResolvedPath.unresolveN)__name__
__module____qualname____doc__str__annotations__r.   r,   r,   r,   r-   r      s   
 r   c                       s  e Zd ZdZdZdZddddee dee f fdd	Zd
ededee de	e
ee f fddZdd Zd4dedee defddZd4dee ddfddZ		d5dededee ddfddZd4dedee ddfddZ			d6dede
d ee dee ddf
d!d"Z	d7ded$e
d%e
dee deeeeeef f  f
d&d'Zd4dedee fd(d)Zd4d*ed+edee ddfd,d-Zdedefd.d/Zdedeeef f fd0d1Z	d8deeee f de
d ee dee fd2d3Z  Z S )9HfFileSystema  
    Access a remote Hugging Face Hub repository as if were a local file system.

    Args:
        endpoint (`str`, *optional*):
            The endpoint to use. If not provided, the default one (https://huggingface.co) is used.
        token (`str`, *optional*):
            Authentication token, obtained with [`HfApi.login`] method. Will default to the stored token.

    Usage:

    ```python
    >>> from huggingface_hub import HfFileSystem

    >>> fs = HfFileSystem()

    >>> # List files
    >>> fs.glob("my-username/my-model/*.bin")
    ['my-username/my-model/pytorch_model.bin']
    >>> fs.ls("datasets/my-username/my-dataset", detail=False)
    ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json']

    >>> # Read/write files
    >>> with fs.open("my-username/my-model/pytorch_model.bin") as f:
    ...     data = f.read()
    >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f:
    ...     f.write(data)
    ```
    r$   ZhfNendpointtokenr7   r8   c                   s:   t  j|i | |pt| _|| _t||d| _i | _d S )Nr6   )super__init__r   r7   r8   r   _api_repo_and_revision_exists_cache)r+   r7   r8   argsZstorage_options	__class__r,   r-   r:   M   s   
zHfFileSystem.__init__r   r    r!   r#   c              
   C   s   |||f| j vroz| jj|||d W nK ttfy; } zd|f| j |||f< d|f| j ||d f< W Y d }~n8d }~w ty^ } zd|f| j |||f< d| j ||d f< W Y d }~nd }~ww d| j |||f< d| j ||d f< | j |||f S )N)r!   r   F)TN)r<   r;   Z	repo_infor   r   r   )r+   r   r    r!   er,   r,   r-   _repo_and_revision_exist_   s    z%HfFileSystem._repo_and_revision_existc              
   K   sL   z| j |fi | W dS  ty% } zt|tr W Y d}~dS d}~ww )a  Is there a file at the given path

        Exact same implementation as in fsspec except that instead of catching all exceptions, we only catch when it's
        not a `NotImplementedError` (which we do want to raise). Catching a `NotImplementedError` can lead to undesired
        behavior.

        Adapted from https://github.com/fsspec/filesystem_spec/blob/f5d24b80a0768bf07a113647d7b4e74a3a2999e0/fsspec/spec.py#L649C1-L656C25
        TNF)info	Exception
isinstanceNotImplementedError)r+   pathkwargsr@   r,   r,   r-   existsp   s   	
zHfFileSystem.existsrF   c                 C   s"  dt t dt t dt t fdd}| |}|std|dd d t v r>d|vr1td|dd	\}}t| }nt}|	ddkrd
|v r||d
d	\}}d|v r`|dd	\}}nd}t
|}|||}| |||\}}	|s{t||	nd|dd d }
d|ddd  }|dd }d|dd	d  }|
}|}| |||\}}	|st|	ttfr|}|}| |||\}}|st||	n/t||	n)|}d}d
|v r|d
d	\}}t
|}|||}| |||\}}|std|d ur|nt}t||||S )Nrevision_in_pathr!   r#   c                 S   s:   |d ur| d ur| |krt d|  d| d|S | }|S )NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.)
ValueError)rI   r!   r,   r,   r-   %_align_revision_in_path_with_revision   s   zHHfFileSystem.resolve_path.<locals>._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r&   r   r   r%   r$      )r   r3   _strip_protocolrE   splitr   valuesr   r   countr   rA   FileNotFoundErrorjoinrD   r   r   r   r   )r+   rF   r!   rK   r   r    rI   r"   Zrepo_and_revision_existerrZrepo_id_with_namespaceZpath_in_repo_with_namespaceZrepo_id_without_namespaceZpath_in_repo_without_namespace_r,   r,   r-   resolve_path   sp   








zHfFileSystem.resolve_pathc                 C   sR   |s| j   | j  d S | | }|r'| j |d  | |}|sd S d S N)dircacheclearZ$_repository_type_and_id_exists_cacherU   r.   pop_parent)r+   rF   r,   r,   r-   invalidate_cache   s   

zHfFileSystem.invalidate_cacherbmodeHfFileSystemFilec                 K   s(   |dkrt dt| |f||d|S )Nabz/Appending to remote files is not yet supported.)r]   r!   )rE   r^   )r+   rF   r]   r!   rG   r,   r,   r-   _open   s   zHfFileSystem._openc              
   K   sR   | j ||d}| jj|j|j| j|j|j|d|dd | j	|
 d d S )Nr!   commit_messagecommit_description)r"   r    r8   r   r!   rb   rc   rF   )rU   r;   Zdelete_filer"   r    r8   r   r!   r'   r[   r.   )r+   rF   r!   rG   resolved_pathr,   r,   r-   _rm   s   	zHfFileSystem._rmF	recursivemaxdepthc              
      s   j ||d}t|jd|j  j||||jd} fdd|D }dd |D }	d| d}
|
|r7d	nd7 }
|
|d urEd
| dnd7 }
jj|j|jj	|	|j|d|
|dd j
| d d S )Nra   r$   )rg   rh   r!   c                    s*   g | ]} |s|t d  d qS )r   N)isdirlen).0rF   	root_pathr+   r,   r-   
<listcomp>      * z#HfFileSystem.rm.<locals>.<listcomp>c                 S   s   g | ]}t |d qS ))r"   )r   )rk   r"   r,   r,   r-   rn          zDelete  zrecursively zup to depth rb   rc   )r    r   r8   
operationsr!   rb   rc   rd   )rU   r   r'   r   r    expand_pathr!   r;   create_commitr8   r[   r.   )r+   rF   rg   rh   r!   rG   re   pathsZpaths_in_reporr   rb   r,   rl   r-   rm   s$   
	zHfFileSystem.rmTdetailrefreshc                    sh  | j ||d}dt|j   |v }| }|| jvs|rt|j|j|jd d }|}	| j|	|jd}
zt	|
}W n t
yX   d|jv rU| |}	| j|	|jd}
n Y nw t|g|
}
g }|
D ]/}||d  |d |d d}|d d	kr||d
 |dt|d d d || qd|| j|	< | |}|s fdd|D }|r|S dd |D S )z!List the contents of a directory.ra   r%   r$   r&   rF   sizetypenamery   rz   fileoidlfsZ
lastCommitdate)Zblob_idr   last_modifiedc              	      s*   g | ]}i |d |d    ddiqS )r|   r$   r   )replacerk   orI   r,   r-   rn   3  ro   z#HfFileSystem.ls.<locals>.<listcomp>c                 S   s   g | ]}|d  qS )r|   r,   r   r,   r,   r-   rn   4  s    )rU   r(   r!   r.   rW   r   r   r    
_iter_treenextr   r"   rZ   	itertoolschainupdater'   r   appendZ_ls_from_cache)r+   rF   rw   rx   r!   rG   re   has_revision_in_pathZpath_prefixZ	tree_pathZ	tree_iterZ	tree_itemZchild_infosZ
child_infooutr,   r   r-   ls  sV   




zHfFileSystem.lsc              	   c   sl    | j ||d}| jj d|j d|j dt|j d|j 	d}| j	 }t
|ddi|dE d H  d S )	Nra   z/api/zs/z/tree/r&   expandT)paramsheaders)rU   r;   r7   r   r    r(   r!   r"   r)   _build_hf_headersr   )r+   rF   r!   re   r   r,   r,   r-   r   6  s   0
zHfFileSystem._iter_treepath1path2c                 K   sD  | j ||d}| j ||d}|j|jko|j|jk}|rQ| j||jdd d urQd| d| }| jj|j|j|j|d||ddt|j	|j	|jdgd	 n?| j
|d
|jd}	|	 }
W d    n1 siw   Y  d| d| }| jj|
|j	|j| j|j|j|d||dd | j| d | j| d d S )Nra   r   zCopy z to rb   rc   r$   )Zsrc_path_in_repor"   Zsrc_revision)r    r   r!   rb   rc   rr   r\   Zpath_or_fileobjr"   r    r8   r   r!   rb   rc   rd   )rU   r   r    rB   r!   r;   rt   r'   r   r"   openreadupload_filer8   r[   r.   )r+   r   r   r!   rG   Zresolved_path1Zresolved_path2Z	same_reporb   fcontentr,   r,   r-   cp_file@  sF   




zHfFileSystem.cp_filec                 K   s*   | j |fi |}d|vrt||d S )Nr   )rB   IsADirectoryError)r+   rF   rG   rB   r,   r,   r-   modifiedj  s   zHfFileSystem.modifiedc                    sd   |  |}|js(dt|j }||v }| }|s ||ddn|}|dddS t j|fi |S )Nr%   r$   r   r   	directoryr{   )rU   r"   r(   r!   r.   r   r9   rB   )r+   rF   rG   re   rI   r   r|   r>   r,   r-   rB   p  s   
zHfFileSystem.infoc           	         s
  |d ur|dk rt dt|tr |g||S t } fdd|D }|D ]Q}t|rQt j|fi |}||O }|rP|t jt|f||d|O }q'|rft j|f|ddd|}||O }||vrx|du ss 	|rx|
| q'|st|tt|S )	Nr   zmaxdepth must be at least 1c                    s   g | ]}  |qS r,   )rM   )rk   pr*   r,   r-   rn     rp   z,HfFileSystem.expand_path.<locals>.<listcomp>)rg   rh   TF)rh   Zwithdirsrw   )rJ   rD   r3   rs   setr   globlistfindrH   addrQ   sorted)	r+   rF   rg   rh   rG   r   r   bitZrecr,   r*   r-   rs   z  s,   
$
zHfFileSystem.expand_pathrV   )r\   N)FNN)TFN)FN)!r/   r0   r1   r2   Zroot_markerprotocolr   r3   r:   r	   boolrC   rA   rH   r   rU   r[   r`   rf   intrv   r   r
   r   r   r   r   r   r   r   rB   rs   __classcell__r,   r,   r>   r-   r5   +   s    
E


1 
*r5   c                       sd   e Zd Zddededee f fddZdeded	efd
dZ	dddZ
dded	dfddZ  ZS )r^   NfsrF   r!   c                    s.   t  j||fi | |  |j||d| _d S )Nra   )r9   r:   rU   re   )r+   r   rF   r!   rG   r>   r,   r-   r:     s   zHfFileSystemFile.__init__startendr#   c                 C   s   dd| d|d  i| j j }| j j dt| jjd| jj  dt	| jj
 dt	| jj }td||d	}t| |jS )
Nrangezbytes=-r   r&   r$   z	/resolve/GET)r   )r   r;   r   r7   r   r'   re   r   r    r(   r!   r"   r   r   r   )r+   r   r   r   urlrr,   r,   r-   _fetch_range  s   
@zHfFileSystemFile._fetch_rangec                 C   s   t jddd| _d S )Nzhffs-F)prefixdelete)tempfileNamedTemporaryFile	temp_filer*   r,   r,   r-   _initiate_upload  s   z!HfFileSystemFile._initiate_uploadFfinalc                 C   s   | j d | j  }| j| |rN| j  | jjj| jj	| j
j| j
j| jj| j
j| j
j| jd| jdd t| jj	 | jj| j
 d d S d S )Nr   rb   rc   r   rd   )bufferseekr   r   writecloser   r;   r   r|   re   r"   r    r8   r   r!   rG   r'   osremover[   r.   )r+   r   blockr,   r,   r-   _upload_chunk  s(   





zHfFileSystemFile._upload_chunkrV   )r#   N)F)r/   r0   r1   r5   r3   r   r:   r   bytesr   r   r   r   r   r,   r,   r>   r-   r^     s
     
r^   sr#   c                 C   s   t | ddS )Nr$   )safe)r   )r   r,   r,   r-   r(     s   r(   )/r   r   r   dataclassesr   r   r   r   typingr   r   r   r   r	   r
   urllib.parser   r   ZfsspecZ_commit_apir   r   	constantsr   r   r   r   r   Zhf_apir   utilsr   r   r   r   r   r   r   r   r   ZAbstractFileSystemr5   specZAbstractBufferedFiler^   r3   r(   r,   r,   r,   r-   <module>   s(     (  o+