o
    eE                     @   s  d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZ dd	lmZ dd
lmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( erxddl)m*Z* e$+e,Z-e.dZ/dZ0dddZ1eG dd dZ2e%	dEdee2 dee3 de3de3dee3 deee4 ee4 f fddZ5G dd deZ6G d d! d!eZ7d"d#d$edee3 ddfd%d&Z8d'e4fd(d)Z9d*e4fd+d,Z:d-e4fd.d/Z;d"d#d0e3ddfd1d2Z<d"d#d3ed4e=d0e3ddf
d5d6Z>d3ed7e2d4e=dee3 fd8d9Z?d:ee d;e3de7fd<d=Z@d"d#d>ee3 d4e=dee fd?d@ZAd"d#d>ee3 d4e=dee fdAdBZBG dCdD dDeZCdS )Fz.Git LFS related type definitions and utilities    N)AbstractContextManager)	dataclass)ceil)getsize)Path)TYPE_CHECKINGBinaryIODictIterableListOptionalTuple	TypedDict)HTTPBasicAuth)ENDPOINTHF_HUB_ENABLE_HF_TRANSFERREPO_TYPES_URL_PREFIXES)get_session   )get_token_to_sendhf_raise_for_statushttp_backoffloggingvalidate_hf_hub_args)sha256sha_fileobj)CommitOperationAddz^[0-9a-f]{40}$zlfs-multipart-uploadzapplication/vnd.git-lfs+json)AcceptzContent-Typec                   @   s`   e Zd ZU dZeed< eed< eed< edefddZ	edefd	d
Z
edefddZdS )
UploadInfoad  
    Dataclass holding required information to determine whether a blob
    should be uploaded to the hub using the LFS protocol or the regular protocol

    Args:
        sha256 (`bytes`):
            SHA256 hash of the blob
        size (`int`):
            Size in bytes of the blob
        sample (`bytes`):
            First 512 bytes of the blob
    r   sizesamplepathc                 C   s\   t |}t|d}|dd d }t|}W d    n1 s"w   Y  | |||dS )Nrb   r   r   r    )r   ioopenpeekr   )clsr!   r   filer    sha r+   CD:\Projects\ConvertPro\env\Lib\site-packages\huggingface_hub/lfs.py	from_pathF   s   
zUploadInfo.from_pathdatac                 C   s&   t | }| t||d d |dS )Nr#   )r   r    r   )r   digestlen)r(   r.   r*   r+   r+   r,   
from_bytesN   s   zUploadInfo.from_bytesfileobjc                 C   sD   | d}|dtj t|}| }|dtj | |||dS )Nr#   r   r$   )readseekr%   SEEK_SETr   tell)r(   r2   r    r*   r   r+   r+   r,   from_fileobjS   s   
zUploadInfo.from_fileobjN)__name__
__module____qualname____doc__bytes__annotations__intclassmethodstrr-   r1   r   r7   r+   r+   r+   r,   r   3   s   
 r   upload_infostoken	repo_typerepo_idendpointreturnc           
      C   s   |dur|nt }d}|tv rt| }| d| | d}t j|tdddgdd	 | D d
dtdt|p4dd}t| | }|	dd}	t
|	tsPtddd	 |	D dd	 |	D fS )a  
    Requests the LFS batch endpoint to retrieve upload instructions

    Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md

    Args:
        upload_infos (`Iterable` of `UploadInfo`):
            `UploadInfo` for the files that are being uploaded, typically obtained
            from `CommitOperationAdd.upload_info`
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        token (`str`, *optional*):
            An authentication token ( See https://huggingface.co/settings/tokens )

    Returns:
        `LfsBatchInfo`: 2-tuple:
            - First element is the list of upload instructions from the server
            - Second element is an list of errors, if any

    Raises:
        `ValueError`: If an argument is invalid or the server response is malformed

        `HTTPError`: If the server returned an error
    N /z.git/info/lfs/objects/batchuploadbasic	multipartc                 S   s   g | ]}|j  |jd qS )oidr   )r   hexr   ).0rI   r+   r+   r,   
<listcomp>   s    z'post_lfs_batch_info.<locals>.<listcomp>r   )	operationZ	transfersobjectsZ	hash_algoZaccess_tokenT)headersjsonauthrR   zMalformed response from serverc                 S   s   g | ]
}d |vrt |qS error)_validate_batch_actionsrO   objr+   r+   r,   rP          c                 S   s   g | ]
}d |v rt |qS rV   )_validate_batch_errorrY   r+   r+   r,   rP      r[   )r   r   r   postLFS_HEADERSr   r   r   rT   get
isinstancelist
ValueError)
rA   rB   rC   rD   rE   Z
url_prefixZ	batch_urlrespZ
batch_inforR   r+   r+   r,   post_lfs_batch_info]   s8   #

rd   c                   @   s   e Zd ZU eed< eed< dS )PayloadPartT
partNumberetagN)r8   r9   r:   r>   r=   r@   r+   r+   r+   r,   re      s   
 re   c                   @   s&   e Zd ZU dZeed< ee ed< dS )CompletionPayloadTz?Payload that will be sent to the Hub when uploading multi-part.rM   partsN)r8   r9   r:   r;   r@   r=   r   re   r+   r+   r+   r,   rh      s   
 rh   rQ   r   lfs_batch_actionc           	   	   C   s>  t | |d}|du rtd| j d dS |d d }t| |d d}|dur2t| |di }|d}|durdzt|}W n ttfyX   td	| d
w t	| |||d d nt
| |d d |durt| t j|d tdt|pdd| jj | jjdd}t| t| j d dS )a  
    Handles uploading a given object to the Hub with the LFS protocol.

    Can be a No-op if the content of the file is already present on the hub large file storage.

    Args:
        operation (`CommitOperationAdd`):
            The add operation triggering this upload.
        lfs_batch_action (`dict`):
            Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
            more details.
        token (`str`, *optional*):
            A [user access token](https://hf.co/settings/tokens) to authenticate requests against the Hub

    Raises:
        - `ValueError` if `lfs_batch_action` is improperly formatted
        - `HTTPError` if the upload resulted in an error
    actionsNzContent of file z. is already present upstream - skipping uploadrI   verifyheader
chunk_sizezTMalformed response from LFS batch endpoint: `chunk_size` should be an integer. Got 'z'.href)rQ   rm   rn   
upload_url)rQ   rp   USERT)usernamepasswordrL   )rU   rT   z: Upload successful)rX   r_   loggerdebugZpath_in_repo_validate_lfs_actionr>   rb   	TypeError_upload_multi_part_upload_single_partr   r]   r   r   upload_infor   rN   r   r   )	rQ   rj   rB   rk   upload_actionverify_actionrm   rn   Zverify_respr+   r+   r,   
lfs_upload   s>   


r}   
lfs_actionc                 C   s:   t | dtr| ddu st | dtstd| S ).validates response from the LFS batch endpointro   rm   Nz"lfs_action is improperly formatted)r`   r_   r@   dictrb   )r~   r+   r+   r,   rv      s
   rv   lfs_batch_actionsc                 C   sp   t | dtrt | dtstd| di d}| di d}|dur.t| |dur6t| | S )r   rM   r   z)lfs_batch_actions is improperly formattedrk   rI   rl   N)r`   r_   r@   r>   rb   rv   )r   r{   r|   r+   r+   r,   rX      s    rX   lfs_batch_errorc                 C   sh   t | dtrt | dtstd| d}t |tr.t |dtr.t |dts2td| S )r   rM   r   z'lfs_batch_error is improperly formattedrW   messagecode)r`   r_   r@   r>   rb   r   )r   
error_infor+   r+   r,   r\     s    
r\   rp   c                 C   sH   | j dd}td||d}t| W d   dS 1 sw   Y  dS )aZ  
    Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)

    Args:
        upload_url (`str`):
            The URL to PUT the file to.
        fileobj:
            The file-like object holding the data to upload.

    Returns: `requests.Response`

    Raises: `requests.HTTPError` if the upload resulted in an error
    TZ	with_tqdmPUTr.   N)as_filer   r   )rQ   rp   r2   responser+   r+   r,   ry     s   
"ry   rm   rn   c                 C   s   t || j|d}t}trt| jtst| jtstd d}|r(t	| ||dnt
| ||d}t j|t|| jj td}t| dS )z@
    Uploads file using HF multipart LFS transfer protocol.
    )rm   rz   rn   zlhf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular uploadF)rQ   sorted_parts_urlsrn   )rT   rS   N)_get_sorted_parts_urlsrz   r   r`   path_or_fileobjr@   r   warningswarn_upload_parts_hf_transfer_upload_parts_iterativelyr   r]   _get_completion_payloadr   rN   r^   r   )rQ   rm   rn   rp   r   Zuse_hf_transferresponse_headersZcompletion_resr+   r+   r,   rx   '  s,   

rx   rz   c                 C   sN   dd t dd |  D dd dD }t|}|t|j| kr%td|S )Nc                 S   s   g | ]\}}|qS r+   r+   )rO   _rp   r+   r+   r,   rP   K  s    z*_get_sorted_parts_urls.<locals>.<listcomp>c                 S   s2   g | ]\}}|  rt|d krt|d|fqS )r   
   )isdigitr0   r>   )rO   Zpart_numrp   r+   r+   r,   rP   N  s    c                 S   s   | d S Nr   r+   )tr+   r+   r,   <lambda>S  s    z(_get_sorted_parts_urls.<locals>.<lambda>)keyz0Invalid server response to upload large LFS file)sorteditemsr0   r   r   rb   )rm   rz   rn   Zsorted_part_upload_urlsZ	num_partsr+   r+   r,   r   J  s   r   r   rM   c                 C   sf   g }t | D ]'\}}|d}|d u s|dkr#td| d|d  ||d |d q||dS )Nrg   rG   zInvalid etag (`z`) returned for part r   )rf   rg   )rM   ri   )	enumerater_   rb   append)r   rM   ri   Zpart_numberrm   rg   r+   r+   r,   r   \  s   

r   r   c           	   
   C   s   g }| j dd<}t|D ].\}}t||| |d}td||d}t| ||j W d    n1 s6w   Y  qW d    |S 1 sGw   Y  |S )NTr   )	seek_from
read_limitr   r   )r   r   SliceFileObjr   r   r   rS   )	rQ   r   rn   rS   r2   Zpart_idxZpart_upload_urlZfileobj_sliceZpart_upload_resr+   r+   r,   r   k  s&   


r   c              
   C   sd   zddl m} W n ty   tdw z|| j||ddddW S  ty1 } ztd|d }~ww )	Nr   )multipart_uploadzFast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is not available in your environment. Try `pip install hf_transfer`.         )	file_pathZ
parts_urlsrn   Z	max_filesZparallel_failuresmax_retriesz~An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.)Zhf_transferr   ImportErrorrb   r   	ExceptionRuntimeError)rQ   r   rn   r   er+   r+   r,   r   |  s0   r   c                   @   sx   e Zd ZdZdededefddZdd Zd	d
 ZddefddZ	defddZ
ejfdededefddZdd ZdS )r   a  
    Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object.

    This is NOT thread safe

    Inspired by stackoverflow.com/a/29838711/593036

    Credits to @julien-c

    Args:
        fileobj (`BinaryIO`):
            A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course).
            `fileobj` will be reset to its original position when exiting the context manager.
        seek_from (`int`):
            The start of the slice (offset from position 0 in bytes).
        read_limit (`int`):
            The maximum number of bytes to read from the slice.

    Attributes:
        previous_position (`int`):
            The previous position

    Examples:

    Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327):
    ```python
    >>> with open("path/to/file", "rb") as file:
    ...     with SliceFileObj(file, seek_from=128, read_limit=200) as fslice:
    ...         fslice.read(...)
    ```

    Reading a file in chunks of 512 bytes
    ```python
    >>> import os
    >>> chunk_size = 512
    >>> file_size = os.getsize("path/to/file")
    >>> with open("path/to/file", "rb") as file:
    ...     for chunk_idx in range(ceil(file_size / chunk_size)):
    ...         with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice:
    ...             chunk = fslice.read(...)

    ```
    r2   r   r   c                 C   s   || _ || _|| _d S N)r2   r   r   )selfr2   r   r   r+   r+   r,   __init__  s   
zSliceFileObj.__init__c                 C   sF   | j  | _| j dtj}t| j|| j | _	| j | jt
j | S r   )r2   r6   _previous_positionr4   osSEEK_ENDminr   r   _lenr%   r5   )r   Zend_of_streamr+   r+   r,   	__enter__  s
   zSliceFileObj.__enter__c                 C   s   | j | jtj d S r   )r2   r4   r   r%   r5   )r   exc_type	exc_value	tracebackr+   r+   r,   __exit__  s   zSliceFileObj.__exit__nc                 C   sH   |   }|| jkrdS | j| }| j|dk r|}|S t||}|S )N    r   )r6   r   r2   r3   r   )r   r   posZremaining_amountr.   r+   r+   r,   r3     s   

zSliceFileObj.readrF   c                 C   s   | j  | j S r   )r2   r6   r   r   r+   r+   r,   r6     s   zSliceFileObj.telloffsetwhencec                 C   s   | j }|| j }|tjtjfv r)|tjkr|| n|| }t|t||}tj}n|tjkr@| j	 }t|| t||| }nt
d| d| j||| j  S )Nzwhence value z is not supported)r   r   r   r5   r   maxr   SEEK_CURr2   r6   rb   r4   )r   r   r   startendZcur_posr+   r+   r,   r4     s   


zSliceFileObj.seekc                 c   s    | j ddV  d S )Ni  @ )r   )r3   r   r+   r+   r,   __iter__  s   zSliceFileObj.__iter__N)r   )r8   r9   r:   r;   r   r>   r   r   r   r3   r6   r   r5   r4   r   r+   r+   r+   r,   r     s    ,r   r   )Dr;   r%   r   rer   
contextlibr   dataclassesr   mathr   Zos.pathr   pathlibr   typingr   r   r	   r
   r   r   r   r   Zrequests.authr   Zhuggingface_hub.constantsr   r   r   Zhuggingface_hub.utilsr   utilsr   r   r   r   r   Z	utils.shar   r   Z_commit_apir   Z
get_loggerr8   rt   compileZ	OID_REGEXZLFS_MULTIPART_UPLOAD_COMMANDr^   r   r@   r   rd   re   rh   r}   rv   rX   r\   ry   r>   rx   r   r   r   r   r   r+   r+   r+   r,   <module>   s   (

)H<
#

