o
    ef                     @   sX  d dl Z d dlZd dlZd dlZd dlZzd dlZd dlmZ dZW n e	y-   dZY nw zd dl
mZ d dlmZ d dlmZ d dlmZ dZW n e	yU   dZY nw d	Zd
d ZG dd deZe ZG dd deZede  G dd deZdd ZG dd deZG dd deZG dd deZdd Zdd Z dd  Z!d!d" Z"dS )#    N)	HdfsErrorTF)	BosClient)	exception)BceClientConfiguration)BceCredentialsi   c                 C   s    t  }||  t| S N)hashlibmd5updatebase64standard_b64encodedigest)bufferr	    r   AD:\Projects\ConvertPro\env\Lib\site-packages\visualdl/io/bfile.pycontent_md5+   s   
r   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )FileFactoryc                 C   s
   i | _ d S r   )_register_factoriesselfr   r   r   __init__2   s   
zFileFactory.__init__c                 C   s   | j ||i d S r   )r   r
   )r   path
filesystemr   r   r   register_filesystem5      zFileFactory.register_filesystemc                 C   s   | dr'd| jvr'tstdz	tdt  W n$ tjj	y&   tdw | dr>d| jvr>t
s7tdtdt  d}|d	}|d
krO|d | }| j|d }|d u r`td| |S )Nhdfs://hdfsz#Please install module named "hdfs".z,Please initialize `~/.hdfscli.cfg` for HDFS.bos://Zbosz-Please install module named "bce-python-sdk". z://r   z&No recognized filesystem for prefix %s)
startswithr   HDFS_ENABLEDRuntimeErrordefault_file_factoryr   HDFileSystemr   utilr   BOS_ENABLEDBosFileSystemfindget
ValueError)r   r   prefixindexfsr   r   r   get_filesystem8   s@   


zFileFactory.get_filesystemN)__name__
__module____qualname__r   r   r-   r   r   r   r   r   1   s    r   c                   @   sx   e Zd Zdd Zedd Zedd Zedd Zd	d
 ZdddZ	dddZ
dd ZdddZdddZdd ZdS )LocalFileSystemc                 C   s   d S r   r   r   r   r   r   r   X      zLocalFileSystem.__init__c                 C   s   t j| S r   )osr   existsr   r   r   r   r4   [   s   zLocalFileSystem.existsc                 C   s   t |  d S r   )r3   makedirsr5   r   r   r   r6   _   s   zLocalFileSystem.makedirsc                 G   s   t jj| g|R  S r   )r3   r   joinr   pathsr   r   r   r7   c   s   zLocalFileSystem.joinc                 C   s   t j|S r   )r3   r   isfiler   filenamer   r   r   r:   g      zLocalFileSystem.isfileTc                 C   sD   |rdnd}t ||}| }W d    |S 1 sw   Y  |S )Nrbr)openread)r   r<   binary_modemodereaderdatar   r   r   	read_filej   s   

zLocalFileSystem.read_fileFNc                 C   s   |rdnd}|r
d nd}d }|d ur| dd }t|||d }|d ur)|| ||}	d| i}
|	|
fW  d    S 1 sBw   Y  d S )Nr>   r?   utf-8last_offset)rC   encoding)r(   r@   seekrA   tell)r   r<   rB   sizecontinue_fromrC   rI   offsetfprE   continue_from_tokenr   r   r   rA   p   s   

$zLocalFileSystem.readc                 C   sN   d|v rd nd}t |||d}|| W d    d S 1 s w   Y  d S )NbrG   )rI   )r@   write)r   r<   file_contentrC   rI   rO   r   r   r   _write}   s   "zLocalFileSystem._writec                 C   D   z|  |||r	dnd W d S  ty!   | tj| Y d S w NabarT   FileNotFoundErrorr6   r3   r   dirnamer   r<   rS   rB   r   r   r   append   
   zLocalFileSystem.appendc                 C   rU   rV   rY   r\   r   r   r   rR      r^   zLocalFileSystem.writec                 C   s"   dt jkrt j|ddS t |S )NposixT)followlinks)r3   namewalk)r   dirr   r   r   rb      s   

zLocalFileSystem.walkT)FNNF)r.   r/   r0   r   staticmethodr4   r6   r7   r:   rF   rA   rT   r]   rR   rb   r   r   r   r   r1   W   s    






r1   r   c                   @   sh   e Zd Zdd Zdd Zdd Zddd	Zd
d Zedd Z	dddZ
dddZdddZdd ZdS )r#   c                 C   s   t j d| _d S )Ndev)r   configZConfigZ
get_clientclir   r   r   r   r      s   zHDFileSystem.__init__c                 C   s$   | j j|dd  ddd u rdS dS )N   F)	hdfs_pathstrictT)ri   statusr   r   r   r   r   r4      s   zHDFileSystem.existsc                 C      t |S r   r4   r;   r   r   r   r:         zHDFileSystem.isfileTc                 C   sD   | j j|dd  d}| }W d    |S 1 sw   Y  |S Nrj   rk   )ri   rA   )r   r<   rB   rD   rE   r   r   r   rF      s   

zHDFileSystem.read_filec                 C   s   | j j|dd  d d S rr   )ri   r6   rn   r   r   r   r6      s   zHDFileSystem.makedirsc                 G   $   t jj| g|R  }|dd |S N\/r3   r   r7   replacer   r9   resultr   r   r   r7         zHDFileSystem.joinFr   Nc           
      C   s   d}|d ur| dd}|rd nd}z1| jj|dd  ||d}| }d|t| i}	||	fW  d    W S 1 s<w   Y  W d S  tyN   tdw )Nr   rH   rG   rj   )rk   rN   rI   zNo more events to read on HDFS.)r(   ri   rA   lenr   EOFError)
r   r<   rB   rL   rM   rN   rI   rD   rE   rP   r   r   r   rA      s"   (zHDFileSystem.readc                 C      | j j|dd  |dd d S Nrj   T)rk   rE   r]   ri   rR   r\   r   r   r   r]         zHDFileSystem.appendc                 C   r   r   r   r\   r   r   r   rR      r   zHDFileSystem.writec                 C   s$   | j j|dd  d}dd |D S )Nrj   rs   c                 s   s$    | ]\}}}d | ||gV  qdS )r   Nr   ).0rootdirsfilesr   r   r   	<genexpr>   s   " z$HDFileSystem.walk.<locals>.<genexpr>)ri   rb   )r   rc   Zwalksr   r   r   rb      s   zHDFileSystem.walkrd   Fr   Nre   )r.   r/   r0   r   r4   r:   rF   r6   rf   r7   rA   r]   rR   rb   r   r   r   r   r#      s    




r#   c                 C   s:   | dd  } |  d}| d| }| |d d  }||fS )N   rw   r      )r+   )r   r+   bucket_name
object_keyr   r   r   get_object_info   s
   
r   c                   @   s:   e Zd ZdddZdd Zdd Zedd	 Zd
d ZdS )BosConfigClientbj.bcebos.comc                 C   &   t t||||d| _t| j| _d S NcredentialsZendpointZsecurity_tokenr   r   rh   r   
bos_clientr   Zbos_akZbos_skbos_stsbos_hostr   r   r   r      s   zBosConfigClient.__init__c                 C   8   t |\}}z
| j|| W dS  tjy   Y dS w NTFr   r   get_object_meta_datar   BceErrorr   r   r   r   r   r   r   r4         zBosConfigClient.existsc                 C   d   | ds	|d7 }| |rd S t|\}}| ds|d7 }d}| jj|||t|t|d d S Nrw       r   keyrE   r   content_lengthendswithr4   r   r   append_objectr   r}   r   r   r   r   	init_datar   r   r   r6         



zBosConfigClient.makedirsc                 G   rt   ru   rx   rz   r   r   r   r7      r|   zBosConfigClient.joinc                 C   sN   |  |s
| | t|\}}| ||}td|  | jj|||d d S )NzUploading file `%s`)bucketr   	file_name)r4   r6   r   r7   printr   Zput_object_from_file)r   r   r<   r   r   r   r   r   upload_object_from_file   s   


z'BosConfigClient.upload_object_from_fileNr   )	r.   r/   r0   r   r4   r6   rf   r7   r   r   r   r   r   r      s    

r   c                   @   s   e Zd Zd$ddZdd Z	d%ddZd	d
 Zdd Zd$ddZdd Z	dd Z
dd Zedd Zd&ddZdd Zd'ddZd(d d!Zd"d# ZdS ))r&   Tc                 C   sH   |r"d| _ d| _|   t| j| _i | _d| _d| _t		 | _
d S d S )Nr   r   r   )max_contents_countmax_contents_timeget_bos_configr   rh   r   Zfile_length_map_file_contents_to_add_file_contents_counttime_start_append_time)r   Z
write_flagr   r   r   r     s   zBosFileSystem.__init__c                 C   s   t d}|stdt d}|stdt d}|s!tdtt dd| _tt d	d| _t d
}tt||||d| _d S )NZBOS_HOSTz${BOS_HOST} is not found.ZBOS_AKz${BOS_AK} is not found.ZBOS_SKz${BOS_SK} is not found.ZBOS_CACHE_COUNTr   ZBOS_CACHE_TIMEZBOS_STSr   )	r3   getenvKeyErrorintr   r   r   r   rh   )r   r   Zaccess_key_idZsecret_access_keyr   r   r   r   r     s"   



zBosFileSystem.get_bos_configr   c                 C   r   r   r   r   r   r   r   set_bos_config)  s   zBosFileSystem.set_bos_configc                 C   s   dd l }dd l}ddlm} t|d}||d }W d    n1 s%w   Y  |d }|j|d }|d}|d}	d	|kr\|	d
}
|	d}|	d}| |
|| d S t	d
|	 d S )Nr   )CONFIG_PATHr?   
server_urlz/sts/)urlcodemsg000000sts_aksts_sktokenz%Renew bos client error. Error msg: {})requestsjsonZvisualdl.utils.dirr   r@   loadpostr(   r   r   format)r   r   r   r   rO   r   r   resZerr_coder   r   r   Z	sts_tokenr   r   r   renew_bos_client_from_server1  s"   




z*BosFileSystem.renew_bos_client_from_serverc                 C   ro   r   rp   r;   r   r   r   r:   D  rq   zBosFileSystem.isfilec                 C   s   t |\}}| j||}|S r   )r   r   get_object_as_string)r   r<   binaryr   r   r{   r   r   r   rF   G  s   zBosFileSystem.read_filec                 C   r   r   r   r   r   r   r   r4   L  r   zBosFileSystem.existsc                 C      | j ||S r   )r   r   )r   r   r   r   r   r   get_metaT     zBosFileSystem.get_metac                 C   r   r   r   r   r   r   r   r6   W  r   zBosFileSystem.makedirsc                 G   rt   ru   rx   rz   r   r   r   r7   g  r|   zBosFileSystem.joinFr   Nc                 C   sp   t |\}}d}|d ur|dd}t| ||jj}||k r.| jj||||d gd}	nd}	d|i}
|	|
fS )Nr   rH   r   )r   r   ranger   )r   r(   r   r   metadatar   r   r   )r   r<   rB   rL   rM   r   r   rN   lengthrE   rP   r   r   r   rA   m  s    
zBosFileSystem.readc                 C   s(   | j | jkst | j | jkrdS dS r   )r   r   r   r   r   r   r   r   r   ready_to_append  s   zBosFileSystem.ready_to_appendc                 C   s  |  j |7  _ |  jd7  _|s|  sd S | j }t|\}}| |sd}z| jj|||t|t|d W nI t	j
t	jfy } z9|dkrt|   z	| j|| W n! t	jys   | jj|||t|t|d Y W Y d }~d S w |W Y d }~nd }~ww t|}	z| ||jj}
| jj|||t||	|
d W n7 t	j
t	jfy } z'|dkr|   | ||jj}
| jj|||t||	|
d n|W Y d }~nd }~ww d| _ d| _t | _d S )Nr   r   r   zvisualdl-server)r   r   rE   r   r   rN   r   )r   r   r   r   r4   r   r   r   r}   r   ZBceServerErrorZBceHttpClientErrorr   r   r   r   r   r   r   r   )r   r<   rS   rB   forcer   r   r   er   rN   r   r   r   r]     s   



zBosFileSystem.appendc                 C   s   | j ||dd d S )NF)rB   )r]   r\   r   r   r   rR     r   zBosFileSystem.writec                 C   sh   G dd d}t |\}}|dv rd }n|dr|n|d }| jj||d}dd |jD }|||S )Nc                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
z)BosFileSystem.walk.<locals>.WalkGeneratorc                 S   s&   d | _ d| _|| _| | d| _d S Nr   )contentsr   r   handle_contentscount)r   r   r   r   r   r   r     s
   

z2BosFileSystem.walk.<locals>.WalkGenerator.__init__c              	   S   s   i }|D ]8}z| d}|d| }||d d  }W n ty(   d}|}Y nw || v r7|| | q|g||< qg }| D ]\}}|td| j |g |g qCt|| _	|| _
d S )Nrw   r   r   .r   )rindexr)   keysr]   itemsr&   r7   r   r}   r   r   )r   r   Zcontents_mapitemr   r   valueZ	temp_walkr   r   r   r     s*   


z9BosFileSystem.walk.<locals>.WalkGenerator.handle_contentsc                 S      | S r   r   r   r   r   r   __iter__  r2   z2BosFileSystem.walk.<locals>.WalkGenerator.__iter__c                 S   s.   | j | jk r|  j d7  _ | j| j d  S t)Nr   )r   r   r   StopIterationr   r   r   r   __next__  s   z2BosFileSystem.walk.<locals>.WalkGenerator.__next__N)r.   r/   r0   r   r   r   r   r   r   r   r   WalkGenerator  s
    r   )r   z./rw   )r*   c                 S   s   g | ]}|j qS r   )r   )r   contentr   r   r   
<listcomp>  s    z&BosFileSystem.walk.<locals>.<listcomp>)r   r   r   Zlist_objectsr   )r   rc   r   r   r   r*   responser   r   r   r   rb     s   )
zBosFileSystem.walkrd   r   r   )FFre   )r.   r/   r0   r   r   r   r   r:   rF   r4   r   r6   rf   r7   rA   r   r]   rR   rb   r   r   r   r   r&   
  s$    






Cr&   c                   @   sx   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdddZ	dddZ
dd Zdd Zdd Zdd Zdd ZdS )BFilec                 C   sx   |dvrt d||| _t|| _t| jd| _d | _t	| _
d| _d | _d | _d| _d|v | _d|v | _d| _d S )N)r?   r>   brwwbbwzmode {} not supported by BFile.r]   r   FrQ   r   )NotImplementedErrorr   	_filenamer"   r-   r,   hasattrfs_supports_appendbuff_DEFAULT_BLOCK_SIZEbuff_chunk_sizebuff_offsetcontinuation_token
write_tempwrite_startedrB   
write_modeclosed)r   r<   rC   r   r   r   r     s    


zBFile.__init__c                 C   r   r   r   r   r   r   r   	__enter__  r2   zBFile.__enter__c                 C   s   |    d | _d| _d | _d S r   )closer   r   r   )r   exc_typeexc_valexc_tbr   r   r   __exit__  s   
zBFile.__exit__c                 C   r   r   r   r   r   r   r   r   %  r2   zBFile.__iter__c                 C   s   | j |S r   )r,   r:   r;   r   r   r   r:   (  r=   zBFile.isfilec                 C   s:   | j }tt| j|| }|  j |7  _ | j|||  S )af  Read buffer from index self.buffer_offset to index new_buff_offset.

        self.buff_offset marks the last position of the last read,
        new_buff_offset indicates the last position of this read.
        self.buff_offset will be updated by new_buff_offset after this read.

        Returns:
            self.buff[i1: i2]: Content of self.buff.
        )r   minr}   r   )r   Znew_buff_offsetZold_buff_offset	read_sizer   r   r   _read_buffer_to_offset+  s   
zBFile._read_buffer_to_offsetTc                 C   r   r   )r,   rF   )r   r<   Zbinnaryr   r   r   rF   :  r   zBFile.read_fileNc                 C   s   d}| j r2t| j | jkr2|dur*| | j| }t||kr!|S |}|t|8 }n| t| j }|dur<t| j|nd}| j| j| j	|| j
\| _ | _
d| _|dur[| |}n| t| j }|rk|| }|S |}|S )zzRead `n` or all contents of self.buff or file.

        Returns:
            result: Data from self.buff or file.
        Nr   )r   r}   r   r  maxr   r,   rA   r   rB   r   )r   nr{   chunkr  r   r   r   rA   =  s,   
z
BFile.readc                 C   s   | j std| jrtd| jr/| js#| j| j|| j d| _dS | j	| j|| j dS | j
du rA| jr9dnd}t|| _
| j
| dS )zqWrite contents to file.

        Args:
            file_content: Contents waiting to be written to file.
        zFile not opened in write modeFile already closedTNzw+bzw+)r   r!   r   r   r   r,   rR   r   rB   r]   r   tempfileTemporaryFile)r   rS   rC   r   r   r   rR   k  s   

zBFile.writec                 C   s   d }	 | j s| d}|r|d dks| j s|S | j st nJ| j d| j}|dkrA| |d | j }|r=|| }|S |}|S | t| j d | j }|rT|| n|}|rc|d dksa| j sc|S | j sit q)NTr   
)r   rA   r   r'   r   r}   )r   liner+   r  r   r   r   r     s.   
zBFile.__next__c                 C   s   |   S r   )r   r   r   r   r   next  rq   z
BFile.nextc                 C   sz   | j rtd| js7| jdur9| j  | jd | j }|dur;| j| j	|| j
 | jt| dS dS dS dS )zFlush data to disk.
        r  Nr   )r   r!   r   r   flushrJ   rA   r,   rR   r   rB   r}   )r   r  r   r   r   r    s   


zBFile.flushc                 C   sn   t | jtrz| jj| jd| jdd W n	 ty   Y nw |   | jd ur2| j	  d | _d| _
d| _d S )Nr   T)r   F)
isinstancer,   r&   r]   r   rB   	Exceptionr  r   r   r   r   r   r   r   r   r     s   



zBFile.closerd   r   )r.   r/   r0   r   r   r   r   r:   r  rF   rA   rR   r   r  r  r   r   r   r   r   r   
  s    

.r   c                 C      t | | S r   )r"   r-   r4   r5   r   r   r   r4        r4   c                 C   r  r   )r"   r-   r6   r5   r   r   r   r6     r  r6   c                 G   s   t | j| g|R  S r   )r"   r-   r7   r8   r   r   r   r7     s   r7   c                 C   r  r   )r"   r-   rb   )rc   r   r   r   rb     r  rb   )#r3   r  r   r   r   r   Z	hdfs.utilr   r    ImportErrorZ baidubce.services.bos.bos_clientr   Zbaidubcer   Z!baidubce.bce_client_configurationr   Zbaidubce.auth.bce_credentialsr   r%   r   r   objectr   r"   r1   r   r#   r   r   r&   r   r4   r6   r7   rb   r   r   r   r   <module>   sN   #>73   6