o
    eRO                     @   s  d Z ddlmZ ddlmZ ddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZzddlmZ W n! eyg   zddlmZ W n eyd   ddlmZ Y nw Y nw ddlZdZdadadZd	d
 Zdd Zdd Zdd Zdd Zdd Zdd Z dd Z!dd Z"dd Z#dd Z$dd  Z%d!d" Z&d#d$ Z'd%d& Z(d'Z)d(d) Z*G d*d+ d+e+Z,d,d- Z-d.d/ Z.d0d1 Z/d2d3 Z0d4d5 Z1d6d7 Z2d8d9 Z3dCd;d<Z4d=d> Z5dDd?d@Z6e7dAkre6ej8dBd  dS dS )Ea8  
Basic tools for working with LMDB.

    copy: Consistent high speed backup an environment.
        %prog copy -e source.lmdb target.lmdb

    copyfd: Consistent high speed backup an environment to stdout.
        %prog copyfd -e source.lmdb > target.lmdb/data.mdb

    drop: Delete one or more sub-databases.
        %prog drop db1

    dump: Dump one or more databases to disk in 'cdbmake' format.
        Usage: dump [db1=file1.cdbmake db2=file2.cdbmake]

        If no databases are given, dumps the main database to 'main.cdbmake'.

    edit: Add/delete/replace values from a database.
        %prog edit --set key=value --set-file key=/path \
                   --add key=value --add-file key=/path/to/file \
                   --delete key

    get: Read one or more values from a database.
        %prog get [<key1> [<keyN> [..]]]

    readers: Display readers in the lock table
        %prog readers -e /path/to/db [-c]

        If -c is specified, clear stale readers.

    restore: Read one or more database from disk in 'cdbmake' format.
        %prog restore db1=file1.cdbmake db2=file2.cdbmake

        The special db name ":main:" may be used to indicate the main DB.

    rewrite: Re-create an environment using MDB_APPEND
        %prog rewrite -e src.lmdb -E dst.lmdb [<db1> [<dbN> ..]]

        If no databases are given, rewrites only the main database.

    shell: Open interactive console with ENV set to the open environment.

    stat: Print environment statistics.

    warm: Read environment into page cache sequentially.

    watch: Show live environment statistics
    )absolute_import)with_statementN)BytesIO)StringIOi   zutf-8c                 C   s   t  dt| tS )zlGiven either a Python 2.x or 3.x str, return either a str (Python 2.x)
    or a bytes instance (Python 3.x).unicode)globalsgetstrencodeENCODING)s r   9D:\Projects\ConvertPro\env\Lib\site-packages\lmdb/tool.py	_to_bytesk   s   r   c                 C   s   | t jv o
t| dkS )zReturn ``True`` if the character `c` can be printed visibly and without
    adversely affecting printing position (e.g. newline).   )string	printableord)cr   r   r   isprintq   s   r   c                 C   s<  t  }td}t| D ]Z\}}tjd dkrt|}|d s@|r3|td || |td |td|  td}|d sK|td	 |td
t|f  |t|r`t|ntd7 }q|d rd|d  }|td|  |td	|d   |td || |td |	 
tS )zQReturn a vaguely /usr/bin/xxd formatted representation of the bytestring
    `s`. r      r   z  
z%07x:    z%02x.   )r   r   	enumeratesysversion_infochrwriter   r   getvaluedecoder   )r   siopridxchZneedr   r   r   xxdw   s0   

r(   c                  C   s  t  } d| _dt  | _| jdddd | jddd	d | jd
ddd | jdddddd | jddddd | jdddd | jddddd | jdddd d!d | jd"dd#d$d | d%}|jd&dd'd(d) | d*}|jd+d,d-d |jd.d,d/d |jd0d,d1d |jd2d,d/d |jd3d,d4d | d5}|jd6d7dd8d | d9}|jd:dd;d |jd<dd#d=d |jd>dd?d@d | S )ANzpython -mlmdbz%prog [options] <command>
z-ez--envzEnvironment file to open)helpz-dz--dbz Database to open (default: main)z-rz--readzOpen environment read-onlyz-Sz
--map_sizeint10z#Map size in megabytes (default: 10))typedefaultr)   z-sz--use-single-file
store_truez@The database was created as a single file and not a subdirectory)actionr)   z-Ez--target_envz$Target environment file for "dumpfd"z-xz--xxdzPrint values in xxd formatz-Mz	--max-dbs   zMaximum open DBs (default: 128)z--out-fd   z"copyfd" command target fdzOptions for "copy" commandz	--compactFz!Perform compaction while copying.)r/   r-   r)   zOptions for "edit" commandz--setappendzList of key=value pairs to set.z
--set-filez%List of key pairs to read from files.z--addzList of key=value pairs to add.z
--add-filez--deletez"List of key=value pairs to delete.zOptions for "readers" commandz-cz--cleanz"Clean stale readers? (default: no)zOptions for "watch" commandz--csvz(Generate CSV instead of terminal output.z
--intervalzInterval size (default: 1sec)z--window
   z!Average window size (default: 10))optparseOptionParserprog__doc__rstripusage
add_optionadd_option_group)parsergroupr   r   r   make_parser   sv   










r>   c                 G   s&   |r| |; } t jd| f  td)Nzlmdb.tool: %s
r1   )r   stderrr!   
SystemExit)fmtargsr   r   r   die   s   rC   c                 C   sn   | D ]+\}}| tdt|t|f  | | | td | | | td q| td d S )Nz+%d,%d:z->r   )r!   r   len)cursorfpkeyvaluer   r   r   dump_cursor_to_fp   s   

rI   c                 C   s   i }| D ]/}| d\}}}|std| |dkrd }||v r$td| t|r,t|nd |f||< q|s?td df|d< |S )N=z DB specification missing "=": %r:main:zDB specified twice: %rzmain.cdbmake)	partitionrC   ENVopen_dbr   )rB   db_mapargdbnameseppathr   r   r   db_map_from_args   s   

 rT   c                 C   sf   t |dkr
td |d }tj|rtd| t|tdd td|f  tj	|| j
d d S )	Nr1   z,Please specify output directory (see --help)r   z#Output directory %r already exists.Z0755   zRunning copy to %r....)compact)rD   rC   osrS   existsmakedirsr*   printrM   copyrV   )optsrB   
output_dirr   r   r   cmd_copy   s   
r^   c                 C   s^   |rt d z
t| jdd W n ty&   t d }t d| j| Y nw t| j d S )Nz0"copyfd" command takes no arguments (see --help)wr   r1   zBad --out-fd %d: %s)	rC   rW   fdopenZout_fdOSErrorr   exc_inforM   Zcopyfd)r\   rB   er   r   r   
cmd_copyfd   s   rd   c           	   
   C   s   t |}tjdd<}| D ].\}\}}t|dt}td|f  |j|d}t|| W d    n1 s8w   Y  qW d    d S 1 sIw   Y  d S )NT)bufferswbzDumping to %r...db)	rT   rM   beginitemsopenBUF_SIZErZ   rE   rI   )	r\   rB   rO   txnrQ   rh   rS   rF   rE   r   r   r   cmd_dump  s   "rn   c                    s  |j }t|d  fdd}d}	 |d7 }|d}|dkr!	 |S |dkr*td| zt|d	d
}t|dd
}W n tyH   td| Y nw ||}	|ddkrXtd| ||}
t|	t|
 || krmtd| |ddkrxtd| | j|	|
|d q)Nr1   c                    s   d t | S )N    )joiniter)rR   read1r   r   <lambda>  s    z(restore_cursor_from_fp.<locals>.<lambda>r   T   
   +z$bad or missing plus, line/record #%d   ,r3      :z&bad or missing length, line/record #%dr   s   ->z)bad or missing separator, line/record #%dz"short key or data, line/record #%dz bad line ending, line/record #%drg   )read	functoolspartialrC   r*   
ValueErrorrD   put)rm   rF   rh   ry   Z
read_untilZrec_nrplusZklendlenrG   datar   rr   r   restore_cursor_from_fp  s8   



r   c              	   C   s   |st d ttjtt|}t|D ]2\}}|| }|dkr#t d td|f  tjdd}|| W d    n1 s@w   Y  qd S )Nz3Must specify at least one sub-database (see --help)rK   zCannot drop main DBzDropping DB %r...Tr!   )	rC   maprM   rN   r   r   rZ   ri   Zdrop)r\   rB   dbsr&   rh   namerm   r   r   r   cmd_drop5  s   r   c                 C   s(   | j rtdt f  tt  d S )NzCleaned %d stale entries.)cleanrZ   rM   Zreader_checkreadersr\   rB   r   r   r   cmd_readersC  s   r   c           	   
   C   s   t |}tjddd?}| D ]1\}\}}t|dt}td|f  t|||}td||f  W d    n1 s<w   Y  qW d    d S 1 sMw   Y  d S )NT)re   r!   rbzRestoring from %r...zLoaded %d keys from %r)rT   rM   ri   rj   rk   rl   rZ   r   )	r\   rB   rO   rm   rQ   rh   rS   rF   countr   r   r   cmd_restoreI  s   "r   c                    s    fddt dt D S )Nc                    s    g | ]} |  |d    qS )r1   r   ).0iZhstr   r   
<listcomp>T  s     zdelta.<locals>.<listcomp>r1   )rangerD   r   r   r   r   deltaS  s   r   z
/sys/blockc                    s   t jtsd S t | }d|jd? |jd@ f   fdd}t tD ].}t jt|}||}|r7|  S t |D ]}t j||}||}|rQ|    S q<q$d S )Nz%s:%srU      c                    sx   t j| d}t j|r:t|}|   kr(t j| dW  d    S W d    d S 1 s3w   Y  d S d S )Ndevstat)rW   rS   rp   rX   rk   ry   strip)ZrootpathdpathrF   Zdevsr   r   maybe`  s   
"z_find_diskstat.<locals>.maybe)rW   rS   rX   	SYS_BLOCKr   st_devlistdirrp   )rS   str   r   ZbasepathstatpathZ	base2pathr   r   r   _find_diskstatZ  s$   
r   c                   @   s    e Zd ZdZdd Zdd ZdS )DiskStatter)ZreadsZreads_mergedsectors_readZread_msZwritesZwrites_mergedsectors_writtenZwrite_msZio_countZio_msZtotal_msc                 C   s   t || _|   d S N)rk   rF   refresh)selfrS   r   r   r   __init__  s   
zDiskStatter.__init__c                    s:    j d t  fddt j   D  d S )Nr   c                 3   s&    | ]\}} j | t|fV  qd S r   )FIELDSr*   )r   r   r   r   r   r   	<genexpr>  s    z&DiskStatter.refresh.<locals>.<genexpr>)rF   seekvarsupdater   ry   splitr   r   r   r   r     s   zDiskStatter.refreshN)__name__
__module____qualname__r   r   r   r   r   r   r   r   s  s    r   c                    s  d  d fdd} fdd}ddfddfddfd	dfdd
fddfdd|fddfddfddfdd|fddfddfddfdd|fddfdd fddfdd|fdd||fdd fddfdd| fd dfg}d t t }|rt||dd!|fd"dfdd#|fd$dfg7 }d%}d&d' |D }jrtjtjtjd(}|	d)d' |D  d%}	z	 t
 t  rԈ  g }
t|D ]\}\}}}||  }|
| t|| t|||< qڈjr|	|
 nK|tks|	td+  s/t|D ]\}\}}}tj||| d,  qtjd- t}t|
D ]\}}tj||| d,  q3tjd- tj |	d,7 }	q tya   Y d S w ).Nc                    s   t   fdd}|S )Nc                     sX       tjkr  tdkrdS ttttd  } | j S )Nr1   r   )r2   rD   windowpopleftsumr   floatinterval)n)funchistoryr\   r   r   
windowfunc  s   
z-cmd_watch.<locals>.window.<locals>.windowfunc)collectionsdeque)r   r   )r\   )r   r   r   r     s   zcmd_watch.<locals>.windowc                      s    d d  d S )N	last_pgnopsize      0Ar   r   )infor   r   r   rt     s    zcmd_watch.<locals>.<lambda>z%dZDepthc                          d S )Ndepthr   r   r   r   r   rt         ZBranchc                      r   )NZbranch_pagesr   r   r   r   r   rt     r   ZLeafc                      r   NZ
leaf_pagesr   r   r   r   r   rt     r   z%+dzLeaf/sc                      r   r   r   r   r   r   r   rt     r   ZOflowc                      r   NZoverflow_pagesr   r   r   r   r   rt     r   zOflow/sc                      r   r   r   r   r   r   r   rt     r   ZRecsc                      r   Nentriesr   r   r   r   r   rt     r   zRecs/sc                      r   r   r   r   r   r   r   rt     r   ZRdrsc                      r   )NZnum_readersr   r   r   r   r   rt     r   z%.2fZEnvMbz%+.2fzEnvMb/sZTxsc                      r   NZ
last_txnidr   r   r   r   r   rt     r   zTxs/sc                      r   r   r   r   r   r   r   rt     r   zSctRd/sc                          j S r   )r   r   statterr   r   rt         zSctWr/sc                      r   r   )r   r   r   r   r   rt     r   r   c                 S   s   g | ]	\}}}t |qS r   )rD   r   _headr   r   r   r     s    zcmd_watch.<locals>.<listcomp>)quotingc                 S   s   g | ]\}}}|qS r   r   r   r   r   r   r     s    Tr   r1   r   )r   rM   rS   r   csvwriterr   stdout	QUOTE_ALLwriterowr   r   r   r   r2   maxrD   _TERM_WIDTH_TERM_HEIGHTr!   rjusttimesleepr   KeyboardInterrupt)r\   rB   r   Zenvmbcolsr   Z
term_widthwidthsr   Zcntvalsr   rA   r   r   valr   )r   r\   r   r   r   	cmd_watch  sv   

r   c           	      C   s   t  }t  }d}|d |d  }tdtd| }t }| jr+t| jd|}n	t| jd d|}|	 |k rE|
| |	 |k s:td|d	 d
t |  f  d S )Ni   r   r   B r   z	/data.mdbzWarmed %.2fmb in %dmsr   i  )rM   r   r   arrayr   r   use_single_filerk   envtellreadintorZ   )	r\   rB   r   r   bufsizeZlast_offsetbuft0rF   r   r   r   cmd_warm  s   

r   c              
   C   sV  | j std t }tj| j |d d | jddddd}g }|D ] }|dkr)d n|}tt|}|t|}|	|||f q!|sR|	dtd |d f |D ]K\}}}t
d|f  |j|dd	0}	tj|dd
}
|
 D ]\}}|	j||dd qtW d    n1 sw   Y  W d    n1 sw   Y  qTt
d |d d S )Nz,Must specify target environment path with -Emap_sizer   FT)r   max_dbssyncZwritemapZ	map_asyncZmetasyncrK   zWriting %r...)rh   r!   )rh   re   )r2   z	Syncing..)
target_envrC   rM   r   lmdbrk   r   rN   r   r2   rZ   ri   rE   r}   r   )r\   rB   Zsrc_infor   r   rP   r   Zsrc_dbZdst_dbZwtxnZrtxnrG   rH   r   r   r   cmd_rewrite  s<   
r   c                 C   s   t |dk}tjdtd:}|D ].}|t|}|d u r%td|f  q|r.td|f  | jr8tt| qtt| qW d    d S 1 sJw   Y  d S )Nr1   T)re   rh   z%r: missingz%r:)	rD   rM   ri   DBr   r   rZ   r(   bytes)r\   rB   Zprint_headerrm   rP   rH   r   r   r   cmd_get  s   "r   c           
   
   C   s  |rt d tjdd}|jtd}| jpg D ]}t|td\}}}|j||dd q| j	p3g D ]}t|td\}}}||| q4| j
pMg D ]}|j
t|td qN| jp^g D ]-}t|td\}}}t|d}	|j||	 dd W d    n1 sw   Y  q_| jpg D ]+}t|td\}}}t|d}	|||	  W d    n1 sw   Y  qW d    d S 1 sw   Y  d S )	Nz;Edit command only takes options, not arguments (see --help)Tr   rg   rJ   F)	overwriter   )rC   rM   ri   rE   r   addr   rL   r}   setdeleteZadd_filerk   ry   Zset_file)
r\   rB   rm   rE   elemrG   r   rH   rS   rF   r   r   r   cmd_edit#  s4   "r   c                 C   s$   dd l }dd l}|t   d S )Nr   )codereadlineZInteractiveConsoler   Zinteract)r\   rB   r   r   r   r   r   	cmd_shell?  s   r   c                 C   s    t  t  t  t  d S r   )pprintrM   r   r   r   r   r   r   cmd_statE  s   r  P      c                 C   sX   z dd l }dd l}|tj |jd}td|\}}||fW S  t	y+   |  Y S w )Nr   1234hh)
fcntltermiosioctlr   r   filenoZ
TIOCGWINSZstructunpack	Exception)r-   r  r	  r   heightwidthr   r   r   _get_term_widthJ  s   
r  c                  G   s   t  \aad S r   )r  r   r   )rB   r   r   r   _on_sigwinchU  s   r  c                 C   s   t  }|| \}}|std |jstd tj|j|jd |j |jd|j	dkda
|jr7t
t|jattdrCttjt t  t d|d	  }|s[td
|d	 f  |||dd   d S )Nz%Please specify a command (see --help)z"Please specify environment (--env)i   FREAD)r   subdirr   createreadonlySIGWINCHZcmd_r   zNo such command: %rr1   )r>   
parse_argsrC   r   r   rk   r   r   r   ry   rM   rh   rN   r   r   hasattrsignalr  r  r   r   )argvr<   r\   rB   r   r   r   r   mainZ  s$   
r  __main__r1   )r  r   )9r7   
__future__r   r   r   r   r   rz   r4   rW   r  r  r   r  r   r   ior   r   ImportError	cStringIOr   rl   rM   r   r   r   r   r(   r>   rC   rI   rT   r^   rd   rn   r   r   r   r   r   r   r   objectr   r   r   r   r   r   r   r  r  r  r  r   r  r   r   r   r   <module>   sz   1 1

%
S 

