o re g @s&ddlZddlZddlZddlZddlZddlZddlZddlmZe dZ Gddde Z Gddde Z Gdd d e ZGd d d e ZGd d d e ZGddde ZGddde ZGddde ZGdddZGddde Zde iZdddZe e eeeeeeef D]ZeeqdS)N)ThreadPoolExecutorZfsspecc@s$eZdZdZdZddZddZdS) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is nonecCs||_||_||_dSN) blocksizefetchersizeselfrrrr >D:\Projects\ConvertPro\env\Lib\site-packages\fsspec/caching.py__init__s zBaseCache.__init__cCs<|durd}|dur |j}||jks||krdS|||SNr)rr)r startstopr r r _fetch$s zBaseCache._fetchN)__name__ __module__ __qualname____doc__namer rr r r r r s  rcsFeZdZdZdZdfdd ZddZdd Zd d Zd d Z Z S) MMapCachezmemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix mmapNcs8t||||durtn||_||_||_dSr)superr setblockslocation _makefilecache)r rrrrr __class__r r r 9szMMapCache.__init__cCsddl}ddl}|jdkrtS|jdustj|jsB|jdur*|}t |_ nt |jd}| |jd| d|nt |jd}|||jS)Nrzwb+1zrb+)rtempfiler bytearrayrospathexists TemporaryFilerropenseekwriteflushfileno)r rr$fdr r r r?s       zMMapCache._makefilec std|d||durd}|durj}|jks!||kr#dS|j}|j}fddt||dD}|rq|d}|j}t|jj}td|d |d|||j||<j ||s=j||S) NzMMap cache fetching -rrcsg|] }|jvr|qSr )r).0ir r r _sz$MMapCache._fetch..r"zMMap get block #z () loggerdebugrrrangepopminrrradd) r rendZ start_blockZ end_blockZneedr2Zsstartsendr r3r rUs&      zMMapCache._fetchcCs|j}|d=|S)Nr)__dict__copyr stater r r __getstate__ls zMMapCache.__getstate__cCs|j|||_dSr)r=updaterrr?r r r __setstate__rs zMMapCache.__setstate__)NN) rrrrrr rrrArC __classcell__r r r r r.src,eZdZdZdZfddZddZZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). Z readaheadcs&t|||d|_d|_d|_dS)Nrr)rr rrr;r r r r r s zReadAheadCache.__init__cCs|durd}|dus||jkr|j}||jks||krdS||}||jkr8||jkr8|j||j||jS|j|krD|jkrZnn|j||jd}|t|8}|j}nd}t|j||j}||||_||_|jt|j|_||jd|Sr)rrr;rlenr9rr)r rr;lpartr r r rs& zReadAheadCache._fetchrrrrrr rrDr r r r rFxs  rFcrE)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstcst|||d|_dSr)rr rr r r r r s zFirstChunkCache.__init__cCs|pd}|p|j}||jkrK|jdur4||jkr,|d|}|d|j|_||dS|d|j|_|j||}||jkrI|||j|7}|S|||SNr)rrrr)r rr;datarIr r r rs        zFirstChunkCache._fetchrJr r r r rKs  rKcsbeZdZdZdZdfdd ZddZdd Zd d Zd d Z ddZ fddZ ddZ Z S) BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. Z blockcache cs<t|||t|||_||_t||j|_ dSr) rr mathceilnblocks maxblocks functools lru_cache _fetch_block_fetch_block_cachedr rrrrTr r r r szBlockCache.__init__cCd|j|j|jS)Nz.formatrrrSr3r r r __repr__ zBlockCache.__repr__cC |jSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. rX cache_infor3r r r rb zBlockCache.cache_infocCs|j}|d=|S)NrXr=r?r r r rAszBlockCache.__getstate__cCs&|j|t|d|j|_dS)NrT)r=rBrUrVrWrXr?r r r rCs   zBlockCache.__setstate__cCst|durd}|dur |j}||jks||krdS||j}||j}t||dD]}||q)|j||||dS)Nrrr"start_block_numberend_block_number)rrr7rX _read_cache)r rr;rfrg block_numberr r r rs    zBlockCache._fetchcsN||jkrtd||j||j}||j}td|t||}|S)= Fetch the block of data for `block_number`. ;'block_number={}' is greater than the number of blocks ({})zBlockCache fetching block %drS ValueErrorr\rr5inforr)r rirr;block_contentsr r r rWs    zBlockCache._fetch_blockc C||j}||j}||kr||}|||Sg}||||dt|d|D] } ||| q.|||d|d|Sz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. Nr"rrrXappendr7join r rr;rfrg start_posend_posblockoutrir r r rh#    zBlockCache._read_cacherPrrrrrr r]rbrArCrrWrhrDr r r r rOs  rOcs6eZdZdZdZd fdd ZddZdd ZZS) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. bytesTcs,t|||d|_d|_d|_||_dS)Nr)rr rrr;trim)r rrrrr r r r Us  zBytesCache.__init__cCs|durd}|dur |j}||jks||krdS|jdur<||jkr<|jdur<||jkr<||j}|j||||S|jrIt|j||j}n|}||ksT||jkrVdS|jdus`||jkru|jdusj||jkru||||_||_nU||jkr|j||jkr||||_||_n=|||j}||_||j|_n,||jkr|j|jkrn ||j|jkr||||_||_n ||j|}|j||_|jt|j|_||j}|j||||}|jr |j|j|jd}|dkr |j|j|7_|j|j|d|_|S)Nrrr") rrr;rrr9rrGr)r rr;offsetZbendnewrynumr r r r\sZ           zBytesCache._fetchcCs t|jSr)rGrr3r r r __len__s zBytesCache.__len__)T) rrrrrr rrrDr r r r r}Fs  =r}cs.eZdZdZdZdfdd ZddZZS) AllBytesz!Cache entire contents of the fileallNcs0t||||dur|d|j}||_dSrM)rr rrrN)r rrrrNr r r r s zAllBytes.__init__cCs|j||Sr)rN)r rr;r r r rszAllBytes._fetch)NNNNrJr r r r rs rcs6eZdZdZdZidffdd ZfddZZS)KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsTc stt||||||_|rhtt|}|dg}||dg} |ddD]3\} } |d\} } | | krL| | f|d<| d|| | f7<q*|| | f| || | fq*t t || |_ dS||_ dS)Nrr") rrr strictsortedlistkeysr8rsdictziprN)r rrrrNr_Z old_offsetsoffsetsrrrZstart0Zstop0r r r r s    zKnownPartsOfAFile.__init__csd}|jD]C\\}}}||kr|krJnq||}|||||}|jr6||kr4|krFnn|d||t|7}|S|}nq|jdurZtd||fdtd||fdt d|d||t ||S)Nrz&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching r0) rNitemsrrGrrmwarningswarnr5r6rr)r rrryZloc0loc1rNoffr r r rs$ zKnownPartsOfAFile._fetchrJr r r r rs rc@sJeZdZdZedgdZdddZddZd d Z d d Z d dZ dS) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache CacheInfo)hitsmissesmaxsizecurrsizecCs0t|_||_||_d|_d|_t|_ dSrM) collections OrderedDict_cache_func _max_size_hits_misses threadingLock_lock)r funcmax_sizer r r r s zUpdatableLRU.__init__cGs|j&||jvr"|j||jd7_|j|WdSWdn1s,wY|j|}|j,||j|<|jd7_t|j|jkr]|jjddWd|SWd|S1shwY|S)Nr"Flast) rr move_to_endrrrrGrpopitem)r argsresultr r r __call__s*      zUpdatableLRU.__call__cGs4|j ||jvWdS1swYdSr)rr)r rr r r is_key_cached%s$zUpdatableLRU.is_key_cachedcGsd|j%||j|<t|j|jkr |jjddWddSWddS1s+wYdS)NFr)rrrGrr)r rrr r r add_key)s "zUpdatableLRU.add_keycCsH|j|j|jt|j|j|jdWdS1swYdS)N)rrrr)rrrrGrrrr3r r r rb/s$zUpdatableLRU.cache_infoN)r) rrrrr namedtuplerr rrrrbr r r r rs  rcsdeZdZdZdZdfdd ZddZdd Zd d Zd d Z ddZ dfdd Z ddZ Z S)BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrPcsZt|||t|||_||_t|j||_t dd|_ d|_ d|_ t |_dS)Nr" max_workers)rr rQrRrSrTrrWrXr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrYr r r r Us zBackgroundBlockCache.__init__cCrZ)Nz8r[r3r r r r]`r^zBackgroundBlockCache.__repr__cCr_r`rar3r r r rberczBackgroundBlockCache.cache_infocCs(|j}|d=|d=|d=|d=|d=|S)NrXrrrrrdr?r r r rApsz!BackgroundBlockCache.__getstate__cCsD|j|t|j|d|_tdd|_d|_d|_t |_ dS)NrTr"r) r=rBrrWrXrrrrrrrr?r r r rCys  z!BackgroundBlockCache.__setstate__c Cs|durd}|dur |j}||jks||krdS||j}||j}d}d}|jF|jdurh|jrKtd|j|j |j d|_ d|_nt ||j koV|kn}|rh|j }|j}d|_ d|_Wdn1srwY|durtd|j| |t ||dD]}||q|d} |j%|jdur| |j kr|j| s| |_ |j|j| d|_Wdn1swY|j||||dS)Nrrz3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.r"asyncre)rrrrdoner5rnrXrrrboolr7rSrrsubmitrWrh) r rr;rfrgZfetch_future_block_numberZ fetch_futureZ must_joinriZend_block_plus_1r r r rst            zBackgroundBlockCache._fetchsynccsP||jkrtd||j||j}||j}td||t||}|S)rjrkz!BlockCache fetching block (%s) %drl)r riZlog_inforr;ror r r rWs   z!BackgroundBlockCache._fetch_blockc Crprqrrrur r r rhrzz BackgroundBlockCache._read_cacher{)rr|r r r r r9s   KrFcCs6|j}|s|tvrtd|dt||t|<dS)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rcachesrm)clsclobberrr r r register_caches  r)F)rrUloggingrQr&rrconcurrent.futuresr getLoggerr5objectrrrFrKrOr}rrrrrrcr r r r sH  !J*WU8J