o re1V@s^ddlmZmZmZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZddlmZddlmZmZddlmZmZmZmZed ZGd d d ZGd d d eZ          d)ddZ ddZ!ddZ"      d*ddZ#d+ddZ$ddZ%ddZ&dd Z'd!d"Z(      d,d#d$Z)d%d&Z*Gd'd(d(ej+Z,dS)-)absolute_importdivisionprint_functionN) has_magic) BaseCache BlockCache BytesCache MMapCacheReadAheadCachecaches)compr) filesystemget_filesystem_class)_unstrip_protocolbuild_name_functioninfer_compressionstringify_pathZfsspecc@s`eZdZdZ     dddZddZdd Zd d Zd d Ze ddZ ddZ ddZ dS)OpenFileaF File-like object to be used in a context Can layer (buffered) text-mode and compression over any file-system, which are typically binary-only. These instances are safe to serialize, as the low-level file object is not created until invoked using ``with``. Parameters ---------- fs: FileSystem The file system to use for opening the file. Should be a subclass or duck-type with ``fsspec.spec.AbstractFileSystem`` path: str Location to open mode: str like 'rb', optional Mode of the opened file compression: str or None, optional Compression to apply encoding: str or None, optional The encoding to use if opened in text mode. errors: str or None, optional How to handle encoding errors if opened in text mode. newline: None or str Passed to TextIOWrapper in text mode, how to handle line endings. autoopen: bool If True, calls open() immediately. Mostly used by pickle pos: int If given and autoopen is True, seek to this location immediately rbNcCs:||_||_||_t|||_||_||_||_g|_dSN) fspathmodeget_compression compressionencodingerrorsnewlinefobjects)selfrrrrrrrr!;D:\Projects\ConvertPro\env\Lib\site-packages\fsspec/core.py__init__?s   zOpenFile.__init__cCs$t|j|j|j|j|j|j|jffSr)rrrrrrrrr r!r!r" __reduce__RszOpenFile.__reduce__cCs d|jS)Nz)formatrr$r!r!r"__repr__` zOpenFile.__repr__cCs|jddddd}|jj|j|d}|g|_|jdur2t|j}|||dd}|j|d|jvrHt ||j |j |j d}|j||jdS)Ntb)rr)rrr) rreplaceropenrrrr appendPickleableTextIOWrapperrrr)r rfcompressr!r!r" __enter__cs      zOpenFile.__enter__cGs |dSr)close)r argsr!r!r"__exit__xr(zOpenFile.__exit__cCst|j|jSr)rrrr$r!r!r" full_name{szOpenFile.full_namecCs|S)aMaterialise this as a real open file without context The OpenFile object should be explicitly closed to avoid enclosed file instances persisting. You must, therefore, keep a reference to the OpenFile during the life of the file-like it generates. r3r$r!r!r"r.sz OpenFile.opencCs>t|jD]}d|jvr|js||q|jdS)z#Close all encapsulated file objectsrN)reversedrrclosedflushr4clear)r r1r!r!r"r4s  zOpenFile.close)rNNNN) __name__ __module__ __qualname____doc__r#r%r'r3r6propertyr7r.r4r!r!r!r"rs $   rcsLeZdZdZdddfdd ZddZd d Zfd d Zd dZZ S) OpenFilesaList of OpenFile instances Can be used in a single context, which opens and closes all of the contained files. Normal list access to get the elements works as normal. A special case is made for caching filesystems - the files will be down/uploaded together at the start or end of the context, and this may happen concurrently, if the target filesystem supports it. rNrrcs"||_||_g|_tj|dSr)rrfilessuperr#)r rrr5 __class__r!r"r#szOpenFiles.__init__cCsd|jdur td|j} t|dr|||_|jSt|dr)|jdur)|j}nnq dd|DS)NzContext has already been usedT open_manyrcSsg|]}|qSr!r8.0sr!r!r" z'OpenFiles.__enter__..)r ValueErrorhasattrrIrE)r rr!r!r"r3s    zOpenFiles.__enter__csd|j}fdd|Dd|jvr0 t|dr||jdSt|dr-|jdur-|j}ndSqdS)Ncsg|]}|jqSr!)r6rJr5r!r"rMz&OpenFiles.__exit__..r9TrIr)rrrPZ commit_manyrE)r r5rr!rQr"r6s   zOpenFiles.__exit__cs,t|}t|trt||j|jdS|S)NrD)rF __getitem__ isinstanceslicerCrr)r itemoutrGr!r"rSs  zOpenFiles.__getitem__cCs dt|S)Nz)lenr$r!r!r"r'r(zOpenFiles.__repr__) r>r?r@rAr#r3r6rSr' __classcell__r!r!rGr"rCs  rCrutf8Tc  srt|||| || d\} } dvr&| r&fdd| D}fdd|Dtfdd| DdS) av Given a path or paths, return a list of ``OpenFile`` objects. For writing, a str path must contain the "*" character, which will be filled in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. For either reading or writing, can instead provide explicit list of paths. Parameters ---------- urlpath: string or list Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol. mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode name_function: function or None if opening a set of files for writing, those files do not yet exist, so we need to generate their names by formatting the urlpath for each sequence number num: int [1] if writing mode, number of files we expect to create (passed to name+function) protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. auto_mkdir: bool (True) If in write mode, this will ensure the target directory exists before writing, by calling ``fs.mkdirs(exist_ok=True)``. expand: bool **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> files = open_files('2015-*-*.csv') # doctest: +SKIP >>> files = open_files( ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' ... ) # doctest: +SKIP Returns ------- An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can be used as a single context Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations )num name_functionstorage_optionsprotocolexpandr9csh|]}|qSr!)_parentrKrrr!r" $rRzopen_files..csg|] }j|ddqS)T)exist_ok)makedirs)rKparentrbr!r"rM%zopen_files..c s"g|] }t|dqS))rrrrr)rrarrrrrrr!r"rM's rD)get_fs_token_pathsrC)urlpathrrrrr\r[r^r auto_mkdirr_kwargsZfs_tokenpathsparentsr!rhr" open_filess&O   roc stdd|vrfdd|dDn|g}g}d}|}t|D]L}|ddp3t|dp3d}t|}||}||i} ||durN| |t d i|| } | |}|dvrfd | vrf|}| ||| f|}q$t t|}|S) Nz .*[^a-z]+.*z::cs*g|]}d|vs |r|n|dqS)://)matchrKpxr!r"rM;s*z_un_chain..r^rfile> filecacheZ blockcacheZ simplecachetarget_protocolr!)recompilesplitcopyr:popsplit_protocolrZ_get_kwargs_from_urlsupdatedict_strip_protocolr/list) rrlbitsrWZ previous_bitbitr^cls extra_kwargskwskwr!rtr" _un_chain8s0        rc  shdfdd|D}t||}i}tt|D].\}}|\}}}|t|dkr7td i||}qtd i|||d<||d<||d<q|d\} }} t|fi|} | | fS) a Turn fully-qualified and potentially chained URL into filesystem instance Parameters ---------- url : str The fsspec-compatible URL **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Returns ------- filesystem : FileSystem The new filesystem discovered from ``url`` and created with ``**kwargs``. urlpath : str The file-systems-specific URL for ``url``. > r_rrrr\rr[rkrcsi|] \}}|vr||qSr!r!)rKkvZ known_kwargsr!r" xszurl_to_fs..rtarget_optionsrxforNr!)itemsr enumerater:rXrr) urlrlchaininkwargsichurlsr^rrj_rr!rr" url_to_fsWs   rc Ks6td|g||||||dd|}|st||dS)aGiven a path or paths, return one ``OpenFile`` object. Parameters ---------- urlpath: string or list Absolute or relative filepath. Prefix with a protocol like ``s3://`` to read from alternative filesystems. Should not include glob character(s). mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> openfile = open('2015-01-01.csv') # doctest: +SKIP >>> openfile = open( ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' ... ) # doctest: +SKIP >>> with openfile as f: ... df = pd.read_csv(f) # doctest: +SKIP ... Returns ------- ``OpenFile`` object. Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations F)rjrrrrr^rr_rNr!)roFileNotFoundError) rjrrrrr^rrlrWr!r!r"r.s<  r.cKsd|vrtdt|fd|i|}t|djddstd|}dd |D}Wd n1s3wYt|trEt|sE|dS|S) aKOpen file(s) which can be resolved to local For files which either are local, or get downloaded upon open (e.g., by file caching) Parameters ---------- url: str or list(str) mode: str Must be read mode storage_options: passed on to FS for or used by open_files (e.g., compression) r9z(Can only ensure local files when readingrr local_fileFzOopen_local can only be used on a filesystem which has attribute local_file=TruecSsg|]}|jqSr!)namerKr1r!r!r"rMszopen_local..N)rOrogetattrrrTstrr)rrr]ZofrErmr!r!r" open_localsrcCs0|dkrt|}|dur|tvrtd||S)NZinferz!Compression type %s not supported)rr rO)rjrr!r!r"rs  rcCs<t|}d|vr|dd\}}t|dkr||fSd|fS)zReturn protocol, path pairrprN)rr{rX)rjr^rr!r!r"r~s  r~cCst|\}}t|}||S)zCReturn only path part of full URL, according to appropriate backend)r~rr)rjr^rrr!r!r"strip_protocols  rcCsg}t|}d|vrFtdd|Ddkrtdt|t|}|D]}d|vr2|t|||q"||q"t||krD|d|}|S|D]}t|rW|| |qH||qH|S)a(Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` in them (read mode). :param paths: list of paths mode: str Mode in which to open files. num: int If opening in writing mode, number of files we expect to create. fs: filesystem object name_function: callable If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. :return: list of paths wcSsg|]}d|vrdqS)*rr!rrr!r!r"rM sz*expand_paths_if_needed..rz;When writing data, only one filename mask can be specified.rN) rsumrOmaxrXextend _expand_pathsr/rglob)rmrr[rr\Zexpanded_pathsZ curr_pathr!r!r"expand_paths_if_needed s(     rcst|tttfr|stdtt|d}nt|}pi|r%|d<t|p*i}i} tt|D].\} } | \} } }| t |dkrOt di|| } q4t di|| | d<| | d<| | d<q4|d\}}}t |fi| t|tttfrfdd |D}t td d |Ddkrtd |d d |D}n |}t|tttfrt ||||}n0d|vr|rt|||}n#d|vr|rt|||}nd|vrfdd t|D}n|g}j|fS)aFilesystem, deterministic token, and paths from a urlpath and options. Parameters ---------- urlpath: string or iterable Absolute or relative filepath, URL (may include protocols like ``s3://``), or globstring pointing to data. mode: str, optional Mode in which to open files. num: int, optional If opening in writing mode, number of files we expect to create. name_function: callable, optional If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. storage_options: dict, optional Additional keywords to pass to the filesystem class. protocol: str or None To override the protocol specifier in the URL expand: bool Expand string paths for writing, assuming the path is a directory zempty urlpath sequencerr^rrrxrcs"g|] }tt|p idqSr)rr)rKu)r]r!r"rMqsz&get_fs_token_paths..css|]}|dVqdS)rNr!rKZpcr!r!r" tsz%get_fs_token_paths..z$Protocol mismatch getting fs from %scSsg|]}|dqSrr!rr!r!r"rMvrNrrurcsg|] }|s|qSr!)isdirrrbr!r"rMrgNr!)rTrtuplesetrOrrrr:rXrrrrrsortedrZ _fs_token)rjrr[r\r]r^r_Zurlpath0rrrrrZnested_protocolrrmrZpchainsr!)rr]r"ri;sL        ricsttr>ddkrtddvrtjddur%t|dfddt|D}|t |kr.zqIn order to preserve order between partitions paths created with ``name_function`` should sort to partition orderzPath should be either 1. A list of paths: ['foo.json', 'bar.json', ...] 2. A directory: 'foo/ 3. A path with a '*' in it: 'foo.*.json')rTrcountrOosrjoinrrangerloggerwarningrrrX)rr\r[rmr!rr"rs(   rcs4eZdZdZ     dfdd ZddZZS) r0zTextIOWrapper cannot be pickled. This solves it. Requires that ``buffer`` be pickleable, which all instances of AbstractBufferedFile are. NFcs$||||||f|_tj|jdSr)r5rFr#)r bufferrrrline_buffering write_throughrGr!r"r#s z PickleableTextIOWrapper.__init__cCs t|jfSr)r0r5r$r!r!r"r%s z"PickleableTextIOWrapper.__reduce__)NNNFF)r>r?r@rAr#r%rYr!r!rGr"r0s  r0) rNrZNNrNNTT)rNrZNNN)r)rrNNNT)- __future__rrriologgingrryrrZcachingrrr r r r rr registryrrutilsrrrr getLoggerrrrrCrorrr.rrr~rrrir TextIOWrapperr0r!r!r!r"s^    s< m4 L 1 M