o MeG@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m mZddlZddlZddl Z gZejdZeeejsS eZejedddZddZeed d Zdd d Zd dZdejfddZ ej!fddZ"dddZ#dS))print_functionN~z.cachepaddledatasetc CsFzttWdSty"}z|jtjkrWYd}~dSd}~wwN)osmakedirs DATA_HOMEOSErrorerrnoEEXIST)pathexcrED:\Projects\ConvertPro\env\Lib\site-packages\paddle/dataset/common.py must_mkdirs6s rcsDt}t|dtfdddD]}||q|S)Nrbcs dS)N)readrfrrEs zmd5file..)hashlibmd5openiterupdateclose hexdigest)fnameZhash_md5chunkrrrmd5fileBs   r"c CsXtjt|}tj|st|tj||dur!|ddn|}tj|r2t||kr2|Sd}d}tj|rCt||kstj|rUtj dt||f||kr^|d7}nt d ||tj d||ftj d zt j|d d }|jd } | durt|d } t|j| Wdn1swYnYt|d L} d} t| } | | d} | dkr| dnd} d}tjjj| dd}|j| dD]}| ||d7}||i|| dkr||qWdn1swYWnty}zWYd}~q6d}~wwtj|rCt||krCtj dtj|S)N/rzfile %s md5 %s z*Cannot download {0} within retry limit {1}z)Cache file %s not found, downloading %s zBegin to download T)streamzcontent-lengthwbritem)name) chunk_sizez Download finished )rr joinr existsrsplitr"sysstderrwrite RuntimeErrorformatrequestsgetheadersrshutil copyfileobjrawintrZhapiZ progressbar ProgressBar iter_contentr Exceptionstdoutflush)url module_nameZmd5sumZ save_namedirnamefilenameretryZ retry_limitr total_lengthrr,Z total_iterZ log_intervalZ log_indexbardataerrrdownloadKsx                & rKcCsJddttjDD]}dttd|vr"ttd|dq dS)NcSsg|] }|ds|qS)__) startswith).0xrrr s  zfetch_all..fetchzpaddle.dataset.%s)dirrr importlib import_modulegetattr)rBrrr fetch_alls  rVz %05d.picklec Cst|stdg}d}t|D]5\}}||||krF||dkrFt||d}|||g}|d7}Wdn1sAwYq|rht||d}|||WddS1sawYdSdS)a you can call the function as: split(paddle.dataset.cifar.train10(), line_count=1000, suffix="imikolov-train-%05d.pickle") the output files as: |-imikolov-train-00000.pickle |-imikolov-train-00001.pickle |- ... |-imikolov-train-00480.pickle :param reader: is a reader creator :param line_count: line count for each file :param suffix: the suffix for the output files, should contain "%d" means the id for each file. Default is "%05d.pickle" :param dumper: is a callable function that dump object to file, this function will be called as dumper(obj, f) and obj is the object will be dumped, f is a file object. Default is cPickle.dump. zdumper should be callable.rwr&N)callable TypeError enumerateappendr) reader line_countsuffixZdumperlinesZindx_fidrrrrr/s$    "r/csfdd}|S)a Create a reader that yield element from the given files, select a file set according trainer count and trainer_id :param files_pattern: the files which generating by split(...) :param trainer_count: total trainer count :param trainer_id: the trainer rank id :param loader: is a callable function that load object from file, this function will be called as loader(f) and f is a file object. Default is cPickle.load c 3sts tdt}|g}t|D]\}}|kr-td|||q|D]#}t|d}|}|D]}|Vq>Wdn1sNwYq0dS)Nzloader should be callable.zappend file: %srF)rXrYglobsortrZprintr[r) file_listZ my_file_listidxfnrr_line files_patternloader trainer_count trainer_idrrr\s(     z$cluster_files_reader..readerr)rjrlrmrkr\rrircluster_files_readersrnTcCs8|r tj|r |S|rtjj|||Std|)Nz({} not exists and auto download disabled) rr r.rrcommonrK ValueErrorr4)r rArrBrKrrr_check_exists_and_downloadsrqr)T)$ __future__rr5rrr r8sixr0rSZpaddle.datasetrZsix.moves.cPicklemovescPicklepickletempfilerb__all__r expanduserHOMEaccessW_OK gettempdirr-r rr"rKrVdumpr/loadrnrqrrrrs:    9 ) #