o Me@sdZddlmZddlZddlZddlZddlZddlm Z ddl m m Z gZdZdZdZdZdZd Zd Zd Zd Zd ZddZddZe dddddddZe dddddddZe dddddddZe dddddd"ddZe dddddd d!ZdS)#a WMT14 dataset. The original WMT14 dataset is too large and a small set of data for set is provided. This module will download dataset from http://paddlepaddle.bj.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz and parse training set and test set into paddle reader creators. )print_functionNzJhttp://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgzZ 7d7897317ddd8ba0ae5c5fa7248d3ff5z/http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgzZ 0791583d57d5beb693b9414c5b36798cz1http://paddlemodels.bj.bcebos.com/wmt%2Fwmt14.tgzZ 0cb4a5366189b6acba876491c8724fa3zzzcCsdd}tj|dd>}dd|D}t|dksJ|||d|}d d|D}t|dks4J|||d|}||fWdS1sLwYdS) NcSs<t}t|D]\}}||kr||t|<q|S|S)N)dict enumeratecptto_textstrip)fdsizeZout_dict line_countliner DD:\Projects\ConvertPro\env\Lib\site-packages\paddle/dataset/wmt14.py __to_dict6s z!__read_to_dict..__to_dictrmodecSg|] }|jdr|jqS)zsrc.dictnameendswith.0Z each_itemr r r @  z"__read_to_dict..rcSr)ztrg.dictrrr r rrFr)tarfileopenlen extractfile)tar_file dict_sizerfnamessrc_dicttrg_dictr r r__read_to_dict4s $r&csfdd}|S)Nc 3s(t\tjdd{}fdd|D}|D]f}||D]^}t|}|d}t|dkr7q"|d}|}fddt g|t gD}|d }|} fd d| D} t|d kskt| d krlq"| t g} t g| } || | fVq"qWddS1swYdS) Nrrcsg|] }|jr|jqSr rr) file_namer rrTrz2reader_creator..reader.. rrcg|]}|tqSr getUNK_IDXrw)r$r rr`s rcr)r r*r-)r%r rrgsP) r&rrrrrrsplitrSTARTEND) r"r#rr Z line_splitZsrc_seqZ src_wordsZsrc_idsZtrg_seqZ trg_wordsZtrg_idsZ trg_ids_nextr!r'r )r$r%rreaderQs8    "zreader_creator..readerr )r r'r!r4r r3rreader_creatorOs r5z2.0.0zpaddle.text.datasets.WMT14rz>Please use new dataset API which supports paddle.io.DataLoader)ZsinceZ update_tolevelreasoncCttjjtdtd|S)a WMT14 training set creator. It returns a reader creator, each sample in the reader is source language word ID sequence, target language word ID sequence and next word ID sequence. :return: Training reader creator :rtype: callable wmt14z train/trainr5paddledatasetcommondownload URL_TRAIN MD5_TRAINr!r r rtraintrBcCr8)z WMT14 test set creator. It returns a reader creator, each sample in the reader is source language word ID sequence, target language word ID sequence and next word ID sequence. :return: Test reader creator :rtype: callable r9z test/testr:rAr r rtestrCrDcCr8)Nr9zgen/genr:rAr r rgensrETcCsTtjjtdt}t||\}}|r&ddt|D}ddt|D}||fS)Nr9cSi|]\}}||qSr r rkvr r r zget_dict..cSrFr r rGr r rrJrK) r;r<r=r>r?r@r&six iteritems)r!reverser r$r%r r rget_dicts rOcCs(tjjtdttjjtdtdS)Nr9)r;r<r=r>r?r@ URL_MODEL MD5_MODELr r r rfetchsrR)T) __doc__ __future__rrLrgzipZpaddle.dataset.commonr;Z paddle.compatcompatrZpaddle.utils.deprecatedutils deprecated__all__Z URL_DEV_TESTZ MD5_DEV_TESTr?r@rPrQr1r2ZUNKr,r&r5rBrDrErOrRr r r rsp  %