o
    Ne}                     @   s\   d dl Z ddlmZ d dlmZmZ daG dd deZG dd	 d	eZ	G d
d deZ
dS )    N   )
get_logger)SequenceMappingTc                   @   s&   e Zd Zdd ZdddZdd ZdS )	_DatasetFetcherc                 C   s   || _ || _|| _|| _d S N)datasetauto_collate_batch
collate_fn	drop_lastselfr   r	   r
   r    r   OD:\Projects\ConvertPro\env\Lib\site-packages\paddle/fluid/dataloader/fetcher.py__init__   s   
z_DatasetFetcher.__init__Nc                 C   s   t d| jj)Nz"'fetch' not implement for class {})NotImplementedErrorformat	__class____name__)r   batch_indices
done_eventr   r   r   fetch'   s   z_DatasetFetcher.fetchc                 C   sZ   ddl m}m} | dkr| dkrd S d}|d7 }|d7 }tdtjd	d
}|| d S )N   )get_world_sizeget_rankr   r   a  Detect dataset only contains single fileds, return format changed since Paddle 2.1. In Paddle <= 2.0, DataLoader add a list surround output data(e.g. return [data]), and in Paddle >= 2.1, DataLoader return the single filed directly (e.g. return data). For example, in following code: 

aR  import numpy as np
from paddle.io import DataLoader, Dataset

class RandomDataset(Dataset):
    def __getitem__(self, idx):
        data = np.random.random((2, 3)).astype('float32')

        return data

    def __len__(self):
        return 10

dataset = RandomDataset()
loader = DataLoader(dataset, batch_size=1)
data = next(loader())

zIn Paddle <= 2.0, data is in format '[Tensor(shape=(1, 2, 3), dtype=float32)]', and in Paddle >= 2.1, data is in format 'Tensor(shape=(1, 2, 3), dtype=float32)'
Z
DataLoaderz%(levelname)s: %(message)s)fmt)distributedr   r   r   loggingINFOwarning)r   r   r   Zwarn_strloggerr   r   r   _log_warning+   s   z_DatasetFetcher._log_warningr   )r   
__module____qualname__r   r   r!   r   r   r   r   r      s    
r   c                       &   e Zd Z fddZdddZ  ZS )_IterableDatasetFetcherc                    s$   t t| |||| t|| _d S r   )superr%   r   iterdataset_iterr   r   r   r   r   O   s
   z _IterableDatasetFetcher.__init__Nc              	   C   s   | j rNg }|D ]!}|d u s| s&z
|t| j W q ty%   Y  nw  d S t|dks:| jr<t|t|k r<tt|d t	t
fsMtrM|   dant| j}| jr[| |}|S Nr   F)r	   is_setappendnextr(   StopIterationlenr   
isinstancer   r   _WARNING_TO_LOGr!   r
   )r   r   r   data_r   r   r   r   T   s.   

z_IterableDatasetFetcher.fetchr   r   r"   r#   r   r   __classcell__r   r   r)   r   r%   M   s    r%   c                       r$   )_MapDatasetFetcherc                    s   t t| |||| d S r   )r&   r6   r   r   r)   r   r   r   t   s   z_MapDatasetFetcher.__init__Nc                 C   s|   | j r/g }|D ]}|d u s| s|| j|  q d S t|d ttfs.tr.|   dan| j| }| j	r<| 	|}|S r*   )
r	   r+   r,   r   r0   r   r   r1   r!   r
   )r   r   r   r2   idxr   r   r   r   x   s    

z_MapDatasetFetcher.fetchr   r4   r   r   r)   r   r6   r   s    r6   )r   Z
log_helperr   collections.abcr   r   r1   objectr   r%   r6   r   r   r   r   <module>   s   7%