o QeA@sddlZddlZddlmZddlZddlmZddlmZgZ ddZ dddZ Gd d d eZ d Z d d ZddZddZGdddeZdS)N)Image)Dataset) try_importcCs6t|ttfs Jdtdd|D}||S)zChecks if a file is a vilid extension. Args: filename (str): path to a file extensions (list[str]|tuple[str]): extensions to consider Returns: bool: True if the filename ends with one of given extensions z#`extensions` must be list or tuple.cSsg|]}|qS)lower).0xrrMD:\Projects\ConvertPro\env\Lib\site-packages\paddle/vision/datasets/folder.py &z'has_valid_extension..) isinstancelisttuplerendswith)filename extensionsrrr has_valid_extensions rc sg}tj|}durfdd}t|D]<}tj||}tj|s(qttj|ddD]"\}}} t| D]} tj|| } || rR| ||f} || q:q1q|S)Nc t|SNrrrrr is_valid_file0 z#make_dataset..is_valid_fileT followlinks) ospath expandusersortedkeysjoinisdirwalkappend) dir class_to_idxrrZimagestargetdroot_fnamesfnameritemrrr make_dataset*s$      r.c@s:eZdZdZ    d ddZddZddZd d ZdS) DatasetFoldera)A generic data loader where the samples are arranged in this way: .. code-block:: text root/class_a/1.ext root/class_a/2.ext root/class_a/3.ext root/class_b/123.ext root/class_b/456.ext root/class_b/789.ext Args: root (str): Root directory path. loader (Callable, optional): A function to load a sample given its path. Default: None. extensions (list[str]|tuple[str], optional): A list of allowed extensions. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. If this value is not set, the default is to use ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp'). Default: None. transform (Callable, optional): A function/transform that takes in a sample and returns a transformed version. Default: None. is_valid_file (Callable, optional): A function that takes path of a file and check if the file is a valid file. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. Default: None. Returns: :ref:`api_paddle_io_Dataset`. An instance of DatasetFolder. Attributes: classes (list[str]): List of the class names. class_to_idx (dict[str, int]): Dict with items (class_name, class_index). samples (list[tuple[str, int]]): List of (sample_path, class_index) tuples. targets (list[int]): The class_index value for each image in the dataset. Example: .. code-block:: python import shutil import tempfile import cv2 import numpy as np import paddle.vision.transforms as T from pathlib import Path from paddle.vision.datasets import DatasetFolder def make_fake_file(img_path: str): if img_path.endswith((".jpg", ".png", ".jpeg")): fake_img = np.random.randint(0, 256, (32, 32, 3), dtype=np.uint8) cv2.imwrite(img_path, fake_img) elif img_path.endswith(".txt"): with open(img_path, "w") as f: f.write("This is a fake file.") def make_directory(root, directory_hierarchy, file_maker=make_fake_file): root = Path(root) root.mkdir(parents=True, exist_ok=True) for subpath in directory_hierarchy: if isinstance(subpath, str): filepath = root / subpath file_maker(str(filepath)) else: dirname = list(subpath.keys())[0] make_directory(root / dirname, subpath[dirname]) directory_hirerarchy = [ {"class_0": [ "abc.jpg", "def.png"]}, {"class_1": [ "ghi.jpeg", "jkl.png", {"mno": [ "pqr.jpeg", "stu.jpg"]}]}, "this_will_be_ignored.txt", ] # You can replace this with any directory to explore the structure # of generated data. e.g. fake_data_dir = "./temp_dir" fake_data_dir = tempfile.mkdtemp() make_directory(fake_data_dir, directory_hirerarchy) data_folder_1 = DatasetFolder(fake_data_dir) print(data_folder_1.classes) # ['class_0', 'class_1'] print(data_folder_1.class_to_idx) # {'class_0': 0, 'class_1': 1} print(data_folder_1.samples) # [('./temp_dir/class_0/abc.jpg', 0), ('./temp_dir/class_0/def.png', 0), # ('./temp_dir/class_1/ghi.jpeg', 1), ('./temp_dir/class_1/jkl.png', 1), # ('./temp_dir/class_1/mno/pqr.jpeg', 1), ('./temp_dir/class_1/mno/stu.jpg', 1)] print(data_folder_1.targets) # [0, 0, 1, 1, 1, 1] print(len(data_folder_1)) # 6 for i in range(len(data_folder_1)): img, label = data_folder_1[i] # do something with img and label print(type(img), img.size, label) # (32, 32) 0 transform = T.Compose( [ T.Resize(64), T.ToTensor(), T.Normalize( mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], to_rgb=True, ), ] ) data_folder_2 = DatasetFolder( fake_data_dir, loader=lambda x: cv2.imread(x), # load image with OpenCV extensions=(".jpg",), # only load *.jpg files transform=transform, # apply transform to every image ) print([img_path for img_path, label in data_folder_2.samples]) # ['./temp_dir/class_0/abc.jpg', './temp_dir/class_1/mno/stu.jpg'] print(len(data_folder_2)) # 2 for img, label in iter(data_folder_2): # do something with img and label print(type(img), img.shape, label) # [3, 64, 64] 0 shutil.rmtree(fake_data_dir) Nc Cs||_||_|dur t}||j\}}t|j|||}t|dkr0td|jdd||dur6tn||_ ||_ ||_ ||_ ||_ dd|D|_t|_dS)Nrz&Found 0 directories in subfolders of:  Supported extensions are: ,cSsg|]}|dqS)r)rsrrr r r z*DatasetFolder.__init__..)r) transformIMG_EXTENSIONS _find_classesr.len RuntimeErrorr!default_loaderloaderrclassesr&samplestargetspaddleZget_default_dtypeZdtype) selfr)r:rr4rr;r&r<rrr __init__s.  zDatasetFolder.__init__csbtjdkrddtDn fddtDfddttD}|fS)a Finds the class folders in a dataset. Args: dir (string): Root directory path. Returns: tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary. )cSsg|] }|r|jqSr)is_dirnamerr(rrr r sz/DatasetFolder._find_classes..cs&g|]}tjtj|r|qSr)rrr"r!rE)r%rr r s csi|]}||qSrr)ri)r;rr sz/DatasetFolder._find_classes..)sys version_inforscandirlistdirsortranger7)r?r%r&r)r;r%r r6s  zDatasetFolder._find_classescCs4|j|\}}||}|jdur||}||fS)z Args: index (int): Index Returns: tuple: (sample, target) where target is class_index of the target class. Nr<r:r4)r?indexrr'samplerrr __getitem__s    zDatasetFolder.__getitem__cC t|jSrr7r<r?rrr __len__rzDatasetFolder.__len__NNNN)__name__ __module__ __qualname____doc__r@r6rQrUrrrr r/As   r/) z.jpgz.jpegz.pngz.ppmz.bmpz.pgmz.tifz.tiffz.webpcCsBt|d}t|}|dWdS1swYdS)NrbRGB)openrconvert)rfZimgrrr pil_loaders  $r`cCstd}||||jS)Ncv2)rZcvtColorZimreadZ COLOR_BGR2RGB)rrarrr cv2_loadersrbcCs&ddlm}|dkrt|St|S)Nr)get_image_backendra)Z paddle.visionrcrbr`)rrcrrr r9!s  r9c@s2eZdZdZ    d ddZddZddZdS) ImageFolderaA generic data loader where the samples are arranged in this way: .. code-block:: text root/1.ext root/2.ext root/sub_dir/3.ext Args: root (str): Root directory path. loader (Callable, optional): A function to load a sample given its path. Default: None. extensions (list[str]|tuple[str], optional): A list of allowed extensions. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. If this value is not set, the default is to use ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp'). Default: None. transform (Callable, optional): A function/transform that takes in a sample and returns a transformed version. Default: None. is_valid_file (Callable, optional): A function that takes path of a file and check if the file is a valid file. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. Default: None. Returns: :ref:`api_paddle_io_Dataset`. An instance of ImageFolder. Attributes: samples (list[str]): List of sample path. Example: .. code-block:: python import shutil import tempfile import cv2 import numpy as np import paddle.vision.transforms as T from pathlib import Path from paddle.vision.datasets import ImageFolder def make_fake_file(img_path: str): if img_path.endswith((".jpg", ".png", ".jpeg")): fake_img = np.random.randint(0, 256, (32, 32, 3), dtype=np.uint8) cv2.imwrite(img_path, fake_img) elif img_path.endswith(".txt"): with open(img_path, "w") as f: f.write("This is a fake file.") def make_directory(root, directory_hierarchy, file_maker=make_fake_file): root = Path(root) root.mkdir(parents=True, exist_ok=True) for subpath in directory_hierarchy: if isinstance(subpath, str): filepath = root / subpath file_maker(str(filepath)) else: dirname = list(subpath.keys())[0] make_directory(root / dirname, subpath[dirname]) directory_hirerarchy = [ "abc.jpg", "def.png", {"ghi": [ "jkl.jpeg", {"mno": [ "pqr.jpg"]}]}, "this_will_be_ignored.txt", ] # You can replace this with any directory to explore the structure # of generated data. e.g. fake_data_dir = "./temp_dir" fake_data_dir = tempfile.mkdtemp() make_directory(fake_data_dir, directory_hirerarchy) image_folder_1 = ImageFolder(fake_data_dir) print(image_folder_1.samples) # ['./temp_dir/abc.jpg', './temp_dir/def.png', # './temp_dir/ghi/jkl.jpeg', './temp_dir/ghi/mno/pqr.jpg'] print(len(image_folder_1)) # 4 for i in range(len(image_folder_1)): (img,) = image_folder_1[i] # do something with img print(type(img), img.size) # (32, 32) transform = T.Compose( [ T.Resize(64), T.ToTensor(), T.Normalize( mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], to_rgb=True, ), ] ) image_folder_2 = ImageFolder( fake_data_dir, loader=lambda x: cv2.imread(x), # load image with OpenCV extensions=(".jpg",), # only load *.jpg files transform=transform, # apply transform to every image ) print(image_folder_2.samples) # ['./temp_dir/abc.jpg', './temp_dir/ghi/mno/pqr.jpg'] print(len(image_folder_2)) # 2 for (img,) in iter(image_folder_2): # do something with img print(type(img), img.shape) # [3, 64, 64] shutil.rmtree(fake_data_dir) Nc s||_dur tg}tj|}durfdd}ttj|ddD]\}}} t| D]} tj|| } || r?|| q-q$t |dkrUt d|jdd|dur[t n||_ |_ ||_||_dS) Ncrrrrrrr rrz+ImageFolder.__init__..is_valid_fileTrrz Found 0 files in subfolders of: r0r1)r)r5rrrrr#r!r$r7r8r9r:rr<r4) r?r)r:rr4rr<rr*r+r,r_rrr r@s0       zImageFolder.__init__cCs.|j|}||}|jdur||}|gS)zn Args: index (int): Index Returns: sample of specific index. NrN)r?rOrrPrrr rQs    zImageFolder.__getitem__cCrRrrSrTrrr rUrzImageFolder.__len__rV)rWrXrYrZr@rQrUrrrr rd)sy # rdr)rrHZPILrr>Z paddle.iorZ paddle.utilsr__all__rr.r/r5r`rbr9rdrrrr s     R