*FileLoader -> *FileOpener (pytorch#5128)

* *FileLoader -> *FileOpener * fix opening mode Co-authored-by: Kevin Tse <NivekT@users.noreply.github.com>
jmargeta · Jan 6, 2022 · 578c154 · 578c154
1 parent 93ec8bf
commit 578c154
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 8 deletions.
diff --git a/torchvision/prototype/datasets/_folder.py b/torchvision/prototype/datasets/_folder.py
@@ -7,8 +7,7 @@
 from typing import Union, Tuple, List, Dict, Any
 
 import torch
-from torch.utils.data import IterDataPipe
-from torch.utils.data.datapipes.iter import FileLister, FileLoader, Mapper, Shuffler, Filter
+from torchdata.datapipes.iter import IterDataPipe, FileLister, FileOpener, Mapper, Shuffler, Filter
 from torchvision.prototype.datasets.decoder import pil
 from torchvision.prototype.datasets.utils._internal import INFINITE_BUFFER_SIZE, hint_sharding
 
@@ -54,7 +53,7 @@ def from_data_folder(
     dp: IterDataPipe = Filter(dp, functools.partial(_is_not_top_level_file, root=root))
     dp = hint_sharding(dp)
     dp = Shuffler(dp, buffer_size=INFINITE_BUFFER_SIZE)
-    dp = FileLoader(dp)
+    dp = FileOpener(dp, mode="rb")
     return (
         Mapper(dp, functools.partial(_collate_and_decode_data, root=root, categories=categories, decoder=decoder)),
         categories,

diff --git a/torchvision/prototype/datasets/utils/_internal.py b/torchvision/prototype/datasets/utils/_internal.py
@@ -30,7 +30,7 @@
 import torch
 import torch.distributed as dist
 import torch.utils.data
-from torchdata.datapipes.iter import IoPathFileLister, IoPathFileLoader, IterDataPipe, ShardingFilter, Shuffler
+from torchdata.datapipes.iter import IoPathFileLister, IoPathFileOpener, IterDataPipe, ShardingFilter, Shuffler
 from torchdata.datapipes.utils import StreamWrapper
 
 
@@ -254,7 +254,7 @@ def _make_sharded_datapipe(root: str, dataset_size: int) -> IterDataPipe[Dict[st
     dp = IoPathFileLister(root=root)
     dp = SharderDataPipe(dp)
     dp = dp.shuffle(buffer_size=INFINITE_BUFFER_SIZE)
-    dp = IoPathFileLoader(dp, mode="rb")
+    dp = IoPathFileOpener(dp, mode="rb")
     dp = PicklerDataPipe(dp)
     # dp = dp.cycle(2)
     dp = TakerDataPipe(dp, dataset_size)

diff --git a/torchvision/prototype/datasets/utils/_resource.py b/torchvision/prototype/datasets/utils/_resource.py
@@ -9,7 +9,7 @@
 from torchdata.datapipes.iter import (
     IterableWrapper,
     FileLister,
-    FileLoader,
+    FileOpener,
     IterDataPipe,
     ZipArchiveReader,
     TarArchiveReader,
@@ -62,9 +62,9 @@ def _decompress(file: pathlib.Path) -> pathlib.Path:
 
     def _default_loader(self, path: pathlib.Path) -> IterDataPipe[Tuple[str, IO]]:
         if path.is_dir():
-            return FileLoader(FileLister(str(path), recursive=True))
+            return FileOpener(FileLister(str(path), recursive=True), mode="rb")
 
-        dp = FileLoader(IterableWrapper((str(path),)))
+        dp = FileOpener(IterableWrapper((str(path),)), mode="rb")
 
         archive_loader = self._guess_archive_loader(path)
         if archive_loader: