Faster fetch dandisets (NeurodataWithoutBorders#292)

* faster fetching metadata from dandisets * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * req * Update nwbwidgets/panel.py * Update nwbwidgets/panel.py --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ben Dichter <ben.dichter@gmail.com>
h-mayorquin · Jun 16, 2023 · b99aa49 · b99aa49
1 parent da0c7ac
commit b99aa49
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 14 deletions.
diff --git a/nwbwidgets/panel.py b/nwbwidgets/panel.py
@@ -1,3 +1,4 @@
+import concurrent.futures
 from pathlib import Path
 
 import fsspec
@@ -290,19 +291,29 @@ def load_local_file(self, args=None):
         nwb = io.read()
         self.widgets_panel.children = [nwb2widget(nwb)]
 
+    def process_dandiset(self, dandiset):
+        try:
+            metadata = dandiset.get_metadata()
+            if has_nwb(metadata):
+                return metadata
+        except:
+            pass
+        return None
+
     def get_all_dandisets_metadata(self):
         with DandiAPIClient() as client:
-            all_metadata = list()
-            dandisets_iter = tqdm(list(client.get_dandisets()), desc="Loading dandiset metadata")
-            self.source_changing_panel.children = [dandisets_iter.container]
-            for ii, dandiset in enumerate(dandisets_iter):
-                if 1 < ii < 560:
-                    try:
-                        metadata = dandiset.get_metadata()
-                        if has_nwb(metadata):
-                            all_metadata.append(metadata)
-                    except:
-                        pass
-                else:
-                    pass
+            all_metadata = []
+            dandisets = list(client.get_dandisets())
+            total_dandisets = len(dandisets)
+            pbar = tqdm(total=total_dandisets, desc="Loading dandiset metadata")
+            self.source_changing_panel.children = [pbar.container]
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                futures = [executor.submit(self.process_dandiset, dandiset) for dandiset in dandisets]
+
+                for future in concurrent.futures.as_completed(futures):
+                    metadata = future.result()
+                    if metadata:
+                        all_metadata.append(metadata)
+                    pbar.update(1)
+                pbar.close()
         return all_metadata
diff --git a/requirements.txt b/requirements.txt
@@ -14,7 +14,7 @@ ndx-spectrum
 ndx-icephys-meta
 ndx-grayscalevolume
 trimesh
-dandi
+dandi>=0.55.0
 importlib-metadata <5.0
 fsspec
 requests