Replace ChunkedLilcomHdf5Writer with LilcomChunkyWriter. (k2-fsa#411)

yaozengwei · Jun 9, 2022 · ed66877 · ed66877
1 parent 8512aaf
commit ed66877
Show file tree

Hide file tree

Showing 12 changed files with 25 additions and 25 deletions.
diff --git a/egs/aidatatang_200zh/ASR/local/compute_fbank_aidatatang_200zh.py b/egs/aidatatang_200zh/ASR/local/compute_fbank_aidatatang_200zh.py
@@ -29,7 +29,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
+from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
 from lhotse.recipes.utils import read_manifests_if_cached
 
 from icefall.utils import get_executor
@@ -90,7 +90,7 @@ def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
                 # when an executor is specified, make more partitions
                 num_jobs=num_jobs if ex is None else 80,
                 executor=ex,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
 
             cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

diff --git a/egs/aishell/ASR/local/compute_fbank_aidatatang_200zh.py b/egs/aishell/ASR/local/compute_fbank_aidatatang_200zh.py
@@ -29,7 +29,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
+from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
 from lhotse.recipes.utils import read_manifests_if_cached
 
 from icefall.utils import get_executor
@@ -90,7 +90,7 @@ def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
                 # when an executor is specified, make more partitions
                 num_jobs=num_jobs if ex is None else 80,
                 executor=ex,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
 
             cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

diff --git a/egs/aishell/ASR/local/compute_fbank_aishell.py b/egs/aishell/ASR/local/compute_fbank_aishell.py
@@ -29,7 +29,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
+from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
 from lhotse.recipes.utils import read_manifests_if_cached
 
 from icefall.utils import get_executor
@@ -86,7 +86,7 @@ def compute_fbank_aishell(num_mel_bins: int = 80):
                 # when an executor is specified, make more partitions
                 num_jobs=num_jobs if ex is None else 80,
                 executor=ex,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
             cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")
 

diff --git a/egs/alimeeting/ASR/local/compute_fbank_alimeeting.py b/egs/alimeeting/ASR/local/compute_fbank_alimeeting.py
@@ -29,7 +29,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
+from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
 from lhotse.recipes.utils import read_manifests_if_cached
 
 from icefall.utils import get_executor
@@ -90,7 +90,7 @@ def compute_fbank_alimeeting(num_mel_bins: int = 80):
                 # when an executor is specified, make more partitions
                 num_jobs=cur_num_jobs,
                 executor=ex,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
 
             logging.info("About splitting cuts into smaller chunks")

diff --git a/egs/librispeech/ASR/local/compute_fbank_librispeech.py b/egs/librispeech/ASR/local/compute_fbank_librispeech.py
@@ -28,7 +28,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
+from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
 from lhotse.recipes.utils import read_manifests_if_cached
 
 from icefall.utils import get_executor
@@ -91,7 +91,7 @@ def compute_fbank_librispeech():
                 # when an executor is specified, make more partitions
                 num_jobs=num_jobs if ex is None else 80,
                 executor=ex,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
             cut_set.to_file(output_dir / cuts_filename)
 

diff --git a/egs/librispeech/ASR/local/compute_fbank_musan.py b/egs/librispeech/ASR/local/compute_fbank_musan.py
@@ -28,7 +28,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig, combine
+from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter, combine
 from lhotse.recipes.utils import read_manifests_if_cached
 
 from icefall.utils import get_executor
@@ -92,7 +92,7 @@ def compute_fbank_musan():
                 storage_path=f"{output_dir}/musan_feats",
                 num_jobs=num_jobs if ex is None else 80,
                 executor=ex,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
         )
         musan_cuts.to_file(musan_cuts_path)

diff --git a/egs/spgispeech/ASR/local/compute_fbank_musan.py b/egs/spgispeech/ASR/local/compute_fbank_musan.py
@@ -27,7 +27,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, combine
+from lhotse import CutSet, LilcomChunkyWriter, combine
 from lhotse.features.kaldifeat import (
     KaldifeatFbank,
     KaldifeatFbankConfig,
@@ -91,7 +91,7 @@ def compute_fbank_musan():
             storage_path=output_dir / "feats_musan",
             batch_duration=500,
             num_workers=4,
-            storage_type=ChunkedLilcomHdf5Writer,
+            storage_type=LilcomChunkyWriter,
         )
     )
 

diff --git a/egs/spgispeech/ASR/local/compute_fbank_spgispeech.py b/egs/spgispeech/ASR/local/compute_fbank_spgispeech.py
@@ -27,7 +27,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, load_manifest_lazy
+from lhotse import LilcomChunkyWriter, load_manifest_lazy
 from lhotse.features.kaldifeat import (
     KaldifeatFbank,
     KaldifeatFbankConfig,
@@ -118,7 +118,7 @@ def compute_fbank_spgispeech(args):
                 storage_path=output_dir / f"feats_train_{idx}",
                 batch_duration=500,
                 num_workers=4,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
             cs.to_file(cuts_train_idx_path)
 
@@ -137,7 +137,7 @@ def compute_fbank_spgispeech(args):
                 manifest_path=src_dir / f"cuts_{partition}.jsonl.gz",
                 batch_duration=500,
                 num_workers=4,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
 
 

diff --git a/egs/tedlium3/ASR/local/compute_fbank_tedlium.py b/egs/tedlium3/ASR/local/compute_fbank_tedlium.py
@@ -27,7 +27,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
+from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
 from lhotse.recipes.utils import read_manifests_if_cached
 
 from icefall.utils import get_executor
@@ -89,7 +89,7 @@ def compute_fbank_tedlium():
                 # when an executor is specified, make more partitions
                 num_jobs=cur_num_jobs,
                 executor=ex,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
             # Split long cuts into many short and un-overlapping cuts
             cut_set = cut_set.trim_to_supervisions(keep_overlapping=False)

diff --git a/egs/timit/ASR/local/compute_fbank_timit.py b/egs/timit/ASR/local/compute_fbank_timit.py
@@ -29,7 +29,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
+from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
 from lhotse.recipes.utils import read_manifests_if_cached
 
 from icefall.utils import get_executor
@@ -88,7 +88,7 @@ def compute_fbank_timit():
                 # when an executor is specified, make more partitions
                 num_jobs=num_jobs if ex is None else 80,
                 executor=ex,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
             cut_set.to_file(cuts_file)
 

diff --git a/egs/wenetspeech/ASR/local/compute_fbank_wenetspeech_splits.py b/egs/wenetspeech/ASR/local/compute_fbank_wenetspeech_splits.py
@@ -23,10 +23,10 @@
 
 import torch
 from lhotse import (
-    ChunkedLilcomHdf5Writer,
     CutSet,
     KaldifeatFbank,
     KaldifeatFbankConfig,
+    LilcomChunkyWriter,
     set_audio_duration_mismatch_tolerance,
     set_caching_enabled,
 )
@@ -135,7 +135,7 @@ def compute_fbank_wenetspeech_splits(args):
             storage_path=f"{output_dir}/feats_{subset}_{idx}",
             num_workers=args.num_workers,
             batch_duration=args.batch_duration,
-            storage_type=ChunkedLilcomHdf5Writer,
+            storage_type=LilcomChunkyWriter,
         )
 
         logging.info("About to split cuts into smaller chunks.")

diff --git a/egs/yesno/ASR/local/compute_fbank_yesno.py b/egs/yesno/ASR/local/compute_fbank_yesno.py
@@ -12,7 +12,7 @@
 from pathlib import Path
 
 import torch
-from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
+from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
 from lhotse.recipes.utils import read_manifests_if_cached
 
 from icefall.utils import get_executor
@@ -74,7 +74,7 @@ def compute_fbank_yesno():
                 # when an executor is specified, make more partitions
                 num_jobs=num_jobs if ex is None else 1,  # use one job
                 executor=ex,
-                storage_type=ChunkedLilcomHdf5Writer,
+                storage_type=LilcomChunkyWriter,
             )
             cut_set.to_file(cuts_file)