Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[cli/paraformer] ali-paraformer inference #2067

Merged
merged 21 commits into from
Oct 30, 2023
Merged

[cli/paraformer] ali-paraformer inference #2067

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
000c7af
[cli/paraformer] ali-paraformer load and infer work
Mddct Oct 16, 2023
6967ef5
fix lint
Mddct Oct 20, 2023
c8cccdc
export jit and load work
Mddct Oct 23, 2023
852165c
reuse init_model.py
Mddct Oct 23, 2023
e8fa013
mv the intermediate files to the assets directory
Mddct Oct 23, 2023
0a98847
merge main
Mddct Oct 23, 2023
e987b00
model.decodde work && recognize.py work
Mddct Oct 23, 2023
3d25e2e
rm positionwise_feed_forward.py/lfr.py
Mddct Oct 23, 2023
aedd43b
Merge branch 'main' into Mddct-cli-paraformer
Mddct Oct 24, 2023
d264297
refactor search
Mddct Oct 24, 2023
3f45af7
merge main
Mddct Oct 24, 2023
30b5677
merge main
Mddct Oct 24, 2023
20146f4
cli work
Mddct Oct 24, 2023
e3ec8e7
fix lint
Mddct Oct 24, 2023
b1b44df
fix att mask && batch infer
Mddct Oct 30, 2023
cd9c659
search confidence works
Mddct Oct 30, 2023
0b0eea7
merge main
Mddct Oct 30, 2023
c11aefe
merge main
Mddct Oct 30, 2023
daff617
fix linux dtype
Mddct Oct 30, 2023
9e810d1
fix label type
Mddct Oct 30, 2023
b765c12
revert init_model.py and add init_model in export_jit
Mddct Oct 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
cli work
  • Loading branch information
@Mddct
Mddct committed Oct 24, 2023
commit 20146f45c8c4f366939f9e3f02787c61450e9eb1
42 changes: 42 additions & 0 deletions wenet/cli/paraformer_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import os

import torch
import torchaudio
import torchaudio.compliance.kaldi as kaldi

from wenet.paraformer.search import paraformer_beam_search, paraformer_greedy_search
from wenet.utils.file_utils import read_symbol_table


class Paraformer:

def __init__(self, model_dir: str) -> None:

model_path = os.path.join(model_dir, 'final.zip')
units_path = os.path.join(model_dir, 'units.txt')
self.model = torch.jit.load(model_path)
symbol_table = read_symbol_table(units_path)
self.char_dict = {v: k for k, v in symbol_table.items()}
self.eos = 2

def transcribe(self, audio_file: str):
waveform, sample_rate = torchaudio.load(audio_file, normalize=False)
waveform = waveform.to(torch.float)
feats = kaldi.fbank(waveform,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The default window in the FunASR frontend is hamming. You can find more details here. However, the default window in kaldi.fbank is povey, as specified here. This different window maybe a little mismatch. As mentioned in line 44 of this document:

"povey" is a window I made to be similar to Hamming but to go to zero at the edges, it's pow((0.5 - 0.5cos(n/N2*pi)), 0.85) I just don't think the Hamming window makes sense as a windowing function.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pr welcome

num_mel_bins=80,
frame_length=25,
frame_shift=10,
energy_floor=0.0,
sample_frequency=16000)
feats = feats.unsqueeze(0)
feats_lens = torch.tensor([feats.size(1)], dtype=torch.int64)

decoder_out, token_num = self.model.forward_paraformer(
feats, feats_lens)

results = paraformer_greedy_search(decoder_out, token_num)
hyp = [self.char_dict[x] for x in results[0].tokens]

# TODO(Mddct): deal with '@@'
result = ''.join(hyp)
return result
9 changes: 8 additions & 1 deletion wenet/cli/transcribe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
import argparse

from wenet.cli.model import Model
from wenet.cli.paraformer_model import Paraformer


def get_args():
parser = argparse.ArgumentParser(description='')
Expand All @@ -23,17 +25,22 @@ def get_args():
choices=[
'chinese',
'english',
'chinese-paraformer',
],
default='chinese',
help='language type')
parser.add_argument('--model_dir', default='', help='wenet jit model dirs')

args = parser.parse_args()
return args


def main():
args = get_args()
model = Model(args.language)
if args.language == 'chinese-paraformer':
model = Paraformer(args.model_dir)
else:
model = Model(args.language)
result = model.transcribe(args.audio_file)
print(result)

Expand Down
2 changes: 0 additions & 2 deletions wenet/paraformer/ali_paraformer/export_jit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@

import argparse
import torch
import torchaudio
import torchaudio.compliance.kaldi as kaldi
import yaml
from wenet.utils.checkpoint import load_checkpoint
from wenet.utils.file_utils import read_symbol_table
Expand Down
5 changes: 1 addition & 4 deletions wenet/paraformer/ali_paraformer/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,9 @@
MultiHeadAttentionCross,
MultiHeadedAttentionSANM
)
from wenet.paraformer.paraformer import Paraformer
from wenet.paraformer.search import paraformer_beam_search, paraformer_greedy_search
from wenet.transducer.predictor import PredictorBase
from wenet.transformer.ctc import CTC
from wenet.transformer.search import DecodeResult
from wenet.transformer.encoder import BaseEncoder, TransformerEncoder
from wenet.transformer.encoder import BaseEncoder
from wenet.transformer.decoder import TransformerDecoder
from wenet.transformer.decoder_layer import DecoderLayer
from wenet.transformer.encoder_layer import TransformerEncoderLayer
Expand Down
Loading