vocab = onmt.Dict([onmt.constants.PAD_WORD, onmt.constants.UNK_WORD,

onmt.constants.BOS_WORD, onmt.constants.EOS_WORD],

lower=opt.lower)

for filename in filenames:

print("Generating vocabulary from file %s ... " % filename)

onmt.Dict.gen_dict_from_file(filename, vocab, tokenizer, num_workers=num_workers)

original_size = vocab.size()

vocab = vocab.prune(size)

print('Created dictionary of size %d (pruned from %d)' %

(vocab.size(), original_size))

vocab = None

if vocab_file is not None:

# If given, load existing word dictionary.

print('Reading ' + name + ' vocabulary from \'' + vocab_file + '\'...')

if not opt.load_bpe_voc:

vocab = onmt.Dict()

else:

vocab = onmt.Dict([onmt.constants.PAD_WORD, onmt.constants.UNK_WORD,

onmt.constants.BOS_WORD, onmt.constants.EOS_WORD],

lower=opt.lower)

vocab.loadFile(vocab_file)

print('Loaded ' + str(vocab.size()) + ' ' + name + ' words')

if vocab is None:

print('Building ' + name + ' vocabulary...')

gen_word_vocab = make_vocab(data_files, vocab_size, tokenizer, num_workers=num_workers)

vocab = gen_word_vocab

print()

tgt = []

sizes = []

count, ignored = 0, 0

print('Processing %s ...' % (tgt_file))

tgtf = open(tgt_file)

eos = torch.LongTensor(1).fill_(onmt.constants.EOS)

# print(eos.size())

tensors = [eos]

# find the number of words in the sentence

while True:

tline = tgtf.readline()

# normal end of file

if tline == "":

break

tline = tline.strip()

# source and/or target are empty

if tline == "":

print('WARNING: ignoring an empty line (' + str(count + 1) + ')')

continue

if input_type == 'word':

tgt_words = tline.split()

elif input_type == 'char':

tgt_words = split_line_by_char(tline)

tensor = tgt_dicts.convertToIdx(tgt_words,

onmt.constants.UNK_WORD,

None,

onmt.constants.EOS_WORD,

type=data_type)

# print(tensor.size())

tensors.append(tensor)

count = count + 1

if count % opt.report_every == 0:

print('... %d sentences prepared' % count)

tgtf.close()

# concatenate all tensors into one

tensor = torch.cat(tensors, dim=-1)

add_bos=True, data_type='int64', num_workers=1, verbose=False):

src, tgt = [], []

src_sizes = []

tgt_sizes = []

print("[INFO] Binarizing file %s ..." % src_file)

binarized_src = Binarizer.binarize_file(src_file, src_dicts, tokenizer,

bos_word=None, eos_word=None,

data_type=data_type,

num_workers=num_workers, verbose=verbose)

if add_bos:

tgt_bos_word = onmt.constants.BOS_WORD

else:

tgt_bos_word = None

print("[INFO] Binarizing file %s ..." % tgt_file)

binarized_tgt = Binarizer.binarize_file(tgt_file, tgt_dicts, tokenizer,

bos_word=tgt_bos_word, eos_word=onmt.constants.EOS_WORD,

data_type=data_type,

num_workers=num_workers, verbose=verbose)

src = binarized_src['data']

src_sizes = binarized_src['sizes']

tgt = binarized_tgt['data']

tgt_sizes = binarized_tgt['sizes']

# currently we don't ignore anything :D

ignored = 0

print(('Prepared %d sentences ' +

'(%d ignored due to length == 0 or src len > %d or tgt len > %d)') %

(len(src), ignored, max_src_length, max_tgt_length))

max_src_length=64, max_tgt_length=64, add_bos=True, data_type='int64', num_workers=1, verbose=False,

input_type='word', stride=1, concat=4, prev_context=0, fp16=False, reshape=True,

asr_format="h5", output_format="raw"):

src, tgt = [], []

src_sizes = []

tgt_sizes = []

count, ignored = 0, 0

n_unk_words = 0

print('[INFO] Processing %s ...' % src_file)

binarized_src = SpeechBinarizer.binarize_file(src_file, input_format=asr_format,

output_format=output_format, concat=concat,

stride=stride, fp16=fp16, prev_context=prev_context,

num_workers=num_workers)

src = binarized_src['data']

src_sizes = binarized_src['sizes']

if add_bos:

tgt_bos_word = onmt.constants.BOS_WORD

else:

tgt_bos_word = None

print("[INFO] Binarizing file %s ..." % tgt_file)

binarized_tgt = Binarizer.binarize_file(tgt_file, tgt_dicts, tokenizer,

bos_word=tgt_bos_word, eos_word=onmt.constants.EOS_WORD,

data_type=data_type,

num_workers=num_workers, verbose=verbose)

tgt = binarized_tgt['data']

tgt_sizes = binarized_tgt['sizes']

ignored = 0

if len(src_sizes) != len(tgt_sizes):

print("Warning: data size mismatched.")

print(('Prepared %d sentences ' +

'(%d ignored due to length == 0 or src len > %d or tgt len > %d)') %

(len(src), ignored, max_src_length, max_tgt_length))

# Each dataset is comprised of the following components:

# src: tensors for the source vectors, or the scp_path (in ASR case)

# tgt: tensors for the target vectors

# src_lang: tensors for the source language ids (simplified)

# tgt_lang: tensors for the target language ids (simplified)

# convert all datasets to pytorch tensors and save to .pt

if format in ['raw', 'bin']:

print('Saving data to ' + os.path.join(path, 'data.pt') + '...')

save_data = {'type': opt.src_type,

'data': data}

torch.save(save_data, os.path.join(path, 'data.pt'))

print("Done")

# for ASR only

elif format in ['scp', 'scpmem']:

print('Saving target data to memory indexed data files. Source data is stored only as scp path.')

from onmt.data.mmap_indexed_dataset import MMapIndexedDatasetBuilder

assert opt.asr, "ASR data format is required for this memory indexed format"

# TODO: changing this to before saving everything

# torch.save(dicts, opt.save_data + '.dict.pt')

# binarize the training set first

for set_ in ['tgt', 'src_lang', 'tgt_lang']:

if data[set_] is None:

continue

if opt.data_type == 'int64':

dtype = np.int64

else:

dtype = np.int32

indexed_data = MMapIndexedDatasetBuilder(os.path.join(path, "data.%s.bin" % set_), dtype=dtype)

# add item from training data to the indexed data

for tensor in data[set_]:

indexed_data.add_item(tensor)

indexed_data.finalize(os.path.join(path, "data.%s.idx" % set_))

del indexed_data

for set_ in ['src_sizes', 'tgt_sizes']:

if data[set_] is not None:

np_array = np.asarray(data[set_])

np.save(os.path.join(path, "data.%s.npy") % set_, np_array)

else:

print("Training %s not found " % set_)

# Finally save the audio path

# save_data = {'data': data['src']}

torch.save(data['src'], os.path.join(path, 'data.scp_path.pt'))

print("Done")

elif opt.format in ['mmap', 'mmem']:

# print('Saving data to memory indexed data files')

from onmt.data.mmap_indexed_dataset import MMapIndexedDatasetBuilder

if opt.asr:

print("ASR data format isn't compatible with memory indexed format")

raise AssertionError

# save dicts in this format

# torch.save(dicts, opt.save_data + '.dict.pt')

# binarize the training set firstd

for set_ in ['src', 'tgt', 'src_lang', 'tgt_lang']:

if data[set_] is None:

continue

if opt.data_type == 'int64':

dtype = np.int64

else:

dtype = np.int32

indexed_data = MMapIndexedDatasetBuilder(os.path.join(path, "data.%s.bin" % set_), dtype=dtype)

# add item from training data to the indexed data

for tensor in data[set_]:

indexed_data.add_item(tensor)

indexed_data.finalize(os.path.join(path, "data.%s.idx" % set_))

del indexed_data

for set_ in ['src_sizes', 'tgt_sizes']:

if data[set_] is not None:

np_array = np.asarray(data[set_])

np.save(os.path.join(path, "data.%s.npy" % set_), np_array)

else:

dicts = {}

tokenizer = onmt.Tokenizer(opt.input_type, opt.lower)

# construct set of languages from the training languages

src_langs = opt.train_src_lang.split("|")

tgt_langs = opt.train_tgt_lang.split("|")

langs = (src_langs + tgt_langs)

langs = list(set(langs))

if opt.load_dict is not None:

loaded_dict = torch.load(opt.load_dict)

new_languages = list()

for lang in langs:

if lang not in loaded_dict['langs']:

new_languages.append(lang)

dicts['langs'] = loaded_dict['langs']

print("Loaded dictionary for languages: ", dicts['langs'])

if len(new_languages) > 0:

for lang in new_languages:

idx = len(dicts['langs'])

dicts['langs'][lang] = idx

print("Added new languages: ", new_languages)

# dicts['tgt'] = loaded_dict['tgt']

# dicts['src'] = loaded_dict['src'] if 'src' in loaded_dict else None

else:

dicts['langs'] = dict()

for lang in langs:

idx = len(dicts['langs'])

dicts['langs'][lang] = idx

print(dicts['langs'])

start = time.time()

src_train_files = opt.train_src.split("|")

tgt_train_files = opt.train_tgt.split("|")

# for ASR and LM we only need to build vocab for the 'target' language

# TODO: create separate dictionary for each language if needed

if not opt.separate_dictionary:

src = 'src'

tgt = 'tgt'

if opt.asr or opt.lm:

dicts[tgt] = init_vocab('target', tgt_train_files, opt.tgt_vocab,

opt.tgt_vocab_size, tokenizer, num_workers=opt.num_threads)

elif opt.join_vocab:

dicts[src] = init_vocab('source', set(src_train_files + tgt_train_files), opt.src_vocab,

opt.tgt_vocab_size, tokenizer, num_workers=opt.num_threads)

dicts[tgt] = dicts[src]

else:

dicts[src] = init_vocab('source', src_train_files, opt.src_vocab,

opt.src_vocab_size, tokenizer, num_workers=opt.num_threads)

dicts[tgt] = init_vocab('target', tgt_train_files, opt.tgt_vocab,

opt.tgt_vocab_size, tokenizer, num_workers=opt.num_threads)

else:

print("START")

assert opt.asr, "Only ASR is supported at the moment. " # currently only supports ASR

tgt_train_files = dict()

all_files = opt.train_tgt.split("|")

tgt_langs = opt.train_tgt_lang.split("|")

for (lang, tgt_file) in zip(tgt_langs, all_files):

assert lang != "langs", "Use a different name for this language, not 'langs'. "

if lang not in tgt_train_files:

tgt_train_files[lang] = list()

tgt_train_files[lang].append(tgt_file)

for lang in tgt_train_files:

# pre existed vocab file is None?

dicts[lang] = init_vocab(lang, tgt_train_files[lang], None,

opt.tgt_vocab_size, tokenizer, num_workers=opt.num_threads)

elapse = str(datetime.timedelta(seconds=int(time.time() - start)))

print("Vocabulary generated after %s" % elapse)

# DATA GENERATION starts from here

if opt.lm:

raise NotImplementedError

elif opt.asr:

print('Preparing training acoustic model ...')

src_input_files = opt.train_src.split("|")

tgt_input_files = opt.train_tgt.split("|")

src_langs = opt.train_src_lang.split("|")

tgt_langs = opt.train_tgt_lang.split("|")

assert len(src_input_files) == len(src_langs)

assert len(src_input_files) == len(tgt_input_files)

assert len(tgt_input_files) == len(tgt_langs)

n_input_files = len(src_input_files)

idx = opt.starting_train_idx

for (src_file, tgt_file, src_lang, tgt_lang) in zip(src_input_files, tgt_input_files, src_langs, tgt_langs):

# First, read and convert data to tensor format

if opt.separate_dictionary:

dict_obj = dicts[tgt_lang]

else:

dict_obj = dicts['tgt']

src_data, tgt_data, src_sizes, tgt_sizes = make_asr_data(src_file, tgt_file,

dict_obj, tokenizer,

max_src_length=opt.src_seq_length,

max_tgt_length=opt.tgt_seq_length,

input_type=opt.input_type,

stride=opt.stride, concat=opt.concat,

prev_context=opt.previous_context,

fp16=opt.fp16,

asr_format=opt.asr_format,

output_format=opt.format,

num_workers=opt.num_threads)

# save each dataset as bilingual (no multi parallel data)

# we only need to have 1 language per file

# which will be broadcasted

n_samples = len(src_data)

src_lang_data = [torch.Tensor([dicts['langs'][src_lang]])]

tgt_lang_data = [torch.Tensor([dicts['langs'][tgt_lang]])]

data = dict()

data['src'] = src_data

data['tgt'] = tgt_data

data['src_sizes'] = src_sizes

data['tgt_sizes'] = tgt_sizes

data['src_lang'] = src_lang_data

data['tgt_lang'] = tgt_lang_data

print("Saving training set %i %s-%s to disk ..." % (idx, src_lang, tgt_lang))

# take basedir from opt.save_data

path = os.path.join(dirname(opt.save_data), "train.%i.%s-%s" % (idx, src_lang, tgt_lang))

os.makedirs(path, exist_ok=True)

# save data immediately

save_dataset(path, data, opt.format, dicts, opt.src_type)

idx = idx + 1

print('Preparing validation ...')

src_input_files = opt.valid_src.split("|")

tgt_input_files = opt.valid_tgt.split("|")

src_langs = opt.valid_src_lang.split("|")

tgt_langs = opt.valid_tgt_lang.split("|")

assert len(src_input_files) == len(src_langs)

assert len(src_input_files) == len(tgt_input_files)

assert len(tgt_input_files) == len(tgt_langs)

n_input_files = len(src_input_files)

idx = opt.starting_valid_idx

for (src_file, tgt_file, src_lang, tgt_lang) in zip(src_input_files, tgt_input_files, src_langs, tgt_langs):

if opt.separate_dictionary:

dict_obj = dicts[tgt_lang]

else:

dict_obj = dicts['tgt']

src_data, tgt_data, src_sizes, tgt_sizes = make_asr_data(src_file, tgt_file,

dict_obj, tokenizer,

max_src_length=max(1024, opt.src_seq_length),

max_tgt_length=max(1024, opt.tgt_seq_length),

input_type=opt.input_type,

stride=opt.stride, concat=opt.concat,

prev_context=opt.previous_context,

fp16=opt.fp16,

asr_format=opt.asr_format,

output_format=opt.format)

# save each dataset as bilingual (no multi parallel data)

# we only need to have 1 language per file

# which will be broadcasted

n_samples = len(src_data)

src_lang_data = [torch.Tensor([dicts['langs'][src_lang]])]

tgt_lang_data = [torch.Tensor([dicts['langs'][tgt_lang]])]

data = dict()

data['src'] = src_data

data['tgt'] = tgt_data

data['src_sizes'] = src_sizes

data['tgt_sizes'] = tgt_sizes

data['src_lang'] = src_lang_data

data['tgt_lang'] = tgt_lang_data

print("Saving validation set %i %s-%s to disk ..." % (idx, src_lang, tgt_lang))

# take basedir from opt.save_data

path = os.path.join(dirname(opt.save_data), "valid.%i.%s-%s" % (idx, src_lang, tgt_lang))

os.makedirs(path, exist_ok=True)

# save data immediately

save_dataset(path, data, opt.format, dicts, opt.src_type)

idx = idx + 1

else:

# Translation dataset

src_input_files = opt.train_src.split("|")

tgt_input_files = opt.train_tgt.split("|")

src_langs = opt.train_src_lang.split("|")

tgt_langs = opt.train_tgt_lang.split("|")

assert len(src_input_files) == len(src_langs)

assert len(src_input_files) == len(tgt_input_files)

assert len(tgt_input_files) == len(tgt_langs)

n_input_files = len(src_input_files)

start = time.time()

print('Binarizing data to train translation models...')

idx = opt.starting_train_idx

for (src_file, tgt_file, src_lang, tgt_lang) in zip(src_input_files, tgt_input_files, src_langs, tgt_langs):

src_data, tgt_data, src_sizes, tgt_sizes = make_translation_data(src_file, tgt_file,

dicts['src'], dicts['tgt'], tokenizer,

max_src_length=opt.src_seq_length,

max_tgt_length=opt.tgt_seq_length,

add_bos=(not opt.no_bos),

data_type=opt.data_type,

num_workers=opt.num_threads,

verbose=opt.verbose)

# save each dataset as bilingual (no multi parallel data)

# we only need to have 1 language per file

# which will be broadcasted

n_samples = len(src_data)

src_lang_data = [torch.Tensor([dicts['langs'][src_lang]])]

tgt_lang_data = [torch.Tensor([dicts['langs'][tgt_lang]])]

data = dict()

data['src'] = src_data

data['tgt'] = tgt_data

data['src_sizes'] = src_sizes

data['tgt_sizes'] = tgt_sizes

data['src_lang'] = src_lang_data

data['tgt_lang'] = tgt_lang_data

print("Saving training set %i %s-%s to disk ..." % (idx, src_lang, tgt_lang))

# take basedir from opt.save_data

path = os.path.join(dirname(opt.save_data), "train.%i.%s-%s" % (idx, src_lang, tgt_lang))

os.makedirs(path, exist_ok=True)

# save data immediately

save_dataset(path, data, opt.format, dicts, opt.src_type)

idx = idx + 1

print('Preparing validation ...')

src_input_files = opt.valid_src.split("|")

tgt_input_files = opt.valid_tgt.split("|")

src_langs = opt.valid_src_lang.split("|")

tgt_langs = opt.valid_tgt_lang.split("|")

assert len(src_input_files) == len(src_langs)

assert len(src_input_files) == len(tgt_input_files)

assert len(tgt_input_files) == len(tgt_langs)

n_input_files = len(src_input_files)

idx = opt.starting_valid_idx

for (src_file, tgt_file, src_lang, tgt_lang) in zip(src_input_files, tgt_input_files, src_langs, tgt_langs):

src_data, tgt_data, src_sizes, tgt_sizes = make_translation_data(src_file, tgt_file,

dicts['src'], dicts['tgt'], tokenizer,

max_src_length=max(1024,

opt.src_seq_length),

max_tgt_length=max(1024,

opt.tgt_seq_length),

add_bos=(not opt.no_bos),

data_type=opt.data_type,

num_workers=opt.num_threads,

verbose=opt.verbose)

# save each dataset as bilingual (no multi parallel data)

# we only need to have 1 language per file

# which will be broadcasted

n_samples = len(src_data)

src_lang_data = [torch.Tensor([dicts['langs'][src_lang]])]

tgt_lang_data = [torch.Tensor([dicts['langs'][tgt_lang]])]

data = dict()

data['src'] = src_data

data['tgt'] = tgt_data

data['src_sizes'] = src_sizes

data['tgt_sizes'] = tgt_sizes

data['src_lang'] = src_lang_data

data['tgt_lang'] = tgt_lang_data

print("Saving validation set %i %s-%s to disk ..." % (idx, src_lang, tgt_lang))

# take basedir from opt.save_data

path = os.path.join(dirname(opt.save_data), "valid.%i.%s-%s" % (idx, src_lang, tgt_lang))

os.makedirs(path, exist_ok=True)

# save data immediately

save_dataset(path, data, opt.format, dicts, opt.src_type)

idx = idx + 1

elapse = str(datetime.timedelta(seconds=int(time.time() - start)))

print("Binarization finished after %s" % elapse)

print("Saving dictionary to %s" % (opt.save_data + '.dict.pt'))

torch.save(dicts, opt.save_data + '.dict.pt')

if not opt.separate_dictionary:

if opt.src_vocab is None and opt.asr == False and opt.lm == False:

save_vocabulary('source', dicts['src'], opt.save_data + '.src.dict')

if opt.tgt_vocab is None:

save_vocabulary('target', dicts['tgt'], opt.save_data + '.tgt.dict')

else:

if opt.save_separate_dictionary:

for lang in dicts:

if lang != 'langs':

save_vocabulary(lang, dicts[lang], opt.save_data + '.%s.dict' % lang)