Skip to content

Commit

Permalink
optimize print log
Browse files Browse the repository at this point in the history
  • Loading branch information
Qznan committed Oct 26, 2020
1 parent 77f8724 commit 4ce62d9
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 5 deletions.
3 changes: 2 additions & 1 deletion qiznlp/common/tfrecord_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ def byte_feat(value):
value = [value]
return tf.train.Feature(bytesList=tf.train.BytesList(value=value))

print(f'generating tfrecord... filename: {tfrecord_file}')
writer = tf.python_io.TFRecordWriter(tfrecord_file)
count = 0
for item in items:
Expand All @@ -81,7 +82,7 @@ def byte_feat(value):
features = {}
try:
for k, v in item.items():
# maybe_flatten
# maybe should flatten
if isinstance(v, np.ndarray) and v.ndim > 1:
v = v.flatten()
elif isinstance(v, list) and isinstance(v[0], list):
Expand Down
8 changes: 4 additions & 4 deletions qiznlp/common/train_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ def prepare_tfrecord(raw_data_file,
if not all([os.path.exists(train_txt_file), os.path.exists(dev_txt_file)]):
train_data, dev_data, test_data = preprocess_raw_data_fn(raw_data_file, tokenize=tokenize, token2id_dct=token2id_dct, **kwargs)
if train_data:
print(f'generating train txt file... filename: {train_txt_file}')
utils.list2file(train_txt_file, train_data)
print(f'generate train txt file ok! {train_txt_file}')
if dev_data:
print(f'generating dev txt file... filename: {dev_txt_file}')
utils.list2file(dev_txt_file, dev_data)
print(f'generate dev txt file ok! {dev_txt_file}')
if test_data:
print(f'generating test txt file... filename: {test_txt_file}')
utils.list2file(test_txt_file, test_data)
print(f'generate test txt file ok! {test_txt_file}')


if os.path.exists(train_txt_file):
model.generate_tfrecord(train_txt_file, token2id_dct, train_tfrecord_file)
if os.path.exists(dev_txt_file):
Expand Down

0 comments on commit 4ce62d9

Please sign in to comment.