Skip to content
This repository has been archived by the owner on Mar 19, 2024. It is now read-only.

JSON Decoder Error Fix #346

Closed
wants to merge 5 commits into from
Closed

JSON Decoder Error Fix #346

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fbafb5a
JSON Decoder Error Fix
pranavsinghps1 Jun 17, 2021
e9efe76
Added Documentation of save_file() for json files
pranavsinghps1 Jun 18, 2021
896a74b
Fixed Typos
pranavsinghps1 Jun 19, 2021
a8cf899
Tutorial Fix
pranavsinghps1 Jun 21, 2021
a7ebc5c
Tutorial Fix
pranavsinghps1 Jun 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion GETTING_STARTED.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ In your python interpretor:
}
}
>>> from vissl.utils.io import save_file
>>> save_file(json_data, "/tmp/configs/config/dataset_catalog.json")
>>> save_file(json_data, "/tmp/configs/config/dataset_catalog.json",append_to_json=True)
Copy link
Contributor

@iseessel iseessel Jun 21, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These should now be append_to_json=False, since we want to prevent the JSON decoder error.

Also nit: add space after comma:

>>> save_file(json_data, "/tmp/configs/config/dataset_catalog.json", append_to_json=False)

(and for all the others).

>>> from vissl.data.dataset_catalog import VisslDatasetCatalog
>>> print(VisslDatasetCatalog.list())
['imagenet1k_folder']
Expand Down
2 changes: 1 addition & 1 deletion docs/source/getting_started.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ In your python interpretor:
}
}
>>> from vissl.utils.io import save_file
>>> save_file(json_data, "/tmp/configs/config/dataset_catalog.json")
>>> save_file(json_data, "/tmp/configs/config/dataset_catalog.json",append_to_json=True)
>>> from vissl.data.dataset_catalog import VisslDatasetCatalog
>>> print(VisslDatasetCatalog.list())
['imagenet1k_folder']
Expand Down
2 changes: 1 addition & 1 deletion vissl/hooks/log_hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ def _print_and_save_meters(self, task, train_phase_idx):
save_metrics[metric_key] = meter_value
logging.info(f"Rank: {rank}, name: {metric_key}, value: {meter_value}")
meter_file = f"{checkpoint_folder}/metrics.json"
save_file(save_metrics, meter_file)
save_file(save_metrics, meter_file,append_to_json=True)


class LogPerfTimeMetricsHook(ClassyHook):
Expand Down
15 changes: 11 additions & 4 deletions vissl/utils/io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,13 @@ def create_file_symlink(file1, file2):
logging.info(f"Could NOT create symlink. Error: {e}")


def save_file(data, filename):
def save_file(data, filename,append_to_json=True):
"""
Common i/o utility to handle saving data to various file formats.
Supported:
.pkl, .pickle, .npy, .json
Specifically for .json, users have the option to either append (default)
or rewrite by passing in Boolean value to append_to_json.
"""
logging.info(f"Saving data to file: {filename}")
file_ext = os.path.splitext(filename)[1]
Expand All @@ -66,9 +68,14 @@ def save_file(data, filename):
with PathManager.open(filename, "wb") as fopen:
np.save(fopen, data)
elif file_ext == ".json":
with PathManager.open(filename, "a") as fopen:
fopen.write(json.dumps(data, sort_keys=True) + "\n")
fopen.flush()
if append_to_json:
with PathManager.open(filename, "a") as fopen:
fopen.write(json.dumps(data, sort_keys=True) + "\n")
fopen.flush()
else:
with PathManager.open(filename, "w") as fopen:
fopen.write(json.dumps(data, sort_keys=True) + "\n")
fopen.flush()
elif file_ext == ".yaml":
with PathManager.open(filename, "w") as fopen:
dump = yaml.dump(data)
Expand Down