Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes for windows issues #528

Merged
merged 5 commits into from
Feb 4, 2021
Merged

Fixes for windows issues #528

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5701cf1
fixes windows issues
AbhinavTuli Feb 3, 2021
7a0030e
cleanup
AbhinavTuli Feb 3, 2021
d11b061
adds tests for other environments
AbhinavTuli Feb 4, 2021
daa6a4b
reverting circleci config
AbhinavTuli Feb 4, 2021
911185d
fixing test due to wasabi issue
AbhinavTuli Feb 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fixes windows issues
  • Loading branch information
@AbhinavTuli
AbhinavTuli committed Feb 3, 2021
commit 5701cf12e187524007f208f651bf11e9e9ce4940
5 changes: 3 additions & 2 deletions hub/compute/transform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@
from pathos.pools import ProcessPool, ThreadPool
from hub.schema.sequence import Sequence
from hub.schema.features import featurify
import posixpath
import os
from hub.defaults import OBJECT_CHUNK



def get_sample_size(schema, workers):
"""Given Schema, decides how many samples to take at once and returns it"""
schema = featurify(schema)
Expand Down Expand Up @@ -123,7 +124,7 @@ def __getitem__(self, slice_):
num, ofs = slice_extract_info(slice_list[0], self.shape[0])
ds_view = self._ds[slice_list[0]]

path = posixpath.expanduser("~/.activeloop/tmparray")
path = os.path.expanduser("~/.activeloop/tmparray")
new_ds = self.store(path, length=num, ds=ds_view, progressbar=False)

index = 1 if len(slice_) > 1 else 0
Expand Down
4 changes: 2 additions & 2 deletions hub/schema/tests/test_features.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@ def test_hub_feature_flatten():


def test_feature_dict_str():
input_dict = {"myint": int, "mystr": str}
input_dict = {"myint": "int64", "mystr": str}
feature_dict_object = SchemaDict(input_dict)
expected_output = "SchemaDict({'myint': 'int64', 'mystr': '<U0'})"
assert expected_output == feature_dict_object.__str__()


def test_feature_dict_repr():
input_dict = {"myint": int, "mystr": str}
input_dict = {"myint": "int64", "mystr": str}
feature_dict_object = SchemaDict(input_dict)
expected_output = "SchemaDict({'myint': 'int64', 'mystr': '<U0'})"
assert expected_output == feature_dict_object.__repr__()
Expand Down
4 changes: 2 additions & 2 deletions hub/schema/tests/test_tensor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,12 @@ def test_tensor_flattening():


def test_primitive_str():
primitve_object = Primitive(int)
primitve_object = Primitive("int64")
assert "'int64'" == primitve_object.__str__()


def test_primitive_repr():
primitve_object = Primitive(int)
primitve_object = Primitive("int64")
assert "'int64'" == primitve_object.__repr__()


Expand Down
2 changes: 2 additions & 0 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,5 @@ azure-storage-blob==12.6.0
pathos>=0.2.2
psutil>=5.7.3
Pillow>=8.0.1
cloudpickle==1.6.0
msgpack==1.0.2
1 change: 1 addition & 0 deletions test/filtering/meta.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"shape": [100], "schema": {"type": "SchemaDict", "items": {"fname": {"dtype": "int64", "shape": [null], "max_shape": [10], "chunks": null, "compressor": "lz4", "type": "Text"}, "lname": {"dtype": "int64", "shape": [null], "max_shape": [10], "chunks": null, "compressor": "lz4", "type": "Text"}}}, "version": 1, "meta_info": {"description": "This is my description"}, "name": null, ".zarray": {"/fname": {"chunks": [209716, 10], "compressor": {"acceleration": 1, "id": "lz4"}, "dtype": "<i8", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 10], "zarr_format": 2}, "/lname": {"chunks": [209716, 10], "compressor": {"acceleration": 1, "id": "lz4"}, "dtype": "<i8", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 10], "zarr_format": 2}}, "--dynamic--/.zarray": {"/fname": {"chunks": [100, 1], "compressor": null, "dtype": "<i4", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 1], "zarr_format": 2}, "/lname": {"chunks": [100, 1], "compressor": null, "dtype": "<i4", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 1], "zarr_format": 2}}, ".hub.dynamic_tensor": {"/fname": {"shape": [100, null]}, "/lname": {"shape": [100, null]}}}
1 change: 1 addition & 0 deletions test/filtering2/meta.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"shape": [100], "schema": {"type": "SchemaDict", "items": {"fname": {"dtype": "int64", "shape": [null], "max_shape": [10], "chunks": null, "compressor": "lz4", "type": "Text"}, "lname": {"dtype": "int64", "shape": [null], "max_shape": [10], "chunks": null, "compressor": "lz4", "type": "Text"}, "image": {"dtype": "uint8", "shape": [1920, 1080, 3], "max_shape": [1920, 1080, 3], "chunks": null, "compressor": "lz4", "type": "Image"}}}, "version": 1, "meta_info": {"description": "This is my description"}, "name": null, ".zarray": {"/fname": {"chunks": [209716, 10], "compressor": {"acceleration": 1, "id": "lz4"}, "dtype": "<i8", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 10], "zarr_format": 2}, "/lname": {"chunks": [209716, 10], "compressor": {"acceleration": 1, "id": "lz4"}, "dtype": "<i8", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 10], "zarr_format": 2}, "/image": {"chunks": [3, 1920, 1080, 3], "compressor": {"acceleration": 1, "id": "lz4"}, "dtype": "|u1", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 1920, 1080, 3], "zarr_format": 2}}, "--dynamic--/.zarray": {"/fname": {"chunks": [100, 1], "compressor": null, "dtype": "<i4", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 1], "zarr_format": 2}, "/lname": {"chunks": [100, 1], "compressor": null, "dtype": "<i4", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 1], "zarr_format": 2}}, ".hub.dynamic_tensor": {"/fname": {"shape": [100, null]}, "/lname": {"shape": [100, null]}, "/image": {"shape": [100, 1920, 1080, 3]}}}
1 change: 1 addition & 0 deletions test/filtering_3/meta.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"shape": [100], "schema": {"type": "SchemaDict", "items": {"img": {"dtype": "uint8", "shape": [null, null, 3], "max_shape": [100, 100, 3], "chunks": null, "compressor": "lz4", "type": "Image"}, "cl": {"shape": [], "dtype": "int64", "max_shape": [], "chunks": null, "compressor": "lz4", "_num_classes": 3, "_str2int": {"cat": 0, "dog": 1, "horse": 2}, "_int2str": ["cat", "dog", "horse"], "_names": ["cat", "dog", "horse"], "type": "ClassLabel"}}}, "version": 1, "meta_info": {"description": "This is my description"}, "name": null, ".zarray": {"/img": {"chunks": [560, 100, 100, 3], "compressor": {"acceleration": 1, "id": "lz4"}, "dtype": "|u1", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 100, 100, 3], "zarr_format": 2}, "/cl": {"chunks": [2097152], "compressor": {"acceleration": 1, "id": "lz4"}, "dtype": "<i8", "fill_value": 0, "filters": null, "order": "C", "shape": [100], "zarr_format": 2}}, "--dynamic--/.zarray": {"/img": {"chunks": [100, 2], "compressor": null, "dtype": "<i4", "fill_value": 0, "filters": null, "order": "C", "shape": [100, 2], "zarr_format": 2}}, ".hub.dynamic_tensor": {"/img": {"shape": [100, null, null, 3]}, "/cl": {"shape": [100]}}}