From aa3e67a7ca8e3609c216aeed3a169c856f219ed6 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 25 Oct 2020 21:59:38 +0200 Subject: [PATCH 1/8] Add license field and Trove classifiers --- setup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup.py b/setup.py index c58878a2d0..2fdcd6d606 100644 --- a/setup.py +++ b/setup.py @@ -22,11 +22,21 @@ long_description_content_type="text/markdown", author="Snark AI Inc.", author_email="support@activeloop.ai", + license="MPL 2.0", url="https://github.com/snarkai/hub", packages=find_packages(), include_package_data=True, zip_safe=False, keywords="snark-hub", + classifiers=[ + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3 :: Only", + ], python_requires=">=3", install_requires=requirements, setup_requires=[], From da1c508a43a5c6bc8f74295647e9aa659d1f6d7d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 25 Oct 2020 22:00:15 +0200 Subject: [PATCH 2/8] Allow install on Python 3.6+ --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2fdcd6d606..2f7534211d 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3 :: Only", ], - python_requires=">=3", + python_requires=">=3.6", install_requires=requirements, setup_requires=[], dependency_links=[], From b9443fa880f578ae5b95faf04ed1cead7909bd23 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 25 Oct 2020 22:03:55 +0200 Subject: [PATCH 3/8] Update URL --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2f7534211d..0b67ec972a 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ author="Snark AI Inc.", author_email="support@activeloop.ai", license="MPL 2.0", - url="https://github.com/snarkai/hub", + url="https://github.com/activeloopai/Hub", packages=find_packages(), include_package_data=True, zip_safe=False, From c161a9a7ab04929acf9b11ed87e6740f00f6372b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 25 Oct 2020 22:05:55 +0200 Subject: [PATCH 4/8] Add project URLs for PyPI --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 0b67ec972a..ccefa16b84 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,10 @@ include_package_data=True, zip_safe=False, keywords="snark-hub", + project_urls={ + "Documentation": "https://docs.activeloop.ai/", + "Source": "https://github.com/activeloopai/Hub", + }, classifiers=[ "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Programming Language :: Python :: 3", From f08c26f1a55af4ed60ed292d5c77d33804e05499 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 25 Oct 2020 21:49:22 +0200 Subject: [PATCH 5/8] Upgrade syntax for Python 3.6+ --- examples/coco/upload_coco2017.py | 1 - examples/fashion-mnist/train_pytorch.py | 6 ++--- .../fashion-mnist/train_tf_gradient_tape.py | 8 +++--- hub/cli/command.py | 2 +- hub/client/auth.py | 8 +++--- hub/client/base.py | 6 ++--- hub/client/hub_control.py | 4 +-- hub/client/token_manager.py | 10 +++---- hub/exceptions.py | 26 +++++++++---------- setup.py | 2 +- test/benchmark/old/baseline.py | 16 ++++++------ test/benchmark/old/imagenet_train.py | 24 ++++++++--------- test/benchmark/old/multiprocess_benchmark.py | 24 ++++++++--------- 13 files changed, 68 insertions(+), 69 deletions(-) diff --git a/examples/coco/upload_coco2017.py b/examples/coco/upload_coco2017.py index c5aff4369c..f1afe41b73 100644 --- a/examples/coco/upload_coco2017.py +++ b/examples/coco/upload_coco2017.py @@ -77,7 +77,6 @@ def get_image_name(args, tag, id): def load_dataset(args, tag): with open( os.path.join(args.dataset_path, f"annotations/instances_{tag}{args.year}.json"), - "r", ) as f: instances = json.load(f) # print(instances.keys()) diff --git a/examples/fashion-mnist/train_pytorch.py b/examples/fashion-mnist/train_pytorch.py index 63d22ae0a6..b993e13f05 100644 --- a/examples/fashion-mnist/train_pytorch.py +++ b/examples/fashion-mnist/train_pytorch.py @@ -7,7 +7,7 @@ class CNN(nn.Module): def __init__(self): - super(CNN, self).__init__() + super().__init__() self.conv1 = nn.Conv2d(1, 10, kernel_size=5) self.conv2 = nn.Conv2d(10, 20, kernel_size=5) self.conv2_drop = nn.Dropout2d() @@ -85,9 +85,9 @@ def main(): optimizer = optim.SGD(model.parameters(), lr=LEARNING_RATE, momentum=MOMENTUM) for epoch in range(EPOCHS): - print("Starting Training Epoch {}".format(epoch)) + print(f"Starting Training Epoch {epoch}") train(model, train_loader, optimizer) - print("Training Epoch {} finished\n".format(epoch)) + print(f"Training Epoch {epoch} finished\n") test(model, test_loader) # sanity check to see outputs of model diff --git a/examples/fashion-mnist/train_tf_gradient_tape.py b/examples/fashion-mnist/train_tf_gradient_tape.py index 803eb7317c..5e4e331925 100644 --- a/examples/fashion-mnist/train_tf_gradient_tape.py +++ b/examples/fashion-mnist/train_tf_gradient_tape.py @@ -32,7 +32,7 @@ def train(model, train_dataset, optimizer, loss_fn, train_acc_metric): train_acc_metric.update_state(batch["labels"], pred) train_acc = train_acc_metric.result() - print("Training acc: %.4f" % (float(train_acc),)) + print("Training acc: {:.4f}".format(float(train_acc))) train_acc_metric.reset_states() @@ -43,7 +43,7 @@ def test(model, test_dataset, test_acc_metric): test_acc_metric.update_state(batch["labels"], pred) test_acc = test_acc_metric.result() - print("Test acc: %.4f" % (float(test_acc),)) + print("Test acc: {:.4f}".format(float(test_acc))) test_acc_metric.reset_states() @@ -74,9 +74,9 @@ def main(): # model.summary() for epoch in range(EPOCHS): - print("\nStarting Training Epoch {}".format(epoch)) + print(f"\nStarting Training Epoch {epoch}") train(model, train_dataset, optimizer, loss_fn, train_acc_metric) - print("Training Epoch {} finished\n".format(epoch)) + print(f"Training Epoch {epoch} finished\n") test(model, test_dataset, test_acc_metric) # sanity check to see outputs of model diff --git a/hub/cli/command.py b/hub/cli/command.py index fc2d7559fc..3ca17bce1c 100644 --- a/hub/cli/command.py +++ b/hub/cli/command.py @@ -9,7 +9,7 @@ @click.option( "-h", "--host", - default="{}".format(config.HUB_REST_ENDPOINT), + default=f"{config.HUB_REST_ENDPOINT}", help="Hub rest endpoint", ) @click.option("-v", "--verbose", count=True, help="Devel debugging") diff --git a/hub/client/auth.py b/hub/client/auth.py index e57d159d12..89ec1d2143 100644 --- a/hub/client/auth.py +++ b/hub/client/auth.py @@ -10,10 +10,10 @@ class AuthClient(HubHttpClient): """ def __init__(self): - super(AuthClient, self).__init__() + super().__init__() def check_token(self, access_token): - auth = "Bearer {}".format(access_token) + auth = f"Bearer {access_token}" response = self.request( "GET", config.CHECK_TOKEN_REST_SUFFIX, headers={"Authorization": auth} ) @@ -22,7 +22,7 @@ def check_token(self, access_token): response_dict = response.json() is_valid = response_dict["is_valid"] except Exception as e: - logger.error("Exception occured while validating token: {}.".format(e)) + logger.error(f"Exception occured while validating token: {e}.") raise HubException( "Error while validating the token. \ Please try logging in using username ans password." @@ -41,7 +41,7 @@ def get_access_token(self, username, password): token_dict = response.json() token = token_dict["token"] except Exception as e: - logger.error("Exception occured while getting token: {}.".format(e)) + logger.error(f"Exception occured while getting token: {e}.") raise HubException( "Error while loggin in. \ Please try logging in using access token." diff --git a/hub/client/base.py b/hub/client/base.py index ff03fd77e0..9907b8291c 100644 --- a/hub/client/base.py +++ b/hub/client/base.py @@ -28,7 +28,7 @@ def urljoin(*args): return "/".join(map(lambda x: str(x).strip("/"), args)) -class HubHttpClient(object): +class HubHttpClient: """ Basic communication with Hub AI Controller rest API """ @@ -60,7 +60,7 @@ def request( headers["Authorization"] = self.auth_header try: - logger.debug("Sending: Headers {}, Json: {}".format(headers, json)) + logger.debug(f"Sending: Headers {headers}, Json: {json}") response = requests.request( method, request_url, @@ -100,7 +100,7 @@ def check_response_status(self, response): message = " " logger.debug( - 'Error received: status code: {}, message: "{}"'.format(code, message) + f'Error received: status code: {code}, message: "{message}"' ) if code == 400: raise BadRequestException(response) diff --git a/hub/client/hub_control.py b/hub/client/hub_control.py index d284370921..4b7bc1ba47 100644 --- a/hub/client/hub_control.py +++ b/hub/client/hub_control.py @@ -15,7 +15,7 @@ class HubControlClient(HubHttpClient): """ def __init__(self): - super(HubControlClient, self).__init__() + super().__init__() self.details = self.get_config() def get_dataset_path(self, tag): @@ -60,7 +60,7 @@ def get_config(self, reset=False): if not os.path.isfile(config.STORE_CONFIG_PATH) or self.auth_header is None: self.get_credentials() - with open(config.STORE_CONFIG_PATH, "r") as file: + with open(config.STORE_CONFIG_PATH) as file: details = file.readlines() details = json.loads("".join(details)) diff --git a/hub/client/token_manager.py b/hub/client/token_manager.py index c02180713f..db532d3fd6 100644 --- a/hub/client/token_manager.py +++ b/hub/client/token_manager.py @@ -4,7 +4,7 @@ from hub.log import logger -class TokenManager(object): +class TokenManager: """ manages access tokens """ @classmethod @@ -14,7 +14,7 @@ def is_authenticated(cls): @classmethod def set_token(cls, token): logger.debug( - "Putting the key {} into {}.".format(token, config.TOKEN_FILE_PATH) + f"Putting the key {token} into {config.TOKEN_FILE_PATH}." ) path = Path(config.TOKEN_FILE_PATH) os.makedirs(path.parent, exist_ok=True) @@ -26,9 +26,9 @@ def get_token(cls): logger.debug("Getting token...") if not os.path.exists(config.TOKEN_FILE_PATH): return None - with open(config.TOKEN_FILE_PATH, "r") as f: + with open(config.TOKEN_FILE_PATH) as f: token = f.read() - logger.debug("Got the key {} from {}.".format(token, config.TOKEN_FILE_PATH)) + logger.debug(f"Got the key {token} from {config.TOKEN_FILE_PATH}.") return token @classmethod @@ -36,7 +36,7 @@ def get_auth_header(cls): logger.debug("Constructing auth header...") token = cls.get_token() if token: - return "Bearer {}".format(token) + return f"Bearer {token}" return None @classmethod diff --git a/hub/exceptions.py b/hub/exceptions.py index 81ec375496..60963ee314 100644 --- a/hub/exceptions.py +++ b/hub/exceptions.py @@ -69,12 +69,12 @@ class S3CredsParseException(Exception): class HubException(ClickException): def __init__(self, message=None, code=None): - super(HubException, self).__init__(message) + super().__init__(message) class AuthenticationException(HubException): def __init__(self, message="Authentication failed. Please login again."): - super(AuthenticationException, self).__init__(message=message) + super().__init__(message=message) class AuthorizationException(HubException): @@ -83,7 +83,7 @@ def __init__(self, response): message = response.json()["message"] except (KeyError, AttributeError): message = "You are not authorized to access this resource on Snark AI." - super(AuthorizationException, self).__init__(message=message) + super().__init__(message=message) class NotFoundException(HubException): @@ -91,7 +91,7 @@ def __init__( self, message="The resource you are looking for was not found. Check if the name or id is correct.", ): - super(NotFoundException, self).__init__(message=message) + super().__init__(message=message) class BadRequestException(HubException): @@ -104,7 +104,7 @@ def __init__(self, response): message = "One or more request parameters is incorrect, %s" % str( response.content ) - super(BadRequestException, self).__init__(message=message) + super().__init__(message=message) class OverLimitException(HubException): @@ -112,41 +112,41 @@ def __init__( self, message="You are over the allowed limits for this operation. Consider upgrading your account.", ): - super(OverLimitException, self).__init__(message=message) + super().__init__(message=message) class ServerException(HubException): def __init__(self, message="Internal Snark AI server error."): - super(ServerException, self).__init__(message=message) + super().__init__(message=message) class BadGatewayException(HubException): def __init__(self, message="Invalid response from Snark AI server."): - super(BadGatewayException, self).__init__(message=message) + super().__init__(message=message) class GatewayTimeoutException(HubException): def __init__(self, message="Snark AI server took too long to respond."): - super(GatewayTimeoutException, self).__init__(message=message) + super().__init__(message=message) class WaitTimeoutException(HubException): def __init__(self, message="Timeout waiting for server state update."): - super(WaitTimeoutException, self).__init__(message=message) + super().__init__(message=message) class LockedException(HubException): def __init__(self, message="Resource locked."): - super(LockedException, self).__init__(message=message) + super().__init__(message=message) class DatasetNotFound(HubException): def __init__(self, response): message = f"The dataset with tag {response} was not found" - super(DatasetNotFound, self).__init__(message=message) + super().__init__(message=message) class PermissionException(HubException): def __init__(self, response): message = f"No permision to store the dataset at {response}" - super(PermissionException, self).__init__(message=message) + super().__init__(message=message) diff --git a/setup.py b/setup.py index ccefa16b84..11cbf17cf7 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ with open(os.path.join(this_directory, "README.md")) as f: long_description = f.read() -with open(os.path.join(this_directory, "requirements.txt"), "r") as f: +with open(os.path.join(this_directory, "requirements.txt")) as f: requirements = f.readlines() setup( diff --git a/test/benchmark/old/baseline.py b/test/benchmark/old/baseline.py index 77f9f87ba8..c10628e708 100644 --- a/test/benchmark/old/baseline.py +++ b/test/benchmark/old/baseline.py @@ -117,7 +117,7 @@ def main_worker(gpu, ngpus_per_node, args): args.gpu = gpu if args.gpu is not None: - print("Use GPU: {} for training".format(args.gpu)) + print(f"Use GPU: {args.gpu} for training") if args.distributed: if args.dist_url == "env://" and args.rank == -1: @@ -130,10 +130,10 @@ def main_worker(gpu, ngpus_per_node, args): world_size=args.world_size, rank=args.rank) # create model if args.pretrained: - print("=> using pre-trained model '{}'".format(args.arch)) + print(f"=> using pre-trained model '{args.arch}'") model = models.__dict__[args.arch](pretrained=True) else: - print("=> creating model '{}'".format(args.arch)) + print(f"=> creating model '{args.arch}'") model = models.__dict__[args.arch]() if args.distributed: @@ -175,7 +175,7 @@ def main_worker(gpu, ngpus_per_node, args): # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): - print("=> loading checkpoint '{}'".format(args.resume)) + print(f"=> loading checkpoint '{args.resume}'") checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_acc1 = checkpoint['best_acc1'] @@ -187,7 +187,7 @@ def main_worker(gpu, ngpus_per_node, args): print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: - print("=> no checkpoint found at '{}'".format(args.resume)) + print(f"=> no checkpoint found at '{args.resume}'") cudnn.benchmark = True @@ -264,7 +264,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args): progress = ProgressMeter( len(train_loader), [batch_time, data_time, losses, top1, top5], - prefix="Epoch: [{}]".format(epoch)) + prefix=f"Epoch: [{epoch}]") # switch to train mode model.train() @@ -351,7 +351,7 @@ def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): shutil.copyfile(filename, 'model_best.pth.tar') -class AverageMeter(object): +class AverageMeter: """Computes and stores the average and current value""" def __init__(self, name, fmt=':f'): self.name = name @@ -375,7 +375,7 @@ def __str__(self): return fmtstr.format(**self.__dict__) -class ProgressMeter(object): +class ProgressMeter: def __init__(self, num_batches, meters, prefix=""): self.batch_fmtstr = self._get_batch_fmtstr(num_batches) self.meters = meters diff --git a/test/benchmark/old/imagenet_train.py b/test/benchmark/old/imagenet_train.py index 3c6f8444df..074d427a9d 100644 --- a/test/benchmark/old/imagenet_train.py +++ b/test/benchmark/old/imagenet_train.py @@ -31,7 +31,7 @@ h_num_threads = 1 chunk_size = 20 -class StandardTransform(object): +class StandardTransform: def __init__(self, transform=None, target_transform=None): self.transform = transform self.target_transform = target_transform @@ -90,9 +90,9 @@ def __len__(self): def __repr__(self): head = "Dataset " + self.__class__.__name__ - body = ["Number of datapoints: {}".format(self.__len__())] + body = [f"Number of datapoints: {self.__len__()}"] if self.root is not None: - body.append("Root location: {}".format(self.root)) + body.append(f"Root location: {self.root}") body += self.extra_repr().splitlines() if hasattr(self, "transforms") and self.transforms is not None: body += [repr(self.transforms)] @@ -123,7 +123,7 @@ class FakeData(VisionDataset): def __init__(self, size=1000, image_size=(3, 224, 224), num_classes=10, transform=None, target_transform=None, random_offset=0): - super(FakeData, self).__init__(None, transform=transform, + super().__init__(None, transform=transform, target_transform=target_transform) self.size = size self.num_classes = num_classes @@ -293,7 +293,7 @@ def main_worker(gpu, ngpus_per_node, args): args.gpu = gpu if args.gpu is not None: - print("Use GPU: {} for training".format(args.gpu)) + print(f"Use GPU: {args.gpu} for training") if args.distributed: if args.dist_url == "env://" and args.rank == -1: @@ -306,10 +306,10 @@ def main_worker(gpu, ngpus_per_node, args): world_size=args.world_size, rank=args.rank) # create model if args.pretrained: - print("=> using pre-trained model '{}'".format(args.arch)) + print(f"=> using pre-trained model '{args.arch}'") model = models.__dict__[args.arch](pretrained=True) else: - print("=> creating model '{}'".format(args.arch)) + print(f"=> creating model '{args.arch}'") model = models.__dict__[args.arch]() if args.distributed: @@ -351,7 +351,7 @@ def main_worker(gpu, ngpus_per_node, args): # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): - print("=> loading checkpoint '{}'".format(args.resume)) + print(f"=> loading checkpoint '{args.resume}'") checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_acc1 = checkpoint['best_acc1'] @@ -363,7 +363,7 @@ def main_worker(gpu, ngpus_per_node, args): print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: - print("=> no checkpoint found at '{}'".format(args.resume)) + print(f"=> no checkpoint found at '{args.resume}'") cudnn.benchmark = True @@ -440,7 +440,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args): progress = ProgressMeter( len(train_loader), [batch_time, data_time, losses, top1, top5], - prefix="Epoch: [{}]".format(epoch)) + prefix=f"Epoch: [{epoch}]") # switch to train mode model.train() @@ -527,7 +527,7 @@ def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): shutil.copyfile(filename, 'model_best.pth.tar') -class AverageMeter(object): +class AverageMeter: """Computes and stores the average and current value""" def __init__(self, name, fmt=':f'): self.name = name @@ -551,7 +551,7 @@ def __str__(self): return fmtstr.format(**self.__dict__) -class ProgressMeter(object): +class ProgressMeter: def __init__(self, num_batches, meters, prefix=""): self.batch_fmtstr = self._get_batch_fmtstr(num_batches) self.meters = meters diff --git a/test/benchmark/old/multiprocess_benchmark.py b/test/benchmark/old/multiprocess_benchmark.py index c2ea626395..d83db32f63 100644 --- a/test/benchmark/old/multiprocess_benchmark.py +++ b/test/benchmark/old/multiprocess_benchmark.py @@ -32,7 +32,7 @@ h_num_threads = 8 chunk_size = 80 # 20 or 1 -class StandardTransform(object): +class StandardTransform: def __init__(self, transform=None, target_transform=None): self.transform = transform self.target_transform = target_transform @@ -91,9 +91,9 @@ def __len__(self): def __repr__(self): head = "Dataset " + self.__class__.__name__ - body = ["Number of datapoints: {}".format(self.__len__())] + body = [f"Number of datapoints: {self.__len__()}"] if self.root is not None: - body.append("Root location: {}".format(self.root)) + body.append(f"Root location: {self.root}") body += self.extra_repr().splitlines() if hasattr(self, "transforms") and self.transforms is not None: body += [repr(self.transforms)] @@ -124,7 +124,7 @@ class FakeData(VisionDataset): def __init__(self, size=1000, image_size=(3, 224, 224), num_classes=10, transform=None, target_transform=None, random_offset=0): - super(FakeData, self).__init__(None, transform=transform, + super().__init__(None, transform=transform, target_transform=target_transform) self.size = size self.num_classes = num_classes @@ -294,7 +294,7 @@ def main_worker(gpu, ngpus_per_node, args): args.gpu = gpu if args.gpu is not None: - print("Use GPU: {} for training".format(args.gpu)) + print(f"Use GPU: {args.gpu} for training") if args.distributed: if args.dist_url == "env://" and args.rank == -1: @@ -307,10 +307,10 @@ def main_worker(gpu, ngpus_per_node, args): world_size=args.world_size, rank=args.rank) # create model if args.pretrained: - print("=> using pre-trained model '{}'".format(args.arch)) + print(f"=> using pre-trained model '{args.arch}'") model = models.__dict__[args.arch](pretrained=True) else: - print("=> creating model '{}'".format(args.arch)) + print(f"=> creating model '{args.arch}'") model = models.__dict__[args.arch]() if args.distributed: @@ -352,7 +352,7 @@ def main_worker(gpu, ngpus_per_node, args): # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): - print("=> loading checkpoint '{}'".format(args.resume)) + print(f"=> loading checkpoint '{args.resume}'") checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_acc1 = checkpoint['best_acc1'] @@ -364,7 +364,7 @@ def main_worker(gpu, ngpus_per_node, args): print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: - print("=> no checkpoint found at '{}'".format(args.resume)) + print(f"=> no checkpoint found at '{args.resume}'") cudnn.benchmark = True @@ -441,7 +441,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args): progress = ProgressMeter( len(train_loader), [batch_time, data_time, losses, top1, top5], - prefix="Epoch: [{}]".format(epoch)) + prefix=f"Epoch: [{epoch}]") # switch to train mode model.train() @@ -529,7 +529,7 @@ def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): shutil.copyfile(filename, 'model_best.pth.tar') -class AverageMeter(object): +class AverageMeter: """Computes and stores the average and current value""" def __init__(self, name, fmt=':f'): self.name = name @@ -553,7 +553,7 @@ def __str__(self): return fmtstr.format(**self.__dict__) -class ProgressMeter(object): +class ProgressMeter: def __init__(self, num_batches, meters, prefix=""): self.batch_fmtstr = self._get_batch_fmtstr(num_batches) self.meters = meters From 63575f0499ce059b08e270dc8f8d5ba30d31891c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 25 Oct 2020 23:01:45 +0200 Subject: [PATCH 6/8] Add JSON code formatting And fix a typo --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9697e178e8..e72effd881 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,14 +15,14 @@ To contribute a feature/fix: ## How you can help * Adding new datasets following [this example](https://docs.activeloop.ai/en/latest/concepts/dataset.html#how-to-upload-a-dataset) -* Fix an issue from Github Issues +* Fix an issue from GitHub Issues * Add a feature. For an extended feature please create an issue to discuss. ## Formatting and Linting Hub uses Black and Flake8 to ensure a consistent code format throughout the project. if you are using vscode then Replace `.vscode/settings.json` content with the following: -``` +```json { "[py]": { "editor.formatOnSave": true From 61562f03efe2155cb1c6ce7b8d9d21c863a3da2e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 25 Oct 2020 22:15:44 +0200 Subject: [PATCH 7/8] Lint with Black on GitHub Actions --- .github/workflows/lint.yml | 12 ++++++++++++ .pre-commit-config.yaml | 6 ++++++ 2 files changed, 18 insertions(+) create mode 100644 .github/workflows/lint.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..bda0c64e26 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,12 @@ +name: Lint + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..c6b925bd89 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: + - repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + args: ["--target-version", "py36"] From 501a9276a0982790932e65a3d0c5827977ea47eb Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 26 Oct 2020 00:22:15 +0200 Subject: [PATCH 8/8] Format with Black --- benchmarks/areal_caching_benchmark.py | 2 +- examples/fashion-mnist/train_pytorch.py | 26 +- examples/fashion-mnist/train_tf_fit.py | 30 +- .../fashion-mnist/train_tf_gradient_tape.py | 29 +- examples/fashion-mnist/upload.py | 27 +- examples/mnist/upload.py | 6 +- hub/areal/tests/test_storage_tensor.py | 2 +- hub/cli/auth.py | 2 +- hub/cli/utils.py | 6 +- hub/client/base.py | 4 +- hub/client/hub_control.py | 4 +- hub/client/token_manager.py | 4 +- hub/codec/image.py | 1 - hub/collections/_store_version.py | 1 - hub/collections/client_manager.py | 30 +- hub/collections/dataset/__init__.py | 10 +- hub/collections/dataset/core.py | 189 +++---- hub/collections/tensor/__init__.py | 8 +- hub/collections/tensor/core.py | 6 +- hub/log.py | 6 +- hub/utils.py | 2 +- test/benchmark/basic_train.py | 8 +- test/benchmark/old/baseline.py | 358 +++++++++----- test/benchmark/old/chunk_benchmark.py | 37 +- test/benchmark/old/imagenet_train.py | 460 +++++++++++------- test/benchmark/old/multiprocess_benchmark.py | 460 +++++++++++------- test/benchmark/transfer.py | 1 - 27 files changed, 1060 insertions(+), 659 deletions(-) diff --git a/benchmarks/areal_caching_benchmark.py b/benchmarks/areal_caching_benchmark.py index b5bb8b2d57..02558a2720 100644 --- a/benchmarks/areal_caching_benchmark.py +++ b/benchmarks/areal_caching_benchmark.py @@ -24,4 +24,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/examples/fashion-mnist/train_pytorch.py b/examples/fashion-mnist/train_pytorch.py index b993e13f05..81475db925 100644 --- a/examples/fashion-mnist/train_pytorch.py +++ b/examples/fashion-mnist/train_pytorch.py @@ -49,13 +49,19 @@ def test(model, test_loader): labels = batch["labels"] labels = labels.type(torch.LongTensor) output = model(data) - test_loss += F.nll_loss(output, labels, reduction='sum').item() + test_loss += F.nll_loss(output, labels, reduction="sum").item() pred = output.data.max(1, keepdim=True)[1] correct += pred.eq(labels.data.view_as(pred)).sum() test_loss /= len(test_loader.dataset) - print('Test set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( - test_loss, correct, len(test_loader.dataset), 100. * correct / len(test_loader.dataset))) + print( + "Test set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( + test_loss, + correct, + len(test_loader.dataset), + 100.0 * correct / len(test_loader.dataset), + ) + ) def main(): @@ -78,8 +84,12 @@ def main(): train_dataset = torch.utils.data.Subset(ds, range(60000)) test_dataset = torch.utils.data.Subset(ds, range(60000, 70000)) - train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn) - test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn) + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn + ) + test_loader = torch.utils.data.DataLoader( + test_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn + ) model = CNN() optimizer = optim.SGD(model.parameters(), lr=LEARNING_RATE, momentum=MOMENTUM) @@ -92,15 +102,15 @@ def main(): # sanity check to see outputs of model for batch in test_loader: - print("\nNamed Labels:",dataset.get_text(batch["named_labels"])) - print("\nLabels:",batch["labels"]) + print("\nNamed Labels:", dataset.get_text(batch["named_labels"])) + print("\nLabels:", batch["labels"]) data = batch["data"] data = torch.unsqueeze(data, 1) output = model(data) pred = output.data.max(1)[1] - print("\nPredictions:",pred) + print("\nPredictions:", pred) break diff --git a/examples/fashion-mnist/train_tf_fit.py b/examples/fashion-mnist/train_tf_fit.py index 1e0440d3b0..36726c1484 100644 --- a/examples/fashion-mnist/train_tf_fit.py +++ b/examples/fashion-mnist/train_tf_fit.py @@ -4,16 +4,28 @@ def create_CNN(): model = tf.keras.Sequential() - model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(28, 28, 1))) + model.add( + tf.keras.layers.Conv2D( + filters=64, + kernel_size=2, + padding="same", + activation="relu", + input_shape=(28, 28, 1), + ) + ) model.add(tf.keras.layers.MaxPooling2D(pool_size=2)) model.add(tf.keras.layers.Dropout(0.3)) - model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', activation='relu')) + model.add( + tf.keras.layers.Conv2D( + filters=32, kernel_size=2, padding="same", activation="relu" + ) + ) model.add(tf.keras.layers.MaxPooling2D(pool_size=2)) model.add(tf.keras.layers.Dropout(0.3)) model.add(tf.keras.layers.Flatten()) - model.add(tf.keras.layers.Dense(256, activation='relu')) + model.add(tf.keras.layers.Dense(256, activation="relu")) model.add(tf.keras.layers.Dropout(0.5)) - model.add(tf.keras.layers.Dense(10, activation='softmax')) + model.add(tf.keras.layers.Dense(10, activation="softmax")) return model @@ -33,7 +45,7 @@ def main(): # transform into Tensorflow dataset # max_text_len is an optional argument that fixes the maximum length of text labels - ds = ds.to_tensorflow(max_text_len = 15) + ds = ds.to_tensorflow(max_text_len=15) # converting ds so that it can be directly used in model.fit ds = ds.map(lambda x: to_model_fit(x)) @@ -47,8 +59,12 @@ def main(): model = create_CNN() # model.summary() - model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy']) - model.fit(train_dataset, epochs=EPOCHS, validation_data=test_dataset, validation_steps=1) + model.compile( + loss="sparse_categorical_crossentropy", optimizer="adam", metrics=["accuracy"] + ) + model.fit( + train_dataset, epochs=EPOCHS, validation_data=test_dataset, validation_steps=1 + ) if __name__ == "__main__": diff --git a/examples/fashion-mnist/train_tf_gradient_tape.py b/examples/fashion-mnist/train_tf_gradient_tape.py index 5e4e331925..76fef654ef 100644 --- a/examples/fashion-mnist/train_tf_gradient_tape.py +++ b/examples/fashion-mnist/train_tf_gradient_tape.py @@ -7,16 +7,28 @@ def create_CNN(): model = tf.keras.Sequential() - model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(28, 28, 1))) + model.add( + tf.keras.layers.Conv2D( + filters=64, + kernel_size=2, + padding="same", + activation="relu", + input_shape=(28, 28, 1), + ) + ) model.add(tf.keras.layers.MaxPooling2D(pool_size=2)) model.add(tf.keras.layers.Dropout(0.3)) - model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', activation='relu')) + model.add( + tf.keras.layers.Conv2D( + filters=32, kernel_size=2, padding="same", activation="relu" + ) + ) model.add(tf.keras.layers.MaxPooling2D(pool_size=2)) model.add(tf.keras.layers.Dropout(0.3)) model.add(tf.keras.layers.Flatten()) - model.add(tf.keras.layers.Dense(256, activation='relu')) + model.add(tf.keras.layers.Dense(256, activation="relu")) model.add(tf.keras.layers.Dropout(0.5)) - model.add(tf.keras.layers.Dense(10, activation='softmax')) + model.add(tf.keras.layers.Dense(10, activation="softmax")) return model @@ -61,7 +73,7 @@ def main(): # transform into Tensorflow dataset # max_text_len is an optional argument that sets the maximum length of text labels, default is 30 - ds = ds.to_tensorflow(max_text_len = 15) + ds = ds.to_tensorflow(max_text_len=15) # Splitting back into the original train and test sets train_dataset = ds.take(60000) @@ -81,14 +93,15 @@ def main(): # sanity check to see outputs of model for batch in test_dataset: - print("\nNamed Labels:",dataset.get_text(batch["named_labels"])) - print("\nLabels:",batch["labels"]) + print("\nNamed Labels:", dataset.get_text(batch["named_labels"])) + print("\nLabels:", batch["labels"]) output = model(tf.expand_dims(batch["data"], axis=3), training=False) print(type(output)) pred = np.argmax(output, axis=-1) - print("\nPredictions:",pred) + print("\nPredictions:", pred) break + if __name__ == "__main__": main() diff --git a/examples/fashion-mnist/upload.py b/examples/fashion-mnist/upload.py index d79c3debcf..cb92506ee1 100644 --- a/examples/fashion-mnist/upload.py +++ b/examples/fashion-mnist/upload.py @@ -46,17 +46,18 @@ def main(): dicts = [] # required to generate named labels - mapping = {0: "T-shirt/top", - 1: "Trouser", - 2: "Pullover", - 3: "Dress", - 4: "Coat", - 5: "Sandal", - 6: "Shirt", - 7: "Sneaker", - 8: "Bag", - 9: "Ankle boot" - } + mapping = { + 0: "T-shirt/top", + 1: "Trouser", + 2: "Pullover", + 3: "Dress", + 4: "Coat", + 5: "Sandal", + 6: "Shirt", + 7: "Sneaker", + 8: "Bag", + 9: "Ankle boot", + } for f in files: images, labels = load_fashion_mnist(f, path="./data/fashion-mnist") @@ -71,7 +72,9 @@ def main(): labels_t = tensor.from_array(labels, dtag="text") named_labels_t = tensor.from_array(named_labels, dtag="text") - ds = dataset.from_tensors({"data": images_t, "labels": labels_t, "named_labels": named_labels_t}) + ds = dataset.from_tensors( + {"data": images_t, "labels": labels_t, "named_labels": named_labels_t} + ) ds.store("mnist/fashion-mnist") diff --git a/examples/mnist/upload.py b/examples/mnist/upload.py index 58c5a0529a..67baaea473 100644 --- a/examples/mnist/upload.py +++ b/examples/mnist/upload.py @@ -52,7 +52,11 @@ def main(): default="./data/mnist", ) parser.add_argument( - "-o", "--output_name", type=str, help="Dataset output name", default="mnist", + "-o", + "--output_name", + type=str, + help="Dataset output name", + default="mnist", ) args = parser.parse_args() files = ["training", "testing"] diff --git a/hub/areal/tests/test_storage_tensor.py b/hub/areal/tests/test_storage_tensor.py index 487c1d7876..0e7accdfaa 100644 --- a/hub/areal/tests/test_storage_tensor.py +++ b/hub/areal/tests/test_storage_tensor.py @@ -96,4 +96,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/hub/cli/auth.py b/hub/cli/auth.py index 41a1480ead..cfe1918754 100644 --- a/hub/cli/auth.py +++ b/hub/cli/auth.py @@ -71,4 +71,4 @@ def register(username, email, password): AuthClient().register(username, email, password) token = AuthClient().get_access_token(username, password) - TokenManager.set_token(token) \ No newline at end of file + TokenManager.set_token(token) diff --git a/hub/cli/utils.py b/hub/cli/utils.py index fb7ce58900..460dd58387 100644 --- a/hub/cli/utils.py +++ b/hub/cli/utils.py @@ -42,8 +42,10 @@ def get_proxy_command(proxy): ssh_proxy = "" if proxy and proxy != " " and proxy != "None" and proxy != "": if check_program_exists("ncat"): - ssh_proxy = '-o "ProxyCommand=ncat --proxy-type socks5 --proxy {} %h %p"'.format( - proxy + ssh_proxy = ( + '-o "ProxyCommand=ncat --proxy-type socks5 --proxy {} %h %p"'.format( + proxy + ) ) else: raise HubException( diff --git a/hub/client/base.py b/hub/client/base.py index 9907b8291c..89594f0b3f 100644 --- a/hub/client/base.py +++ b/hub/client/base.py @@ -99,9 +99,7 @@ def check_response_status(self, response): except Exception: message = " " - logger.debug( - f'Error received: status code: {code}, message: "{message}"' - ) + logger.debug(f'Error received: status code: {code}, message: "{message}"') if code == 400: raise BadRequestException(response) elif response.status_code == 401: diff --git a/hub/client/hub_control.py b/hub/client/hub_control.py index 4b7bc1ba47..f1e87d27a8 100644 --- a/hub/client/hub_control.py +++ b/hub/client/hub_control.py @@ -38,7 +38,9 @@ def get_credentials(self): self.auth_header = f"Bearer {token}" r = self.request( - "GET", config.GET_CREDENTIALS_SUFFIX, endpoint=config.HUB_REST_ENDPOINT, + "GET", + config.GET_CREDENTIALS_SUFFIX, + endpoint=config.HUB_REST_ENDPOINT, ).json() details = { diff --git a/hub/client/token_manager.py b/hub/client/token_manager.py index db532d3fd6..345d8852b3 100644 --- a/hub/client/token_manager.py +++ b/hub/client/token_manager.py @@ -13,9 +13,7 @@ def is_authenticated(cls): @classmethod def set_token(cls, token): - logger.debug( - f"Putting the key {token} into {config.TOKEN_FILE_PATH}." - ) + logger.debug(f"Putting the key {token} into {config.TOKEN_FILE_PATH}.") path = Path(config.TOKEN_FILE_PATH) os.makedirs(path.parent, exist_ok=True) with open(config.TOKEN_FILE_PATH, "w") as f: diff --git a/hub/codec/image.py b/hub/codec/image.py index f7535ab102..2086331de7 100644 --- a/hub/codec/image.py +++ b/hub/codec/image.py @@ -53,4 +53,3 @@ def decode(self, content: bytes) -> np.ndarray: arr = np.asarray(img) array[index] = arr return array - diff --git a/hub/collections/_store_version.py b/hub/collections/_store_version.py index 3b1f19ac3b..a96c8a5e67 100644 --- a/hub/collections/_store_version.py +++ b/hub/collections/_store_version.py @@ -1,2 +1 @@ CURRENT_STORE_VERSION = 1 - diff --git a/hub/collections/client_manager.py b/hub/collections/client_manager.py index 54961c8a32..b5a9d5a440 100644 --- a/hub/collections/client_manager.py +++ b/hub/collections/client_manager.py @@ -36,18 +36,18 @@ def init( ): """Initializes cluster either local or on the cloud - Parameters - ---------- - token: str - token provided by snark - cache: float - Amount on local memory to cache locally, default 2e9 (2GB) - cloud: bool - Should be run locally or on the cloud - n_workers: int - number of concurrent workers, default to1 - threads_per_worker: int - Number of threads per each worker + Parameters + ---------- + token: str + token provided by snark + cache: float + Amount on local memory to cache locally, default 2e9 (2GB) + cloud: bool + Should be run locally or on the cloud + n_workers: int + number of concurrent workers, default to1 + threads_per_worker: int + Number of threads per each worker """ print("initialized") global _client @@ -69,9 +69,9 @@ def init( ) local_directory = os.path.join( - os.path.expanduser('~'), - '.activeloop', - 'tmp', + os.path.expanduser("~"), + ".activeloop", + "tmp", ) if not os.path.exists(local_directory): os.makedirs(local_directory) diff --git a/hub/collections/dataset/__init__.py b/hub/collections/dataset/__init__.py index 4a947f4acc..e4e7af7770 100644 --- a/hub/collections/dataset/__init__.py +++ b/hub/collections/dataset/__init__.py @@ -34,7 +34,7 @@ def _meta_preprocess(meta: dict): def generate(generator: DatasetGenerator, input) -> Dataset: - """ Generates dataset based on DatabaseGenerator class instance and iterable input + """Generates dataset based on DatabaseGenerator class instance and iterable input For every element in input runs generators __call__ function. That function should return dict of numpy arrays containing single or multiple outputs for axis 0 of generating dataset """ @@ -80,8 +80,7 @@ def from_tensors( citation: str = None, howtoload: str = None, ) -> Dataset: - """ Creates a dataset from dict of tensors - """ + """Creates a dataset from dict of tensors""" return Dataset( tensors, metainfo={ @@ -109,7 +108,7 @@ def _meta_concat(metas: Tuple[Dict[str, object]]): def concat(datasets: Iterable[Dataset]) -> Dataset: - """ Concats multiple datasets into one along axis 0 + """Concats multiple datasets into one along axis 0 This is equivalent to concat every tensor with the same key """ keys = [sorted(dataset._tensors.keys()) for dataset in datasets] @@ -138,8 +137,7 @@ def concat(datasets: Iterable[Dataset]) -> Dataset: def merge(datasets: Iterable[Dataset]) -> Dataset: - """ Merges multiple datasets that have distinct keys into one big datasets containing all keys - """ + """Merges multiple datasets that have distinct keys into one big datasets containing all keys""" tensors = {} for dataset in datasets: for tname, tvalue in dataset._tensors.items(): diff --git a/hub/collections/dataset/core.py b/hub/collections/dataset/core.py index 8657d5ffef..d3d3f02f1e 100644 --- a/hub/collections/dataset/core.py +++ b/hub/collections/dataset/core.py @@ -25,7 +25,7 @@ def _flatten(l): """ - Helper function to flatten the list + Helper function to flatten the list """ return [item for sublist in l for item in sublist] @@ -40,21 +40,21 @@ def __call__(self, input): def meta(self): """ Provides the metadata for all tensors including shapes, dtypes, dtags and chunksize for each array in the form - + Returns ------- - returns + returns dict of tensor - + Examples ------- >>> def meta() >>> return { >>> ... >>> "tesnor_name":{ - >>> "shape": (1,256,256), - >>> "dtype": "uint8", - >>> "chunksize": 100, + >>> "shape": (1,256,256), + >>> "dtype": "uint8", + >>> "chunksize": 100, >>> "dtag": "segmentation" >>> } >>> ... @@ -66,17 +66,17 @@ def meta(self): def forward(input): """ Takes a an element of a list or sample from dataset and returns sample of the dataset - + Parameters ------- input - an element of list or dict of arrays - + an element of list or dict of arrays + Returns ------- dict dict of numpy arrays - + Examples ------- >>> def forward(input): @@ -92,16 +92,14 @@ def forward(input): def _numpy_to_tuple(arr: np.ndarray): - """ Converts numpy array to tuple of numpy arrays - """ + """Converts numpy array to tuple of numpy arrays""" return [np.array([t]) for t in arr] def _numpy_saver( fs: fsspec.AbstractFileSystem, filepath: str, array: np.ndarray, codec: BaseCodec ): - """ Saves a single numpy array into filepath given specific filesystem - """ + """Saves a single numpy array into filepath given specific filesystem""" with fs.open(filepath, "wb") as f: f.write(codec.encode(array)) @@ -121,14 +119,12 @@ def _preprocess_meta_before_save(meta: dict): def _dask_shape(input_shape: Tuple[int]): - """ Dask accept np.nan value in shape if the axis length is not known, our API uses -1 for that, this function converts -1 to np.nan - """ + """Dask accept np.nan value in shape if the axis length is not known, our API uses -1 for that, this function converts -1 to np.nan""" return (np.nan,) + input_shape[1:] if input_shape[0] == -1 else input_shape def _dict_to_tuple(d: dict): - """ Converts dict of lists into (flattened list of values, list of keys) - """ + """Converts dict of lists into (flattened list of values, list of keys)""" keys = sorted(d.keys()) lens = {len(d[key]) for key in keys} assert len(lens) == 1 @@ -137,15 +133,14 @@ def _dict_to_tuple(d: dict): def _tuple_to_dict(t: tuple, keys: tuple): - """ Converts (flattened list of values, list of keys) into dict of lists - """ + """Converts (flattened list of values, list of keys) into dict of lists""" cnt = len(keys) assert len(t) % cnt == 0 return {key: [t[i] for i in range(j, len(t), cnt)] for j, key in enumerate(keys)} def _load_creds(creds): - """ Loads credentials from "{creds}" cfg file if such exists + """Loads credentials from "{creds}" cfg file if such exists if creds is dict, then dict will be returned, assuming all credential data is in dict """ if creds is None: @@ -159,8 +154,7 @@ def _load_creds(creds): def _connect(tag): - """ Connects to the backend and receive credentials - """ + """Connects to the backend and receive credentials""" creds = HubControlClient().get_config() dataset = HubControlClient().get_dataset_path(tag) @@ -179,8 +173,7 @@ def _connect(tag): def _load_fs_and_path(path, creds=None, session_creds=True, google_cloud_project=""): - """ Given url(path) and creds returns filesystem required for accessing that file + url's filepath in that filesystem - """ + """Given url(path) and creds returns filesystem required for accessing that file + url's filepath in that filesystem""" if ( path.startswith("./") or path.startswith("/") @@ -217,7 +210,9 @@ def _load_fs_and_path(path, creds=None, session_creds=True, google_cloud_project elif creds is not None: return ( fsspec.filesystem( - "s3", key=creds.get("access_key"), secret=creds.get("secret_key"), + "s3", + key=creds.get("access_key"), + secret=creds.get("secret_key"), ), path, ) @@ -232,8 +227,7 @@ def _load_fs_and_path(path, creds=None, session_creds=True, google_cloud_project class Dataset: def __init__(self, tensors: Dict[str, Tensor], metainfo=dict()): - """ Creates dict given dict of tensors (name -> Tensor key value pairs) - """ + """Creates dict given dict of tensors (name -> Tensor key value pairs)""" self._tensors = tensors self._metainfo = metainfo shape = None @@ -243,7 +237,7 @@ def __init__(self, tensors: Dict[str, Tensor], metainfo=dict()): self._len = tensor.count def __len__(self) -> int: - """ len of dataset (len of tensors across axis 0, yes, they all should be = to each other) + """len of dataset (len of tensors across axis 0, yes, they all should be = to each other) Raises Exception if length is unknown """ if self._len == -1: @@ -254,58 +248,50 @@ def __len__(self) -> int: @property def license(self) -> str: - """ Dataset license - """ + """Dataset license""" return self._metainfo.get("license") if self._metainfo else None @property def description(self) -> str: - """ Dataset description - """ + """Dataset description""" return self._metainfo.get("description") if self._metainfo else None @property def citation(self) -> str: - """ Dataset citation - """ + """Dataset citation""" return self._metainfo.get("citation") if self._metainfo else None @property def howtoload(self) -> str: - """ Dataset howtoload - """ + """Dataset howtoload""" return self._metainfo.get("howtoload") if self._metainfo else None @property def count(self) -> int: - """ len of dataset (len of tensors across axis 0, yes, they all should be = to each other) + """len of dataset (len of tensors across axis 0, yes, they all should be = to each other) Returns -1 if length is unknown """ return self._len def __iter__(self): - """ Iterates over axis 0 return dict of Tensors - """ + """Iterates over axis 0 return dict of Tensors""" for i in range(len(self)): yield {key: t._array[i] for key, t in self._tensors.items()} def keys(self): - """ Returns names of tensors - """ + """Returns names of tensors""" yield from self._tensors.keys() def values(self): - """ Returns tensors - """ + """Returns tensors""" yield from self._tensors.values() def items(self): - """ Returns tensors - """ + """Returns tensors""" yield from self._tensors.items() def __getitem__(self, slices) -> "Dataset": - """ Returns a slice of dataset + """Returns a slice of dataset slices can be 1) List of strs (slicing horizontally) 2) List of slices or ints (slicing vertically) @@ -491,7 +477,7 @@ def _store_known_sized_ds(self, fs: fsspec.AbstractFileSystem, path: str) -> int @property def meta(self) -> dict: - """ Dict of meta's of each tensor + """Dict of meta's of each tensor meta of tensor contains all metadata for tensor storage """ tensor_meta = { @@ -502,8 +488,7 @@ def meta(self) -> dict: return ds_meta def delete(self, tag, creds=None, session_creds=True) -> bool: - """ Deletes dataset given tag(filepath) and credentials (optional) - """ + """Deletes dataset given tag(filepath) and credentials (optional)""" fs, path = _load_fs_and_path(tag, creds, session_creds=session_creds) fs: fsspec.AbstractFileSystem = fs if fs.exists(path): @@ -512,8 +497,7 @@ def delete(self, tag, creds=None, session_creds=True) -> bool: return False def store(self, tag, creds=None, session_creds=True) -> "Dataset": - """ Stores dataset by tag(filepath) given credentials (can be omitted) - """ + """Stores dataset by tag(filepath) given credentials (can be omitted)""" fs, path = _load_fs_and_path(tag, creds, session_creds=session_creds) fs: fsspec.AbstractFileSystem = fs @@ -559,27 +543,27 @@ def store(self, tag, creds=None, session_creds=True) -> "Dataset": return load(tag, creds) - def to_pytorch(self, transform = None, max_text_len = 30): + def to_pytorch(self, transform=None, max_text_len=30): """ - Transforms into pytorch dataset - - Parameters - ---------- - transform: func - any transform that takes input a dictionary of a sample and returns transformed dictionary - max_text_len: integer - the maximum length of text strings that would be stored. Strings longer than this would be snipped + Transforms into pytorch dataset + + Parameters + ---------- + transform: func + any transform that takes input a dictionary of a sample and returns transformed dictionary + max_text_len: integer + the maximum length of text strings that would be stored. Strings longer than this would be snipped """ return TorchDataset(self, transform, max_text_len) - def to_tensorflow(self, max_text_len = 30): + def to_tensorflow(self, max_text_len=30): """ - Transforms into tensorflow dataset - - Parameters - ---------- - max_text_len: integer - the maximum length of text strings that would be stored. Strings longer than this would be snipped + Transforms into tensorflow dataset + + Parameters + ---------- + max_text_len: integer + the maximum length of text strings that would be stored. Strings longer than this would be snipped """ try: @@ -598,8 +582,21 @@ def tf_gen(step=4): arrs = arrs.compute() for ind, arr in enumerate(arrs): if arr.dtype.type is np.str_: - arr = [([ ord(x) for x in sample.tolist()[0:max_text_len] ] ) for sample in arr] - arr = np.array([np.pad(sample, (0, max_text_len-len(sample)), 'constant', constant_values=(32)) for sample in arr]) + arr = [ + ([ord(x) for x in sample.tolist()[0:max_text_len]]) + for sample in arr + ] + arr = np.array( + [ + np.pad( + sample, + (0, max_text_len - len(sample)), + "constant", + constant_values=(32), + ) + for sample in arr + ] + ) arrs[ind] = arr for i in range(step): @@ -614,11 +611,11 @@ def tf_dtype(np_dtype): except Exception as e: return tf.variant - output_shapes={} - output_types={} + output_shapes = {} + output_types = {} for key in self.keys(): output_types[key] = tf_dtype(self._tensors[key].dtype) - output_shapes[key] = self._tensors[key].shape[1:] + output_shapes[key] = self._tensors[key].shape[1:] # if this is a string, we change the type to int, as it's going to become ascii. shape is also set to None if output_types[key] == tf.dtypes.as_dtype("string"): @@ -637,8 +634,7 @@ def tf_dtype(np_dtype): def _numpy_load( fs: fsspec.AbstractFileSystem, filepath: str, codec: BaseCodec ) -> np.ndarray: - """ Given filesystem and filepath, loads numpy array - """ + """Given filesystem and filepath, loads numpy array""" # assert fs.exists( # filepath # ), f"Dataset file {filepath} does not exists. Your dataset data is likely to be corrupted" @@ -652,30 +648,35 @@ def _numpy_load( f"Dataset file {filepath} does not exists. Your dataset data is likely to be corrupted" ) + def get_text(input): - """ Converts strings stored as ascii value tensors back into strings - """ - if input.ndim==1: - try: + """Converts strings stored as ascii value tensors back into strings""" + if input.ndim == 1: + try: text = "".join([chr(x) for x in input]).rstrip() return text except Exception as e: logger.error(traceback.format_exc() + str(e)) - raise Exception("get_text can only be called on a tensor of text or a batch of tensors of text") - elif input.ndim==2: + raise Exception( + "get_text can only be called on a tensor of text or a batch of tensors of text" + ) + elif input.ndim == 2: try: - text= ["".join([chr(x) for x in sample]).rstrip() for sample in input] + text = ["".join([chr(x) for x in sample]).rstrip() for sample in input] return text except Exception as e: logger.error(traceback.format_exc() + str(e)) - raise Exception("get_text can only be called on a tensor of text or a batch of tensors of text") + raise Exception( + "get_text can only be called on a tensor of text or a batch of tensors of text" + ) else: - raise Exception(f"Got input of dimension {input.ndim} for get_text. Expected dimension of 1 or 2") + raise Exception( + f"Got input of dimension {input.ndim} for get_text. Expected dimension of 1 or 2" + ) def load(tag, creds=None, session_creds=True) -> Dataset: - """ Load a dataset from repository using given url and credentials (optional) - """ + """Load a dataset from repository using given url and credentials (optional)""" fs, path = _load_fs_and_path(tag, creds, session_creds=session_creds) fs: fsspec.AbstractFileSystem = fs path_2 = f"{path}/meta.json" @@ -750,7 +751,7 @@ def _is_tensor_dynamic(tensor): class TorchDataset: - def __init__(self, ds, transform = None, max_text_len = 30): + def __init__(self, ds, transform=None, max_text_len=30): self._ds = ds self._transform = transform self._dynkeys = { @@ -793,8 +794,15 @@ def __iter__(self): def _to_tensor(self, key, sample): if key not in self._dynkeys: if isinstance(sample, np.str_): - sample = np.array([ ord(x) for x in sample.tolist()[0:self._max_text_len] ]) - sample=np.pad(sample, (0, self._max_text_len-len(sample)), 'constant',constant_values=(32)) + sample = np.array( + [ord(x) for x in sample.tolist()[0 : self._max_text_len]] + ) + sample = np.pad( + sample, + (0, self._max_text_len - len(sample)), + "constant", + constant_values=(32), + ) return torch.tensor(sample) else: return [torch.tensor(item) for item in sample] @@ -810,6 +818,7 @@ def collate_fn(self, batch): return ans + def _dask_concat(arr): if len(arr) == 1: return arr[0] diff --git a/hub/collections/tensor/__init__.py b/hub/collections/tensor/__init__.py index 58821d29bc..2d038f92e1 100644 --- a/hub/collections/tensor/__init__.py +++ b/hub/collections/tensor/__init__.py @@ -5,7 +5,7 @@ def from_array(array, dtag=None, dcompress=None, chunksize=None) -> Tensor: - """ Generates tensor from arraylike object + """Generates tensor from arraylike object Parameters ---------- array : np.ndarray @@ -36,21 +36,21 @@ def from_array(array, dtag=None, dcompress=None, chunksize=None) -> Tensor: def concat(tensors, axis=0, chunksize=-1): - """ Concats multiple tensors on axis into one tensor + """Concats multiple tensors on axis into one tensor All input tensors should have same dtag, dtype, dcompress """ raise NotImplementedError() def stack(tensors, axis=0, chunksize=-1): - """ Stack multiple tesnors into new axis + """Stack multiple tesnors into new axis All input tensors should have same dtag, dtype, dcompress """ raise NotImplementedError() def from_zeros(shape, dtype, dtag=None, dcompress=None, chunksize=-1) -> Tensor: - """ Generates tensor from 0 filled array + """Generates tensor from 0 filled array Parameters ---------- shape : Iterable diff --git a/hub/collections/tensor/core.py b/hub/collections/tensor/core.py index ca0e32b268..7826d2b910 100644 --- a/hub/collections/tensor/core.py +++ b/hub/collections/tensor/core.py @@ -71,7 +71,7 @@ def meta(self): @property def shape(self): - """ + """ Returns ------- tuple @@ -141,7 +141,7 @@ def chunksize(self): return self._chunksize def __getitem__(self, slices) -> "Tensor": - """ Slices tensor + """Slices tensor Parameters ---------- slices @@ -167,7 +167,7 @@ def __iter__(self): yield self._array[i] def compute(self): - """ Does lazy computation and converts data to numpy array + """Does lazy computation and converts data to numpy array Returns ------- np.ndarray diff --git a/hub/log.py b/hub/log.py index add362d972..3e910e5763 100644 --- a/hub/log.py +++ b/hub/log.py @@ -1,14 +1,12 @@ import logging import sys -logger = logging.getLogger('hub') +logger = logging.getLogger("hub") def configure_logger(debug=0): log_level = logging.DEBUG if debug == 1 else logging.INFO - logging.basicConfig(format='%(message)s', - level=log_level, - stream=sys.stdout) + logging.basicConfig(format="%(message)s", level=log_level, stream=sys.stdout) configure_logger(0) diff --git a/hub/utils.py b/hub/utils.py index b55742e7eb..1515682744 100644 --- a/hub/utils.py +++ b/hub/utils.py @@ -1,5 +1,5 @@ def _flatten(l): """ - Helper function to flatten the list + Helper function to flatten the list """ return [item for sublist in l for item in sublist] diff --git a/test/benchmark/basic_train.py b/test/benchmark/basic_train.py index 8292aa670a..0708169025 100644 --- a/test/benchmark/basic_train.py +++ b/test/benchmark/basic_train.py @@ -116,11 +116,15 @@ def empty_train_hub(samples=100, backend="hub:pytorch", read_from_fs=False): """ if "hub" in backend: ds = get_dataset_from_hub( - samples=samples, read_from_fs=read_from_fs, pytorch="pytorch" in backend, + samples=samples, + read_from_fs=read_from_fs, + pytorch="pytorch" in backend, ) else: ds = dataset_loader( - samples=samples, read_from_fs=read_from_fs, pytorch="pytorch" in backend, + samples=samples, + read_from_fs=read_from_fs, + pytorch="pytorch" in backend, ) if "pytorch" in backend: diff --git a/test/benchmark/old/baseline.py b/test/benchmark/old/baseline.py index c10628e708..4e76231f2b 100644 --- a/test/benchmark/old/baseline.py +++ b/test/benchmark/old/baseline.py @@ -18,61 +18,124 @@ import torchvision.datasets as datasets import torchvision.models as models -model_names = sorted(name for name in models.__dict__ - if name.islower() and not name.startswith("__") - and callable(models.__dict__[name])) - -parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') -parser.add_argument('data', metavar='DIR', - help='path to dataset') -parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', - choices=model_names, - help='model architecture: ' + - ' | '.join(model_names) + - ' (default: resnet18)') -parser.add_argument('-j', '--workers', default=8, type=int, metavar='N', - help='number of data loading workers (default: 4)') -parser.add_argument('--epochs', default=90, type=int, metavar='N', - help='number of total epochs to run') -parser.add_argument('--start-epoch', default=0, type=int, metavar='N', - help='manual epoch number (useful on restarts)') -parser.add_argument('-b', '--batch-size', default=64, type=int, - metavar='N', - help='mini-batch size (default: 256), this is the total ' - 'batch size of all GPUs on the current node when ' - 'using Data Parallel or Distributed Data Parallel') -parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, - metavar='LR', help='initial learning rate', dest='lr') -parser.add_argument('--momentum', default=0.9, type=float, metavar='M', - help='momentum') -parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, - metavar='W', help='weight decay (default: 1e-4)', - dest='weight_decay') -parser.add_argument('-p', '--print-freq', default=10, type=int, - metavar='N', help='print frequency (default: 10)') -parser.add_argument('--resume', default='', type=str, metavar='PATH', - help='path to latest checkpoint (default: none)') -parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', - help='evaluate model on validation set') -parser.add_argument('--pretrained', dest='pretrained', action='store_true', - help='use pre-trained model') -parser.add_argument('--world-size', default=-1, type=int, - help='number of nodes for distributed training') -parser.add_argument('--rank', default=-1, type=int, - help='node rank for distributed training') -parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, - help='url used to set up distributed training') -parser.add_argument('--dist-backend', default='nccl', type=str, - help='distributed backend') -parser.add_argument('--seed', default=None, type=int, - help='seed for initializing training. ') -parser.add_argument('--gpu', default=None, type=int, - help='GPU id to use.') -parser.add_argument('--multiprocessing-distributed', action='store_true', - help='Use multi-processing distributed training to launch ' - 'N processes per node, which has N GPUs. This is the ' - 'fastest way to use PyTorch for either single node or ' - 'multi node data parallel training') +model_names = sorted( + name + for name in models.__dict__ + if name.islower() and not name.startswith("__") and callable(models.__dict__[name]) +) + +parser = argparse.ArgumentParser(description="PyTorch ImageNet Training") +parser.add_argument("data", metavar="DIR", help="path to dataset") +parser.add_argument( + "-a", + "--arch", + metavar="ARCH", + default="resnet18", + choices=model_names, + help="model architecture: " + " | ".join(model_names) + " (default: resnet18)", +) +parser.add_argument( + "-j", + "--workers", + default=8, + type=int, + metavar="N", + help="number of data loading workers (default: 4)", +) +parser.add_argument( + "--epochs", default=90, type=int, metavar="N", help="number of total epochs to run" +) +parser.add_argument( + "--start-epoch", + default=0, + type=int, + metavar="N", + help="manual epoch number (useful on restarts)", +) +parser.add_argument( + "-b", + "--batch-size", + default=64, + type=int, + metavar="N", + help="mini-batch size (default: 256), this is the total " + "batch size of all GPUs on the current node when " + "using Data Parallel or Distributed Data Parallel", +) +parser.add_argument( + "--lr", + "--learning-rate", + default=0.1, + type=float, + metavar="LR", + help="initial learning rate", + dest="lr", +) +parser.add_argument("--momentum", default=0.9, type=float, metavar="M", help="momentum") +parser.add_argument( + "--wd", + "--weight-decay", + default=1e-4, + type=float, + metavar="W", + help="weight decay (default: 1e-4)", + dest="weight_decay", +) +parser.add_argument( + "-p", + "--print-freq", + default=10, + type=int, + metavar="N", + help="print frequency (default: 10)", +) +parser.add_argument( + "--resume", + default="", + type=str, + metavar="PATH", + help="path to latest checkpoint (default: none)", +) +parser.add_argument( + "-e", + "--evaluate", + dest="evaluate", + action="store_true", + help="evaluate model on validation set", +) +parser.add_argument( + "--pretrained", dest="pretrained", action="store_true", help="use pre-trained model" +) +parser.add_argument( + "--world-size", + default=-1, + type=int, + help="number of nodes for distributed training", +) +parser.add_argument( + "--rank", default=-1, type=int, help="node rank for distributed training" +) +parser.add_argument( + "--dist-url", + default="tcp://224.66.41.62:23456", + type=str, + help="url used to set up distributed training", +) +parser.add_argument( + "--dist-backend", default="nccl", type=str, help="distributed backend" +) +parser.add_argument( + "--seed", default=None, type=int, help="seed for initializing training. " +) +parser.add_argument("--gpu", default=None, type=int, help="GPU id to use.") +parser.add_argument( + "--multiprocessing-distributed", + action="store_true", + help="Use multi-processing distributed training to launch " + "N processes per node, which has N GPUs. This is the " + "fastest way to use PyTorch for either single node or " + "multi node data parallel training", +) best_acc1 = 0 @@ -84,15 +147,19 @@ def main(): random.seed(args.seed) torch.manual_seed(args.seed) cudnn.deterministic = True - warnings.warn('You have chosen to seed training. ' - 'This will turn on the CUDNN deterministic setting, ' - 'which can slow down your training considerably! ' - 'You may see unexpected behavior when restarting ' - 'from checkpoints.') + warnings.warn( + "You have chosen to seed training. " + "This will turn on the CUDNN deterministic setting, " + "which can slow down your training considerably! " + "You may see unexpected behavior when restarting " + "from checkpoints." + ) if args.gpu is not None: - warnings.warn('You have chosen a specific GPU. This will completely ' - 'disable data parallelism.') + warnings.warn( + "You have chosen a specific GPU. This will completely " + "disable data parallelism." + ) if args.dist_url == "env://" and args.world_size == -1: args.world_size = int(os.environ["WORLD_SIZE"]) @@ -126,8 +193,12 @@ def main_worker(gpu, ngpus_per_node, args): # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu - dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, - world_size=args.world_size, rank=args.rank) + dist.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) # create model if args.pretrained: print(f"=> using pre-trained model '{args.arch}'") @@ -148,7 +219,9 @@ def main_worker(gpu, ngpus_per_node, args): # ourselves based on the total number of GPUs we have args.batch_size = int(args.batch_size / ngpus_per_node) args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) - model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[args.gpu] + ) else: model.cuda() # DistributedDataParallel will divide and allocate batch_size to all @@ -159,7 +232,7 @@ def main_worker(gpu, ngpus_per_node, args): model = model.cuda(args.gpu) else: # DataParallel will divide and allocate batch_size to all available GPUs - if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + if args.arch.startswith("alexnet") or args.arch.startswith("vgg"): model.features = torch.nn.DataParallel(model.features) model.cuda() else: @@ -168,43 +241,53 @@ def main_worker(gpu, ngpus_per_node, args): # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda(args.gpu) - optimizer = torch.optim.SGD(model.parameters(), args.lr, - momentum=args.momentum, - weight_decay=args.weight_decay) + optimizer = torch.optim.SGD( + model.parameters(), + args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay, + ) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print(f"=> loading checkpoint '{args.resume}'") checkpoint = torch.load(args.resume) - args.start_epoch = checkpoint['epoch'] - best_acc1 = checkpoint['best_acc1'] + args.start_epoch = checkpoint["epoch"] + best_acc1 = checkpoint["best_acc1"] if args.gpu is not None: # best_acc1 may be from a checkpoint from a different GPU best_acc1 = best_acc1.to(args.gpu) - model.load_state_dict(checkpoint['state_dict']) - optimizer.load_state_dict(checkpoint['optimizer']) - print("=> loaded checkpoint '{}' (epoch {})" - .format(args.resume, checkpoint['epoch'])) + model.load_state_dict(checkpoint["state_dict"]) + optimizer.load_state_dict(checkpoint["optimizer"]) + print( + "=> loaded checkpoint '{}' (epoch {})".format( + args.resume, checkpoint["epoch"] + ) + ) else: print(f"=> no checkpoint found at '{args.resume}'") cudnn.benchmark = True # Data loading code - traindir = os.path.join(args.data, 'train') - valdir = os.path.join(args.data, 'val') - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) + traindir = os.path.join(args.data, "train") + valdir = os.path.join(args.data, "val") + normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) train_dataset = datasets.ImageFolder( traindir, - transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - normalize, - ])) + transforms.Compose( + [ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ] + ), + ) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) @@ -212,18 +295,31 @@ def main_worker(gpu, ngpus_per_node, args): train_sampler = None train_loader = torch.utils.data.DataLoader( - train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), - num_workers=args.workers, pin_memory=True, sampler=train_sampler) + train_dataset, + batch_size=args.batch_size, + shuffle=(train_sampler is None), + num_workers=args.workers, + pin_memory=True, + sampler=train_sampler, + ) val_loader = torch.utils.data.DataLoader( - datasets.ImageFolder(valdir, transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - normalize, - ])), - batch_size=args.batch_size, shuffle=False, - num_workers=args.workers, pin_memory=True) + datasets.ImageFolder( + valdir, + transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ] + ), + ), + batch_size=args.batch_size, + shuffle=False, + num_workers=args.workers, + pin_memory=True, + ) if args.evaluate: validate(val_loader, model, criterion, args) @@ -244,27 +340,32 @@ def main_worker(gpu, ngpus_per_node, args): is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) - if not args.multiprocessing_distributed or (args.multiprocessing_distributed - and args.rank % ngpus_per_node == 0): - save_checkpoint({ - 'epoch': epoch + 1, - 'arch': args.arch, - 'state_dict': model.state_dict(), - 'best_acc1': best_acc1, - 'optimizer' : optimizer.state_dict(), - }, is_best) + if not args.multiprocessing_distributed or ( + args.multiprocessing_distributed and args.rank % ngpus_per_node == 0 + ): + save_checkpoint( + { + "epoch": epoch + 1, + "arch": args.arch, + "state_dict": model.state_dict(), + "best_acc1": best_acc1, + "optimizer": optimizer.state_dict(), + }, + is_best, + ) def train(train_loader, model, criterion, optimizer, epoch, args): - batch_time = AverageMeter('Time', ':6.3f') - data_time = AverageMeter('Data', ':6.3f') - losses = AverageMeter('Loss', ':.4e') - top1 = AverageMeter('Acc@1', ':6.2f') - top5 = AverageMeter('Acc@5', ':6.2f') + batch_time = AverageMeter("Time", ":6.3f") + data_time = AverageMeter("Data", ":6.3f") + losses = AverageMeter("Loss", ":.4e") + top1 = AverageMeter("Acc@1", ":6.2f") + top5 = AverageMeter("Acc@5", ":6.2f") progress = ProgressMeter( len(train_loader), [batch_time, data_time, losses, top1, top5], - prefix=f"Epoch: [{epoch}]") + prefix=f"Epoch: [{epoch}]", + ) # switch to train mode model.train() @@ -302,14 +403,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args): def validate(val_loader, model, criterion, args): - batch_time = AverageMeter('Time', ':6.3f') - losses = AverageMeter('Loss', ':.4e') - top1 = AverageMeter('Acc@1', ':6.2f') - top5 = AverageMeter('Acc@5', ':6.2f') + batch_time = AverageMeter("Time", ":6.3f") + losses = AverageMeter("Loss", ":.4e") + top1 = AverageMeter("Acc@1", ":6.2f") + top5 = AverageMeter("Acc@5", ":6.2f") progress = ProgressMeter( - len(val_loader), - [batch_time, losses, top1, top5], - prefix='Test: ') + len(val_loader), [batch_time, losses, top1, top5], prefix="Test: " + ) # switch to evaluate mode model.eval() @@ -339,21 +439,23 @@ def validate(val_loader, model, criterion, args): progress.display(i) # TODO: this should also be done with the ProgressMeter - print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' - .format(top1=top1, top5=top5)) + print( + " * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}".format(top1=top1, top5=top5) + ) return top1.avg -def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): +def save_checkpoint(state, is_best, filename="checkpoint.pth.tar"): torch.save(state, filename) if is_best: - shutil.copyfile(filename, 'model_best.pth.tar') + shutil.copyfile(filename, "model_best.pth.tar") class AverageMeter: """Computes and stores the average and current value""" - def __init__(self, name, fmt=':f'): + + def __init__(self, name, fmt=":f"): self.name = name self.fmt = fmt self.reset() @@ -371,7 +473,7 @@ def update(self, val, n=1): self.avg = self.sum / self.count def __str__(self): - fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + fmtstr = "{name} {val" + self.fmt + "} ({avg" + self.fmt + "})" return fmtstr.format(**self.__dict__) @@ -384,19 +486,19 @@ def __init__(self, num_batches, meters, prefix=""): def display(self, batch): entries = [self.prefix + self.batch_fmtstr.format(batch)] entries += [str(meter) for meter in self.meters] - print('\t'.join(entries)) + print("\t".join(entries)) def _get_batch_fmtstr(self, num_batches): num_digits = len(str(num_batches // 1)) - fmt = '{:' + str(num_digits) + 'd}' - return '[' + fmt + '/' + fmt.format(num_batches) + ']' + fmt = "{:" + str(num_digits) + "d}" + return "[" + fmt + "/" + fmt.format(num_batches) + "]" def adjust_learning_rate(optimizer, epoch, args): """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" lr = args.lr * (0.1 ** (epoch // 30)) for param_group in optimizer.param_groups: - param_group['lr'] = lr + param_group["lr"] = lr def accuracy(output, target, topk=(1,)): @@ -416,5 +518,5 @@ def accuracy(output, target, topk=(1,)): return res -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/test/benchmark/old/chunk_benchmark.py b/test/benchmark/old/chunk_benchmark.py index 07d393e40c..ab71d07940 100644 --- a/test/benchmark/old/chunk_benchmark.py +++ b/test/benchmark/old/chunk_benchmark.py @@ -3,27 +3,48 @@ import numpy as np -chunk_sizes = [32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 8192*2, 8192*4, 8192*8] +chunk_sizes = [ + 32, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 8192, + 8192 * 2, + 8192 * 4, + 8192 * 8, +] download_time = [] upload_time = [] for cs in chunk_sizes: - x = HubArray((cs,cs), key="test/benchmark:t{}".format(str(cs)), chunk_shape=(cs, cs), dtype='uint8', compression=None) - arr = (255*np.random.rand(cs, cs)).astype('uint8') + x = HubArray( + (cs, cs), + key="test/benchmark:t{}".format(str(cs)), + chunk_shape=(cs, cs), + dtype="uint8", + compression=None, + ) + arr = (255 * np.random.rand(cs, cs)).astype("uint8") # Upload t1 = time.time() - x[:] = arr + x[:] = arr t2 = time.time() - upload_time.append(t2-t1) + upload_time.append(t2 - t1) # Download t3 = time.time() x[:] t4 = time.time() - download_time.append(t4-t3) + download_time.append(t4 - t3) - print('chunk size {} download in {} and uploaded in {}'.format(cs, t4-t3, t2-t1)) + print( + "chunk size {} download in {} and uploaded in {}".format(cs, t4 - t3, t2 - t1) + ) print(download_time) -print(upload_time) \ No newline at end of file +print(upload_time) diff --git a/test/benchmark/old/imagenet_train.py b/test/benchmark/old/imagenet_train.py index 074d427a9d..fe2bebea82 100644 --- a/test/benchmark/old/imagenet_train.py +++ b/test/benchmark/old/imagenet_train.py @@ -31,6 +31,7 @@ h_num_threads = 1 chunk_size = 20 + class StandardTransform: def __init__(self, transform=None, target_transform=None): self.transform = transform @@ -45,19 +46,20 @@ def __call__(self, input, target): def _format_transform_repr(self, transform, head): lines = transform.__repr__().splitlines() - return (["{}{}".format(head, lines[0])] + - ["{}{}".format(" " * len(head), line) for line in lines[1:]]) + return ["{}{}".format(head, lines[0])] + [ + "{}{}".format(" " * len(head), line) for line in lines[1:] + ] def __repr__(self): body = [self.__class__.__name__] if self.transform is not None: - body += self._format_transform_repr(self.transform, - "Transform: ") + body += self._format_transform_repr(self.transform, "Transform: ") if self.target_transform is not None: - body += self._format_transform_repr(self.target_transform, - "Target transform: ") + body += self._format_transform_repr( + self.target_transform, "Target transform: " + ) - return '\n'.join(body) + return "\n".join(body) class VisionDataset(data.Dataset): @@ -71,8 +73,10 @@ def __init__(self, root, transforms=None, transform=None, target_transform=None) has_transforms = transforms is not None has_separate_transform = transform is not None or target_transform is not None if has_transforms and has_separate_transform: - raise ValueError("Only transforms or transform/target_transform can " - "be passed as argument") + raise ValueError( + "Only transforms or transform/target_transform can " + "be passed as argument" + ) # for backwards-compatibility self.transform = transform @@ -97,16 +101,18 @@ def __repr__(self): if hasattr(self, "transforms") and self.transforms is not None: body += [repr(self.transforms)] lines = [head] + [" " * self._repr_indent + line for line in body] - return '\n'.join(lines) + return "\n".join(lines) def _format_transform_repr(self, transform, head): lines = transform.__repr__().splitlines() - return (["{}{}".format(head, lines[0])] + - ["{}{}".format(" " * len(head), line) for line in lines[1:]]) + return ["{}{}".format(head, lines[0])] + [ + "{}{}".format(" " * len(head), line) for line in lines[1:] + ] def extra_repr(self): return "" + class FakeData(VisionDataset): """A fake dataset that returns randomly generated images and returns them as PIL images Args: @@ -121,10 +127,16 @@ class FakeData(VisionDataset): generate each image. Default: 0 """ - def __init__(self, size=1000, image_size=(3, 224, 224), num_classes=10, - transform=None, target_transform=None, random_offset=0): - super().__init__(None, transform=transform, - target_transform=target_transform) + def __init__( + self, + size=1000, + image_size=(3, 224, 224), + num_classes=10, + transform=None, + target_transform=None, + random_offset=0, + ): + super().__init__(None, transform=transform, target_transform=target_transform) self.size = size self.num_classes = num_classes self.image_size = image_size @@ -132,11 +144,11 @@ def __init__(self, size=1000, image_size=(3, 224, 224), num_classes=10, self.x = None self.batch = None self.indexbeta = 0 - self.batch_size = chunk_size*h_num_threads - #self.avgTime = AverageMeter('Time', ':6.3f') - #self.progress = ProgressMeter(1, - #[self.avgTime], - #prefix="Epoch: ") + self.batch_size = chunk_size * h_num_threads + # self.avgTime = AverageMeter('Time', ':6.3f') + # self.progress = ProgressMeter(1, + # [self.avgTime], + # prefix="Epoch: ") def __getitem__(self, index): """ @@ -146,42 +158,41 @@ def __getitem__(self, index): tuple: (image, target) where target is class_index of the target class. """ # create random image that is consistent with the index id - #if index >= len(self): + # if index >= len(self): # raise IndexError("{} index out of range".format(self.__class__.__name__)) - #rng_state = torch.get_rng_state() - #torch.manual_seed(index + self.random_offset) - #img = torch.randn(*self.image_size) + # rng_state = torch.get_rng_state() + # torch.manual_seed(index + self.random_offset) + # img = torch.randn(*self.image_size) target = torch.randint(0, self.num_classes, size=(1,), dtype=torch.long)[0] - #torch.set_rng_state(rng_state) + # torch.set_rng_state(rng_state) # convert to PIL Image - #from PIL import Image + # from PIL import Image - - #img = np.array(img) - #t1 = time.time() + # img = np.array(img) + # t1 = time.time() if not self.x: - self.x = hub.load('imagenet/benchmark:{}'.format(str(chunk_size))) - #print('loaded {}s'.format(time.time()-t1)) + self.x = hub.load("imagenet/benchmark:{}".format(str(chunk_size))) + # print('loaded {}s'.format(time.time()-t1)) if self.batch is None or self.indexbeta == self.batch_size: self.indexbeta = 0 - #t3 = time.time() - self.batch = self.x[:self.batch_size] - #t4 = time.time() - #self.avgTime.update(self.batch.shape[0]/(t4-t3)) - #if index % 10 == 0: + # t3 = time.time() + self.batch = self.x[: self.batch_size] + # t4 = time.time() + # self.avgTime.update(self.batch.shape[0]/(t4-t3)) + # if index % 10 == 0: # print(self.batch.shape) # self.progress.display(index) # print('{} img/sec'.format(self.batch.shape[0]/(t4-t3))) - - #if i % args.print_freq == 0: - + + # if i % args.print_freq == 0: + img = self.batch[self.indexbeta] self.indexbeta += 1 img = transforms.ToPILImage()(img) - + if self.transform is not None: img = self.transform(img) if self.target_transform is not None: @@ -193,62 +204,124 @@ def __len__(self): return self.size - -model_names = sorted(name for name in models.__dict__ - if name.islower() and not name.startswith("__") - and callable(models.__dict__[name])) - -parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') -parser.add_argument('data', metavar='DIR', - help='path to dataset') -parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', - choices=model_names, - help='model architecture: ' + - ' | '.join(model_names) + - ' (default: resnet18)') -parser.add_argument('-j', '--workers', default=h_num_process, type=int, metavar='N', - help='number of data loading workers (default: 4)') -parser.add_argument('--epochs', default=90, type=int, metavar='N', - help='number of total epochs to run') -parser.add_argument('--start-epoch', default=0, type=int, metavar='N', - help='manual epoch number (useful on restarts)') -parser.add_argument('-b', '--batch-size', default=256, type=int, - metavar='N', - help='mini-batch size (default: 256), this is the total ' - 'batch size of all GPUs on the current node when ' - 'using Data Parallel or Distributed Data Parallel') -parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, - metavar='LR', help='initial learning rate', dest='lr') -parser.add_argument('--momentum', default=0.9, type=float, metavar='M', - help='momentum') -parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, - metavar='W', help='weight decay (default: 1e-4)', - dest='weight_decay') -parser.add_argument('-p', '--print-freq', default=10, type=int, - metavar='N', help='print frequency (default: 10)') -parser.add_argument('--resume', default='', type=str, metavar='PATH', - help='path to latest checkpoint (default: none)') -parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', - help='evaluate model on validation set') -parser.add_argument('--pretrained', dest='pretrained', action='store_true', - help='use pre-trained model') -parser.add_argument('--world-size', default=-1, type=int, - help='number of nodes for distributed training') -parser.add_argument('--rank', default=-1, type=int, - help='node rank for distributed training') -parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, - help='url used to set up distributed training') -parser.add_argument('--dist-backend', default='nccl', type=str, - help='distributed backend') -parser.add_argument('--seed', default=None, type=int, - help='seed for initializing training. ') -parser.add_argument('--gpu', default=None, type=int, - help='GPU id to use.') -parser.add_argument('--multiprocessing-distributed', action='store_true', - help='Use multi-processing distributed training to launch ' - 'N processes per node, which has N GPUs. This is the ' - 'fastest way to use PyTorch for either single node or ' - 'multi node data parallel training') +model_names = sorted( + name + for name in models.__dict__ + if name.islower() and not name.startswith("__") and callable(models.__dict__[name]) +) + +parser = argparse.ArgumentParser(description="PyTorch ImageNet Training") +parser.add_argument("data", metavar="DIR", help="path to dataset") +parser.add_argument( + "-a", + "--arch", + metavar="ARCH", + default="resnet18", + choices=model_names, + help="model architecture: " + " | ".join(model_names) + " (default: resnet18)", +) +parser.add_argument( + "-j", + "--workers", + default=h_num_process, + type=int, + metavar="N", + help="number of data loading workers (default: 4)", +) +parser.add_argument( + "--epochs", default=90, type=int, metavar="N", help="number of total epochs to run" +) +parser.add_argument( + "--start-epoch", + default=0, + type=int, + metavar="N", + help="manual epoch number (useful on restarts)", +) +parser.add_argument( + "-b", + "--batch-size", + default=256, + type=int, + metavar="N", + help="mini-batch size (default: 256), this is the total " + "batch size of all GPUs on the current node when " + "using Data Parallel or Distributed Data Parallel", +) +parser.add_argument( + "--lr", + "--learning-rate", + default=0.1, + type=float, + metavar="LR", + help="initial learning rate", + dest="lr", +) +parser.add_argument("--momentum", default=0.9, type=float, metavar="M", help="momentum") +parser.add_argument( + "--wd", + "--weight-decay", + default=1e-4, + type=float, + metavar="W", + help="weight decay (default: 1e-4)", + dest="weight_decay", +) +parser.add_argument( + "-p", + "--print-freq", + default=10, + type=int, + metavar="N", + help="print frequency (default: 10)", +) +parser.add_argument( + "--resume", + default="", + type=str, + metavar="PATH", + help="path to latest checkpoint (default: none)", +) +parser.add_argument( + "-e", + "--evaluate", + dest="evaluate", + action="store_true", + help="evaluate model on validation set", +) +parser.add_argument( + "--pretrained", dest="pretrained", action="store_true", help="use pre-trained model" +) +parser.add_argument( + "--world-size", + default=-1, + type=int, + help="number of nodes for distributed training", +) +parser.add_argument( + "--rank", default=-1, type=int, help="node rank for distributed training" +) +parser.add_argument( + "--dist-url", + default="tcp://224.66.41.62:23456", + type=str, + help="url used to set up distributed training", +) +parser.add_argument( + "--dist-backend", default="nccl", type=str, help="distributed backend" +) +parser.add_argument( + "--seed", default=None, type=int, help="seed for initializing training. " +) +parser.add_argument("--gpu", default=None, type=int, help="GPU id to use.") +parser.add_argument( + "--multiprocessing-distributed", + action="store_true", + help="Use multi-processing distributed training to launch " + "N processes per node, which has N GPUs. This is the " + "fastest way to use PyTorch for either single node or " + "multi node data parallel training", +) best_acc1 = 0 @@ -260,15 +333,19 @@ def main(): random.seed(args.seed) torch.manual_seed(args.seed) cudnn.deterministic = True - warnings.warn('You have chosen to seed training. ' - 'This will turn on the CUDNN deterministic setting, ' - 'which can slow down your training considerably! ' - 'You may see unexpected behavior when restarting ' - 'from checkpoints.') + warnings.warn( + "You have chosen to seed training. " + "This will turn on the CUDNN deterministic setting, " + "which can slow down your training considerably! " + "You may see unexpected behavior when restarting " + "from checkpoints." + ) if args.gpu is not None: - warnings.warn('You have chosen a specific GPU. This will completely ' - 'disable data parallelism.') + warnings.warn( + "You have chosen a specific GPU. This will completely " + "disable data parallelism." + ) if args.dist_url == "env://" and args.world_size == -1: args.world_size = int(os.environ["WORLD_SIZE"]) @@ -302,8 +379,12 @@ def main_worker(gpu, ngpus_per_node, args): # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu - dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, - world_size=args.world_size, rank=args.rank) + dist.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) # create model if args.pretrained: print(f"=> using pre-trained model '{args.arch}'") @@ -324,7 +405,9 @@ def main_worker(gpu, ngpus_per_node, args): # ourselves based on the total number of GPUs we have args.batch_size = int(args.batch_size / ngpus_per_node) args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) - model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[args.gpu] + ) else: model.cuda() # DistributedDataParallel will divide and allocate batch_size to all @@ -335,7 +418,7 @@ def main_worker(gpu, ngpus_per_node, args): model = model.cuda(args.gpu) else: # DataParallel will divide and allocate batch_size to all available GPUs - if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + if args.arch.startswith("alexnet") or args.arch.startswith("vgg"): model.features = torch.nn.DataParallel(model.features) model.cuda() else: @@ -344,62 +427,87 @@ def main_worker(gpu, ngpus_per_node, args): # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda(args.gpu) - optimizer = torch.optim.SGD(model.parameters(), args.lr, - momentum=args.momentum, - weight_decay=args.weight_decay) + optimizer = torch.optim.SGD( + model.parameters(), + args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay, + ) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print(f"=> loading checkpoint '{args.resume}'") checkpoint = torch.load(args.resume) - args.start_epoch = checkpoint['epoch'] - best_acc1 = checkpoint['best_acc1'] + args.start_epoch = checkpoint["epoch"] + best_acc1 = checkpoint["best_acc1"] if args.gpu is not None: # best_acc1 may be from a checkpoint from a different GPU best_acc1 = best_acc1.to(args.gpu) - model.load_state_dict(checkpoint['state_dict']) - optimizer.load_state_dict(checkpoint['optimizer']) - print("=> loaded checkpoint '{}' (epoch {})" - .format(args.resume, checkpoint['epoch'])) + model.load_state_dict(checkpoint["state_dict"]) + optimizer.load_state_dict(checkpoint["optimizer"]) + print( + "=> loaded checkpoint '{}' (epoch {})".format( + args.resume, checkpoint["epoch"] + ) + ) else: print(f"=> no checkpoint found at '{args.resume}'") cudnn.benchmark = True # Data loading code - traindir = os.path.join(args.data, 'train') - valdir = os.path.join(args.data, 'val') - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) + traindir = os.path.join(args.data, "train") + valdir = os.path.join(args.data, "val") + normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) train_dataset = FakeData( - size=256*100, image_size=(3,224,224), num_classes=10, - transform=transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - normalize, - ])) + size=256 * 100, + image_size=(3, 224, 224), + num_classes=10, + transform=transforms.Compose( + [ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ] + ), + ) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None - + train_loader = torch.utils.data.DataLoader( - train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), - num_workers=args.workers, pin_memory=True, sampler=train_sampler) + train_dataset, + batch_size=args.batch_size, + shuffle=(train_sampler is None), + num_workers=args.workers, + pin_memory=True, + sampler=train_sampler, + ) val_loader = torch.utils.data.DataLoader( - datasets.ImageFolder(valdir, transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - normalize, - ])), - batch_size=args.batch_size, shuffle=False, - num_workers=args.workers, pin_memory=True) + datasets.ImageFolder( + valdir, + transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ] + ), + ), + batch_size=args.batch_size, + shuffle=False, + num_workers=args.workers, + pin_memory=True, + ) if args.evaluate: validate(val_loader, model, criterion, args) @@ -420,27 +528,32 @@ def main_worker(gpu, ngpus_per_node, args): is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) - if not args.multiprocessing_distributed or (args.multiprocessing_distributed - and args.rank % ngpus_per_node == 0): - save_checkpoint({ - 'epoch': epoch + 1, - 'arch': args.arch, - 'state_dict': model.state_dict(), - 'best_acc1': best_acc1, - 'optimizer' : optimizer.state_dict(), - }, is_best) + if not args.multiprocessing_distributed or ( + args.multiprocessing_distributed and args.rank % ngpus_per_node == 0 + ): + save_checkpoint( + { + "epoch": epoch + 1, + "arch": args.arch, + "state_dict": model.state_dict(), + "best_acc1": best_acc1, + "optimizer": optimizer.state_dict(), + }, + is_best, + ) def train(train_loader, model, criterion, optimizer, epoch, args): - batch_time = AverageMeter('Time', ':6.3f') - data_time = AverageMeter('Data', ':6.3f') - losses = AverageMeter('Loss', ':.4e') - top1 = AverageMeter('Acc@1', ':6.2f') - top5 = AverageMeter('Acc@5', ':6.2f') + batch_time = AverageMeter("Time", ":6.3f") + data_time = AverageMeter("Data", ":6.3f") + losses = AverageMeter("Loss", ":.4e") + top1 = AverageMeter("Acc@1", ":6.2f") + top5 = AverageMeter("Acc@5", ":6.2f") progress = ProgressMeter( len(train_loader), [batch_time, data_time, losses, top1, top5], - prefix=f"Epoch: [{epoch}]") + prefix=f"Epoch: [{epoch}]", + ) # switch to train mode model.train() @@ -449,7 +562,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args): for i, (images, target) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) - + if args.gpu is not None: images = images.cuda(args.gpu, non_blocking=True) target = target.cuda(args.gpu, non_blocking=True) @@ -478,14 +591,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args): def validate(val_loader, model, criterion, args): - batch_time = AverageMeter('Time', ':6.3f') - losses = AverageMeter('Loss', ':.4e') - top1 = AverageMeter('Acc@1', ':6.2f') - top5 = AverageMeter('Acc@5', ':6.2f') + batch_time = AverageMeter("Time", ":6.3f") + losses = AverageMeter("Loss", ":.4e") + top1 = AverageMeter("Acc@1", ":6.2f") + top5 = AverageMeter("Acc@5", ":6.2f") progress = ProgressMeter( - len(val_loader), - [batch_time, losses, top1, top5], - prefix='Test: ') + len(val_loader), [batch_time, losses, top1, top5], prefix="Test: " + ) # switch to evaluate mode model.eval() @@ -515,21 +627,23 @@ def validate(val_loader, model, criterion, args): progress.display(i) # TODO: this should also be done with the ProgressMeter - print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' - .format(top1=top1, top5=top5)) + print( + " * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}".format(top1=top1, top5=top5) + ) return top1.avg -def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): +def save_checkpoint(state, is_best, filename="checkpoint.pth.tar"): torch.save(state, filename) if is_best: - shutil.copyfile(filename, 'model_best.pth.tar') + shutil.copyfile(filename, "model_best.pth.tar") class AverageMeter: """Computes and stores the average and current value""" - def __init__(self, name, fmt=':f'): + + def __init__(self, name, fmt=":f"): self.name = name self.fmt = fmt self.reset() @@ -547,7 +661,7 @@ def update(self, val, n=1): self.avg = self.sum / self.count def __str__(self): - fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + fmtstr = "{name} {val" + self.fmt + "} ({avg" + self.fmt + "})" return fmtstr.format(**self.__dict__) @@ -560,19 +674,19 @@ def __init__(self, num_batches, meters, prefix=""): def display(self, batch): entries = [self.prefix + self.batch_fmtstr.format(batch)] entries += [str(meter) for meter in self.meters] - print('\t'.join(entries)) + print("\t".join(entries)) def _get_batch_fmtstr(self, num_batches): num_digits = len(str(num_batches // 1)) - fmt = '{:' + str(num_digits) + 'd}' - return '[' + fmt + '/' + fmt.format(num_batches) + ']' + fmt = "{:" + str(num_digits) + "d}" + return "[" + fmt + "/" + fmt.format(num_batches) + "]" def adjust_learning_rate(optimizer, epoch, args): """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" lr = args.lr * (0.1 ** (epoch // 30)) for param_group in optimizer.param_groups: - param_group['lr'] = lr + param_group["lr"] = lr def accuracy(output, target, topk=(1,)): @@ -592,5 +706,5 @@ def accuracy(output, target, topk=(1,)): return res -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/test/benchmark/old/multiprocess_benchmark.py b/test/benchmark/old/multiprocess_benchmark.py index d83db32f63..ab48ced72e 100644 --- a/test/benchmark/old/multiprocess_benchmark.py +++ b/test/benchmark/old/multiprocess_benchmark.py @@ -27,10 +27,11 @@ import torch.utils.data as data import hub -# Training is turned off, only downloads the data all the time and compute average images downloaded per process per second +# Training is turned off, only downloads the data all the time and compute average images downloaded per process per second h_num_process = 8 h_num_threads = 8 -chunk_size = 80 # 20 or 1 +chunk_size = 80 # 20 or 1 + class StandardTransform: def __init__(self, transform=None, target_transform=None): @@ -46,19 +47,20 @@ def __call__(self, input, target): def _format_transform_repr(self, transform, head): lines = transform.__repr__().splitlines() - return (["{}{}".format(head, lines[0])] + - ["{}{}".format(" " * len(head), line) for line in lines[1:]]) + return ["{}{}".format(head, lines[0])] + [ + "{}{}".format(" " * len(head), line) for line in lines[1:] + ] def __repr__(self): body = [self.__class__.__name__] if self.transform is not None: - body += self._format_transform_repr(self.transform, - "Transform: ") + body += self._format_transform_repr(self.transform, "Transform: ") if self.target_transform is not None: - body += self._format_transform_repr(self.target_transform, - "Target transform: ") + body += self._format_transform_repr( + self.target_transform, "Target transform: " + ) - return '\n'.join(body) + return "\n".join(body) class VisionDataset(data.Dataset): @@ -72,8 +74,10 @@ def __init__(self, root, transforms=None, transform=None, target_transform=None) has_transforms = transforms is not None has_separate_transform = transform is not None or target_transform is not None if has_transforms and has_separate_transform: - raise ValueError("Only transforms or transform/target_transform can " - "be passed as argument") + raise ValueError( + "Only transforms or transform/target_transform can " + "be passed as argument" + ) # for backwards-compatibility self.transform = transform @@ -98,16 +102,18 @@ def __repr__(self): if hasattr(self, "transforms") and self.transforms is not None: body += [repr(self.transforms)] lines = [head] + [" " * self._repr_indent + line for line in body] - return '\n'.join(lines) + return "\n".join(lines) def _format_transform_repr(self, transform, head): lines = transform.__repr__().splitlines() - return (["{}{}".format(head, lines[0])] + - ["{}{}".format(" " * len(head), line) for line in lines[1:]]) + return ["{}{}".format(head, lines[0])] + [ + "{}{}".format(" " * len(head), line) for line in lines[1:] + ] def extra_repr(self): return "" + class FakeData(VisionDataset): """A fake dataset that returns randomly generated images and returns them as PIL images Args: @@ -122,10 +128,16 @@ class FakeData(VisionDataset): generate each image. Default: 0 """ - def __init__(self, size=1000, image_size=(3, 224, 224), num_classes=10, - transform=None, target_transform=None, random_offset=0): - super().__init__(None, transform=transform, - target_transform=target_transform) + def __init__( + self, + size=1000, + image_size=(3, 224, 224), + num_classes=10, + transform=None, + target_transform=None, + random_offset=0, + ): + super().__init__(None, transform=transform, target_transform=target_transform) self.size = size self.num_classes = num_classes self.image_size = image_size @@ -133,11 +145,9 @@ def __init__(self, size=1000, image_size=(3, 224, 224), num_classes=10, self.x = None self.batch = None self.indexbeta = 0 - self.batch_size = h_num_threads*chunk_size - self.avgTime = AverageMeter('Time', ':6.3f') - self.progress = ProgressMeter(1, - [self.avgTime], - prefix="Epoch: ") + self.batch_size = h_num_threads * chunk_size + self.avgTime = AverageMeter("Time", ":6.3f") + self.progress = ProgressMeter(1, [self.avgTime], prefix="Epoch: ") def __getitem__(self, index): """ @@ -147,42 +157,41 @@ def __getitem__(self, index): tuple: (image, target) where target is class_index of the target class. """ # create random image that is consistent with the index id - #if index >= len(self): + # if index >= len(self): # raise IndexError("{} index out of range".format(self.__class__.__name__)) - #rng_state = torch.get_rng_state() - #torch.manual_seed(index + self.random_offset) - #img = torch.randn(*self.image_size) + # rng_state = torch.get_rng_state() + # torch.manual_seed(index + self.random_offset) + # img = torch.randn(*self.image_size) target = torch.randint(0, self.num_classes, size=(1,), dtype=torch.long)[0] - #torch.set_rng_state(rng_state) + # torch.set_rng_state(rng_state) # convert to PIL Image - #from PIL import Image + # from PIL import Image - - #img = np.array(img) - #t1 = time.time() + # img = np.array(img) + # t1 = time.time() if not self.x: - self.x = hub.load('imagenet/benchmark:{}'.format(str(chunk_size))) - #print('loaded {}s'.format(time.time()-t1)) + self.x = hub.load("imagenet/benchmark:{}".format(str(chunk_size))) + # print('loaded {}s'.format(time.time()-t1)) - #if self.batch is None or self.indexbeta == self.batch_size: + # if self.batch is None or self.indexbeta == self.batch_size: self.indexbeta = 0 t3 = time.time() - self.batch = self.x[:self.batch_size] + self.batch = self.x[: self.batch_size] t4 = time.time() - self.avgTime.update(self.batch.shape[0]/(t4-t3)) + self.avgTime.update(self.batch.shape[0] / (t4 - t3)) if index % 10 == 0: print(self.batch.shape) self.progress.display(index) - #print('{} img/sec'.format(self.batch.shape[0]/(t4-t3))) - - #if i % args.print_freq == 0: - + # print('{} img/sec'.format(self.batch.shape[0]/(t4-t3))) + + # if i % args.print_freq == 0: + img = self.batch[self.indexbeta] self.indexbeta += 1 img = transforms.ToPILImage()(img) - + if self.transform is not None: img = self.transform(img) if self.target_transform is not None: @@ -194,62 +203,124 @@ def __len__(self): return self.size - -model_names = sorted(name for name in models.__dict__ - if name.islower() and not name.startswith("__") - and callable(models.__dict__[name])) - -parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') -parser.add_argument('data', metavar='DIR', - help='path to dataset') -parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', - choices=model_names, - help='model architecture: ' + - ' | '.join(model_names) + - ' (default: resnet18)') -parser.add_argument('-j', '--workers', default=h_num_process, type=int, metavar='N', - help='number of data loading workers (default: 4)') -parser.add_argument('--epochs', default=90, type=int, metavar='N', - help='number of total epochs to run') -parser.add_argument('--start-epoch', default=0, type=int, metavar='N', - help='manual epoch number (useful on restarts)') -parser.add_argument('-b', '--batch-size', default=256, type=int, - metavar='N', - help='mini-batch size (default: 256), this is the total ' - 'batch size of all GPUs on the current node when ' - 'using Data Parallel or Distributed Data Parallel') -parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, - metavar='LR', help='initial learning rate', dest='lr') -parser.add_argument('--momentum', default=0.9, type=float, metavar='M', - help='momentum') -parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, - metavar='W', help='weight decay (default: 1e-4)', - dest='weight_decay') -parser.add_argument('-p', '--print-freq', default=10, type=int, - metavar='N', help='print frequency (default: 10)') -parser.add_argument('--resume', default='', type=str, metavar='PATH', - help='path to latest checkpoint (default: none)') -parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', - help='evaluate model on validation set') -parser.add_argument('--pretrained', dest='pretrained', action='store_true', - help='use pre-trained model') -parser.add_argument('--world-size', default=-1, type=int, - help='number of nodes for distributed training') -parser.add_argument('--rank', default=-1, type=int, - help='node rank for distributed training') -parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, - help='url used to set up distributed training') -parser.add_argument('--dist-backend', default='nccl', type=str, - help='distributed backend') -parser.add_argument('--seed', default=None, type=int, - help='seed for initializing training. ') -parser.add_argument('--gpu', default=None, type=int, - help='GPU id to use.') -parser.add_argument('--multiprocessing-distributed', action='store_true', - help='Use multi-processing distributed training to launch ' - 'N processes per node, which has N GPUs. This is the ' - 'fastest way to use PyTorch for either single node or ' - 'multi node data parallel training') +model_names = sorted( + name + for name in models.__dict__ + if name.islower() and not name.startswith("__") and callable(models.__dict__[name]) +) + +parser = argparse.ArgumentParser(description="PyTorch ImageNet Training") +parser.add_argument("data", metavar="DIR", help="path to dataset") +parser.add_argument( + "-a", + "--arch", + metavar="ARCH", + default="resnet18", + choices=model_names, + help="model architecture: " + " | ".join(model_names) + " (default: resnet18)", +) +parser.add_argument( + "-j", + "--workers", + default=h_num_process, + type=int, + metavar="N", + help="number of data loading workers (default: 4)", +) +parser.add_argument( + "--epochs", default=90, type=int, metavar="N", help="number of total epochs to run" +) +parser.add_argument( + "--start-epoch", + default=0, + type=int, + metavar="N", + help="manual epoch number (useful on restarts)", +) +parser.add_argument( + "-b", + "--batch-size", + default=256, + type=int, + metavar="N", + help="mini-batch size (default: 256), this is the total " + "batch size of all GPUs on the current node when " + "using Data Parallel or Distributed Data Parallel", +) +parser.add_argument( + "--lr", + "--learning-rate", + default=0.1, + type=float, + metavar="LR", + help="initial learning rate", + dest="lr", +) +parser.add_argument("--momentum", default=0.9, type=float, metavar="M", help="momentum") +parser.add_argument( + "--wd", + "--weight-decay", + default=1e-4, + type=float, + metavar="W", + help="weight decay (default: 1e-4)", + dest="weight_decay", +) +parser.add_argument( + "-p", + "--print-freq", + default=10, + type=int, + metavar="N", + help="print frequency (default: 10)", +) +parser.add_argument( + "--resume", + default="", + type=str, + metavar="PATH", + help="path to latest checkpoint (default: none)", +) +parser.add_argument( + "-e", + "--evaluate", + dest="evaluate", + action="store_true", + help="evaluate model on validation set", +) +parser.add_argument( + "--pretrained", dest="pretrained", action="store_true", help="use pre-trained model" +) +parser.add_argument( + "--world-size", + default=-1, + type=int, + help="number of nodes for distributed training", +) +parser.add_argument( + "--rank", default=-1, type=int, help="node rank for distributed training" +) +parser.add_argument( + "--dist-url", + default="tcp://224.66.41.62:23456", + type=str, + help="url used to set up distributed training", +) +parser.add_argument( + "--dist-backend", default="nccl", type=str, help="distributed backend" +) +parser.add_argument( + "--seed", default=None, type=int, help="seed for initializing training. " +) +parser.add_argument("--gpu", default=None, type=int, help="GPU id to use.") +parser.add_argument( + "--multiprocessing-distributed", + action="store_true", + help="Use multi-processing distributed training to launch " + "N processes per node, which has N GPUs. This is the " + "fastest way to use PyTorch for either single node or " + "multi node data parallel training", +) best_acc1 = 0 @@ -261,15 +332,19 @@ def main(): random.seed(args.seed) torch.manual_seed(args.seed) cudnn.deterministic = True - warnings.warn('You have chosen to seed training. ' - 'This will turn on the CUDNN deterministic setting, ' - 'which can slow down your training considerably! ' - 'You may see unexpected behavior when restarting ' - 'from checkpoints.') + warnings.warn( + "You have chosen to seed training. " + "This will turn on the CUDNN deterministic setting, " + "which can slow down your training considerably! " + "You may see unexpected behavior when restarting " + "from checkpoints." + ) if args.gpu is not None: - warnings.warn('You have chosen a specific GPU. This will completely ' - 'disable data parallelism.') + warnings.warn( + "You have chosen a specific GPU. This will completely " + "disable data parallelism." + ) if args.dist_url == "env://" and args.world_size == -1: args.world_size = int(os.environ["WORLD_SIZE"]) @@ -303,8 +378,12 @@ def main_worker(gpu, ngpus_per_node, args): # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu - dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, - world_size=args.world_size, rank=args.rank) + dist.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) # create model if args.pretrained: print(f"=> using pre-trained model '{args.arch}'") @@ -325,7 +404,9 @@ def main_worker(gpu, ngpus_per_node, args): # ourselves based on the total number of GPUs we have args.batch_size = int(args.batch_size / ngpus_per_node) args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) - model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[args.gpu] + ) else: model.cuda() # DistributedDataParallel will divide and allocate batch_size to all @@ -336,7 +417,7 @@ def main_worker(gpu, ngpus_per_node, args): model = model.cuda(args.gpu) else: # DataParallel will divide and allocate batch_size to all available GPUs - if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + if args.arch.startswith("alexnet") or args.arch.startswith("vgg"): model.features = torch.nn.DataParallel(model.features) model.cuda() else: @@ -345,62 +426,87 @@ def main_worker(gpu, ngpus_per_node, args): # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda(args.gpu) - optimizer = torch.optim.SGD(model.parameters(), args.lr, - momentum=args.momentum, - weight_decay=args.weight_decay) + optimizer = torch.optim.SGD( + model.parameters(), + args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay, + ) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print(f"=> loading checkpoint '{args.resume}'") checkpoint = torch.load(args.resume) - args.start_epoch = checkpoint['epoch'] - best_acc1 = checkpoint['best_acc1'] + args.start_epoch = checkpoint["epoch"] + best_acc1 = checkpoint["best_acc1"] if args.gpu is not None: # best_acc1 may be from a checkpoint from a different GPU best_acc1 = best_acc1.to(args.gpu) - model.load_state_dict(checkpoint['state_dict']) - optimizer.load_state_dict(checkpoint['optimizer']) - print("=> loaded checkpoint '{}' (epoch {})" - .format(args.resume, checkpoint['epoch'])) + model.load_state_dict(checkpoint["state_dict"]) + optimizer.load_state_dict(checkpoint["optimizer"]) + print( + "=> loaded checkpoint '{}' (epoch {})".format( + args.resume, checkpoint["epoch"] + ) + ) else: print(f"=> no checkpoint found at '{args.resume}'") cudnn.benchmark = True # Data loading code - traindir = os.path.join(args.data, 'train') - valdir = os.path.join(args.data, 'val') - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) + traindir = os.path.join(args.data, "train") + valdir = os.path.join(args.data, "val") + normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) train_dataset = FakeData( - size=1000000, image_size=(3,224,224), num_classes=10, - transform=transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - normalize, - ])) + size=1000000, + image_size=(3, 224, 224), + num_classes=10, + transform=transforms.Compose( + [ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ] + ), + ) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None - + train_loader = torch.utils.data.DataLoader( - train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), - num_workers=args.workers, pin_memory=True, sampler=train_sampler) + train_dataset, + batch_size=args.batch_size, + shuffle=(train_sampler is None), + num_workers=args.workers, + pin_memory=True, + sampler=train_sampler, + ) val_loader = torch.utils.data.DataLoader( - datasets.ImageFolder(valdir, transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - normalize, - ])), - batch_size=args.batch_size, shuffle=False, - num_workers=args.workers, pin_memory=True) + datasets.ImageFolder( + valdir, + transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ] + ), + ), + batch_size=args.batch_size, + shuffle=False, + num_workers=args.workers, + pin_memory=True, + ) if args.evaluate: validate(val_loader, model, criterion, args) @@ -421,27 +527,32 @@ def main_worker(gpu, ngpus_per_node, args): is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) - if not args.multiprocessing_distributed or (args.multiprocessing_distributed - and args.rank % ngpus_per_node == 0): - save_checkpoint({ - 'epoch': epoch + 1, - 'arch': args.arch, - 'state_dict': model.state_dict(), - 'best_acc1': best_acc1, - 'optimizer' : optimizer.state_dict(), - }, is_best) + if not args.multiprocessing_distributed or ( + args.multiprocessing_distributed and args.rank % ngpus_per_node == 0 + ): + save_checkpoint( + { + "epoch": epoch + 1, + "arch": args.arch, + "state_dict": model.state_dict(), + "best_acc1": best_acc1, + "optimizer": optimizer.state_dict(), + }, + is_best, + ) def train(train_loader, model, criterion, optimizer, epoch, args): - batch_time = AverageMeter('Time', ':6.3f') - data_time = AverageMeter('Data', ':6.3f') - losses = AverageMeter('Loss', ':.4e') - top1 = AverageMeter('Acc@1', ':6.2f') - top5 = AverageMeter('Acc@5', ':6.2f') + batch_time = AverageMeter("Time", ":6.3f") + data_time = AverageMeter("Data", ":6.3f") + losses = AverageMeter("Loss", ":.4e") + top1 = AverageMeter("Acc@1", ":6.2f") + top5 = AverageMeter("Acc@5", ":6.2f") progress = ProgressMeter( len(train_loader), [batch_time, data_time, losses, top1, top5], - prefix=f"Epoch: [{epoch}]") + prefix=f"Epoch: [{epoch}]", + ) # switch to train mode model.train() @@ -451,7 +562,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args): continue # measure data loading time data_time.update(time.time() - end) - + if args.gpu is not None: images = images.cuda(args.gpu, non_blocking=True) target = target.cuda(args.gpu, non_blocking=True) @@ -480,14 +591,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args): def validate(val_loader, model, criterion, args): - batch_time = AverageMeter('Time', ':6.3f') - losses = AverageMeter('Loss', ':.4e') - top1 = AverageMeter('Acc@1', ':6.2f') - top5 = AverageMeter('Acc@5', ':6.2f') + batch_time = AverageMeter("Time", ":6.3f") + losses = AverageMeter("Loss", ":.4e") + top1 = AverageMeter("Acc@1", ":6.2f") + top5 = AverageMeter("Acc@5", ":6.2f") progress = ProgressMeter( - len(val_loader), - [batch_time, losses, top1, top5], - prefix='Test: ') + len(val_loader), [batch_time, losses, top1, top5], prefix="Test: " + ) # switch to evaluate mode model.eval() @@ -517,21 +627,23 @@ def validate(val_loader, model, criterion, args): progress.display(i) # TODO: this should also be done with the ProgressMeter - print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' - .format(top1=top1, top5=top5)) + print( + " * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}".format(top1=top1, top5=top5) + ) return top1.avg -def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): +def save_checkpoint(state, is_best, filename="checkpoint.pth.tar"): torch.save(state, filename) if is_best: - shutil.copyfile(filename, 'model_best.pth.tar') + shutil.copyfile(filename, "model_best.pth.tar") class AverageMeter: """Computes and stores the average and current value""" - def __init__(self, name, fmt=':f'): + + def __init__(self, name, fmt=":f"): self.name = name self.fmt = fmt self.reset() @@ -549,7 +661,7 @@ def update(self, val, n=1): self.avg = self.sum / self.count def __str__(self): - fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + fmtstr = "{name} {val" + self.fmt + "} ({avg" + self.fmt + "})" return fmtstr.format(**self.__dict__) @@ -562,19 +674,19 @@ def __init__(self, num_batches, meters, prefix=""): def display(self, batch): entries = [self.prefix + self.batch_fmtstr.format(batch)] entries += [str(meter) for meter in self.meters] - print('\t'.join(entries)) + print("\t".join(entries)) def _get_batch_fmtstr(self, num_batches): num_digits = len(str(num_batches // 1)) - fmt = '{:' + str(num_digits) + 'd}' - return '[' + fmt + '/' + fmt.format(num_batches) + ']' + fmt = "{:" + str(num_digits) + "d}" + return "[" + fmt + "/" + fmt.format(num_batches) + "]" def adjust_learning_rate(optimizer, epoch, args): """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" lr = args.lr * (0.1 ** (epoch // 30)) for param_group in optimizer.param_groups: - param_group['lr'] = lr + param_group["lr"] = lr def accuracy(output, target, topk=(1,)): @@ -594,5 +706,5 @@ def accuracy(output, target, topk=(1,)): return res -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/test/benchmark/transfer.py b/test/benchmark/transfer.py index 1c8adb77c8..4bf5e3805a 100644 --- a/test/benchmark/transfer.py +++ b/test/benchmark/transfer.py @@ -53,4 +53,3 @@ def upload_and_download(samples=30, chunksize=None, name="hub"): r2 = aws_cli_copy(samples, chunksize=chunksize) report([r1, r2]) -