From 607f01a46a447d83c12d56eb9f699d69422ebd0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guanghui=20Ren=28=E4=BB=BB=E5=B9=BF=E8=BE=89=29?= Date: Thu, 1 Apr 2021 13:31:58 +0800 Subject: [PATCH] Fix bug: optimize bn when distributed is True --- det3d/torchie/apis/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/det3d/torchie/apis/train.py b/det3d/torchie/apis/train.py index ad48ef06..bf49820c 100644 --- a/det3d/torchie/apis/train.py +++ b/det3d/torchie/apis/train.py @@ -265,7 +265,8 @@ def train_detector(model, dataset, cfg, distributed=False, validate=False, logge total_steps = cfg.total_epochs * len(data_loaders[0]) # print(f"total_steps: {total_steps}") - + if distributed: + model = apex.parallel.convert_syncbn_model(model) if cfg.lr_config.type == "one_cycle": # build trainer optimizer = build_one_cycle_optimizer(model, cfg.optimizer) @@ -281,7 +282,6 @@ def train_detector(model, dataset, cfg, distributed=False, validate=False, logge # put model on gpus if distributed: - model = apex.parallel.convert_syncbn_model(model) model = DistributedDataParallel( model.cuda(cfg.local_rank), device_ids=[cfg.local_rank],