Merge pull request hpcaitech#409 from 1SAA/develop

[hotfix] fixed error when no collective communication in CommProfiler
twenty-zp · Mar 14, 2022 · 32296cf · 32296cf
2 parents 62b08ac + 907ac4a
commit 32296cf
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 8 deletions.
diff --git a/colossalai/utils/profiler/comm_profiler.py b/colossalai/utils/profiler/comm_profiler.py
@@ -93,16 +93,16 @@ def disable(self):
         dist.reduce = torch_reduce
 
     def to_tensorboard(self, writer):
-        writer.add_text(tag="Collective Communication", text_string=self.result_list("\n\n"))
+        writer.add_text(tag="Collective Communication", text_string=self.result_str("\n\n"))
 
     def to_file(self, filename: Path):
         with open(filename, "w") as f:
-            f.write(self.result_list())
+            f.write(self.result_str())
 
     def show(self):
-        print(self.result_list())
+        print(self.result_str())
 
-    def result_list(self, sep: str = "\n"):
+    def result_str(self, sep: str = "\n"):
         res = []
 
         def append(s: str = None):
@@ -114,6 +114,9 @@ def append(s: str = None):
             append("Warnning: there exists multiple communication operations in the same time. As a result, "
                    "the profiling result is not accurate.")
 
+        if self.total_cuda_time == 0:
+            return "No collective communication has been called yet!"
+
         append("Collective communication profiling result:")
         append("total cuda time: {}".format(_format_time(self.total_cuda_time)))
         append("average bandwidth: {}".format(_format_bandwidth(self.total_comm_vol, self.total_cuda_time)))

diff --git a/colossalai/utils/profiler/pcie_profiler.py b/colossalai/utils/profiler/pcie_profiler.py
@@ -105,16 +105,16 @@ def disable(self):
         self.profiler = None
 
     def to_tensorboard(self, writer):
-        writer.add_text(tag="Data Transmission", text_string=self.result_list("\n\n"))
+        writer.add_text(tag="Data Transmission", text_string=self.result_str("\n\n"))
 
     def to_file(self, filename: Path):
         with open(filename, "w") as f:
-            f.write(self.result_list())
+            f.write(self.result_str())
 
     def show(self):
-        print(self.result_list())
+        print(self.result_str())
 
-    def result_list(self, sep: str = "\n"):
+    def result_str(self, sep: str = "\n"):
         res = []
 
         def append(s: str = None):