def _bcast_pipe_scalar(self, data, src_rank=None, dtype=torch.float32):
    # Default to last stage (e.g., for broadcasting loss)
    if src_rank is None:
        src_rank = self.grid.stage_to_global(self.num_stages - 1)
    assert src_rank in self.grid.pp_group

    if self.global_rank == src_rank:
        result = data.clone().detach() # f16 tensor
    else:
        result = torch.Tensor([0.]).type(dtype).to(self.device) # f32 tensor

    # trying to broadcast a f16 tensor to f32 tensors here, and the result is noise.
    dist.broadcast(tensor=result, src=src_rank, group=self.mpu.get_pipe_parallel_group())

    return result