# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time import torch.distributed as dist import torch class PerformanceMeter(): def __init__(self, benchmark_mode=True): self.benchmark_mode = benchmark_mode self.reset() def reset(self): if self.benchmark_mode: torch.cuda.synchronize() self.avg = 0 self.count = 0 self.total_time = 0 self.last_update_time = time.time() self.intervals = [] def update(self, n, exclude_from_total=False): if self.benchmark_mode: torch.cuda.synchronize() delta = time.time() - self.last_update_time self.intervals.append(delta) if not exclude_from_total: self.total_time += delta self.count += n self.avg = self.count / self.total_time self.last_update_time = time.time() return n/delta def reset_current_lap(self): if self.benchmark_mode: torch.cuda.synchronize() self.last_update_time = time.time() def p(self, i): assert i <= 100 idx = int(len(self.intervals) * i / 100) return sorted(self.intervals)[idx] def print_once(*args, **kwargs): if not dist.is_initialized() or dist.get_rank() == 0: print(*args, **kwargs)