experimental/tracing/bin/diff_heap_profiler.py

#!/usr/bin/env vpython3
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import absolute_import
from __future__ import print_function
import argparse
import gzip
import json
import os
import shutil
import six
from six.moves import zip

_OUTPUT_DIR = 'output'
_OUTPUT_GRAPH_DIR = os.path.join(_OUTPUT_DIR, 'graph')


class Process(object):

  def __init__(self):
    self.pid = None
    self.name = None
    self.labels = None
    self.types = {}
    self.strings = {}
    self.stackframes = {}
    self.allocators = None
    self.version = None


class Entry(object):

  def __init__(self):
    self.count = None
    self.size = None
    self.type = None
    self.stackframe = None


class GraphDump(object):

  def __init__(self):
    self.pid = None
    self.name = None
    self.labels = None
    self.heap = None
    self.root = ''
    self.leaks = ''
    self.leak_stackframes = 0
    self.leak_objects = 0

def OpenTraceFile(file_path, mode):
  if file_path.endswith('.gz'):
    return gzip.open(file_path, mode + 'b')
  return open(file_path, mode + 't')

def FindMemoryDumps(filename):
  processes = {}

  with OpenTraceFile(filename, 'r') as f:
    data = json.loads(f.read())

    for event in data['traceEvents']:
      pid = event['pid']
      if pid not in processes:
        processes[pid] = Process()
        processes[pid].pid = pid
      process = processes[pid]

      # Retrieve process informations.
      if event['ph'] == 'M':
        if event['name'] == 'process_name' and 'name' in event['args']:
          process.name = event['args']['name']
        if event['name'] == 'process_labels' and 'labels' in event['args']:
          process.labels = event['args']['labels']

      if event['name'] == 'typeNames':
        process.types = {}
        for type_id, t in six.iteritems(event['args']['typeNames']):
          process.types[int(type_id)] = t

      if event['name'] == 'stackFrames':
        process.stackframes = {}
        for stack_id, s in six.iteritems(event['args']['stackFrames']):
          new_stackframe = {}
          new_stackframe['name'] = s['name']
          if 'parent' in s:
            new_stackframe['parent'] = int(s['parent'])
          process.stackframes[int(stack_id)] = new_stackframe

      # Look for a detailed memory dump event.
      if not ((event['name'] == 'periodic_interval' or
               event['name'] == 'explicitly_triggered') and
              event['args']['dumps']['level_of_detail'] == 'detailed'):
        continue

      # Check for a memory dump V1.
      if u'heaps' in event['args']['dumps']:
        # Get the first memory dump.
        if not process.allocators:
          process.version = 1
          process.allocators = event['args']['dumps']['heaps']

      # Check for a memory dump V2.
      # See format: [chromium] src/base/trace_event/heap_profiler_event_writer.h
      if u'heaps_v2' in event['args']['dumps']:
        # Memory dump format V2 is dumping information incrementally. Update
        # the cumulated indexes.
        maps = event['args']['dumps']['heaps_v2']['maps']
        for string in maps['strings']:
          process.strings[string['id']] = string['string']

        for node in maps['nodes']:
          node_v1 = {}
          node_v1['name'] = process.strings[node['name_sid']]
          if 'parent' in node:
            node_v1['parent'] = node['parent']
          process.stackframes[node['id']] = node_v1

        for t in maps['types']:
          process.types[t['id']] = process.strings[t['name_sid']]

        # Get the first memory dump.
        if not process.allocators:
          dump = event['args']['dumps']
          process.version = 2
          process.allocators = dump['heaps_v2']['allocators']

  # Remove processes with incomplete memory dump.
  # Note: Calling list() otherwise we can't modify list while iterating.
  for pid, process in list(processes.items()):
    if not (process.allocators and process.stackframes and process.types):
      del processes[pid]

  return processes


def ResolveMemoryDumpFields(entries, stackframes, types):
  def ResolveStackTrace(stack_id, stackframes):
    stackframe = stackframes[stack_id]
    tail = ()
    if 'parent' in stackframe:
      tail = ResolveStackTrace(stackframe['parent'], stackframes)
    name = stackframe['name'].replace('\r', '').replace('\n', '')
    return (name,) + tail

  def ResolveType(type_id, types):
    return types[type_id]

  for entry in entries:
    # Stackframe may be -1 (18446744073709551615L) when not stackframe are
    # available.
    if entry.stackframe not in stackframes:
      entry.stackframe = []
    else:
      entry.stackframe = ResolveStackTrace(entry.stackframe, stackframes)
    entry.type = ResolveType(entry.type, types)


def IncrementHeapEntry(stack, count, size, typename, root):
  if not stack:
    root['count'] += count
    root['size'] += size
    if typename not in root['count_by_type']:
      root['count_by_type'][typename] = 0
    root['count_by_type'][typename] += count
  else:
    top = stack[-1]
    tail = stack[:-1]

    if top not in root['children']:
      new_node = {}
      new_node['count'] = 0
      new_node['size'] = 0
      new_node['children'] = {}
      new_node['count_by_type'] = {}
      root['children'][top] = new_node

    IncrementHeapEntry(tail, count, size, typename, root['children'][top])


def CanonicalHeapEntries(root):
  total_count = 0
  total_size = 0
  for child in six.itervalues(root['children']):
    total_count += child['count']
    total_size += child['size']
  root['count'] -= total_count
  root['size'] -= total_size

  for typename in root['count_by_type']:
    total_count_for_type = 0
    for child in six.itervalues(root['children']):
      if typename in child['count_by_type']:
        total_count_for_type += child['count_by_type'][typename]
    root['count_by_type'][typename] -= total_count_for_type

  for child in six.itervalues(root['children']):
    CanonicalHeapEntries(child)


def FindLeaks(root, stack, leaks, threshold, size_threshold):
  for frame in root['children']:
    FindLeaks(root['children'][frame], [frame] + stack, leaks, threshold,
              size_threshold)

  if root['count'] > threshold and root['size'] > size_threshold:
    leaks.append({'count': root['count'],
                  'size': root['size'],
                  'count_by_type': root['count_by_type'],
                  'stackframes': stack})

def DumpTree(root, frame, output, threshold, size_threshold):
  output.write('\n{ \"name\": \"%s\",' % frame)
  if root['count'] > threshold and root['count'] > size_threshold:
    output.write(' \"size\": \"%s\",' % root['size'])
    output.write(' \"count\": \"%s\",' % root['count'])
  output.write(' \"children\": [')
  is_first = True
  for child_frame, child in root['children'].items():
    if is_first:
      is_first = False
    else:
      output.write(',')

    DumpTree(child, child_frame, output, threshold, size_threshold)
  output.write(']')
  output.write('}')


def GetEntries(heap, process):
  """
  Returns all entries in a heap, after filtering out unknown entries, and doing
  some post processing to extract the relevant fields.
  """
  if not process:
    return []

  entries = []
  if process.version == 1:
    for raw_entry in process.allocators[heap]['entries']:
      # Cumulative sizes and types are skipped. see:
      # https://chromium.googlesource.com/chromium/src/+/a990af190304be5bf38b120799c594df5a293518/base/trace_event/heap_profiler_heap_dump_writer.cc#294
      if 'type' not in raw_entry or not raw_entry['bt']:
        continue

      entry = Entry()
      entry.count = int(raw_entry['count'], 16)
      entry.size = int(raw_entry['size'], 16)
      entry.type = int(raw_entry['type'])
      entry.stackframe = int(raw_entry['bt'])
      entries.append(entry)

  elif process.version == 2:
    raw_entries = list(zip(process.allocators[heap]['counts'],
                      process.allocators[heap]['sizes'],
                      process.allocators[heap]['types'],
                      process.allocators[heap]['nodes']))
    for (raw_count, raw_size, raw_type, raw_stackframe) in raw_entries:
      entry = Entry()
      entry.count = raw_count
      entry.size = raw_size
      entry.type = raw_type
      entry.stackframe = raw_stackframe
      entries.append(entry)

  # Resolve fields by looking into indexes
  ResolveMemoryDumpFields(entries, process.stackframes, process.types)

  return entries


def FilterProcesses(processes, filter_by_name, filter_by_labels):
  remaining_processes = {}
  for pid, process in six.iteritems(processes):
    if filter_by_name and process.name != filter_by_name:
      continue
    if (filter_by_labels and
        (not process.labels or filter_by_labels not in process.labels)):
      continue
    remaining_processes[pid] = process

  return remaining_processes


def FindRelevantProcesses(start_trace, end_trace,
                          filter_by_name,
                          filter_by_labels,
                          match_by_labels):
  # Retrieve the processes and the associated memory dump.
  end_processes = FindMemoryDumps(end_trace)
  end_processes = FilterProcesses(end_processes, filter_by_name,
                                  filter_by_labels)

  start_processes = None
  if start_trace:
    start_processes = FindMemoryDumps(start_trace)
    start_processes = FilterProcesses(start_processes, filter_by_name,
                                      filter_by_labels)

  # Build a sequence of pair of processes to be compared.
  processes = []
  if not start_processes:
    # Only keep end-processes.
    for _, end_process in six.iteritems(end_processes):
      processes.append((None, end_process))
  elif match_by_labels:
    # Processes are paired based on name/labels.
    for _, end_process in six.iteritems(end_processes):
      matching_start_process = None
      for _, start_process in six.iteritems(start_processes):
        if (start_process.name == end_process.name and
            (start_process.name in ['Browser', 'GPU'] or
             start_process.labels == end_process.labels)):
          matching_start_process = start_process

      if matching_start_process:
        processes.append((matching_start_process, end_process))
  else:
    # Processes are paired based on their PID.
    relevant_pids = set(end_processes.keys()) & set(start_processes.keys())
    for pid in relevant_pids:
      start_process = start_processes[pid]
      end_process = end_processes[pid]
      processes.append((start_process, end_process))

  return processes


def BuildGraphDumps(processes, threshold, size_threshold):
  """
  Build graph for a sequence of pair of processes.
  If start_process is None, counts objects in end_trace.
  Otherwise, counts objects present in end_trace, but not in start_process.
  """

  graph_dumps = []

  for (start_process, end_process) in processes:
    pid = end_process.pid
    name = end_process.name if end_process.name else ''
    labels = end_process.labels if end_process.labels else ''
    print('Process[%d] %s: %s' % (pid, name, labels))

    for heap in end_process.allocators:
      start_entries = GetEntries(heap, start_process)
      end_entries = GetEntries(heap, end_process)

      graph = GraphDump()
      graph.pid = pid
      graph.name = name
      graph.labels = labels
      graph.heap = heap
      graph_dumps.append(graph)

      # Do the math: diffing start and end memory dumps.
      root = {}
      root['count'] = 0
      root['size'] = 0
      root['children'] = {}
      root['count_by_type'] = {}

      for entry in start_entries:
        if entry.type:
          IncrementHeapEntry(entry.stackframe, - entry.count, - entry.size,
                             entry.type, root)
      for entry in end_entries:
        if entry.type:
          IncrementHeapEntry(entry.stackframe, entry.count, entry.size,
                             entry.type, root)

      CanonicalHeapEntries(root)

      graph.root = root

      # Find leaks
      leaks = []
      FindLeaks(root, [], leaks, threshold, size_threshold)
      leaks.sort(reverse=True, key=lambda k: k['size'])

      if leaks:
        print('  %s: %d potential leaks found.' % (heap, len(leaks)))
        graph.leaks = leaks
        graph.leak_stackframes = len(leaks)
        for leak in leaks:
          graph.leak_objects += leak['count']

  return graph_dumps


def WritePotentialLeaks(graph_dumps):
  for graph in graph_dumps:
    if graph.leaks:
      filename = 'process_%d_%s-leaks.json' % (graph.pid, graph.heap)
      output_filename = os.path.join(_OUTPUT_DIR, filename)
      with open(output_filename, 'w') as output:
        json.dump(graph.leaks, output)


def WriteGrahDumps(graph_dumps, threshold, size_threshold):
  for graph in graph_dumps:
    # Dump the remaining allocated objects tree.
    filename = 'process_%d_%s-objects.json' % (graph.pid, graph.heap)
    output_filename = os.path.join(_OUTPUT_GRAPH_DIR, filename)
    if graph.root:
      with open(output_filename, 'w') as output:
        DumpTree(graph.root, '.', output, threshold, size_threshold)
      graph.root = filename


def WriteIndex(graph_dumps):
  output_filename = os.path.join(_OUTPUT_GRAPH_DIR, 'index.json')
  with open(output_filename, 'w') as output:
    json.dump([
        {'pid': graph.pid,
         'heap': graph.heap,
         'name': graph.name,
         'labels': graph.labels,
         'objects': graph.root,
         'potential leaks': graph.leak_stackframes,
         'objects leaked': graph.leak_objects,
        }
        for graph in graph_dumps], output)


def WriteHTML():
  # Copy the HTML page.
  source = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                        'diff_heap_profiler.html')
  destination = os.path.join(_OUTPUT_GRAPH_DIR, 'index.html')
  shutil.copyfile(source, destination)

  # Copy the D3 library file.
  source = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                        os.path.pardir,
                        os.path.pardir,
                        os.path.pardir,
                        'tracing',
                        'third_party',
                        'd3',
                        'd3.min.js')
  destination = os.path.join(_OUTPUT_GRAPH_DIR, 'd3.min.js')
  shutil.copyfile(source, destination)


def Main():
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--flame-graph',
      action='store_true',
      help='Output a flame graph based on stackframe allocations')
  parser.add_argument(
      '--threshold',
      type=int,
      default=0,
      help='Objects threshold for being a potential memory leak')
  parser.add_argument(
      '--size-threshold',
      type=int,
      default=0,
      help='Size threshold for being a potential memory leak')
  parser.add_argument(
      '--filter-by-name',
      type=str,
      help='Only keep processes with name (i.e. Browser, Renderer, ...)')
  parser.add_argument(
      '--filter-by-labels',
      type=str,
      help='Only keep processes with matching labels')
  parser.add_argument(
      '--match-by-labels',
      action='store_true',
      help='Match processes between runs by labels')
  parser.add_argument(
      'trace',
      nargs='+',
      help='Trace files to be processed')
  options = parser.parse_args()

  if options.threshold == 0 and options.size_threshold == 0:
    options.threshold = 1000

  if len(options.trace) == 1:
    end_trace = options.trace[0]
    start_trace = None
  else:
    start_trace = options.trace[0]
    end_trace = options.trace[1]

  if not os.path.exists(_OUTPUT_DIR):
    os.makedirs(_OUTPUT_DIR)

  # Find relevant processes to be processed.
  processes = FindRelevantProcesses(start_trace, end_trace,
                                    options.filter_by_name,
                                    options.filter_by_labels,
                                    options.match_by_labels)

  graph_dumps = BuildGraphDumps(processes, options.threshold,
                                options.size_threshold)

  WritePotentialLeaks(graph_dumps)

  if options.flame_graph:
    if not os.path.exists(_OUTPUT_GRAPH_DIR):
      os.makedirs(_OUTPUT_GRAPH_DIR)
    WriteGrahDumps(graph_dumps, options.threshold, options.size_threshold)
    WriteIndex(graph_dumps)
    WriteHTML()

if __name__ == '__main__':
  Main()