tools/vim/chromium.ycm_extra_conf.py

# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Autocompletion config for YouCompleteMe in Chromium.
#
# USAGE:
#
#   1. Install YCM [https://github.com/Valloric/YouCompleteMe]
#          (Googlers should check out [go/ycm])
#
#   2. Create a symbolic link to this file called .ycm_extra_conf.py in the
#      directory above your Chromium checkout (i.e. next to your .gclient file).
#
#          cd src
#          ln -rs tools/vim/chromium.ycm_extra_conf.py ../.ycm_extra_conf.py
#
#   3. (optional) Whitelist the .ycm_extra_conf.py from step #2 by adding the
#      following to your .vimrc:
#
#          let g:ycm_extra_conf_globlist=['<path to .ycm_extra_conf.py>']
#
#      You can also add other .ycm_extra_conf.py files you want to use to this
#      list to prevent excessive prompting each time you visit a directory
#      covered by a config file.
#
#   4. Profit
#
#
# Usage notes:
#
#   * You must use ninja & clang to build Chromium.
#
#   * You must have run gyp_chromium and built Chromium recently.
#
#
# Hacking notes:
#
#   * The purpose of this script is to construct an accurate enough command line
#     for YCM to pass to clang so it can build and extract the symbols.
#
#   * Right now, we only pull the -I and -D flags. That seems to be sufficient
#     for everything I've used it for.
#
#   * That whole ninja & clang thing? We could support other configs if someone
#     were willing to write the correct commands and a parser.
#
#   * This has only been tested on gPrecise.


import os
import os.path
import re
import shlex
import subprocess
import sys

# A dictionary mapping Clang binary path to a list of Clang command line
# arguments that specify the system include paths. It is used as a cache of the
# system include options since these options aren't expected to change per
# source file for the same clang binary. SystemIncludeDirectoryFlags() updates
# this map each time it runs a Clang binary to determine system include paths.
#
# Entries look like:
#   '/home/username/my-llvm/bin/clang++': ['-isystem',
#        '/home/username/my-llvm/include', '-isystem', '/usr/include']
_clang_system_include_map = {}


# Flags from YCM's default config.
_default_flags = [
  '-DUSE_CLANG_COMPLETER',
  '-std=c++11',
  '-x',
  'c++',
]


def FallbackSystemIncludeDirectoryFlags():
  """Returns a best guess list of system include directory flags for Clang.

  If Ninja doesn't give us a build step that specifies a Clang invocation or if
  something goes wrong while determining the system include paths, then this
  function can be used to determine some set of values that's better than
  nothing.

  Returns:
    (List of Strings) Compiler flags that specify the system include paths.
  """
  if _clang_system_include_map:
    return _clang_system_include_map.itervalues().next()
  return []


def SystemIncludeDirectoryFlags(clang_binary, clang_flags):
  """Determines compile flags for specifying system include directories.

  Use as a workaround for https://github.com/Valloric/YouCompleteMe/issues/303

  Caches the results of determining the system include directories in
  _clang_system_include_map.  Subsequent calls to SystemIncludeDirectoryFlags()
  uses the cached results for the same binary even if |clang_flags| differ.

  Args:
    clang_binary: (String) Path to clang binary.
    clang_flags: (List of Strings) List of additional flags to clang. It may
      affect the choice of system include directories if -stdlib= is specified.
      _default_flags are always included in the list of flags passed to clang.

  Returns:
    (List of Strings) Compile flags to append.
  """

  if clang_binary in _clang_system_include_map:
    return _clang_system_include_map[clang_binary]

  all_clang_flags = [] + _default_flags
  all_clang_flags += [flag for flag in clang_flags
                if flag.startswith('-std=') or flag.startswith('-stdlib=')]
  all_clang_flags += ['-v', '-E', '-']
  try:
    with open(os.devnull, 'rb') as DEVNULL:
      output = subprocess.check_output([clang_binary] + all_clang_flags,
                                       stdin=DEVNULL, stderr=subprocess.STDOUT)
  except:
    # Even though we couldn't figure out the flags for the given binary, if we
    # have results from another one, we'll use that. This logic assumes that the
    # list of default system directories for one binary can be used with
    # another.
    return FallbackSystemIncludeDirectoryFlags()
  includes_regex = r'#include <\.\.\.> search starts here:\s*' \
                   r'(.*?)End of search list\.'
  includes = re.search(includes_regex, output.decode(), re.DOTALL).group(1)
  system_include_flags = []
  for path in includes.splitlines():
    path = path.strip()
    if os.path.isdir(path):
      system_include_flags.append('-isystem')
      system_include_flags.append(path)
  if system_include_flags:
    _clang_system_include_map[clang_binary] = system_include_flags
  return system_include_flags


def PathExists(*args):
  return os.path.exists(os.path.join(*args))


def FindChromeSrcFromFilename(filename):
  """Searches for the root of the Chromium checkout.

  Simply checks parent directories until it finds .gclient and src/.

  Args:
    filename: (String) Path to source file being edited.

  Returns:
    (String) Path of 'src/', or None if unable to find.
  """
  curdir = os.path.normpath(os.path.dirname(filename))
  while not (os.path.basename(os.path.realpath(curdir)) == 'src'
             and PathExists(curdir, 'DEPS')
             and (PathExists(curdir, '..', '.gclient')
                  or PathExists(curdir, '.git'))):
    nextdir = os.path.normpath(os.path.join(curdir, '..'))
    if nextdir == curdir:
      return None
    curdir = nextdir
  return curdir


def GetDefaultSourceFile(chrome_root, filename):
  """Returns the default source file to use as an alternative to |filename|.

  Compile flags used to build the default source file is assumed to be a
  close-enough approximation for building |filename|.

  Args:
    chrome_root: (String) Absolute path to the root of Chromium checkout.
    filename: (String) Absolute path to the source file.

  Returns:
    (String) Absolute path to substitute source file.
  """
  blink_root = os.path.join(chrome_root, 'third_party', 'WebKit')
  if filename.startswith(blink_root):
    return os.path.join(blink_root, 'Source', 'core', 'Init.cpp')
  else:
    return os.path.join(chrome_root, 'base', 'logging.cc')


def GetBuildableSourceFile(chrome_root, filename):
  """Returns a buildable source file corresponding to |filename|.

  A buildable source file is one which is likely to be passed into clang as a
  source file during the build. For .h files, returns the closest matching .cc,
  .cpp or .c file. If no such file is found, returns the same as
  GetDefaultSourceFile().

  Args:
    chrome_root: (String) Absolute path to the root of Chromium checkout.
    filename: (String) Absolute path to the target source file.

  Returns:
    (String) Absolute path to source file.
  """
  if filename.endswith('.h'):
    # Header files can't be built. Instead, try to match a header file to its
    # corresponding source file.
    alternates = ['.cc', '.cpp', '.c']
    for alt_extension in alternates:
      alt_name = filename[:-2] + alt_extension
      if os.path.exists(alt_name):
        return alt_name

    return GetDefaultSourceFile(chrome_root, filename)

  return filename


def GetNinjaBuildOutputsForSourceFile(out_dir, filename):
  """Returns a list of build outputs for filename.

  The list is generated by invoking 'ninja -t query' tool to retrieve a list of
  inputs and outputs of |filename|. This list is then filtered to only include
  .o and .obj outputs.

  Args:
    out_dir: (String) Absolute path to ninja build output directory.
    filename: (String) Absolute path to source file.

  Returns:
    (List of Strings) List of target names. Will return [] if |filename| doesn't
        yield any .o or .obj outputs.
  """
  # Ninja needs the path to the source file relative to the output build
  # directory.
  rel_filename = os.path.relpath(os.path.realpath(filename), out_dir)

  p = subprocess.Popen(['ninja', '-C', out_dir, '-t', 'query', rel_filename],
                       stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  stdout, _ = p.communicate()
  if p.returncode:
    return []

  # The output looks like:
  #   ../../relative/path/to/source.cc:
  #     outputs:
  #       obj/reative/path/to/target.source.o
  #       obj/some/other/target2.source.o
  #       another/target.txt
  #
  outputs_text = stdout.partition('\n  outputs:\n')[2]
  output_lines = [line.strip() for line in outputs_text.split('\n')]
  return [target for target in output_lines
          if target and (target.endswith('.o') or target.endswith('.obj'))]


def GetClangCommandLineForNinjaOutput(out_dir, build_target):
  """Returns the Clang command line for building |build_target|

  Asks ninja for the list of commands used to build |filename| and returns the
  final Clang invocation.

  Args:
    out_dir: (String) Absolute path to ninja build output directory.
    build_target: (String) A build target understood by ninja

  Returns:
    (String or None) Clang command line or None if a Clang command line couldn't
        be determined.
  """
  p = subprocess.Popen(['ninja', '-v', '-C', out_dir,
                        '-t', 'commands', build_target],
                       stdout=subprocess.PIPE)
  stdout, stderr = p.communicate()
  if p.returncode:
    return None

  # Ninja will return multiple build steps for all dependencies up to
  # |build_target|. The build step we want is the last Clang invocation, which
  # is expected to be the one that outputs |build_target|.
  for line in reversed(stdout.split('\n')):
    if 'clang' in line:
      return line
  return None


def GetClangCommandLineFromNinjaForSource(out_dir, filename):
  """Returns a Clang command line used to build |filename|.

  The same source file could be built multiple times using different tool
  chains. In such cases, this command returns the first Clang invocation. We
  currently don't prefer one toolchain over another. Hopefully the tool chain
  corresponding to the Clang command line is compatible with the Clang build
  used by YCM.

  Args:
    out_dir: (String) Absolute path to Chromium checkout.
    filename: (String) Absolute path to source file.

  Returns:
    (String or None): Command line for Clang invocation using |filename| as a
        source. Returns None if no such command line could be found.
  """
  build_targets = GetNinjaBuildOutputsForSourceFile(out_dir, filename)
  for build_target in build_targets:
    command_line = GetClangCommandLineForNinjaOutput(out_dir, build_target)
    if command_line:
      return command_line
  return None


def GetNormalizedClangCommand(command, out_dir):
  """Gets the normalized Clang binary path if |command| is a Clang command.

  Args:
    command: (String) Clang command.
    out_dir: (String) Absolute path the ninja build directory.

  Returns:
    (String or None)
      None : if command is not a clang command.
      Absolute path to clang binary : if |command| is an absolute or relative
          path to clang. If relative, it is assumed to be relative to |out_dir|.
      |command|: if command is a name of a binary.
  """
  if command.endswith('clang++') or command.endswith('clang'):
    if os.path.basename(command) == command:
      return command
    return os.path.normpath(os.path.join(out_dir, command))
  return None


def GetClangOptionsFromCommandLine(clang_commandline, out_dir,
                                   additional_flags):
  """Extracts relevant command line options from |clang_commandline|

  Args:
    clang_commandline: (String) Full Clang invocation.
    out_dir: (String) Absolute path to ninja build directory. Relative paths in
        the command line are relative to |out_dir|.
    additional_flags: (List of String) Additional flags to return.

  Returns:
    ((List of Strings), (List of Strings)) The first item in the tuple is a list
    of command line flags for this source file. The second item in the tuple is
    a list of command line flags that define the system include paths. Either or
    both can be empty.
  """
  chrome_flags = [] + additional_flags
  system_include_flags = []

  # Parse flags that are important for YCM's purposes.
  clang_tokens = shlex.split(clang_commandline)
  for flag in clang_tokens:
    if flag.startswith('-I'):
      # Relative paths need to be resolved, because they're relative to the
      # output dir, not the source.
      if flag[2] == '/':
        chrome_flags.append(flag)
      else:
        abs_path = os.path.normpath(os.path.join(out_dir, flag[2:]))
        chrome_flags.append('-I' + abs_path)
    elif flag.startswith('-std'):
      chrome_flags.append(flag)
    elif flag.startswith('-') and flag[1] in 'DWFfmO':
      if flag == '-Wno-deprecated-register' or flag == '-Wno-header-guard':
        # These flags causes libclang (3.3) to crash. Remove it until things
        # are fixed.
        continue
      chrome_flags.append(flag)

  # Assume that the command for invoking clang++ looks like one of the
  # following:
  #   1) /path/to/clang/clang++ arguments
  #   2) /some/wrapper /path/to/clang++ arguments
  #
  # We'll look at the first two tokens on the command line to see if they look
  # like Clang commands, and if so use it to determine the system include
  # directory flags.
  for command in clang_tokens[0:2]:
    normalized_command = GetNormalizedClangCommand(command, out_dir)
    if normalized_command:
      system_include_flags += SystemIncludeDirectoryFlags(normalized_command,
                                                          chrome_flags)
      break

  return (chrome_flags, system_include_flags)


def GetClangOptionsFromNinjaForFilename(chrome_root, filename):
  """Returns the Clang command line options needed for building |filename|.

  Command line options are based on the command used by ninja for building
  |filename|. If |filename| is a .h file, uses its companion .cc or .cpp file.
  If a suitable companion file can't be located or if ninja doesn't know about
  |filename|, then uses default source files in Blink and Chromium for
  determining the commandline.

  Args:
    chrome_root: (String) Path to src/.
    filename: (String) Absolute path to source file being edited.

  Returns:
    ((List of Strings), (List of Strings)) The first item in the tuple is a list
    of command line flags for this source file. The second item in the tuple is
    a list of command line flags that define the system include paths. Either or
    both can be empty.
  """
  if not chrome_root:
    return ([],[])

  # Generally, everyone benefits from including Chromium's src/, because all of
  # Chromium's includes are relative to that.
  additional_flags = ['-I' + os.path.join(chrome_root)]

  # Version of Clang used to compile Chromium can be newer then version of
  # libclang that YCM uses for completion. So it's possible that YCM's libclang
  # doesn't know about some used warning options, which causes compilation
  # warnings (and errors, because of '-Werror');
  additional_flags.append('-Wno-unknown-warning-option')

  sys.path.append(os.path.join(chrome_root, 'tools', 'vim'))
  from ninja_output import GetNinjaOutputDirectory
  out_dir = os.path.realpath(GetNinjaOutputDirectory(chrome_root))

  clang_line = GetClangCommandLineFromNinjaForSource(
      out_dir, GetBuildableSourceFile(chrome_root, filename))
  if not clang_line:
    # If ninja didn't know about filename or it's companion files, then try a
    # default build target. It is possible that the file is new, or build.ninja
    # is stale.
    clang_line = GetClangCommandLineFromNinjaForSource(
        out_dir, GetDefaultSourceFile(chrome_root, filename))

  if not clang_line:
    return (additional_flags, [])

  return GetClangOptionsFromCommandLine(clang_line, out_dir, additional_flags)


def FlagsForFile(filename):
  """This is the main entry point for YCM. Its interface is fixed.

  Args:
    filename: (String) Path to source file being edited.

  Returns:
    (Dictionary)
      'flags': (List of Strings) Command line flags.
      'do_cache': (Boolean) True if the result should be cached.
  """
  abs_filename = os.path.abspath(filename)
  chrome_root = FindChromeSrcFromFilename(abs_filename)
  (chrome_flags, system_include_flags) = GetClangOptionsFromNinjaForFilename(
      chrome_root, abs_filename)

  # If either chrome_flags or system_include_flags could not be determined, then
  # assume that was due to a transient failure. Preventing YCM from caching the
  # flags allows us to try to determine the flags again.
  should_cache_flags_for_file = \
      bool(chrome_flags) and bool(system_include_flags)

  if not system_include_flags:
    system_include_flags = FallbackSystemIncludeDirectoryFlags()
  final_flags = _default_flags + chrome_flags + system_include_flags

  return {
    'flags': final_flags,
    'do_cache': should_cache_flags_for_file
  }