Copy some of the testing infrastructure from the primary Clang repository into

the new tools repo to preserve history. I'll subsequently be editting these
down to work w/ the tooling test suites.


git-svn-id: https://llvm.org/svn/llvm-project/clang-tools-extra/trunk@161402 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 446 insertions, 0 deletions

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
new file mode 100644
index 00000000..8184c3d3
--- /dev/null
+++ b/test/CMakeLists.txt
@@ -0,0 +1,93 @@
+# Test runner infrastructure for Clang. This configures the Clang test trees
+# for use by Lit, and delegates to LLVM's lit test handlers.
+#
+# If this is a stand-alone Clang build, we fake up our own Lit support here
+# rather than relying on LLVM's.
+
+set(CLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/..")
+set(CLANG_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/..")
+
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
+  )
+
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.in
+  ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg
+  )
+
+if( PATH_TO_LLVM_BUILD )
+  set(CLANG_TEST_EXTRA_ARGS "--path=${CLANG_BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}")
+endif()
+
+option(CLANG_TEST_USE_VG "Run Clang tests under Valgrind" OFF)
+if(CLANG_TEST_USE_VG)
+  set(CLANG_TEST_EXTRA_ARGS ${CLANG_TEST_EXTRA_ARGS} "--vg")
+endif ()
+
+if( NOT CLANG_BUILT_STANDALONE )
+
+  set(CLANG_TEST_DEPS
+    clang clang-headers
+    c-index-test diagtool arcmt-test c-arcmt-test
+    clang-check
+    llvm-dis llc opt FileCheck count not
+    )
+  set(CLANG_TEST_PARAMS
+    clang_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
+    )
+
+  if(LLVM_INCLUDE_TESTS)
+    list(APPEND CLANG_TEST_DEPS ClangUnitTests)
+    list(APPEND CLANG_TEST_PARAMS
+      clang_unit_site_config=${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg
+      )
+  endif()
+  add_lit_testsuite(check-clang "Running the Clang regression tests"
+    ${CMAKE_CURRENT_BINARY_DIR}
+    PARAMS ${CLANG_TEST_PARAMS}
+    DEPENDS ${CLANG_TEST_DEPS}
+    ARGS ${CLANG_TEST_EXTRA_ARGS}
+    )
+  set_target_properties(check-clang PROPERTIES FOLDER "Clang tests")
+
+else()
+
+  include(FindPythonInterp)
+  if(PYTHONINTERP_FOUND)
+    if( LLVM_MAIN_SRC_DIR )
+      set(LIT "${LLVM_MAIN_SRC_DIR}/utils/lit/lit.py")
+    else()
+      set(LIT "${PATH_TO_LLVM_BUILD}/bin/${CMAKE_CFG_INTDIR}/llvm-lit")
+      # Installed LLVM does not contain ${CMAKE_CFG_INTDIR} in paths.
+      if( NOT EXISTS ${LIT} )
+        set(LIT "${PATH_TO_LLVM_BUILD}/bin/llvm-lit")
+      endif()
+    endif()
+
+    set(LIT_ARGS "${CLANG_TEST_EXTRA_ARGS} ${LLVM_LIT_ARGS}")
+    separate_arguments(LIT_ARGS)
+
+    add_custom_target(check-clang
+      COMMAND ${PYTHON_EXECUTABLE}
+              ${LIT}
+              --param clang_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
+              --param build_config=${CMAKE_CFG_INTDIR}
+              --param build_mode=${RUNTIME_BUILD_MODE}
+              ${LIT_ARGS}
+              ${CMAKE_CURRENT_BINARY_DIR}
+      COMMENT "Running Clang regression tests"
+      DEPENDS clang clang-headers
+              c-index-test diagtool arcmt-test c-arcmt-test
+              clang-check
+      )
+    set_target_properties(check-clang PROPERTIES FOLDER "Clang tests")
+  endif()
+
+endif()
+
+# Add a legacy target spelling: clang-test
+add_custom_target(clang-test)
+add_dependencies(clang-test check-clang)
+set_target_properties(clang-test PROPERTIES FOLDER "Clang tests")
diff --git a/test/Makefile b/test/Makefile
new file mode 100644
index 00000000..4fdafe14
--- /dev/null
+++ b/test/Makefile
@@ -0,0 +1,71 @@
+CLANG_LEVEL := ..
+include $(CLANG_LEVEL)/Makefile
+
+# Test in all immediate subdirectories if unset.
+ifdef TESTSUITE
+TESTDIRS := $(TESTSUITE:%=$(PROJ_SRC_DIR)/%)
+else
+TESTDIRS ?= $(PROJ_SRC_DIR)
+endif
+
+# 'lit' wants objdir paths, so it will pick up the lit.site.cfg.
+TESTDIRS := $(TESTDIRS:$(PROJ_SRC_DIR)%=$(PROJ_OBJ_DIR)%)
+
+# Allow EXTRA_TESTDIRS to provide additional test directories.
+TESTDIRS += $(EXTRA_TESTDIRS)
+
+ifndef TESTARGS
+ifdef VERBOSE
+TESTARGS = -v
+else
+TESTARGS = -s -v
+endif
+endif
+
+# Make sure any extra test suites can find the main site config.
+LIT_ARGS := --param clang_site_config=$(PROJ_OBJ_DIR)/lit.site.cfg
+
+ifdef VG
+  LIT_ARGS += "--vg"
+endif
+
+all:: lit.site.cfg Unit/lit.site.cfg
+	@ echo '--- Running clang tests for $(TARGET_TRIPLE) ---'
+	@ $(PYTHON) $(LLVM_SRC_ROOT)/utils/lit/lit.py \
+	  $(LIT_ARGS) $(TESTARGS) $(TESTDIRS)
+
+FORCE:
+
+lit.site.cfg: FORCE
+	@echo "Making Clang 'lit.site.cfg' file..."
+	@$(ECHOPATH) s=@LLVM_SOURCE_DIR@=$(LLVM_SRC_ROOT)=g > lit.tmp
+	@$(ECHOPATH) s=@LLVM_BINARY_DIR@=$(LLVM_OBJ_ROOT)=g >> lit.tmp
+	@$(ECHOPATH) s=@LLVM_TOOLS_DIR@=$(ToolDir)=g >> lit.tmp
+	@$(ECHOPATH) s=@LLVM_LIBS_DIR@=$(LibDir)=g >> lit.tmp
+	@$(ECHOPATH) s=@CLANG_SOURCE_DIR@=$(PROJ_SRC_DIR)/..=g >> lit.tmp
+	@$(ECHOPATH) s=@CLANG_BINARY_DIR@=$(PROJ_OBJ_DIR)/..=g >> lit.tmp
+	@$(ECHOPATH) s=@TARGET_TRIPLE@=$(TARGET_TRIPLE)=g >> lit.tmp
+	@sed -f lit.tmp $(PROJ_SRC_DIR)/lit.site.cfg.in > $@
+	@-rm -f lit.tmp
+
+Unit/lit.site.cfg: FORCE
+	@echo "Making Clang 'Unit/lit.site.cfg' file..."
+	@$(MKDIR) $(dir $@)
+	@$(ECHOPATH) s=@LLVM_SOURCE_DIR@=$(LLVM_SRC_ROOT)=g > unit.tmp
+	@$(ECHOPATH) s=@LLVM_BINARY_DIR@=$(LLVM_OBJ_ROOT)=g >> unit.tmp
+	@$(ECHOPATH) s=@LLVM_TOOLS_DIR@=$(ToolDir)=g >> unit.tmp
+	@$(ECHOPATH) s=@LLVM_LIBS_DIR@=$(LibDir)=g >> unit.tmp
+	@$(ECHOPATH) s=@CLANG_SOURCE_DIR@=$(PROJ_SRC_DIR)/..=g >> unit.tmp
+	@$(ECHOPATH) s=@CLANG_BINARY_DIR@=$(PROJ_OBJ_DIR)/..=g >> unit.tmp
+	@$(ECHOPATH) s=@TARGET_TRIPLE@=$(TARGET_TRIPLE)=g >> unit.tmp
+	@$(ECHOPATH) s=@LLVM_BUILD_MODE@=$(BuildMode)=g >> unit.tmp
+	@$(ECHOPATH) s=@ENABLE_SHARED@=$(ENABLE_SHARED)=g >> unit.tmp
+	@$(ECHOPATH) s=@SHLIBDIR@=$(SharedLibDir)=g >> unit.tmp
+	@$(ECHOPATH) s=@SHLIBPATH_VAR@=$(SHLIBPATH_VAR)=g >> unit.tmp
+	@sed -f unit.tmp $(PROJ_SRC_DIR)/Unit/lit.site.cfg.in > $@
+	@-rm -f unit.tmp
+
+clean::
+	@ find . -name Output | xargs rm -fr
+
+.PHONY: all report clean
diff --git a/test/lit.cfg b/test/lit.cfg
new file mode 100644
index 00000000..1fc60593
--- /dev/null
+++ b/test/lit.cfg
@@ -0,0 +1,261 @@
+# -*- Python -*-
+
+import os
+import platform
+import re
+import subprocess
+
+
+# Configuration file for the 'lit' test runner.
+
+# name: The name of this test suite.
+config.name = 'Clang'
+
+# Tweak PATH for Win32
+if platform.system() == 'Windows':
+    # Seek sane tools in directories and set to $PATH.
+    path = getattr(config, 'lit_tools_dir', None)
+    path = lit.getToolsPath(path,
+                            config.environment['PATH'],
+                            ['cmp.exe', 'grep.exe', 'sed.exe'])
+    if path is not None:
+        path = os.path.pathsep.join((path,
+                                     config.environment['PATH']))
+        config.environment['PATH'] = path
+
+# testFormat: The test format to use to interpret tests.
+#
+# For now we require '&&' between commands, until they get globally killed and
+# the test runner updated.
+execute_external = (platform.system() != 'Windows'
+                    or lit.getBashPath() not in [None, ""])
+config.test_format = lit.formats.ShTest(execute_external)
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.c', '.cpp', '.m', '.mm', '.cu', '.ll', '.cl', '.s']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# test_exec_root: The root path where tests should be run.
+clang_obj_root = getattr(config, 'clang_obj_root', None)
+if clang_obj_root is not None:
+    config.test_exec_root = os.path.join(clang_obj_root, 'test')
+
+# Set llvm_{src,obj}_root for use by others.
+config.llvm_src_root = getattr(config, 'llvm_src_root', None)
+config.llvm_obj_root = getattr(config, 'llvm_obj_root', None)
+
+# Clear some environment variables that might affect Clang.
+#
+# This first set of vars are read by Clang, but shouldn't affect tests
+# that aren't specifically looking for these features, or are required
+# simply to run the tests at all.
+#
+# FIXME: Should we have a tool that enforces this?
+
+# safe_env_vars = ('TMPDIR', 'TEMP', 'TMP', 'USERPROFILE', 'PWD',
+#                  'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET',
+#                  'IOS_SIMULATOR_DEPLOYMENT_TARGET',
+#                  'VCINSTALLDIR', 'VC100COMNTOOLS', 'VC90COMNTOOLS',
+#                  'VC80COMNTOOLS')
+possibly_dangerous_env_vars = ['COMPILER_PATH', 'RC_DEBUG_OPTIONS',
+                               'CINDEXTEST_PREAMBLE_FILE', 'LIBRARY_PATH',
+                               'CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH',
+                               'OBJC_INCLUDE_PATH', 'OBJCPLUS_INCLUDE_PATH',
+                               'LIBCLANG_TIMING', 'LIBCLANG_OBJTRACKING',
+                               'LIBCLANG_LOGGING', 'LIBCLANG_BGPRIO_INDEX',
+                               'LIBCLANG_BGPRIO_EDIT', 'LIBCLANG_NOTHREADS',
+                               'LIBCLANG_RESOURCE_USAGE',
+                               'LIBCLANG_CODE_COMPLETION_LOGGING']
+# Clang/Win32 may refer to %INCLUDE%. vsvarsall.bat sets it.
+if platform.system() != 'Windows':
+    possibly_dangerous_env_vars.append('INCLUDE')
+for name in possibly_dangerous_env_vars:
+  if name in config.environment:
+    del config.environment[name]
+
+# Tweak the PATH to include the tools dir and the scripts dir.
+if clang_obj_root is not None:
+    llvm_tools_dir = getattr(config, 'llvm_tools_dir', None)
+    if not llvm_tools_dir:
+        lit.fatal('No LLVM tools dir set!')
+    path = os.path.pathsep.join((llvm_tools_dir, config.environment['PATH']))
+    config.environment['PATH'] = path
+
+    llvm_libs_dir = getattr(config, 'llvm_libs_dir', None)
+    if not llvm_libs_dir:
+        lit.fatal('No LLVM libs dir set!')
+    path = os.path.pathsep.join((llvm_libs_dir,
+                                 config.environment.get('LD_LIBRARY_PATH','')))
+    config.environment['LD_LIBRARY_PATH'] = path
+
+###
+
+# Check that the object root is known.
+if config.test_exec_root is None:
+    # Otherwise, we haven't loaded the site specific configuration (the user is
+    # probably trying to run on a test file directly, and either the site
+    # configuration hasn't been created by the build system, or we are in an
+    # out-of-tree build situation).
+
+    # Check for 'clang_site_config' user parameter, and use that if available.
+    site_cfg = lit.params.get('clang_site_config', None)
+    if site_cfg and os.path.exists(site_cfg):
+        lit.load_config(config, site_cfg)
+        raise SystemExit
+
+    # Try to detect the situation where we are using an out-of-tree build by
+    # looking for 'llvm-config'.
+    #
+    # FIXME: I debated (i.e., wrote and threw away) adding logic to
+    # automagically generate the lit.site.cfg if we are in some kind of fresh
+    # build situation. This means knowing how to invoke the build system though,
+    # and I decided it was too much magic. We should solve this by just having
+    # the .cfg files generated during the configuration step.
+
+    llvm_config = lit.util.which('llvm-config', config.environment['PATH'])
+    if not llvm_config:
+        lit.fatal('No site specific configuration available!')
+
+    # Get the source and object roots.
+    llvm_src_root = lit.util.capture(['llvm-config', '--src-root']).strip()
+    llvm_obj_root = lit.util.capture(['llvm-config', '--obj-root']).strip()
+    clang_src_root = os.path.join(llvm_src_root, "tools", "clang")
+    clang_obj_root = os.path.join(llvm_obj_root, "tools", "clang")
+
+    # Validate that we got a tree which points to here, using the standard
+    # tools/clang layout.
+    this_src_root = os.path.dirname(config.test_source_root)
+    if os.path.realpath(clang_src_root) != os.path.realpath(this_src_root):
+        lit.fatal('No site specific configuration available!')
+
+    # Check that the site specific configuration exists.
+    site_cfg = os.path.join(clang_obj_root, 'test', 'lit.site.cfg')
+    if not os.path.exists(site_cfg):
+        lit.fatal('No site specific configuration available! You may need to '
+                  'run "make test" in your Clang build directory.')
+
+    # Okay, that worked. Notify the user of the automagic, and reconfigure.
+    lit.note('using out-of-tree build at %r' % clang_obj_root)
+    lit.load_config(config, site_cfg)
+    raise SystemExit
+
+###
+
+# Discover the 'clang' and 'clangcc' to use.
+
+import os
+
+def inferClang(PATH):
+    # Determine which clang to use.
+    clang = os.getenv('CLANG')
+
+    # If the user set clang in the environment, definitely use that and don't
+    # try to validate.
+    if clang:
+        return clang
+
+    # Otherwise look in the path.
+    clang = lit.util.which('clang', PATH)
+
+    if not clang:
+        lit.fatal("couldn't find 'clang' program, try setting "
+                  "CLANG in your environment")
+
+    return clang
+
+# When running under valgrind, we mangle '-vg' onto the end of the triple so we
+# can check it with XFAIL and XTARGET.
+if lit.useValgrind:
+    config.target_triple += '-vg'
+
+config.clang = inferClang(config.environment['PATH']).replace('\\', '/')
+if not lit.quiet:
+    lit.note('using clang: %r' % config.clang)
+
+# Note that when substituting %clang_cc1 also fill in the include directory of
+# the builtin headers. Those are part of even a freestanding environment, but
+# Clang relies on the driver to locate them.
+def getClangBuiltinIncludeDir(clang):
+    # FIXME: Rather than just getting the version, we should have clang print
+    # out its resource dir here in an easy to scrape form.
+    cmd = subprocess.Popen([clang, '-print-file-name=include'],
+                           stdout=subprocess.PIPE)
+    if not cmd.stdout:
+      lit.fatal("Couldn't find the include dir for Clang ('%s')" % clang)
+    return cmd.stdout.read().strip()
+
+config.substitutions.append( ('%clang_cc1', '%s -cc1 -internal-isystem %s'
+                              % (config.clang,
+                                 getClangBuiltinIncludeDir(config.clang))) )
+
+config.substitutions.append( ('%clangxx', ' ' + config.clang +
+                              ' -ccc-clang-cxx -ccc-cxx '))
+config.substitutions.append( ('%clang', ' ' + config.clang + ' ') )
+config.substitutions.append( ('%test_debuginfo', ' ' + config.llvm_src_root + '/utils/test_debuginfo.pl ') )
+
+# FIXME: Find nicer way to prohibit this.
+config.substitutions.append(
+    (' clang ', """*** Do not use 'clang' in tests, use '%clang'. ***""") )
+config.substitutions.append(
+    (' clang\+\+ ', """*** Do not use 'clang++' in tests, use '%clangxx'. ***"""))
+config.substitutions.append(
+    (' clang-cc ',
+     """*** Do not use 'clang-cc' in tests, use '%clang_cc1'. ***""") )
+config.substitutions.append(
+    (' clang -cc1 ',
+     """*** Do not use 'clang -cc1' in tests, use '%clang_cc1'. ***""") )
+config.substitutions.append(
+    (' %clang-cc1 ',
+     """*** invalid substitution, use '%clang_cc1'. ***""") )
+
+###
+
+# Set available features we allow tests to conditionalize on.
+#
+# As of 2011.08, crash-recovery tests still do not pass on FreeBSD.
+if platform.system() not in ['FreeBSD']:
+    config.available_features.add('crash-recovery')
+
+# Shell execution
+if platform.system() not in ['Windows'] or lit.getBashPath() != '':
+    config.available_features.add('shell')
+
+# ANSI escape sequences in non-dump terminal
+if platform.system() not in ['Windows']:
+    config.available_features.add('ansi-escape-sequences')
+
+# Registered Targets
+def get_llc_props(tool):
+    set_of_targets = set()
+    enable_assertions = False
+
+    cmd = subprocess.Popen([tool, '-version'], stdout=subprocess.PIPE)
+
+    # Parse the stdout to get the list of registered targets.
+    parse_targets = False
+    for line in cmd.stdout:
+        if parse_targets:
+            m = re.match( r'(.*) - ', line)
+            if m is not None:
+                set_of_targets.add(m.group(1).strip() + '-registered-target')
+            else:
+                break
+        elif "Registered Targets:" in line:
+            parse_targets = True
+
+        if re.search(r'with assertions', line):
+            enable_assertions = True
+
+    return {"set_of_targets":    set_of_targets,
+            "enable_assertions": enable_assertions}
+
+llc_props = get_llc_props(os.path.join(llvm_tools_dir, 'llc'))
+if len(llc_props['set_of_targets']) > 0:
+    config.available_features.update(llc_props['set_of_targets'])
+else:
+    lit.fatal('No Targets Registered with the LLVM Tools!')
+
+if llc_props['enable_assertions']:
+    config.available_features.add('asserts')
diff --git a/test/lit.site.cfg.in b/test/lit.site.cfg.in
new file mode 100644
index 00000000..df90b810
--- /dev/null
+++ b/test/lit.site.cfg.in
@@ -0,0 +1,21 @@
+## Autogenerated by LLVM/Clang configuration.
+# Do not edit!
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
+config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.clang_obj_root = "@CLANG_BINARY_DIR@"
+config.target_triple = "@TARGET_TRIPLE@"
+
+# Support substitution of the tools and libs dirs with user parameters. This is
+# used when we can't determine the tool dir at configuration time.
+try:
+    config.llvm_tools_dir = config.llvm_tools_dir % lit.params
+    config.llvm_libs_dir = config.llvm_libs_dir % lit.params
+except KeyError,e:
+    key, = e.args
+    lit.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+
+# Let the main config do the real work.
+lit.load_config(config, "@CLANG_SOURCE_DIR@/test/lit.cfg")