Move libcxx/test/libcxx python package into libcxx/utils/libcxx.

This patch merges the test python package with the newly
created package in utils.

git-svn-id: https://llvm.org/svn/llvm-project/libcxx/trunk@294651 91177308-0d34-0410-b5e6-96231b3b80d8

test/libcxx/util.py

@@ -1,55 +0,0 @@
-#===----------------------------------------------------------------------===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is dual licensed under the MIT and the University of Illinois Open
-# Source Licenses. See LICENSE.TXT for details.
-#
-#===----------------------------------------------------------------------===##
-
-from contextlib import contextmanager
-import os
-import tempfile
-
-
-def cleanFile(filename):
-    try:
-        os.remove(filename)
-    except OSError:
-        pass
-
-
-@contextmanager
-def guardedTempFilename(suffix='', prefix='', dir=None):
-    # Creates and yeilds a temporary filename within a with statement. The file
-    # is removed upon scope exit.
-    handle, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir)
-    os.close(handle)
-    yield name
-    cleanFile(name)
-
-
-@contextmanager
-def guardedFilename(name):
-    # yeilds a filename within a with statement. The file is removed upon scope
-    # exit.
-    yield name
-    cleanFile(name)
-
-
-@contextmanager
-def nullContext(value):
-    # yeilds a variable within a with statement. No action is taken upon scope
-    # exit.
-    yield value
-
-
-def makeReport(cmd, out, err, rc):
-    report = "Command: %s\n" % cmd
-    report += "Exit Code: %d\n" % rc
-    if out:
-        report += "Standard Output:\n--\n%s--\n" % out
-    if err:
-        report += "Standard Error:\n--\n%s--\n" % err
-    report += '\n'
-    return report

test/lit.cfg

@@ -3,7 +3,7 @@
 import os
 import site
 
-site.addsitedir(os.path.dirname(__file__))
+site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'utils'))
 
 
 # Tell pylint that we know config and lit_config exist somewhere.

utils/libcxx/compiler.py

@@ -9,7 +9,6 @@
 
 import platform
 import os
-import lit.util
 import libcxx.util
 
 
@@ -154,26 +153,26 @@ class CXXCompiler(object):
 
     def preprocess(self, source_files, out=None, flags=[], cwd=None):
         cmd = self.preprocessCmd(source_files, out, flags)
-        out, err, rc = lit.util.executeCommand(cmd, env=self.compile_env,
+        out, err, rc = libcxx.util.executeCommand(cmd, env=self.compile_env,
                                                   cwd=cwd)
         return cmd, out, err, rc
 
     def compile(self, source_files, out=None, flags=[], cwd=None):
         cmd = self.compileCmd(source_files, out, flags)
-        out, err, rc = lit.util.executeCommand(cmd, env=self.compile_env,
+        out, err, rc = libcxx.util.executeCommand(cmd, env=self.compile_env,
                                                   cwd=cwd)
         return cmd, out, err, rc
 
     def link(self, source_files, out=None, flags=[], cwd=None):
         cmd = self.linkCmd(source_files, out, flags)
-        out, err, rc = lit.util.executeCommand(cmd, env=self.compile_env,
+        out, err, rc = libcxx.util.executeCommand(cmd, env=self.compile_env,
                                                   cwd=cwd)
         return cmd, out, err, rc
 
     def compileLink(self, source_files, out=None, flags=[],
                     cwd=None):
         cmd = self.compileLinkCmd(source_files, out, flags)
-        out, err, rc = lit.util.executeCommand(cmd, env=self.compile_env,
+        out, err, rc = libcxx.util.executeCommand(cmd, env=self.compile_env,
                                                   cwd=cwd)
         return cmd, out, err, rc
 
@@ -216,7 +215,7 @@ class CXXCompiler(object):
 
     def getTriple(self):
         cmd = [self.path] + self.flags + ['-dumpmachine']
-        return lit.util.capture(cmd).strip()
+        return libcxx.util.capture(cmd).strip()
 
     def hasCompileFlag(self, flag):
         if isinstance(flag, list):
@@ -275,8 +274,8 @@ class CXXCompiler(object):
         # TODO(EricWF): Are there other flags we need to worry about?
         if '-v' in cmd:
             cmd.remove('-v')
-        out, err, rc = lit.util.executeCommand(
+        out, err, rc = libcxx.util.executeCommand(
-            cmd, input=lit.util.to_bytes('#error\n'))
+            cmd, input=libcxx.util.to_bytes('#error\n'))
 
         assert rc != 0
         if flag in err:

utils/libcxx/sym_check/extract.py

@@ -14,6 +14,7 @@ import distutils.spawn
 import sys
 import re
 
+import libcxx.util
 from libcxx.sym_check import util
 
 extract_ignore_names = ['_init', '_fini']
@@ -48,7 +49,7 @@ class NMExtractor(object):
         parsed symbols.
         """
         cmd = [self.nm_exe] + self.flags + [lib]
-        out, _, exit_code = util.execute_command_verbose(cmd)
+        out, _, exit_code = util.executeCommandVerbose(cmd)
         if exit_code != 0:
             raise RuntimeError('Failed to run %s on %s' % (self.nm_exe, lib))
         fmt_syms = (self._extract_sym(l)
@@ -134,7 +135,7 @@ class ReadElfExtractor(object):
         parsed symbols.
         """
         cmd = [self.tool] + self.flags + [lib]
-        out, _, exit_code = util.execute_command_verbose(cmd)
+        out, _, exit_code = libcxx.util.executeCommandVerbose(cmd)
         if exit_code != 0:
             raise RuntimeError('Failed to run %s on %s' % (self.nm_exe, lib))
         dyn_syms = self.get_dynsym_table(out)

utils/libcxx/sym_check/util.py

@@ -9,64 +9,9 @@
 
 import ast
 import distutils.spawn
-import signal
-import subprocess
 import sys
 import re
-
+import libcxx.util
-def to_bytes(str):
-    # Encode to UTF-8 to get binary data.
-    if isinstance(str, bytes):
-        return str
-    return str.encode('utf-8')
-
-def to_string(bytes):
-    if isinstance(bytes, str):
-        return bytes
-    return to_bytes(bytes)
-
-def convert_string(bytes):
-    try:
-        return to_string(bytes.decode('utf-8'))
-    except AttributeError: # 'str' object has no attribute 'decode'.
-        return str(bytes)
-    except UnicodeError:
-        return str(bytes)
-
-def execute_command(cmd, input_str=None):
-    """
-    Execute a command, capture and return its output.
-    """
-    kwargs = {
-        'stdin': subprocess.PIPE,
-        'stdout': subprocess.PIPE,
-        'stderr': subprocess.PIPE,
-    }
-    p = subprocess.Popen(cmd, **kwargs)
-    out, err = p.communicate(input=input_str)
-    exitCode = p.wait()
-    if exitCode == -signal.SIGINT:
-        raise KeyboardInterrupt
-    out = convert_string(out)
-    err = convert_string(err)
-    return out, err, exitCode
-
-
-def execute_command_verbose(cmd, input_str=None):
-    """
-    Execute a command and print its output on failure.
-    """
-    out, err, exitCode = execute_command(cmd, input_str=input_str)
-    if exitCode != 0:
-        report = "Command: %s\n" % ' '.join(["'%s'" % a for a in cmd])
-        report += "Exit Code: %d\n" % exitCode
-        if out:
-            report += "Standard Output:\n--\n%s--" % out
-        if err:
-            report += "Standard Error:\n--\n%s--" % err
-        report += "\n\nFailed!"
-        sys.stderr.write('%s\n' % report)
-    return out, err, exitCode
 
 
 def read_syms_from_list(slist):
@@ -118,8 +63,8 @@ _cppfilt_exe = distutils.spawn.find_executable('c++filt')
 def demangle_symbol(symbol):
     if _cppfilt_exe is None:
         return symbol
-    out, _, exit_code = execute_command_verbose(
+    out, _, exit_code = libcxx.util.executeCommandVerbose(
-        [_cppfilt_exe], input_str=symbol)
+        [_cppfilt_exe], input=symbol)
     if exit_code != 0:
         return symbol
     return out

utils/libcxx/test/config.py

@@ -17,10 +17,6 @@ import shlex
 import shutil
 import sys
 
-import lit.Test  # pylint: disable=import-error,no-name-in-module
-import lit.util  # pylint: disable=import-error,no-name-in-module
-
-from libcxx.test.format import LibcxxTestFormat
 from libcxx.compiler import CXXCompiler
 from libcxx.test.target_info import make_target_info
 from libcxx.test.executor import *
@@ -164,6 +160,7 @@ class Configuration(object):
         sys.stderr.flush()  # Force flushing to avoid broken output on Windows
 
     def get_test_format(self):
+        from libcxx.test.format import LibcxxTestFormat
         return LibcxxTestFormat(
             self.cxx,
             self.use_clang_verify,
@@ -200,7 +197,7 @@ class Configuration(object):
         # If no specific cxx_under_test was given, attempt to infer it as
         # clang++.
         if cxx is None or self.cxx_is_clang_cl:
-            clangxx = lit.util.which('clang++',
+            clangxx = libcxx.util.which('clang++',
                                      self.config.environment['PATH'])
             if clangxx:
                 cxx = clangxx
@@ -799,12 +796,12 @@ class Configuration(object):
             # Search for llvm-symbolizer along the compiler path first
             # and then along the PATH env variable.
             symbolizer_search_paths = os.environ.get('PATH', '')
-            cxx_path = lit.util.which(self.cxx.path)
+            cxx_path = libcxx.util.which(self.cxx.path)
             if cxx_path is not None:
                 symbolizer_search_paths = (
                     os.path.dirname(cxx_path) +
                     os.pathsep + symbolizer_search_paths)
-            llvm_symbolizer = lit.util.which('llvm-symbolizer',
+            llvm_symbolizer = libcxx.util.which('llvm-symbolizer',
                                                 symbolizer_search_paths)
 
             def add_ubsan():

utils/libcxx/test/executor.py

@@ -11,8 +11,7 @@ import platform
 import os
 
 from libcxx.test import tracing
-
+from libcxx.util import executeCommand
-from lit.util import executeCommand  # pylint: disable=import-error
 
 
 class Executor(object):

utils/libcxx/test/format.py

@@ -17,8 +17,6 @@ import lit.Test        # pylint: disable=import-error
 import lit.TestRunner  # pylint: disable=import-error
 from lit.TestRunner import ParserKind, IntegratedTestKeywordParser  \
     # pylint: disable=import-error
-import lit.util        # pylint: disable=import-error
-
 
 from libcxx.test.executor import LocalExecutor as LocalExecutor
 import libcxx.util
@@ -164,7 +162,7 @@ class LibcxxTestFormat(object):
         exec_path = tmpBase + '.exe'
         object_path = tmpBase + '.o'
         # Create the output directory if it does not already exist.
-        lit.util.mkdir_p(os.path.dirname(tmpBase))
+        libcxx.util.mkdir_p(os.path.dirname(tmpBase))
         try:
             # Compile the test
             cmd, out, err, rc = test_cxx.compileLinkTwoSteps(

utils/libcxx/test/target_info.py

@@ -221,4 +221,3 @@ def make_target_info(full_config):
     if target_system == 'Linux':   return LinuxLocalTI(full_config)
     if target_system == 'Windows': return WindowsLocalTI(full_config)
     return DefaultTargetInfo(full_config)
-

utils/libcxx/util.py

@@ -0,0 +1,285 @@
+#===----------------------------------------------------------------------===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is dual licensed under the MIT and the University of Illinois Open
+# Source Licenses. See LICENSE.TXT for details.
+#
+#===----------------------------------------------------------------------===##
+
+from contextlib import contextmanager
+import os
+import platform
+import signal
+import subprocess
+import sys
+import tempfile
+
+
+
+# FIXME: Most of these functions are cribbed from LIT
+def to_bytes(str):
+    # Encode to UTF-8 to get binary data.
+    if isinstance(str, bytes):
+        return str
+    return str.encode('utf-8')
+
+def to_string(bytes):
+    if isinstance(bytes, str):
+        return bytes
+    return to_bytes(bytes)
+
+def convert_string(bytes):
+    try:
+        return to_string(bytes.decode('utf-8'))
+    except AttributeError: # 'str' object has no attribute 'decode'.
+        return str(bytes)
+    except UnicodeError:
+        return str(bytes)
+
+
+def cleanFile(filename):
+    try:
+        os.remove(filename)
+    except OSError:
+        pass
+
+
+@contextmanager
+def guardedTempFilename(suffix='', prefix='', dir=None):
+    # Creates and yeilds a temporary filename within a with statement. The file
+    # is removed upon scope exit.
+    handle, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir)
+    os.close(handle)
+    yield name
+    cleanFile(name)
+
+
+@contextmanager
+def guardedFilename(name):
+    # yeilds a filename within a with statement. The file is removed upon scope
+    # exit.
+    yield name
+    cleanFile(name)
+
+
+@contextmanager
+def nullContext(value):
+    # yeilds a variable within a with statement. No action is taken upon scope
+    # exit.
+    yield value
+
+
+def makeReport(cmd, out, err, rc):
+    report = "Command: %s\n" % cmd
+    report += "Exit Code: %d\n" % rc
+    if out:
+        report += "Standard Output:\n--\n%s--\n" % out
+    if err:
+        report += "Standard Error:\n--\n%s--\n" % err
+    report += '\n'
+    return report
+
+
+def capture(args, env=None):
+    """capture(command) - Run the given command (or argv list) in a shell and
+    return the standard output. Raises a CalledProcessError if the command
+    exits with a non-zero status."""
+    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                         env=env)
+    out, err = p.communicate()
+    out = convert_string(out)
+    err = convert_string(err)
+    if p.returncode != 0:
+        raise subprocess.CalledProcessError(cmd=args,
+                                            returncode=p.returncode,
+                                            output="{}\n{}".format(out, err))
+    return out
+
+
+def which(command, paths = None):
+    """which(command, [paths]) - Look up the given command in the paths string
+    (or the PATH environment variable, if unspecified)."""
+
+    if paths is None:
+        paths = os.environ.get('PATH','')
+
+    # Check for absolute match first.
+    if os.path.isfile(command):
+        return command
+
+    # Would be nice if Python had a lib function for this.
+    if not paths:
+        paths = os.defpath
+
+    # Get suffixes to search.
+    # On Cygwin, 'PATHEXT' may exist but it should not be used.
+    if os.pathsep == ';':
+        pathext = os.environ.get('PATHEXT', '').split(';')
+    else:
+        pathext = ['']
+
+    # Search the paths...
+    for path in paths.split(os.pathsep):
+        for ext in pathext:
+            p = os.path.join(path, command + ext)
+            if os.path.exists(p) and not os.path.isdir(p):
+                return p
+
+    return None
+
+
+def checkToolsPath(dir, tools):
+    for tool in tools:
+        if not os.path.exists(os.path.join(dir, tool)):
+            return False
+    return True
+
+
+def whichTools(tools, paths):
+    for path in paths.split(os.pathsep):
+        if checkToolsPath(path, tools):
+            return path
+    return None
+
+def mkdir_p(path):
+    """mkdir_p(path) - Make the "path" directory, if it does not exist; this
+    will also make directories for any missing parent directories."""
+    if not path or os.path.exists(path):
+        return
+
+    parent = os.path.dirname(path)
+    if parent != path:
+        mkdir_p(parent)
+
+    try:
+        os.mkdir(path)
+    except OSError:
+        e = sys.exc_info()[1]
+        # Ignore EEXIST, which may occur during a race condition.
+        if e.errno != errno.EEXIST:
+            raise
+
+
+class ExecuteCommandTimeoutException(Exception):
+    def __init__(self, msg, out, err, exitCode):
+        assert isinstance(msg, str)
+        assert isinstance(out, str)
+        assert isinstance(err, str)
+        assert isinstance(exitCode, int)
+        self.msg = msg
+        self.out = out
+        self.err = err
+        self.exitCode = exitCode
+
+# Close extra file handles on UNIX (on Windows this cannot be done while
+# also redirecting input).
+kUseCloseFDs = not (platform.system() == 'Windows')
+def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
+    """
+        Execute command ``command`` (list of arguments or string)
+        with
+        * working directory ``cwd`` (str), use None to use the current
+          working directory
+        * environment ``env`` (dict), use None for none
+        * Input to the command ``input`` (str), use string to pass
+          no input.
+        * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
+
+        Returns a tuple (out, err, exitCode) where
+        * ``out`` (str) is the standard output of running the command
+        * ``err`` (str) is the standard error of running the command
+        * ``exitCode`` (int) is the exitCode of running the command
+
+        If the timeout is hit an ``ExecuteCommandTimeoutException``
+        is raised.
+    """
+    if input is not None:
+        input = to_bytes(input)
+    p = subprocess.Popen(command, cwd=cwd,
+                         stdin=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE,
+                         env=env, close_fds=kUseCloseFDs)
+    timerObject = None
+    # FIXME: Because of the way nested function scopes work in Python 2.x we
+    # need to use a reference to a mutable object rather than a plain
+    # bool. In Python 3 we could use the "nonlocal" keyword but we need
+    # to support Python 2 as well.
+    hitTimeOut = [False]
+    try:
+        if timeout > 0:
+            def killProcess():
+                # We may be invoking a shell so we need to kill the
+                # process and all its children.
+                hitTimeOut[0] = True
+                killProcessAndChildren(p.pid)
+
+            timerObject = threading.Timer(timeout, killProcess)
+            timerObject.start()
+
+        out,err = p.communicate(input=input)
+        exitCode = p.wait()
+    finally:
+        if timerObject != None:
+            timerObject.cancel()
+
+    # Ensure the resulting output is always of string type.
+    out = convert_string(out)
+    err = convert_string(err)
+
+    if hitTimeOut[0]:
+        raise ExecuteCommandTimeoutException(
+            msg='Reached timeout of {} seconds'.format(timeout),
+            out=out,
+            err=err,
+            exitCode=exitCode
+            )
+
+    # Detect Ctrl-C in subprocess.
+    if exitCode == -signal.SIGINT:
+        raise KeyboardInterrupt
+
+    return out, err, exitCode
+
+
+def killProcessAndChildren(pid):
+    """
+    This function kills a process with ``pid`` and all its
+    running children (recursively). It is currently implemented
+    using the psutil module which provides a simple platform
+    neutral implementation.
+
+    TODO: Reimplement this without using psutil so we can
+          remove our dependency on it.
+    """
+    import psutil
+    try:
+        psutilProc = psutil.Process(pid)
+        # Handle the different psutil API versions
+        try:
+            # psutil >= 2.x
+            children_iterator = psutilProc.children(recursive=True)
+        except AttributeError:
+            # psutil 1.x
+            children_iterator = psutilProc.get_children(recursive=True)
+        for child in children_iterator:
+            try:
+                child.kill()
+            except psutil.NoSuchProcess:
+                pass
+        psutilProc.kill()
+    except psutil.NoSuchProcess:
+        pass
+
+
+def executeCommandVerbose(cmd, *args, **kwargs):
+    """
+    Execute a command and print its output on failure.
+    """
+    out, err, exitCode = executeCommand(cmd, *args, **kwargs)
+    if exitCode != 0:
+        report = makeReport(cmd, out, err, exitCode)
+        report += "\n\nFailed!"
+        sys.stderr.write('%s\n' % report)
+    return out, err, exitCode