Move test/libcxx/android/ to utils/libcxx/android/

This got refactored upstream.

Test: checkbuild and ./run_tests.py
Change-Id: I801869923cf7c51528de354f8a3da705f188ff43

utils/libcxx/android/adb.py

@@ -0,0 +1,24 @@
+import lit.util  # pylint: disable=import-error
+
+
+class AdbError(RuntimeError):
+    def __init__(self, cmd, out, err, exit_code):
+        super(AdbError, self).__init__(err)
+        self.cmd = cmd
+        self.out = out
+        self.err = err
+        self.exit_code = exit_code
+
+
+def mkdir(path):
+    cmd = ['adb', 'shell', 'mkdir', path]
+    out, err, exit_code = lit.util.executeCommand(cmd)
+    if exit_code != 0:
+        raise AdbError(cmd, out, err, exit_code)
+
+
+def push(src, dst):
+    cmd = ['adb', 'push', src, dst]
+    out, err, exit_code = lit.util.executeCommand(cmd)
+    if exit_code != 0:
+        raise AdbError(cmd, out, err, exit_code)

utils/libcxx/android/build.py

@@ -0,0 +1,13 @@
+import os
+import subprocess
+
+
+def mm(path, android_build_top):
+    env = os.environ
+    env['ONE_SHOT_MAKEFILE'] = os.path.join(path, 'Android.mk')
+
+    cmd = [
+        'make', '-C', android_build_top, '-f', 'build/core/main.mk',
+        'MODULES-IN-' + path.replace('/', '-'), '-B'
+    ]
+    return not subprocess.Popen(cmd, stdout=None, stderr=None, env=env).wait()

utils/libcxx/android/compiler.py

@@ -0,0 +1,79 @@
+import copy
+import os
+import re
+import shlex
+import subprocess
+
+import libcxx.compiler
+
+
+class AndroidCXXCompiler(libcxx.compiler.CXXCompiler):
+    def __init__(self, cxx_under_test, cxx_template, link_template):
+        super(AndroidCXXCompiler, self).__init__(cxx_under_test)
+        self.cxx_template = cxx_template
+        self.link_template = link_template
+        self.build_top = os.getenv('ANDROID_BUILD_TOP')
+
+    def copy(self):
+        return copy.deepcopy(self)
+
+    def get_triple(self):
+        if 'clang' in self.path:
+            return self.get_clang_triple()
+        else:
+            return self.get_gcc_triple()
+
+        raise RuntimeError('Could not determine target triple.')
+
+    def get_clang_triple(self):
+        match = re.search(r'-target\s+(\S+)', self.cxx_template)
+        if match:
+            return match.group(1)
+        return None
+
+    def get_gcc_triple(self):
+        proc = subprocess.Popen([self.path, '-v'],
+                                stderr=subprocess.PIPE)
+        _, stderr = proc.communicate()
+        for line in stderr.split('\n'):
+            print 'Checking {}'.format(line)
+            match = re.search(r'^Target: (.+)$', line)
+            if match:
+                return match.group(1)
+        return None
+
+    def compile(self, source_files, out=None, flags=None, cwd=None):
+        flags = [] if flags is None else flags
+        return super(AndroidCXXCompiler, self).compile(source_files, out, flags,
+                                                       self.build_top)
+
+    def link(self, source_files, out=None, flags=None, cwd=None):
+        flags = [] if flags is None else flags
+        return super(AndroidCXXCompiler, self).link(source_files, out, flags,
+                                                    self.build_top)
+
+    def compileCmd(self, source_files, out=None, flags=None):
+        if out is None:
+            raise RuntimeError('The Android compiler requires an out path.')
+
+        if isinstance(source_files, str):
+            source_files = [source_files]
+        cxx_args = self.cxx_template.replace('%OUT%', out)
+        cxx_args = cxx_args.replace('%SOURCE%', ' '.join(source_files))
+        return [self.path] + shlex.split(cxx_args)
+
+    def linkCmd(self, source_files, out=None, flags=None):
+        if out is None:
+            raise RuntimeError('The Android compiler requires an out path.')
+
+        if isinstance(source_files, str):
+            source_files = [source_files]
+        link_args = self.link_template.replace('%OUT%', out)
+        link_args = link_args.replace('%SOURCE%', ' '.join(source_files))
+        return [self.path] + shlex.split(link_args)
+
+    def _basicCmd(self, source_files, out, is_link=False, input_is_cxx=False):
+        raise NotImplementedError()
+
+    def _initTypeAndVersion(self):
+        pass

utils/libcxx/android/executors.py

@@ -0,0 +1,65 @@
+import time
+
+import libcxx.test.executor
+
+from libcxx.android import adb
+from lit.util import executeCommand  # pylint: disable=import-error
+
+
+class AdbExecutor(libcxx.test.executor.RemoteExecutor):
+    def __init__(self, serial=None):
+        # TODO(danalbert): Should factor out the shared pieces of SSHExecutor
+        # so we don't have this ugly parent constructor...
+        super(AdbExecutor, self).__init__()
+        self.serial = serial
+        self.local_run = executeCommand
+
+    def _remote_temp(self, is_dir):
+        dir_arg = '-d' if is_dir else ''
+        cmd = 'mktemp -q {} /data/local/tmp/libcxx.XXXXXXXXXX'.format(dir_arg)
+        _, temp_path, err, exitCode = self._execute_command_remote([cmd])
+        temp_path = temp_path.strip()
+        if exitCode != 0:
+            raise RuntimeError(err)
+        return temp_path
+
+    def _copy_in_file(self, src, dst):  # pylint: disable=no-self-use
+        adb.push(src, dst)
+
+    def _execute_command_remote(self, cmd, remote_work_dir='.', env=None):
+        adb_cmd = ['adb', 'shell']
+        if self.serial:
+            adb_cmd.extend(['-s', self.serial])
+
+        delimiter = 'x'
+        probe_cmd = ' '.join(cmd) + '; echo {}$?'.format(delimiter)
+
+        env_cmd = []
+        if env is not None:
+            env_cmd = ['env'] + ['%s=%s' % (k, v) for k, v in env.items()]
+
+        remote_cmd = ' '.join(env_cmd + [probe_cmd])
+        if remote_work_dir != '.':
+            remote_cmd = 'cd {} && {}'.format(remote_work_dir, remote_cmd)
+
+        adb_cmd.append(remote_cmd)
+
+        # Tests will commonly fail with ETXTBSY. Possibly related to this bug:
+        # https://code.google.com/p/android/issues/detail?id=65857. Work around
+        # it by just waiting a second and then retrying.
+        for _ in range(10):
+            out, err, exit_code = self.local_run(adb_cmd)
+            if 'Text file busy' in out:
+                time.sleep(1)
+            else:
+                out, delim, rc_str = out.rpartition(delimiter)
+                if delim == '':
+                    continue
+
+                out = out.strip()
+                try:
+                    exit_code = int(rc_str)
+                    break
+                except ValueError:
+                    continue
+        return adb_cmd, out, err, exit_code

utils/libcxx/android/test/config.py

@@ -0,0 +1,88 @@
+import os
+import re
+
+import libcxx.test.config
+import libcxx.android.build
+import libcxx.android.compiler
+import libcxx.android.test.format
+
+
+class Configuration(libcxx.test.config.Configuration):
+    def __init__(self, lit_config, config):
+        super(Configuration, self).__init__(lit_config, config)
+        self.build_cmds_dir = None
+
+    def configure(self):
+        self.configure_src_root()
+        self.configure_build_cmds()
+        self.configure_obj_root()
+
+        self.configure_cxx()
+        self.configure_triple()
+        self.configure_features()
+
+    def print_config_info(self):
+        self.lit_config.note(
+            'Using compiler: {}'.format(self.cxx.path))
+        self.lit_config.note(
+            'Using compile template: {}'.format(self.cxx.cxx_template))
+        self.lit_config.note(
+            'Using link template: {}'.format(self.cxx.link_template))
+        self.lit_config.note('Using available_features: %s' %
+                             list(self.config.available_features))
+
+    def configure_obj_root(self):
+        test_config_file = os.path.join(self.build_cmds_dir, 'testconfig.mk')
+        if 'HOST_NATIVE_TEST' in open(test_config_file).read():
+            self.libcxx_obj_root = os.getenv('ANDROID_HOST_OUT')
+        else:
+            self.libcxx_obj_root = os.getenv('ANDROID_PRODUCT_OUT')
+
+    def configure_build_cmds(self):
+        os.chdir(self.config.android_root)
+        self.build_cmds_dir = 'external/libcxx/buildcmds'
+        if not libcxx.android.build.mm(self.build_cmds_dir,
+                                       self.config.android_root):
+            raise RuntimeError('Could not generate build commands.')
+
+    def configure_cxx(self):
+        cxx_under_test_file = os.path.join(self.build_cmds_dir,
+                                           'cxx_under_test')
+        cxx_under_test = open(cxx_under_test_file).read().strip()
+
+        cxx_template_file = os.path.join(self.build_cmds_dir, 'cxx.cmds')
+        cxx_template = open(cxx_template_file).read().strip()
+
+        link_template_file = os.path.join(self.build_cmds_dir, 'link.cmds')
+        link_template = open(link_template_file).read().strip()
+
+        self.cxx = libcxx.android.compiler.AndroidCXXCompiler(
+            cxx_under_test, cxx_template, link_template)
+
+    def configure_triple(self):
+        self.config.target_triple = self.cxx.get_triple()
+
+    def configure_features(self):
+        self.config.available_features.add('long_tests')
+        std_pattern = re.compile(r'-std=(c\+\+\d[0-9x-z])')
+        match = std_pattern.search(self.cxx.cxx_template)
+        if match:
+            self.config.available_features.add(match.group(1))
+
+    def get_test_format(self):
+        mode = self.lit_config.params.get('android_mode', 'device')
+        if mode == 'device':
+            return libcxx.android.test.format.TestFormat(
+                self.cxx,
+                self.libcxx_src_root,
+                self.libcxx_obj_root,
+                getattr(self.config, 'device_dir', '/data/local/tmp/'),
+                getattr(self.config, 'timeout', '60'))
+        elif mode == 'host':
+            return libcxx.android.test.format.HostTestFormat(
+                self.cxx,
+                self.libcxx_src_root,
+                self.libcxx_obj_root,
+                getattr(self.config, 'timeout', '60'))
+        else:
+            raise RuntimeError('Invalid android_mode: {}'.format(mode))

utils/libcxx/android/test/format.py

@@ -0,0 +1,91 @@
+import os
+
+import lit.util  # pylint: disable=import-error
+
+from libcxx.android.executors import AdbExecutor
+from libcxx.test.executor import LocalExecutor, TimeoutExecutor
+import libcxx.test.format
+import libcxx.android.adb as adb
+
+
+class HostTestFormat(libcxx.test.format.LibcxxTestFormat):
+    # pylint: disable=super-init-not-called
+    def __init__(self, cxx, libcxx_src_root, libcxx_obj_root, timeout,
+                 exec_env=None):
+        self.cxx = cxx
+        self.libcxx_src_root = libcxx_src_root
+        self.libcxx_obj_root = libcxx_obj_root
+        self.use_verify_for_fail = False
+        self.executor = TimeoutExecutor(timeout, LocalExecutor())
+
+        # We need to use LD_LIBRARY_PATH because the build system's rpath is
+        # relative, which won't work since we're running from /tmp. We can
+        # either scan `cxx_under_test`/`link_template` to determine whether
+        # we're 32-bit or 64-bit, scan testconfig.mk, or just add both
+        # directories and let the linker sort it out. I'm choosing the lazy
+        # option.
+        outdir = os.getenv('ANDROID_HOST_OUT')
+        libpath = os.pathsep.join([
+            os.path.join(outdir, 'lib'),
+            os.path.join(outdir, 'lib64'),
+        ])
+        default_env = {'LD_LIBRARY_PATH': libpath}
+        self.exec_env = default_env if exec_env is None else exec_env
+
+
+class TestFormat(HostTestFormat):
+    def __init__(self, cxx, libcxx_src_root, libcxx_obj_root, device_dir,
+                 timeout, exec_env=None):
+        HostTestFormat.__init__(
+            self,
+            cxx,
+            libcxx_src_root,
+            libcxx_obj_root,
+            timeout,
+            exec_env)
+        self.device_dir = device_dir
+        self.executor = TimeoutExecutor(timeout, AdbExecutor())
+
+    def _working_directory(self, file_name):
+        return os.path.join(self.device_dir, file_name)
+
+    def _wd_path(self, test_name, file_name):
+        return os.path.join(self._working_directory(test_name), file_name)
+
+    def _build(self, exec_path, source_path, compile_only=False,
+               use_verify=False):
+        # pylint: disable=protected-access
+        cmd, report, rc = libcxx.test.format.LibcxxTestFormat._build(
+            self, exec_path, source_path, compile_only, use_verify)
+        if rc != 0:
+            return cmd, report, rc
+
+        try:
+            exec_file = os.path.basename(exec_path)
+
+            adb.mkdir(self._working_directory(exec_file))
+            adb.push(exec_path, self._wd_path(exec_file, exec_file))
+
+            # Push any .dat files in the same directory as the source to the
+            # working directory.
+            src_dir = os.path.dirname(source_path)
+            data_files = [f for f in os.listdir(src_dir) if f.endswith('.dat')]
+            for data_file in data_files:
+                df_path = os.path.join(src_dir, data_file)
+                df_dev_path = self._wd_path(exec_file, data_file)
+                adb.push(df_path, df_dev_path)
+            return cmd, report, rc
+        except adb.AdbError as ex:
+            return self._make_report(ex.cmd, ex.out, ex.err, ex.exit_code)
+
+    def _clean(self, exec_path):
+        exec_file = os.path.basename(exec_path)
+        cmd = ['adb', 'shell', 'rm', '-rf', self._working_directory(exec_file)]
+        lit.util.executeCommand(cmd)
+        try:
+            os.remove(exec_path)
+        except OSError:
+            pass
+
+    def _run(self, exec_path, _, in_dir=None):
+        raise NotImplementedError()