diff --git a/scripts/cargo2android.py b/scripts/cargo2android.py new file mode 100755 index 000000000..c88536e5f --- /dev/null +++ b/scripts/cargo2android.py @@ -0,0 +1,1123 @@ +#!/usr/bin/env python +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Call cargo -v, parse its output, and generate Android.bp. + +Usage: Run this script in a crate workspace root directory. +The Cargo.toml file should work at least for the host platform. + +(1) Without other flags, "cargo2android.py --run" + calls cargo clean, calls cargo build -v, and generates Android.bp. + The cargo build only generates crates for the host, + without test crates. + +(2) To build crates for both host and device in Android.bp, use the + --device flag, for example: + cargo2android.py --run --device + + This is equivalent to using the --cargo flag to add extra builds: + cargo2android.py --run + --cargo "build" + --cargo "build --target x86_64-unknown-linux-gnu" + + On MacOS, use x86_64-apple-darwin as target triple. + Here the host target triple is used as a fake cross compilation target. + If the crate's Cargo.toml and environment configuration works for an + Android target, use that target triple as the cargo build flag. + +(3) To build default and test crates, for host and device, use both + --device and --tests flags: + cargo2android.py --run --device --tests + + This is equivalent to using the --cargo flag to add extra builds: + cargo2android.py --run + --cargo "build" + --cargo "build --tests" + --cargo "build --target x86_64-unknown-linux-gnu" + --cargo "build --tests --target x86_64-unknown-linux-gnu" + +Since Android rust builds by default treat all warnings as errors, +if there are rustc warning messages, this script will add +deny_warnings:false to the owner crate module in Android.bp. +""" + +from __future__ import print_function + +import argparse +import os +import os.path +import re + +RENAME_MAP = { + # This map includes all changes to the default rust library module + # names to resolve name conflicts or avoid confusion. + 'libbacktrace': 'libbacktrace_rust', + 'libgcc': 'libgcc_rust', + 'liblog': 'liblog_rust', + 'libsync': 'libsync_rust', + 'libx86_64': 'libx86_64_rust', +} + +# Header added to all generated Android.bp files. +ANDROID_BP_HEADER = '// This file is generated by cargo2android.py.\n' + +CARGO_OUT = 'cargo.out' # Name of file to keep cargo build -v output. + +TARGET_TMP = 'target.tmp' # Name of temporary output directory. + +# Message to be displayed when this script is called without the --run flag. +DRY_RUN_NOTE = ( + 'Dry-run: This script uses ./' + TARGET_TMP + ' for output directory,\n' + + 'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' + + 'and writes to Android.bp in the current and subdirectories.\n\n' + + 'To do do all of the above, use the --run flag.\n' + + 'See --help for other flags, and more usage notes in this script.\n') + +# Cargo -v output of a call to rustc. +RUSTC_PAT = re.compile('^ +Running `rustc (.*)`$') + +# Cargo -vv output of a call to rustc could be split into multiple lines. +# Assume that the first line will contain some CARGO_* env definition. +RUSTC_VV_PAT = re.compile('^ +Running `.*CARGO_.*=.*$') +# The combined -vv output rustc command line pattern. +RUSTC_VV_CMD_ARGS = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$') + +# Cargo -vv output of a "cc" or "ar" command; all in one line. +CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$') +# Some package, such as ring-0.13.5, has pattern '... running "cc"'. + +# Rustc output of file location path pattern for a warning message. +WARNING_FILE_PAT = re.compile('^ *--> ([^:]*):[0-9]+') + +# Rust package name with suffix -d1.d2.d3. +VERSION_SUFFIX_PAT = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+$') + + +def altered_name(name): + return RENAME_MAP[name] if (name in RENAME_MAP) else name + + +def is_build_crate_name(name): + # We added special prefix to build script crate names. + return name.startswith('build_script_') + + +def is_dependent_file_path(path): + # Absolute or dependent '.../' paths are not main files of this crate. + return path.startswith('/') or path.startswith('.../') + + +def get_module_name(crate): # to sort crates in a list + return crate.module_name + + +def pkg2crate_name(s): + return s.replace('-', '_').replace('.', '_') + + +def file_base_name(path): + return os.path.splitext(os.path.basename(path))[0] + + +def test_base_name(path): + return pkg2crate_name(file_base_name(path)) + + +def unquote(s): # remove quotes around str + if s and len(s) > 1 and s[0] == '"' and s[-1] == '"': + return s[1:-1] + return s + + +def remove_version_suffix(s): # remove -d1.d2.d3 suffix + if VERSION_SUFFIX_PAT.match(s): + return VERSION_SUFFIX_PAT.match(s).group(1) + return s + + +def short_out_name(pkg, s): # replace /.../pkg-*/out/* with .../out/* + return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s) + + +def escape_quotes(s): # replace '"' with '\\"' + return s.replace('"', '\\"') + + +class Crate(object): + """Information of a Rust crate to collect/emit for an Android.bp module.""" + + def __init__(self, runner, outf_name): + # Remembered global runner and its members. + self.runner = runner + self.debug = runner.args.debug + self.cargo_dir = '' # directory of my Cargo.toml + self.outf_name = outf_name # path to Android.bp + self.outf = None # open file handle of outf_name during dump* + # Variants/results that could be merged from multiple rustc lines. + self.host_supported = False + self.device_supported = False + self.has_warning = False + # Android module properties derived from rustc parameters. + self.module_name = '' # unique in Android build system + self.module_type = '' # rust_{binary,library,test}[_host] etc. + self.root_pkg = '' # parent package name of a sub/test packge, from -L + self.srcs = list() # main_src or merged multiple source files + self.stem = '' # real base name of output file + # Kept parsed status + self.errors = '' # all errors found during parsing + self.line_num = 1 # runner told input source line number + self.line = '' # original rustc command line parameters + # Parameters collected from rustc command line. + self.crate_name = '' # follows --crate-name + self.main_src = '' # follows crate_name parameter, shortened + self.crate_type = '' # bin|lib|test (see --test flag) + self.cfgs = list() # follows --cfg, without feature= prefix + self.features = list() # follows --cfg, name in 'feature="..."' + self.codegens = list() # follows -C, some ignored + self.externs = list() # follows --extern + self.core_externs = list() # first part of self.externs elements + self.static_libs = list() # e.g. -l static=host_cpuid + self.shared_libs = list() # e.g. -l dylib=wayland-client, -l z + self.cap_lints = '' # follows --cap-lints + self.emit_list = '' # e.g., --emit=dep-info,metadata,link + self.edition = '2015' # rustc default, e.g., --edition=2018 + self.target = '' # follows --target + + def write(self, s): + # convenient way to output one line at a time with EOL. + self.outf.write(s + '\n') + + def same_flags(self, other): + # host_supported, device_supported, has_warning are not compared but merged + # target is not compared, to merge different target/host modules + # externs is not compared; only core_externs is compared + return (not self.errors and not other.errors and + self.edition == other.edition and + self.cap_lints == other.cap_lints and + self.emit_list == other.emit_list and + self.core_externs == other.core_externs and + self.codegens == other.codegens and + self.features == other.features and + self.static_libs == other.static_libs and + self.shared_libs == other.shared_libs and self.cfgs == other.cfgs) + + def merge_host_device(self, other): + """Returns true if attributes are the same except host/device support.""" + return (self.crate_name == other.crate_name and + self.crate_type == other.crate_type and + self.main_src == other.main_src and self.stem == other.stem and + self.root_pkg == other.root_pkg and not self.skip_crate() and + self.same_flags(other)) + + def merge_test(self, other): + """Returns true if self and other are tests of same root_pkg.""" + # Before merger, each test has its own crate_name. + # A merged test uses its source file base name as output file name, + # so a test is mergeable only if its base name equals to its crate name. + return (self.crate_type == other.crate_type and + self.crate_type == 'test' and self.root_pkg == other.root_pkg and + not self.skip_crate() and + other.crate_name == test_base_name(other.main_src) and + (len(self.srcs) > 1 or + (self.crate_name == test_base_name(self.main_src)) and + self.host_supported == other.host_supported and + self.device_supported == other.device_supported) and + self.same_flags(other)) + + def merge(self, other, outf_name): + """Try to merge crate into self.""" + should_merge_host_device = self.merge_host_device(other) + should_merge_test = False + if not should_merge_host_device: + should_merge_test = self.merge_test(other) + # A for-device test crate can be merged with its for-host version, + # or merged with a different test for the same host or device. + # Since we run cargo once for each device or host, test crates for the + # first device or host will be merged first. Then test crates for a + # different device or host should be allowed to be merged into a + # previously merged one, maybe for a different device or host. + if should_merge_host_device or should_merge_test: + self.runner.init_bp_file(outf_name) + with open(outf_name, 'a') as outf: # to write debug info + self.outf = outf + other.outf = outf + self.do_merge(other, should_merge_test) + return True + return False + + def do_merge(self, other, should_merge_test): + """Merge attributes of other to self.""" + if self.debug: + self.write('\n// Before merge definition (1):') + self.dump_debug_info() + self.write('\n// Before merge definition (2):') + other.dump_debug_info() + # Merge properties of other to self. + self.host_supported = self.host_supported or other.host_supported + self.device_supported = self.device_supported or other.device_supported + self.has_warning = self.has_warning or other.has_warning + if not self.target: # okay to keep only the first target triple + self.target = other.target + # decide_module_type sets up default self.stem, + # which can be changed if self is a merged test module. + self.decide_module_type() + if should_merge_test: + self.srcs.append(other.main_src) + # use a short unique name as the merged module name. + prefix = self.root_pkg + '_tests' + self.module_name = self.runner.claim_module_name(prefix, self, 0) + self.stem = self.module_name + # This normalized root_pkg name although might be the same + # as other module's crate_name, it is not actually used for + # output file name. A merged test module always have multiple + # source files and each source file base name is used as + # its output file name. + self.crate_name = pkg2crate_name(self.root_pkg) + if self.debug: + self.write('\n// After merge definition (1):') + self.dump_debug_info() + + def find_cargo_dir(self): + """Deepest directory with Cargo.toml and contains the main_src.""" + if not is_dependent_file_path(self.main_src): + dir_name = os.path.dirname(self.main_src) + while dir_name: + if os.path.exists(dir_name + '/Cargo.toml'): + self.cargo_dir = dir_name + return + dir_name = os.path.dirname(dir_name) + + def parse(self, line_num, line): + """Find important rustc arguments to convert to Android.bp properties.""" + self.line_num = line_num + self.line = line + args = line.split() # Loop through every argument of rustc. + i = 0 + while i < len(args): + arg = args[i] + if arg == '--crate-name': + self.crate_name = args[i + 1] + i += 2 + # shorten imported crate main source path + self.main_src = re.sub('^/[^ ]*/registry/src/', '.../', args[i]) + self.main_src = re.sub('^.../github.com-[0-9a-f]*/', '.../', + self.main_src) + self.find_cargo_dir() + if self.cargo_dir and not self.runner.args.onefile: + # Write to Android.bp in the subdirectory with Cargo.toml. + self.outf_name = self.cargo_dir + '/Android.bp' + self.main_src = self.main_src[len(self.cargo_dir) + 1:] + elif arg == '--crate-type': + i += 1 + if self.crate_type: + self.errors += ' ERROR: multiple --crate-type ' + self.errors += self.crate_type + ' ' + args[i] + '\n' + # TODO(chh): handle multiple types, e.g. lexical-core-0.4.6 has + # crate-type = ["lib", "staticlib", "cdylib"] + # output: debug/liblexical_core.{a,so,rlib} + # cargo calls rustc with multiple --crate-type flags. + # rustc can accept: + # --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro] + # Comma separated list of types of crates for the compiler to emit + self.crate_type = args[i] + elif arg == '--test': + # only --test or --crate-type should appear once + if self.crate_type: + self.errors += (' ERROR: found both --test and --crate-type ' + + self.crate_type + '\n') + else: + self.crate_type = 'test' + elif arg == '--target': + i += 1 + self.target = args[i] + elif arg == '--cfg': + i += 1 + if args[i].startswith('\'feature='): + self.features.append(unquote(args[i].replace('\'feature=', '')[:-1])) + else: + self.cfgs.append(args[i]) + elif arg == '--extern': + i += 1 + extern_names = re.sub('=/[^ ]*/deps/', ' = ', args[i]) + self.externs.append(extern_names) + self.core_externs.append(re.sub(' = .*', '', extern_names)) + elif arg == '-C': # codegen options + i += 1 + # ignore options not used in Android + if not (args[i].startswith('debuginfo=') or + args[i].startswith('extra-filename=') or + args[i].startswith('incremental=') or + args[i].startswith('metadata=')): + self.codegens.append(args[i]) + elif arg == '--cap-lints': + i += 1 + self.cap_lints = args[i] + elif arg == '-L': + i += 1 + if args[i].startswith('dependency=') and args[i].endswith('/deps'): + if '/' + TARGET_TMP + '/' in args[i]: + self.root_pkg = re.sub( + '^.*/', '', re.sub('/' + TARGET_TMP + '/.*/deps$', '', args[i])) + else: + self.root_pkg = re.sub('^.*/', '', + re.sub('/[^/]+/[^/]+/deps$', '', args[i])) + self.root_pkg = remove_version_suffix(self.root_pkg) + elif arg == '-l': + i += 1 + if args[i].startswith('static='): + self.static_libs.append(re.sub('static=', '', args[i])) + elif args[i].startswith('dylib='): + self.shared_libs.append(re.sub('dylib=', '', args[i])) + else: + self.shared_libs.append(args[i]) + elif arg == '--out-dir' or arg == '--color': # ignored + i += 1 + elif arg.startswith('--error-format=') or arg.startswith('--json='): + _ = arg # ignored + elif arg.startswith('--emit='): + self.emit_list = arg.replace('--emit=', '') + elif arg.startswith('--edition='): + self.edition = arg.replace('--edition=', '') + else: + self.errors += 'ERROR: unknown ' + arg + '\n' + i += 1 + if not self.crate_name: + self.errors += 'ERROR: missing --crate-name\n' + if not self.main_src: + self.errors += 'ERROR: missing main source file\n' + else: + self.srcs.append(self.main_src) + if not self.crate_type: + # Treat "--cfg test" as "--test" + if 'test' in self.cfgs: + self.crate_type = 'test' + else: + self.errors += 'ERROR: missing --crate-type\n' + if not self.root_pkg: + self.root_pkg = self.crate_name + if self.target: + self.device_supported = True + self.host_supported = True # assume host supported for all builds + self.cfgs = sorted(set(self.cfgs)) + self.features = sorted(set(self.features)) + self.codegens = sorted(set(self.codegens)) + self.externs = sorted(set(self.externs)) + self.core_externs = sorted(set(self.core_externs)) + self.static_libs = sorted(set(self.static_libs)) + self.shared_libs = sorted(set(self.shared_libs)) + self.decide_module_type() + self.module_name = altered_name(self.stem) + return self + + def dump_line(self): + self.write('\n// Line ' + str(self.line_num) + ' ' + self.line) + + def feature_list(self): + """Return a string of main_src + "feature_list".""" + pkg = self.main_src + if pkg.startswith('.../'): # keep only the main package name + pkg = re.sub('/.*', '', pkg[4:]) + if not self.features: + return pkg + return pkg + ' "' + ','.join(self.features) + '"' + + def dump_skip_crate(self, kind): + if self.debug: + self.write('\n// IGNORED: ' + kind + ' ' + self.main_src) + return self + + def skip_crate(self): + """Return crate_name or a message if this crate should be skipped.""" + if is_build_crate_name(self.crate_name): + return self.crate_name + if is_dependent_file_path(self.main_src): + return 'dependent crate' + return '' + + def dump(self): + """Dump all error/debug/module code to the output .bp file.""" + self.runner.init_bp_file(self.outf_name) + with open(self.outf_name, 'a') as outf: + self.outf = outf + if self.errors: + self.dump_line() + self.write(self.errors) + elif self.skip_crate(): + self.dump_skip_crate(self.skip_crate()) + else: + if self.debug: + self.dump_debug_info() + self.dump_android_module() + + def dump_debug_info(self): + """Dump parsed data, when cargo2android is called with --debug.""" + + def dump(name, value): + self.write('//%12s = %s' % (name, value)) + + def opt_dump(name, value): + if value: + dump(name, value) + + def dump_list(fmt, values): + for v in values: + self.write(fmt % v) + + self.dump_line() + dump('module_name', self.module_name) + dump('crate_name', self.crate_name) + dump('crate_type', self.crate_type) + dump('main_src', self.main_src) + dump('has_warning', self.has_warning) + dump('for_host', self.host_supported) + dump('for_device', self.device_supported) + dump('module_type', self.module_type) + opt_dump('target', self.target) + opt_dump('edition', self.edition) + opt_dump('emit_list', self.emit_list) + opt_dump('cap_lints', self.cap_lints) + dump_list('// cfg = %s', self.cfgs) + dump_list('// cfg = \'feature "%s"\'', self.features) + # TODO(chh): escape quotes in self.features, but not in other dump_list + dump_list('// codegen = %s', self.codegens) + dump_list('// externs = %s', self.externs) + dump_list('// -l static = %s', self.static_libs) + dump_list('// -l (dylib) = %s', self.shared_libs) + + def dump_android_module(self): + """Dump one Android module definition.""" + if not self.module_type: + self.write('\nERROR: unknown crate_type ' + self.crate_type) + return + self.write('\n' + self.module_type + ' {') + self.dump_android_core_properties() + if self.edition: + self.write(' edition: "' + self.edition + '",') + self.dump_android_property_list('features', '"%s"', self.features) + cfg_fmt = '"--cfg %s"' + if self.cap_lints: + allowed = '"--cap-lints ' + self.cap_lints + '"' + if not self.cfgs: + self.write(' flags: [' + allowed + '],') + else: + self.write(' flags: [\n ' + allowed + ',') + self.dump_android_property_list_items(cfg_fmt, self.cfgs) + self.write(' ],') + else: + self.dump_android_property_list('flags', cfg_fmt, self.cfgs) + if self.externs: + self.dump_android_externs() + self.dump_android_property_list('static_libs', '"lib%s"', self.static_libs) + self.dump_android_property_list('shared_libs', '"lib%s"', self.shared_libs) + self.write('}') + + def test_module_name(self): + """Return a unique name for a test module.""" + # root_pkg+'_tests_'+(crate_name|source_file_path) + suffix = self.crate_name + if not suffix: + suffix = re.sub('/', '_', re.sub('.rs$', '', self.main_src)) + return self.root_pkg + '_tests_' + suffix + + def decide_module_type(self): + """Decide which Android module type to use.""" + host = '' if self.device_supported else '_host' + if self.crate_type == 'bin': # rust_binary[_host] + self.module_type = 'rust_binary' + host + self.stem = self.crate_name + elif self.crate_type == 'lib': # rust_library[_host]_rlib + self.module_type = 'rust_library' + host + '_rlib' + self.stem = 'lib' + self.crate_name + elif self.crate_type == 'cdylib': # rust_library[_host]_dylib + # TODO(chh): complete and test cdylib module type + self.module_type = 'rust_library' + host + '_dylib' + self.stem = 'lib' + self.crate_name + '.so' + elif self.crate_type == 'test': # rust_test[_host] + self.module_type = 'rust_test' + host + self.stem = self.test_module_name() + # self.stem will be changed after merging with other tests. + # self.stem is NOT used for final test binary name. + # rust_test uses each source file base name as its output file name, + # unless crate_name is specified by user in Cargo.toml. + elif self.crate_type == 'proc-macro': # rust_proc_macro + self.module_type = 'rust_proc_macro' + self.stem = 'lib' + self.crate_name + else: # unknown module type, rust_prebuilt_dylib? rust_library[_host]? + self.module_type = '' + self.stem = '' + + def dump_android_property_list_items(self, fmt, values): + for v in values: + # fmt has quotes, so we need escape_quotes(v) + self.write(' ' + (fmt % escape_quotes(v)) + ',') + + def dump_android_property_list(self, name, fmt, values): + if values: + self.write(' ' + name + ': [') + self.dump_android_property_list_items(fmt, values) + self.write(' ],') + + def dump_android_core_properties(self): + """Dump the module header, name, stem, etc.""" + self.write(' name: "' + self.module_name + '",') + if self.stem != self.module_name: + self.write(' stem: "' + self.stem + '",') + if self.has_warning and not self.cap_lints: + self.write(' deny_warnings: false,') + if self.host_supported and self.device_supported: + self.write(' host_supported: true,') + self.write(' crate_name: "' + self.crate_name + '",') + if len(self.srcs) > 1: + self.srcs = sorted(set(self.srcs)) + self.dump_android_property_list('srcs', '"%s"', self.srcs) + else: + self.write(' srcs: ["' + self.main_src + '"],') + if self.crate_type == 'test': + # self.root_pkg can have multiple test modules, with different *_tests[n] + # names, but their executables can all be installed under the same _tests + # directory. When built from Cargo.toml, all tests should have different + # file or crate names. + self.write(' relative_install_path: "' + self.root_pkg + '_tests",') + self.write(' test_suites: ["general-tests"],') + self.write(' auto_gen_config: true,') + + def dump_android_externs(self): + """Dump the dependent rlibs and dylibs property.""" + so_libs = list() + rust_libs = '' + deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$') + for lib in self.externs: + # normal value of lib: "libc = liblibc-*.rlib" + # strange case in rand crate: "getrandom_package = libgetrandom-*.rlib" + # we should use "libgetrandom", not "lib" + "getrandom_package" + groups = deps_libname.match(lib) + if groups is not None: + lib_name = groups.group(1) + else: + lib_name = re.sub(' .*$', '', lib) + if lib.endswith('.rlib') or lib.endswith('.rmeta'): + # On MacOS .rmeta is used when Linux uses .rlib or .rmeta. + rust_libs += ' "' + altered_name('lib' + lib_name) + '",\n' + elif lib.endswith('.so'): + so_libs.append(lib_name) + else: + rust_libs += ' // ERROR: unknown type of lib ' + lib_name + '\n' + if rust_libs: + self.write(' rlibs: [\n' + rust_libs + ' ],') + # Are all dependent .so files proc_macros? + # TODO(chh): Separate proc_macros and dylib. + self.dump_android_property_list('proc_macros', '"lib%s"', so_libs) + + +class ARObject(object): + """Information of an "ar" link command.""" + + def __init__(self, runner, outf_name): + # Remembered global runner and its members. + self.runner = runner + self.pkg = '' + self.outf_name = outf_name # path to Android.bp + # "ar" arguments + self.line_num = 1 + self.line = '' + self.flags = '' # e.g. "crs" + self.lib = '' # e.g. "/.../out/lib*.a" + self.objs = list() # e.g. "/.../out/.../*.o" + + def parse(self, pkg, line_num, args_line): + """Collect ar obj/lib file names.""" + self.pkg = pkg + self.line_num = line_num + self.line = args_line + args = args_line.split() + num_args = len(args) + if num_args < 3: + print('ERROR: "ar" command has too few arguments', args_line) + else: + self.flags = unquote(args[0]) + self.lib = unquote(args[1]) + self.objs = sorted(set(map(unquote, args[2:]))) + return self + + def write(self, s): + self.outf.write(s + '\n') + + def dump_debug_info(self): + self.write('\n// Line ' + str(self.line_num) + ' "ar" ' + self.line) + self.write('// ar_object for %12s' % self.pkg) + self.write('// flags = %s' % self.flags) + self.write('// lib = %s' % short_out_name(self.pkg, self.lib)) + for o in self.objs: + self.write('// obj = %s' % short_out_name(self.pkg, o)) + + def dump_android_lib(self): + """Write cc_library_static into Android.bp.""" + self.write('\ncc_library_static {') + self.write(' name: "' + file_base_name(self.lib) + '",') + self.write(' host_supported: true,') + if self.flags != 'crs': + self.write(' // ar flags = %s' % self.flags) + if self.pkg not in self.runner.pkg_obj2cc: + self.write(' ERROR: cannot find source files.\n}') + return + self.write(' srcs: [') + obj2cc = self.runner.pkg_obj2cc[self.pkg] + # Note: wflags are ignored. + dflags = list() + fflags = list() + for obj in self.objs: + self.write(' "' + short_out_name(self.pkg, obj2cc[obj].src) + '",') + # TODO(chh): union of dflags and flags of all obj + # Now, just a temporary hack that uses the last obj's flags + dflags = obj2cc[obj].dflags + fflags = obj2cc[obj].fflags + self.write(' ],') + self.write(' cflags: [') + self.write(' "-O3",') # TODO(chh): is this default correct? + self.write(' "-Wno-error",') + for x in fflags: + self.write(' "-f' + x + '",') + for x in dflags: + self.write(' "-D' + x + '",') + self.write(' ],') + self.write('}') + + def dump(self): + """Dump error/debug/module info to the output .bp file.""" + self.runner.init_bp_file(self.outf_name) + with open(self.outf_name, 'a') as outf: + self.outf = outf + if self.runner.args.debug: + self.dump_debug_info() + self.dump_android_lib() + + +class CCObject(object): + """Information of a "cc" compilation command.""" + + def __init__(self, runner, outf_name): + # Remembered global runner and its members. + self.runner = runner + self.pkg = '' + self.outf_name = outf_name # path to Android.bp + # "cc" arguments + self.line_num = 1 + self.line = '' + self.src = '' + self.obj = '' + self.dflags = list() # -D flags + self.fflags = list() # -f flags + self.iflags = list() # -I flags + self.wflags = list() # -W flags + self.other_args = list() + + def parse(self, pkg, line_num, args_line): + """Collect cc compilation flags and src/out file names.""" + self.pkg = pkg + self.line_num = line_num + self.line = args_line + args = args_line.split() + i = 0 + while i < len(args): + arg = args[i] + if arg == '"-c"': + i += 1 + if args[i].startswith('"-o'): + # ring-0.13.5 dumps: ... "-c" "-o/.../*.o" ".../*.c" + self.obj = unquote(args[i])[2:] + i += 1 + self.src = unquote(args[i]) + else: + self.src = unquote(args[i]) + elif arg == '"-o"': + i += 1 + self.obj = unquote(args[i]) + elif arg == '"-I"': + i += 1 + self.iflags.append(unquote(args[i])) + elif arg.startswith('"-D'): + self.dflags.append(unquote(args[i])[2:]) + elif arg.startswith('"-f'): + self.fflags.append(unquote(args[i])[2:]) + elif arg.startswith('"-W'): + self.wflags.append(unquote(args[i])[2:]) + elif not (arg.startswith('"-O') or arg == '"-m64"' or arg == '"-g"' or + arg == '"-g3"'): + # ignore -O -m64 -g + self.other_args.append(unquote(args[i])) + i += 1 + self.dflags = sorted(set(self.dflags)) + self.fflags = sorted(set(self.fflags)) + # self.wflags is not sorted because some are order sensitive + # and we ignore them anyway. + if self.pkg not in self.runner.pkg_obj2cc: + self.runner.pkg_obj2cc[self.pkg] = {} + self.runner.pkg_obj2cc[self.pkg][self.obj] = self + return self + + def write(self, s): + self.outf.write(s + '\n') + + def dump_debug_flags(self, name, flags): + self.write('// ' + name + ':') + for f in flags: + self.write('// %s' % f) + + def dump(self): + """Dump only error/debug info to the output .bp file.""" + if not self.runner.args.debug: + return + self.runner.init_bp_file(self.outf_name) + with open(self.outf_name, 'a') as outf: + self.outf = outf + self.write('\n// Line ' + str(self.line_num) + ' "cc" ' + self.line) + self.write('// cc_object for %12s' % self.pkg) + self.write('// src = %s' % short_out_name(self.pkg, self.src)) + self.write('// obj = %s' % short_out_name(self.pkg, self.obj)) + self.dump_debug_flags('-I flags', self.iflags) + self.dump_debug_flags('-D flags', self.dflags) + self.dump_debug_flags('-f flags', self.fflags) + self.dump_debug_flags('-W flags', self.wflags) + if self.other_args: + self.dump_debug_flags('other args', self.other_args) + + +class Runner(object): + """Main class to parse cargo -v output and print Android module definitions.""" + + def __init__(self, args): + self.bp_files = set() # Remember all output Android.bp files. + self.root_pkg = '' # name of package in ./Cargo.toml + # Saved flags, modes, and data. + self.args = args + self.dry_run = not args.run + self.skip_cargo = args.skipcargo + # All cc/ar objects, crates, dependencies, and warning files + self.cc_objects = list() + self.pkg_obj2cc = {} + # pkg_obj2cc[cc_object[i].pkg][cc_objects[i].obj] = cc_objects[i] + self.ar_objects = list() + self.crates = list() + self.dependencies = list() # dependent and build script crates + self.warning_files = set() + # Keep a unique mapping from (module name) to crate + self.name_owners = {} + # Default action is cargo clean, followed by build or user given actions. + if args.cargo: + self.cargo = ['clean'] + args.cargo + else: + self.cargo = ['clean', 'build'] + default_target = '--target x86_64-unknown-linux-gnu' + if args.device: + self.cargo.append('build ' + default_target) + if args.tests: + self.cargo.append('build --tests') + self.cargo.append('build --tests ' + default_target) + elif args.tests: + self.cargo.append('build --tests') + + def init_bp_file(self, name): + if name not in self.bp_files: + self.bp_files.add(name) + with open(name, 'w') as outf: + outf.write(ANDROID_BP_HEADER) + + def claim_module_name(self, prefix, owner, counter): + """Return prefix if not owned yet, otherwise, prefix+str(counter).""" + while True: + name = prefix + if counter > 0: + name += str(counter) + if name not in self.name_owners: + self.name_owners[name] = owner + return name + if owner == self.name_owners[name]: + return name + counter += 1 + + def find_root_pkg(self): + """Read name of [package] in ./Cargo.toml.""" + if not os.path.exists('./Cargo.toml'): + return + with open('./Cargo.toml', 'r') as inf: + pkg_section = re.compile(r'^ *\[package\]') + name = re.compile('^ *name *= * "([^"]*)"') + in_pkg = False + for line in inf: + if in_pkg: + if name.match(line): + self.root_pkg = name.match(line).group(1) + break + else: + in_pkg = pkg_section.match(line) is not None + + def run_cargo(self): + """Calls cargo -v and save its output to ./cargo.out.""" + if self.skip_cargo: + return self + cargo = './Cargo.toml' + if not os.access(cargo, os.R_OK): + print('ERROR: Cannot find or read', cargo) + return self + if not self.dry_run and os.path.exists('cargo.out'): + os.remove('cargo.out') + cmd_tail = ' --target-dir ' + TARGET_TMP + ' >> cargo.out 2>&1' + for c in self.cargo: + features = '' + if self.args.features and c != 'clean': + features = ' --features ' + self.args.features + cmd = 'cargo -vv ' if self.args.vv else 'cargo -v ' + cmd += c + features + cmd_tail + if self.args.rustflags and c != 'clean': + cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cmd + if self.dry_run: + print('Dry-run skip:', cmd) + else: + if self.args.verbose: + print('Running:', cmd) + with open('cargo.out', 'a') as cargo_out: + cargo_out.write('### Running: ' + cmd + '\n') + os.system(cmd) + return self + + def dump_dependencies(self): + """Append dependencies and their features to Android.bp.""" + if not self.dependencies: + return + dependent_list = list() + for c in self.dependencies: + dependent_list.append(c.feature_list()) + sorted_dependencies = sorted(set(dependent_list)) + self.init_bp_file('Android.bp') + with open('Android.bp', 'a') as outf: + outf.write('\n// dependent_library ["feature_list"]\n') + for s in sorted_dependencies: + outf.write('// ' + s + '\n') + + def dump_pkg_obj2cc(self): + """Dump debug info of the pkg_obj2cc map.""" + if not self.args.debug: + return + self.init_bp_file('Android.bp') + with open('Android.bp', 'a') as outf: + sorted_pkgs = sorted(self.pkg_obj2cc.keys()) + for pkg in sorted_pkgs: + if not self.pkg_obj2cc[pkg]: + continue + outf.write('\n// obj => src for %s\n' % pkg) + obj2cc = self.pkg_obj2cc[pkg] + for obj in sorted(obj2cc.keys()): + outf.write('// ' + short_out_name(pkg, obj) + ' => ' + + short_out_name(pkg, obj2cc[obj].src) + '\n') + + def gen_bp(self): + """Parse cargo.out and generate Android.bp files.""" + if self.dry_run: + print('Dry-run skip: read', CARGO_OUT, 'write Android.bp') + elif os.path.exists(CARGO_OUT): + self.find_root_pkg() + with open(CARGO_OUT, 'r') as cargo_out: + self.parse(cargo_out, 'Android.bp') + self.crates.sort(key=get_module_name) + for obj in self.cc_objects: + obj.dump() + self.dump_pkg_obj2cc() + for crate in self.crates: + crate.dump() + dumped_libs = set() + for lib in self.ar_objects: + if lib.pkg == self.root_pkg: + lib_name = file_base_name(lib.lib) + if lib_name not in dumped_libs: + dumped_libs.add(lib_name) + lib.dump() + if self.args.dependencies and self.dependencies: + self.dump_dependencies() + return self + + def add_ar_object(self, obj): + self.ar_objects.append(obj) + + def add_cc_object(self, obj): + self.cc_objects.append(obj) + + def add_crate(self, crate): + """Merge crate with someone in crates, or append to it. Return crates.""" + if crate.skip_crate(): + if self.args.debug: # include debug info of all crates + self.crates.append(crate) + if self.args.dependencies: # include only dependent crates + if (is_dependent_file_path(crate.main_src) and + not is_build_crate_name(crate.crate_name)): + self.dependencies.append(crate) + else: + for c in self.crates: + if c.merge(crate, 'Android.bp'): + return + self.crates.append(crate) + + def find_warning_owners(self): + """For each warning file, find its owner crate.""" + missing_owner = False + for f in self.warning_files: + cargo_dir = '' # find lowest crate, with longest path + owner = None # owner crate of this warning + for c in self.crates: + if (f.startswith(c.cargo_dir + '/') and + len(cargo_dir) < len(c.cargo_dir)): + cargo_dir = c.cargo_dir + owner = c + if owner: + owner.has_warning = True + else: + missing_owner = True + if missing_owner and os.path.exists('Cargo.toml'): + # owner is the root cargo, with empty cargo_dir + for c in self.crates: + if not c.cargo_dir: + c.has_warning = True + + def rustc_command(self, n, rustc_line, line, outf_name): + """Process a rustc command line from cargo -vv output.""" + # cargo build -vv output can have multiple lines for a rustc command + # due to '\n' in strings for environment variables. + # strip removes leading spaces and '\n' at the end + new_rustc = (rustc_line.strip() + line) if rustc_line else line + # Use an heuristic to detect the completions of a multi-line command. + # This might fail for some very rare case, but easy to fix manually. + if not line.endswith('`\n') or (new_rustc.count('`') % 2) != 0: + return new_rustc + if RUSTC_VV_CMD_ARGS.match(new_rustc): + args = RUSTC_VV_CMD_ARGS.match(new_rustc).group(1) + self.add_crate(Crate(self, outf_name).parse(n, args)) + else: + self.assert_empty_vv_line(new_rustc) + return '' + + def cc_ar_command(self, n, groups, outf_name): + pkg = groups.group(1) + line = groups.group(3) + if groups.group(2) == 'cc': + self.add_cc_object(CCObject(self, outf_name).parse(pkg, n, line)) + else: + self.add_ar_object(ARObject(self, outf_name).parse(pkg, n, line)) + + def assert_empty_vv_line(self, line): + if line: # report error if line is not empty + self.init_bp_file('Android.bp') + with open('Android.bp', 'a') as outf: + outf.write('ERROR -vv line: ', line) + return '' + + def parse(self, inf, outf_name): + """Parse rustc and warning messages in inf, return a list of Crates.""" + n = 0 # line number + prev_warning = False # true if the previous line was warning: ... + rustc_line = '' # previous line(s) matching RUSTC_VV_PAT + for line in inf: + n += 1 + if line.startswith('warning: '): + prev_warning = True + rustc_line = self.assert_empty_vv_line(rustc_line) + continue + new_rustc = '' + if RUSTC_PAT.match(line): + args_line = RUSTC_PAT.match(line).group(1) + self.add_crate(Crate(self, outf_name).parse(n, args_line)) + self.assert_empty_vv_line(rustc_line) + elif rustc_line or RUSTC_VV_PAT.match(line): + new_rustc = self.rustc_command(n, rustc_line, line, outf_name) + elif CC_AR_VV_PAT.match(line): + self.cc_ar_command(n, CC_AR_VV_PAT.match(line), outf_name) + elif prev_warning and WARNING_FILE_PAT.match(line): + self.assert_empty_vv_line(rustc_line) + fpath = WARNING_FILE_PAT.match(line).group(1) + if fpath[0] != '/': # ignore absolute path + self.warning_files.add(fpath) + prev_warning = False + rustc_line = new_rustc + self.find_warning_owners() + + +def parse_args(): + """Parse main arguments.""" + parser = argparse.ArgumentParser('cargo2android') + parser.add_argument( + '--cargo', + action='append', + metavar='args_string', + help=('extra cargo build -v args in a string, ' + + 'each --cargo flag calls cargo build -v once')) + parser.add_argument( + '--debug', + action='store_true', + default=False, + help='dump debug info into Android.bp') + parser.add_argument( + '--dependencies', + action='store_true', + default=False, + help='dump debug info of dependent crates') + parser.add_argument( + '--device', + action='store_true', + default=False, + help='run cargo also for a default device target') + parser.add_argument( + '--features', type=str, help='passing features to cargo build') + parser.add_argument( + '--onefile', + action='store_true', + default=False, + help=('output all into one ./Android.bp, default will generate ' + + 'one Android.bp per Cargo.toml in subdirectories')) + parser.add_argument( + '--run', + action='store_true', + default=False, + help='run it, default is dry-run') + parser.add_argument('--rustflags', type=str, help='passing flags to rustc') + parser.add_argument( + '--skipcargo', + action='store_true', + default=False, + help='skip cargo command, parse cargo.out, and generate Android.bp') + parser.add_argument( + '--tests', + action='store_true', + default=False, + help='run cargo build --tests after normal build') + parser.add_argument( + '--verbose', + action='store_true', + default=False, + help='echo executed commands') + parser.add_argument( + '--vv', + action='store_true', + default=False, + help='run cargo with -vv instead of default -v') + return parser.parse_args() + + +def main(): + args = parse_args() + if not args.run: # default is dry-run + print(DRY_RUN_NOTE) + Runner(args).run_cargo().gen_bp() + + +if __name__ == '__main__': + main()