#!/usr/bin/env python # # Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Call cargo -v, parse its output, and generate Android.bp. Usage: Run this script in a crate workspace root directory. The Cargo.toml file should work at least for the host platform. (1) Without other flags, "cargo2android.py --run" calls cargo clean, calls cargo build -v, and generates Android.bp. The cargo build only generates crates for the host, without test crates. (2) To build crates for both host and device in Android.bp, use the --device flag, for example: cargo2android.py --run --device This is equivalent to using the --cargo flag to add extra builds: cargo2android.py --run --cargo "build" --cargo "build --target x86_64-unknown-linux-gnu" On MacOS, use x86_64-apple-darwin as target triple. Here the host target triple is used as a fake cross compilation target. If the crate's Cargo.toml and environment configuration works for an Android target, use that target triple as the cargo build flag. (3) To build default and test crates, for host and device, use both --device and --tests flags: cargo2android.py --run --device --tests This is equivalent to using the --cargo flag to add extra builds: cargo2android.py --run --cargo "build" --cargo "build --tests" --cargo "build --target x86_64-unknown-linux-gnu" --cargo "build --tests --target x86_64-unknown-linux-gnu" If there are rustc warning messages, this script will add a warning comment to the owner crate module in Android.bp. """ from __future__ import print_function import argparse import os import os.path import re import sys # Some Rust packages include extra unwanted crates. # This set contains all such excluded crate names. EXCLUDED_CRATES = set(['protobuf_bin_gen_rust_do_not_use']) RENAME_MAP = { # This map includes all changes to the default rust module names # to resolve name conflicts, avoid confusion, or work as plugin. 'libbacktrace': 'libbacktrace_rust', 'libgcc': 'libgcc_rust', 'liblog': 'liblog_rust', 'libsync': 'libsync_rust', 'libx86_64': 'libx86_64_rust', 'protoc_gen_rust': 'protoc-gen-rust', } RENAME_STEM_MAP = { # This map includes all changes to the default rust module stem names, # which is used for output files when different from the module name. 'protoc_gen_rust': 'protoc-gen-rust', } RENAME_DEFAULTS_MAP = { # This map includes all changes to the default prefix of rust_default # module names, to avoid conflict with existing Android modules. 'libc': 'rust_libc', } # Header added to all generated Android.bp files. ANDROID_BP_HEADER = '// This file is generated by cargo2android.py {args}.\n' CARGO_OUT = 'cargo.out' # Name of file to keep cargo build -v output. TARGET_TMP = 'target.tmp' # Name of temporary output directory. # Message to be displayed when this script is called without the --run flag. DRY_RUN_NOTE = ( 'Dry-run: This script uses ./' + TARGET_TMP + ' for output directory,\n' + 'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' + 'and writes to Android.bp in the current and subdirectories.\n\n' + 'To do do all of the above, use the --run flag.\n' + 'See --help for other flags, and more usage notes in this script.\n') # Cargo -v output of a call to rustc. RUSTC_PAT = re.compile('^ +Running `rustc (.*)`$') # Cargo -vv output of a call to rustc could be split into multiple lines. # Assume that the first line will contain some CARGO_* env definition. RUSTC_VV_PAT = re.compile('^ +Running `.*CARGO_.*=.*$') # The combined -vv output rustc command line pattern. RUSTC_VV_CMD_ARGS = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$') # Cargo -vv output of a "cc" or "ar" command; all in one line. CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$') # Some package, such as ring-0.13.5, has pattern '... running "cc"'. # Rustc output of file location path pattern for a warning message. WARNING_FILE_PAT = re.compile('^ *--> ([^:]*):[0-9]+') # Rust package name with suffix -d1.d2.d3. VERSION_SUFFIX_PAT = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+$') def altered_name(name): return RENAME_MAP[name] if (name in RENAME_MAP) else name def altered_stem(name): return RENAME_STEM_MAP[name] if (name in RENAME_STEM_MAP) else name def altered_defaults(name): return RENAME_DEFAULTS_MAP[name] if (name in RENAME_DEFAULTS_MAP) else name def is_build_crate_name(name): # We added special prefix to build script crate names. return name.startswith('build_script_') def is_dependent_file_path(path): # Absolute or dependent '.../' paths are not main files of this crate. return path.startswith('/') or path.startswith('.../') def get_module_name(crate): # to sort crates in a list return crate.module_name def pkg2crate_name(s): return s.replace('-', '_').replace('.', '_') def file_base_name(path): return os.path.splitext(os.path.basename(path))[0] def test_base_name(path): return pkg2crate_name(file_base_name(path)) def unquote(s): # remove quotes around str if s and len(s) > 1 and s[0] == '"' and s[-1] == '"': return s[1:-1] return s def remove_version_suffix(s): # remove -d1.d2.d3 suffix if VERSION_SUFFIX_PAT.match(s): return VERSION_SUFFIX_PAT.match(s).group(1) return s def short_out_name(pkg, s): # replace /.../pkg-*/out/* with .../out/* return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s) def escape_quotes(s): # replace '"' with '\\"' return s.replace('"', '\\"') class Crate(object): """Information of a Rust crate to collect/emit for an Android.bp module.""" def __init__(self, runner, outf_name): # Remembered global runner and its members. self.runner = runner self.debug = runner.args.debug self.cargo_dir = '' # directory of my Cargo.toml self.outf_name = outf_name # path to Android.bp self.outf = None # open file handle of outf_name during dump* # Variants/results that could be merged from multiple rustc lines. self.host_supported = False self.device_supported = False self.has_warning = False # Android module properties derived from rustc parameters. self.module_name = '' # unique in Android build system self.module_type = '' # rust_{binary,library,test}[_host] etc. self.defaults = '' # rust_defaults used by rust_test* modules self.root_pkg = '' # parent package name of a sub/test packge, from -L self.srcs = list() # main_src or merged multiple source files self.stem = '' # real base name of output file # Kept parsed status self.errors = '' # all errors found during parsing self.line_num = 1 # runner told input source line number self.line = '' # original rustc command line parameters # Parameters collected from rustc command line. self.crate_name = '' # follows --crate-name self.main_src = '' # follows crate_name parameter, shortened self.crate_types = list() # follows --crate-type self.cfgs = list() # follows --cfg, without feature= prefix self.features = list() # follows --cfg, name in 'feature="..."' self.codegens = list() # follows -C, some ignored self.externs = list() # follows --extern self.core_externs = list() # first part of self.externs elements self.static_libs = list() # e.g. -l static=host_cpuid self.shared_libs = list() # e.g. -l dylib=wayland-client, -l z self.cap_lints = '' # follows --cap-lints self.emit_list = '' # e.g., --emit=dep-info,metadata,link self.edition = '2015' # rustc default, e.g., --edition=2018 self.target = '' # follows --target def write(self, s): # convenient way to output one line at a time with EOL. self.outf.write(s + '\n') def same_flags(self, other): # host_supported, device_supported, has_warning are not compared but merged # target is not compared, to merge different target/host modules # externs is not compared; only core_externs is compared return (not self.errors and not other.errors and self.edition == other.edition and self.cap_lints == other.cap_lints and self.emit_list == other.emit_list and self.core_externs == other.core_externs and self.codegens == other.codegens and self.features == other.features and self.static_libs == other.static_libs and self.shared_libs == other.shared_libs and self.cfgs == other.cfgs) def merge_host_device(self, other): """Returns true if attributes are the same except host/device support.""" return (self.crate_name == other.crate_name and self.crate_types == other.crate_types and self.main_src == other.main_src and # before merge, each test module has an unique module name and stem (self.stem == other.stem or self.crate_types == ['test']) and self.root_pkg == other.root_pkg and not self.skip_crate() and self.same_flags(other)) def merge_test(self, other): """Returns true if self and other are tests of same root_pkg.""" # Before merger, each test has its own crate_name. # A merged test uses its source file base name as output file name, # so a test is mergeable only if its base name equals to its crate name. return (self.crate_types == other.crate_types and self.crate_types == ['test'] and self.root_pkg == other.root_pkg and not self.skip_crate() and other.crate_name == test_base_name(other.main_src) and (len(self.srcs) > 1 or (self.crate_name == test_base_name(self.main_src)) and self.host_supported == other.host_supported and self.device_supported == other.device_supported) and self.same_flags(other)) def merge(self, other, outf_name): """Try to merge crate into self.""" should_merge_host_device = self.merge_host_device(other) should_merge_test = False if not should_merge_host_device: should_merge_test = self.merge_test(other) # A for-device test crate can be merged with its for-host version, # or merged with a different test for the same host or device. # Since we run cargo once for each device or host, test crates for the # first device or host will be merged first. Then test crates for a # different device or host should be allowed to be merged into a # previously merged one, maybe for a different device or host. if should_merge_host_device or should_merge_test: self.runner.init_bp_file(outf_name) with open(outf_name, 'a') as outf: # to write debug info self.outf = outf other.outf = outf self.do_merge(other, should_merge_test) return True return False def do_merge(self, other, should_merge_test): """Merge attributes of other to self.""" if self.debug: self.write('\n// Before merge definition (1):') self.dump_debug_info() self.write('\n// Before merge definition (2):') other.dump_debug_info() # Merge properties of other to self. self.host_supported = self.host_supported or other.host_supported self.device_supported = self.device_supported or other.device_supported self.has_warning = self.has_warning or other.has_warning if not self.target: # okay to keep only the first target triple self.target = other.target # decide_module_type sets up default self.stem, # which can be changed if self is a merged test module. self.decide_module_type() if should_merge_test: self.srcs.append(other.main_src) # use a short unique name as the merged module name. prefix = self.root_pkg + '_tests' self.module_name = self.runner.claim_module_name(prefix, self, 0) self.stem = self.module_name # This normalized root_pkg name although might be the same # as other module's crate_name, it is not actually used for # output file name. A merged test module always have multiple # source files and each source file base name is used as # its output file name. self.crate_name = pkg2crate_name(self.root_pkg) if self.debug: self.write('\n// After merge definition (1):') self.dump_debug_info() def find_cargo_dir(self): """Deepest directory with Cargo.toml and contains the main_src.""" if not is_dependent_file_path(self.main_src): dir_name = os.path.dirname(self.main_src) while dir_name: if os.path.exists(dir_name + '/Cargo.toml'): self.cargo_dir = dir_name return dir_name = os.path.dirname(dir_name) def parse(self, line_num, line): """Find important rustc arguments to convert to Android.bp properties.""" self.line_num = line_num self.line = line args = line.split() # Loop through every argument of rustc. i = 0 while i < len(args): arg = args[i] if arg == '--crate-name': i += 1 self.crate_name = args[i] elif arg == '--crate-type': i += 1 # cargo calls rustc with multiple --crate-type flags. # rustc can accept: # --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro] self.crate_types.append(args[i]) elif arg == '--test': self.crate_types.append('test') elif arg == '--target': i += 1 self.target = args[i] elif arg == '--cfg': i += 1 if args[i].startswith('\'feature='): self.features.append(unquote(args[i].replace('\'feature=', '')[:-1])) else: self.cfgs.append(args[i]) elif arg == '--extern': i += 1 extern_names = re.sub('=/[^ ]*/deps/', ' = ', args[i]) self.externs.append(extern_names) self.core_externs.append(re.sub(' = .*', '', extern_names)) elif arg == '-C': # codegen options i += 1 # ignore options not used in Android if not (args[i].startswith('debuginfo=') or args[i].startswith('extra-filename=') or args[i].startswith('incremental=') or args[i].startswith('metadata=')): self.codegens.append(args[i]) elif arg == '--cap-lints': i += 1 self.cap_lints = args[i] elif arg == '-L': i += 1 if args[i].startswith('dependency=') and args[i].endswith('/deps'): if '/' + TARGET_TMP + '/' in args[i]: self.root_pkg = re.sub( '^.*/', '', re.sub('/' + TARGET_TMP + '/.*/deps$', '', args[i])) else: self.root_pkg = re.sub('^.*/', '', re.sub('/[^/]+/[^/]+/deps$', '', args[i])) self.root_pkg = remove_version_suffix(self.root_pkg) elif arg == '-l': i += 1 if args[i].startswith('static='): self.static_libs.append(re.sub('static=', '', args[i])) elif args[i].startswith('dylib='): self.shared_libs.append(re.sub('dylib=', '', args[i])) else: self.shared_libs.append(args[i]) elif arg == '--out-dir' or arg == '--color': # ignored i += 1 elif arg.startswith('--error-format=') or arg.startswith('--json='): _ = arg # ignored elif arg.startswith('--emit='): self.emit_list = arg.replace('--emit=', '') elif arg.startswith('--edition='): self.edition = arg.replace('--edition=', '') elif not arg.startswith('-'): # shorten imported crate main source paths like $HOME/.cargo/ # registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/lib.rs self.main_src = re.sub(r'^/[^ ]*/registry/src/', '.../', args[i]) self.main_src = re.sub(r'^\.\.\./github.com-[0-9a-f]*/', '.../', self.main_src) self.find_cargo_dir() if self.cargo_dir and not self.runner.args.onefile: # Write to Android.bp in the subdirectory with Cargo.toml. self.outf_name = self.cargo_dir + '/Android.bp' self.main_src = self.main_src[len(self.cargo_dir) + 1:] else: self.errors += 'ERROR: unknown ' + arg + '\n' i += 1 if not self.crate_name: self.errors += 'ERROR: missing --crate-name\n' if not self.main_src: self.errors += 'ERROR: missing main source file\n' else: self.srcs.append(self.main_src) if not self.crate_types: # Treat "--cfg test" as "--test" if 'test' in self.cfgs: self.crate_types.append('test') else: self.errors += 'ERROR: missing --crate-type or --test\n' elif len(self.crate_types) > 1: if 'test' in self.crate_types: self.errors += 'ERROR: cannot handle both --crate-type and --test\n' if 'lib' in self.crate_types and 'rlib' in self.crate_types: self.errors += 'ERROR: cannot generate both lib and rlib crate types\n' if not self.root_pkg: self.root_pkg = self.crate_name if self.target: self.device_supported = True self.host_supported = True # assume host supported for all builds if self.runner.args.no_host: # unless --no-host was specified self.host_supported = False self.cfgs = sorted(set(self.cfgs)) self.features = sorted(set(self.features)) self.codegens = sorted(set(self.codegens)) self.externs = sorted(set(self.externs)) self.core_externs = sorted(set(self.core_externs)) self.static_libs = sorted(set(self.static_libs)) self.shared_libs = sorted(set(self.shared_libs)) self.crate_types = sorted(set(self.crate_types)) self.decide_module_type() self.module_name = altered_name(self.stem) return self def dump_line(self): self.write('\n// Line ' + str(self.line_num) + ' ' + self.line) def feature_list(self): """Return a string of main_src + "feature_list".""" pkg = self.main_src if pkg.startswith('.../'): # keep only the main package name pkg = re.sub('/.*', '', pkg[4:]) if not self.features: return pkg return pkg + ' "' + ','.join(self.features) + '"' def dump_skip_crate(self, kind): if self.debug: self.write('\n// IGNORED: ' + kind + ' ' + self.main_src) return self def skip_crate(self): """Return crate_name or a message if this crate should be skipped.""" if (is_build_crate_name(self.crate_name) or self.crate_name in EXCLUDED_CRATES): return self.crate_name if is_dependent_file_path(self.main_src): return 'dependent crate' return '' def dump(self): """Dump all error/debug/module code to the output .bp file.""" self.runner.init_bp_file(self.outf_name) with open(self.outf_name, 'a') as outf: self.outf = outf if self.errors: self.dump_line() self.write(self.errors) elif self.skip_crate(): self.dump_skip_crate(self.skip_crate()) else: if self.debug: self.dump_debug_info() self.dump_android_module() def dump_debug_info(self): """Dump parsed data, when cargo2android is called with --debug.""" def dump(name, value): self.write('//%12s = %s' % (name, value)) def opt_dump(name, value): if value: dump(name, value) def dump_list(fmt, values): for v in values: self.write(fmt % v) self.dump_line() dump('module_name', self.module_name) dump('crate_name', self.crate_name) dump('crate_types', self.crate_types) dump('main_src', self.main_src) dump('has_warning', self.has_warning) dump('for_host', self.host_supported) dump('for_device', self.device_supported) dump('module_type', self.module_type) opt_dump('target', self.target) opt_dump('edition', self.edition) opt_dump('emit_list', self.emit_list) opt_dump('cap_lints', self.cap_lints) dump_list('// cfg = %s', self.cfgs) dump_list('// cfg = \'feature "%s"\'', self.features) # TODO(chh): escape quotes in self.features, but not in other dump_list dump_list('// codegen = %s', self.codegens) dump_list('// externs = %s', self.externs) dump_list('// -l static = %s', self.static_libs) dump_list('// -l (dylib) = %s', self.shared_libs) def dump_android_module(self): """Dump one or more Android module definition, depending on crate_types.""" if len(self.crate_types) == 1: self.dump_single_type_android_module() return if 'test' in self.crate_types: self.write('\nERROR: multiple crate types cannot include test type') return # Dump one Android module per crate_type. for crate_type in self.crate_types: self.decide_one_module_type(crate_type) self.dump_one_android_module(crate_type) def build_default_name(self): """Return a short and readable name for the rust_defaults module.""" # Choices: (1) root_pkg + '_defaults', # (2) root_pkg + '_defaults_' + crate_name # (3) root_pkg + '_defaults_' + main_src_basename_path # (4) root_pkg + '_defaults_' + a_positive_sequence_number name1 = altered_defaults(self.root_pkg) + '_defaults' if self.runner.try_claim_module_name(name1, self): return name1 name2 = name1 + '_' + self.crate_name if self.runner.try_claim_module_name(name2, self): return name2 name3 = name1 + '_' + self.main_src_basename_path() if self.runner.try_claim_module_name(name3, self): return name3 return self.runner.claim_module_name(name1, self, 0) def dump_defaults_module(self): """Dump a rust_defaults module to be shared by other modules.""" name = self.build_default_name() self.defaults = name self.write('\nrust_defaults {') self.write(' name: "' + name + '",') self.write(' crate_name: "' + self.crate_name + '",') if 'test' in self.crate_types: self.write(' test_suites: ["general-tests"],') self.write(' auto_gen_config: true,') self.dump_edition_flags_libs() self.write('}') def dump_single_type_android_module(self): """Dump one simple Android module, which has only one crate_type.""" crate_type = self.crate_types[0] if crate_type != 'test': # do not change self.stem or self.module_name self.dump_one_android_module(crate_type) return # Dump one test module per source file, and separate host and device tests. # crate_type == 'test' if (self.host_supported and self.device_supported) or len(self.srcs) > 1: self.srcs = sorted(set(self.srcs)) self.dump_defaults_module() saved_srcs = self.srcs for src in saved_srcs: self.srcs = [src] saved_device_supported = self.device_supported saved_host_supported = self.host_supported saved_main_src = self.main_src self.main_src = src if saved_host_supported: self.device_supported = False self.host_supported = True self.module_name = self.test_module_name() self.decide_one_module_type(crate_type) self.dump_one_android_module(crate_type) if saved_device_supported: self.device_supported = True self.host_supported = False self.module_name = self.test_module_name() self.decide_one_module_type(crate_type) self.dump_one_android_module(crate_type) self.host_supported = saved_host_supported self.device_supported = saved_device_supported self.main_src = saved_main_src self.srcs = saved_srcs def dump_one_android_module(self, crate_type): """Dump one Android module definition.""" if not self.module_type: self.write('\nERROR: unknown crate_type ' + crate_type) return self.write('\n' + self.module_type + ' {') self.dump_android_core_properties() if not self.defaults: self.dump_edition_flags_libs() if self.runner.args.host_first_multilib and self.host_supported and crate_type != 'test': self.write(' compile_multilib: "first",') self.write('}') def dump_android_flags(self): """Dump Android module flags property.""" cfg_fmt = '"--cfg %s"' if self.cap_lints: allowed = '"--cap-lints ' + self.cap_lints + '"' if not self.cfgs: self.write(' flags: [' + allowed + '],') else: self.write(' flags: [\n ' + allowed + ',') self.dump_android_property_list_items(cfg_fmt, self.cfgs) self.write(' ],') else: self.dump_android_property_list('flags', cfg_fmt, self.cfgs) def dump_edition_flags_libs(self): if self.edition: self.write(' edition: "' + self.edition + '",') self.dump_android_property_list('features', '"%s"', self.features) self.dump_android_flags() if self.externs: self.dump_android_externs() self.dump_android_property_list('static_libs', '"lib%s"', self.static_libs) self.dump_android_property_list('shared_libs', '"lib%s"', self.shared_libs) def main_src_basename_path(self): return re.sub('/', '_', re.sub('.rs$', '', self.main_src)) def test_module_name(self): """Return a unique name for a test module.""" # root_pkg+(_host|_device) + '_test_'+source_file_name suffix = self.main_src_basename_path() host_device = '_host' if self.device_supported: host_device = '_device' return self.root_pkg + host_device + '_test_' + suffix def decide_module_type(self): # Use the first crate type for the default/first module. crate_type = self.crate_types[0] if self.crate_types else '' self.decide_one_module_type(crate_type) def decide_one_module_type(self, crate_type): """Decide which Android module type to use.""" host = '' if self.device_supported else '_host' if crate_type == 'bin': # rust_binary[_host] self.module_type = 'rust_binary' + host # In rare cases like protobuf-codegen, the output binary name must # be renamed to use as a plugin for protoc. self.stem = altered_stem(self.crate_name) self.module_name = altered_name(self.crate_name) elif crate_type == 'lib': # rust_library[_host] # TODO(chh): should this be rust_library[_host]? # Assuming that Cargo.toml do not use both 'lib' and 'rlib', # because we map them both to rlib. self.module_type = 'rust_library' + host self.stem = 'lib' + self.crate_name self.module_name = altered_name(self.stem) elif crate_type == 'rlib': # rust_library[_host] self.module_type = 'rust_library' + host self.stem = 'lib' + self.crate_name self.module_name = altered_name(self.stem) elif crate_type == 'dylib': # rust_library[_host]_dylib self.module_type = 'rust_library' + host + '_dylib' self.stem = 'lib' + self.crate_name self.module_name = altered_name(self.stem) + '_dylib' elif crate_type == 'cdylib': # rust_library[_host]_shared self.module_type = 'rust_library' + host + '_shared' self.stem = 'lib' + self.crate_name self.module_name = altered_name(self.stem) + '_shared' elif crate_type == 'staticlib': # rust_library[_host]_static self.module_type = 'rust_library' + host + '_static' self.stem = 'lib' + self.crate_name self.module_name = altered_name(self.stem) + '_static' elif crate_type == 'test': # rust_test[_host] self.module_type = 'rust_test' + host # Before do_merge, stem name is based on the --crate-name parameter. # and test module name is based on stem. self.stem = self.test_module_name() # self.stem will be changed after merging with other tests. # self.stem is NOT used for final test binary name. # rust_test uses each source file base name as part of output file name. # In do_merge, this function is called again, with a module_name. # We make sure that the module name is unique in each package. if self.module_name: # Cargo uses "-C extra-filename=..." and "-C metadata=..." to add # different suffixes and distinguish multiple tests of the same # crate name. We ignore -C and use claim_module_name to get # unique sequential suffix. self.module_name = self.runner.claim_module_name( self.module_name, self, 0) # Now the module name is unique, stem should also match and unique. self.stem = self.module_name elif crate_type == 'proc-macro': # rust_proc_macro self.module_type = 'rust_proc_macro' self.stem = 'lib' + self.crate_name self.module_name = altered_name(self.stem) else: # unknown module type, rust_prebuilt_dylib? rust_library[_host]? self.module_type = '' self.stem = '' def dump_android_property_list_items(self, fmt, values): for v in values: # fmt has quotes, so we need escape_quotes(v) self.write(' ' + (fmt % escape_quotes(v)) + ',') def dump_android_property_list(self, name, fmt, values): if values: self.write(' ' + name + ': [') self.dump_android_property_list_items(fmt, values) self.write(' ],') def dump_android_core_properties(self): """Dump the module header, name, stem, etc.""" self.write(' name: "' + self.module_name + '",') # see properties shared by dump_defaults_module if self.defaults: self.write(' defaults: ["' + self.defaults + '"],') if self.stem != self.module_name: self.write(' stem: "' + self.stem + '",') if self.has_warning and not self.cap_lints: self.write(' // has rustc warnings') if self.host_supported and self.device_supported: self.write(' host_supported: true,') if not self.defaults: self.write(' crate_name: "' + self.crate_name + '",') if len(self.srcs) > 1: self.srcs = sorted(set(self.srcs)) self.dump_android_property_list('srcs', '"%s"', self.srcs) else: self.write(' srcs: ["' + self.main_src + '"],') if 'test' in self.crate_types and not self.defaults: # self.root_pkg can have multiple test modules, with different *_tests[n] # names, but their executables can all be installed under the same _tests # directory. When built from Cargo.toml, all tests should have different # file or crate names. So we used (root_pkg + '_tests') name as the # relative_install_path. # However, some package like 'slab' can have non-mergeable tests that # must be separated by different module names. So, here we no longer # emit relative_install_path. # self.write(' relative_install_path: "' + self.root_pkg + '_tests",') self.write(' test_suites: ["general-tests"],') self.write(' auto_gen_config: true,') def dump_android_externs(self): """Dump the dependent rlibs and dylibs property.""" so_libs = list() rust_libs = '' deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$') for lib in self.externs: # normal value of lib: "libc = liblibc-*.rlib" # strange case in rand crate: "getrandom_package = libgetrandom-*.rlib" # we should use "libgetrandom", not "lib" + "getrandom_package" groups = deps_libname.match(lib) if groups is not None: lib_name = groups.group(1) else: lib_name = re.sub(' .*$', '', lib) if lib.endswith('.rlib') or lib.endswith('.rmeta'): # On MacOS .rmeta is used when Linux uses .rlib or .rmeta. rust_libs += ' "' + altered_name('lib' + lib_name) + '",\n' elif lib.endswith('.so'): so_libs.append(lib_name) elif lib != 'proc_macro': # --extern proc_macro is special and ignored rust_libs += ' // ERROR: unknown type of lib ' + lib + '\n' if rust_libs: self.write(' rustlibs: [\n' + rust_libs + ' ],') # Are all dependent .so files proc_macros? # TODO(chh): Separate proc_macros and dylib. self.dump_android_property_list('proc_macros', '"lib%s"', so_libs) class ARObject(object): """Information of an "ar" link command.""" def __init__(self, runner, outf_name): # Remembered global runner and its members. self.runner = runner self.pkg = '' self.outf_name = outf_name # path to Android.bp # "ar" arguments self.line_num = 1 self.line = '' self.flags = '' # e.g. "crs" self.lib = '' # e.g. "/.../out/lib*.a" self.objs = list() # e.g. "/.../out/.../*.o" def parse(self, pkg, line_num, args_line): """Collect ar obj/lib file names.""" self.pkg = pkg self.line_num = line_num self.line = args_line args = args_line.split() num_args = len(args) if num_args < 3: print('ERROR: "ar" command has too few arguments', args_line) else: self.flags = unquote(args[0]) self.lib = unquote(args[1]) self.objs = sorted(set(map(unquote, args[2:]))) return self def write(self, s): self.outf.write(s + '\n') def dump_debug_info(self): self.write('\n// Line ' + str(self.line_num) + ' "ar" ' + self.line) self.write('// ar_object for %12s' % self.pkg) self.write('// flags = %s' % self.flags) self.write('// lib = %s' % short_out_name(self.pkg, self.lib)) for o in self.objs: self.write('// obj = %s' % short_out_name(self.pkg, o)) def dump_android_lib(self): """Write cc_library_static into Android.bp.""" self.write('\ncc_library_static {') self.write(' name: "' + file_base_name(self.lib) + '",') self.write(' host_supported: true,') if self.flags != 'crs': self.write(' // ar flags = %s' % self.flags) if self.pkg not in self.runner.pkg_obj2cc: self.write(' ERROR: cannot find source files.\n}') return self.write(' srcs: [') obj2cc = self.runner.pkg_obj2cc[self.pkg] # Note: wflags are ignored. dflags = list() fflags = list() for obj in self.objs: self.write(' "' + short_out_name(self.pkg, obj2cc[obj].src) + '",') # TODO(chh): union of dflags and flags of all obj # Now, just a temporary hack that uses the last obj's flags dflags = obj2cc[obj].dflags fflags = obj2cc[obj].fflags self.write(' ],') self.write(' cflags: [') self.write(' "-O3",') # TODO(chh): is this default correct? self.write(' "-Wno-error",') for x in fflags: self.write(' "-f' + x + '",') for x in dflags: self.write(' "-D' + x + '",') self.write(' ],') self.write('}') def dump(self): """Dump error/debug/module info to the output .bp file.""" self.runner.init_bp_file(self.outf_name) with open(self.outf_name, 'a') as outf: self.outf = outf if self.runner.args.debug: self.dump_debug_info() self.dump_android_lib() class CCObject(object): """Information of a "cc" compilation command.""" def __init__(self, runner, outf_name): # Remembered global runner and its members. self.runner = runner self.pkg = '' self.outf_name = outf_name # path to Android.bp # "cc" arguments self.line_num = 1 self.line = '' self.src = '' self.obj = '' self.dflags = list() # -D flags self.fflags = list() # -f flags self.iflags = list() # -I flags self.wflags = list() # -W flags self.other_args = list() def parse(self, pkg, line_num, args_line): """Collect cc compilation flags and src/out file names.""" self.pkg = pkg self.line_num = line_num self.line = args_line args = args_line.split() i = 0 while i < len(args): arg = args[i] if arg == '"-c"': i += 1 if args[i].startswith('"-o'): # ring-0.13.5 dumps: ... "-c" "-o/.../*.o" ".../*.c" self.obj = unquote(args[i])[2:] i += 1 self.src = unquote(args[i]) else: self.src = unquote(args[i]) elif arg == '"-o"': i += 1 self.obj = unquote(args[i]) elif arg == '"-I"': i += 1 self.iflags.append(unquote(args[i])) elif arg.startswith('"-D'): self.dflags.append(unquote(args[i])[2:]) elif arg.startswith('"-f'): self.fflags.append(unquote(args[i])[2:]) elif arg.startswith('"-W'): self.wflags.append(unquote(args[i])[2:]) elif not (arg.startswith('"-O') or arg == '"-m64"' or arg == '"-g"' or arg == '"-g3"'): # ignore -O -m64 -g self.other_args.append(unquote(args[i])) i += 1 self.dflags = sorted(set(self.dflags)) self.fflags = sorted(set(self.fflags)) # self.wflags is not sorted because some are order sensitive # and we ignore them anyway. if self.pkg not in self.runner.pkg_obj2cc: self.runner.pkg_obj2cc[self.pkg] = {} self.runner.pkg_obj2cc[self.pkg][self.obj] = self return self def write(self, s): self.outf.write(s + '\n') def dump_debug_flags(self, name, flags): self.write('// ' + name + ':') for f in flags: self.write('// %s' % f) def dump(self): """Dump only error/debug info to the output .bp file.""" if not self.runner.args.debug: return self.runner.init_bp_file(self.outf_name) with open(self.outf_name, 'a') as outf: self.outf = outf self.write('\n// Line ' + str(self.line_num) + ' "cc" ' + self.line) self.write('// cc_object for %12s' % self.pkg) self.write('// src = %s' % short_out_name(self.pkg, self.src)) self.write('// obj = %s' % short_out_name(self.pkg, self.obj)) self.dump_debug_flags('-I flags', self.iflags) self.dump_debug_flags('-D flags', self.dflags) self.dump_debug_flags('-f flags', self.fflags) self.dump_debug_flags('-W flags', self.wflags) if self.other_args: self.dump_debug_flags('other args', self.other_args) class Runner(object): """Main class to parse cargo -v output and print Android module definitions.""" def __init__(self, args): self.bp_files = set() # Remember all output Android.bp files. self.root_pkg = '' # name of package in ./Cargo.toml # Saved flags, modes, and data. self.args = args self.dry_run = not args.run self.skip_cargo = args.skipcargo # All cc/ar objects, crates, dependencies, and warning files self.cc_objects = list() self.pkg_obj2cc = {} # pkg_obj2cc[cc_object[i].pkg][cc_objects[i].obj] = cc_objects[i] self.ar_objects = list() self.crates = list() self.dependencies = list() # dependent and build script crates self.warning_files = set() # Keep a unique mapping from (module name) to crate self.name_owners = {} # Save and dump all errors from cargo to Android.bp. self.errors = '' # Default action is cargo clean, followed by build or user given actions. if args.cargo: self.cargo = ['clean'] + args.cargo else: self.cargo = ['clean', 'build'] if args.no_host: # do not run "cargo build" for host self.cargo = ['clean'] default_target = '--target x86_64-unknown-linux-gnu' if args.device: self.cargo.append('build ' + default_target) if args.tests: if not args.no_host: self.cargo.append('build --tests') self.cargo.append('build --tests ' + default_target) elif args.tests and not args.no_host: self.cargo.append('build --tests') def init_bp_file(self, name): if name not in self.bp_files: self.bp_files.add(name) with open(name, 'w') as outf: outf.write(ANDROID_BP_HEADER.format(args=' '.join(sys.argv[1:]))) def try_claim_module_name(self, name, owner): """Reserve and return True if it has not been reserved yet.""" if name not in self.name_owners or owner == self.name_owners[name]: self.name_owners[name] = owner return True return False def claim_module_name(self, prefix, owner, counter): """Return prefix if not owned yet, otherwise, prefix+str(counter).""" while True: name = prefix if counter > 0: name += '_' + str(counter) if self.try_claim_module_name(name, owner): return name counter += 1 def find_root_pkg(self): """Read name of [package] in ./Cargo.toml.""" if not os.path.exists('./Cargo.toml'): return with open('./Cargo.toml', 'r') as inf: pkg_section = re.compile(r'^ *\[package\]') name = re.compile('^ *name *= * "([^"]*)"') in_pkg = False for line in inf: if in_pkg: if name.match(line): self.root_pkg = name.match(line).group(1) break else: in_pkg = pkg_section.match(line) is not None def run_cargo(self): """Calls cargo -v and save its output to ./cargo.out.""" if self.skip_cargo: return self cargo = './Cargo.toml' if not os.access(cargo, os.R_OK): print('ERROR: Cannot find or read', cargo) return self if not self.dry_run and os.path.exists('cargo.out'): os.remove('cargo.out') cmd_tail = ' --target-dir ' + TARGET_TMP + ' >> cargo.out 2>&1' for c in self.cargo: features = '' if c != 'clean': if self.args.features is not None: features = ' --no-default-features' if self.args.features: features += ' --features ' + self.args.features cmd = 'cargo -vv ' if self.args.vv else 'cargo -v ' cmd += c + features + cmd_tail if self.args.rustflags and c != 'clean': cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cmd if self.dry_run: print('Dry-run skip:', cmd) else: if self.args.verbose: print('Running:', cmd) with open('cargo.out', 'a') as cargo_out: cargo_out.write('### Running: ' + cmd + '\n') os.system(cmd) return self def dump_dependencies(self): """Append dependencies and their features to Android.bp.""" if not self.dependencies: return dependent_list = list() for c in self.dependencies: dependent_list.append(c.feature_list()) sorted_dependencies = sorted(set(dependent_list)) self.init_bp_file('Android.bp') with open('Android.bp', 'a') as outf: outf.write('\n// dependent_library ["feature_list"]\n') for s in sorted_dependencies: outf.write('// ' + s + '\n') def dump_pkg_obj2cc(self): """Dump debug info of the pkg_obj2cc map.""" if not self.args.debug: return self.init_bp_file('Android.bp') with open('Android.bp', 'a') as outf: sorted_pkgs = sorted(self.pkg_obj2cc.keys()) for pkg in sorted_pkgs: if not self.pkg_obj2cc[pkg]: continue outf.write('\n// obj => src for %s\n' % pkg) obj2cc = self.pkg_obj2cc[pkg] for obj in sorted(obj2cc.keys()): outf.write('// ' + short_out_name(pkg, obj) + ' => ' + short_out_name(pkg, obj2cc[obj].src) + '\n') def gen_bp(self): """Parse cargo.out and generate Android.bp files.""" if self.dry_run: print('Dry-run skip: read', CARGO_OUT, 'write Android.bp') elif os.path.exists(CARGO_OUT): self.find_root_pkg() with open(CARGO_OUT, 'r') as cargo_out: self.parse(cargo_out, 'Android.bp') self.crates.sort(key=get_module_name) for obj in self.cc_objects: obj.dump() self.dump_pkg_obj2cc() for crate in self.crates: crate.dump() dumped_libs = set() for lib in self.ar_objects: if lib.pkg == self.root_pkg: lib_name = file_base_name(lib.lib) if lib_name not in dumped_libs: dumped_libs.add(lib_name) lib.dump() if self.args.dependencies and self.dependencies: self.dump_dependencies() if self.errors: self.append_to_bp('\nErrors in ' + CARGO_OUT + ':\n' + self.errors) return self def add_ar_object(self, obj): self.ar_objects.append(obj) def add_cc_object(self, obj): self.cc_objects.append(obj) def add_crate(self, crate): """Merge crate with someone in crates, or append to it. Return crates.""" if crate.skip_crate(): if self.args.debug: # include debug info of all crates self.crates.append(crate) if self.args.dependencies: # include only dependent crates if (is_dependent_file_path(crate.main_src) and not is_build_crate_name(crate.crate_name)): self.dependencies.append(crate) else: for c in self.crates: if c.merge(crate, 'Android.bp'): return # If not merged, decide module type and name now. crate.decide_module_type() self.crates.append(crate) def find_warning_owners(self): """For each warning file, find its owner crate.""" missing_owner = False for f in self.warning_files: cargo_dir = '' # find lowest crate, with longest path owner = None # owner crate of this warning for c in self.crates: if (f.startswith(c.cargo_dir + '/') and len(cargo_dir) < len(c.cargo_dir)): cargo_dir = c.cargo_dir owner = c if owner: owner.has_warning = True else: missing_owner = True if missing_owner and os.path.exists('Cargo.toml'): # owner is the root cargo, with empty cargo_dir for c in self.crates: if not c.cargo_dir: c.has_warning = True def rustc_command(self, n, rustc_line, line, outf_name): """Process a rustc command line from cargo -vv output.""" # cargo build -vv output can have multiple lines for a rustc command # due to '\n' in strings for environment variables. # strip removes leading spaces and '\n' at the end new_rustc = (rustc_line.strip() + line) if rustc_line else line # Use an heuristic to detect the completions of a multi-line command. # This might fail for some very rare case, but easy to fix manually. if not line.endswith('`\n') or (new_rustc.count('`') % 2) != 0: return new_rustc if RUSTC_VV_CMD_ARGS.match(new_rustc): args = RUSTC_VV_CMD_ARGS.match(new_rustc).group(1) self.add_crate(Crate(self, outf_name).parse(n, args)) else: self.assert_empty_vv_line(new_rustc) return '' def cc_ar_command(self, n, groups, outf_name): pkg = groups.group(1) line = groups.group(3) if groups.group(2) == 'cc': self.add_cc_object(CCObject(self, outf_name).parse(pkg, n, line)) else: self.add_ar_object(ARObject(self, outf_name).parse(pkg, n, line)) def append_to_bp(self, line): self.init_bp_file('Android.bp') with open('Android.bp', 'a') as outf: outf.write(line) def assert_empty_vv_line(self, line): if line: # report error if line is not empty self.append_to_bp('ERROR -vv line: ' + line) return '' def parse(self, inf, outf_name): """Parse rustc and warning messages in inf, return a list of Crates.""" n = 0 # line number prev_warning = False # true if the previous line was warning: ... rustc_line = '' # previous line(s) matching RUSTC_VV_PAT for line in inf: n += 1 if line.startswith('warning: '): prev_warning = True rustc_line = self.assert_empty_vv_line(rustc_line) continue new_rustc = '' if RUSTC_PAT.match(line): args_line = RUSTC_PAT.match(line).group(1) self.add_crate(Crate(self, outf_name).parse(n, args_line)) self.assert_empty_vv_line(rustc_line) elif rustc_line or RUSTC_VV_PAT.match(line): new_rustc = self.rustc_command(n, rustc_line, line, outf_name) elif CC_AR_VV_PAT.match(line): self.cc_ar_command(n, CC_AR_VV_PAT.match(line), outf_name) elif prev_warning and WARNING_FILE_PAT.match(line): self.assert_empty_vv_line(rustc_line) fpath = WARNING_FILE_PAT.match(line).group(1) if fpath[0] != '/': # ignore absolute path self.warning_files.add(fpath) elif line.startswith('error: ') or line.startswith('error[E'): self.errors += line prev_warning = False rustc_line = new_rustc self.find_warning_owners() def parse_args(): """Parse main arguments.""" parser = argparse.ArgumentParser('cargo2android') parser.add_argument( '--cargo', action='append', metavar='args_string', help=('extra cargo build -v args in a string, ' + 'each --cargo flag calls cargo build -v once')) parser.add_argument( '--debug', action='store_true', default=False, help='dump debug info into Android.bp') parser.add_argument( '--dependencies', action='store_true', default=False, help='dump debug info of dependent crates') parser.add_argument( '--device', action='store_true', default=False, help='run cargo also for a default device target') parser.add_argument( '--no-host', action='store_true', default=False, help='do not run cargo for the host; only for the device target') parser.add_argument( '--host-first-multilib', action='store_true', default=False, help=('add a compile_multilib:"first" property ' + 'to Android.bp host modules.')) parser.add_argument( '--features', type=str, help=('pass features to cargo build, ' + 'empty string means no default features')) parser.add_argument( '--onefile', action='store_true', default=False, help=('output all into one ./Android.bp, default will generate ' + 'one Android.bp per Cargo.toml in subdirectories')) parser.add_argument( '--run', action='store_true', default=False, help='run it, default is dry-run') parser.add_argument('--rustflags', type=str, help='passing flags to rustc') parser.add_argument( '--skipcargo', action='store_true', default=False, help='skip cargo command, parse cargo.out, and generate Android.bp') parser.add_argument( '--tests', action='store_true', default=False, help='run cargo build --tests after normal build') parser.add_argument( '--verbose', action='store_true', default=False, help='echo executed commands') parser.add_argument( '--vv', action='store_true', default=False, help='run cargo with -vv instead of default -v') return parser.parse_args() def main(): args = parse_args() if not args.run: # default is dry-run print(DRY_RUN_NOTE) Runner(args).run_cargo().gen_bp() if __name__ == '__main__': main()