cec8bac2cb
This is important because: 1. Some crates do not have their own tests enabled in presubmit. 2. As much as possible, we try to stick with one version of each crate. This often results in using different versions of dependencies than a crate has specified in its Cargo.toml. Ensuring that a crate's tests continue to pass when its dependencies are upgraded improves our confidence that the update is safe. The underlying implementation uses the new Bazel queryview to query modules and reverse dependencies. Bug: 168167373 Test: Run cargo2android.py on a number of crates including rusqlite, scopeguard, lock_api. Change-Id: Id24f2d3267cf8d5e0369ece2442f8971d4ab1343
1580 lines
60 KiB
Python
Executable File
1580 lines
60 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) 2019 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Call cargo -v, parse its output, and generate Android.bp.
|
|
|
|
Usage: Run this script in a crate workspace root directory.
|
|
The Cargo.toml file should work at least for the host platform.
|
|
|
|
(1) Without other flags, "cargo2android.py --run"
|
|
calls cargo clean, calls cargo build -v, and generates Android.bp.
|
|
The cargo build only generates crates for the host,
|
|
without test crates.
|
|
|
|
(2) To build crates for both host and device in Android.bp, use the
|
|
--device flag, for example:
|
|
cargo2android.py --run --device
|
|
|
|
Note that cargo build is only called once with the default target
|
|
x86_64-unknown-linux-gnu.
|
|
|
|
(3) To build default and test crates, for host and device, use both
|
|
--device and --tests flags:
|
|
cargo2android.py --run --device --tests
|
|
|
|
This is equivalent to using the --cargo flag to add extra builds:
|
|
cargo2android.py --run
|
|
--cargo "build --target x86_64-unknown-linux-gnu"
|
|
--cargo "build --tests --target x86_64-unknown-linux-gnu"
|
|
|
|
Note that when there are tests for this module or for its reverse
|
|
dependencies, these tests will be added to the TEST_MAPPING file.
|
|
|
|
If there are rustc warning messages, this script will add
|
|
a warning comment to the owner crate module in Android.bp.
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
from update_crate_tests import TestMapping
|
|
|
|
import argparse
|
|
import glob
|
|
import os
|
|
import os.path
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import sys
|
|
|
|
# Some Rust packages include extra unwanted crates.
|
|
# This set contains all such excluded crate names.
|
|
EXCLUDED_CRATES = set(['protobuf_bin_gen_rust_do_not_use'])
|
|
|
|
RENAME_MAP = {
|
|
# This map includes all changes to the default rust module names
|
|
# to resolve name conflicts, avoid confusion, or work as plugin.
|
|
'libbacktrace': 'libbacktrace_rust',
|
|
'libbase': 'libbase_rust',
|
|
'libfuse': 'libfuse_rust',
|
|
'libgcc': 'libgcc_rust',
|
|
'liblog': 'liblog_rust',
|
|
'libminijail': 'libminijail_rust',
|
|
'libsync': 'libsync_rust',
|
|
'libx86_64': 'libx86_64_rust',
|
|
'protoc_gen_rust': 'protoc-gen-rust',
|
|
}
|
|
|
|
RENAME_STEM_MAP = {
|
|
# This map includes all changes to the default rust module stem names,
|
|
# which is used for output files when different from the module name.
|
|
'protoc_gen_rust': 'protoc-gen-rust',
|
|
}
|
|
|
|
RENAME_DEFAULTS_MAP = {
|
|
# This map includes all changes to the default prefix of rust_default
|
|
# module names, to avoid conflict with existing Android modules.
|
|
'libc': 'rust_libc',
|
|
}
|
|
|
|
# Header added to all generated Android.bp files.
|
|
ANDROID_BP_HEADER = '// This file is generated by cargo2android.py {args}.\n'
|
|
|
|
CARGO_OUT = 'cargo.out' # Name of file to keep cargo build -v output.
|
|
|
|
TARGET_TMP = 'target.tmp' # Name of temporary output directory.
|
|
|
|
# Message to be displayed when this script is called without the --run flag.
|
|
DRY_RUN_NOTE = (
|
|
'Dry-run: This script uses ./' + TARGET_TMP + ' for output directory,\n' +
|
|
'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' +
|
|
'and writes to Android.bp in the current and subdirectories.\n\n' +
|
|
'To do do all of the above, use the --run flag.\n' +
|
|
'See --help for other flags, and more usage notes in this script.\n')
|
|
|
|
# Cargo -v output of a call to rustc.
|
|
RUSTC_PAT = re.compile('^ +Running `rustc (.*)`$')
|
|
|
|
# Cargo -vv output of a call to rustc could be split into multiple lines.
|
|
# Assume that the first line will contain some CARGO_* env definition.
|
|
RUSTC_VV_PAT = re.compile('^ +Running `.*CARGO_.*=.*$')
|
|
# The combined -vv output rustc command line pattern.
|
|
RUSTC_VV_CMD_ARGS = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$')
|
|
|
|
# Cargo -vv output of a "cc" or "ar" command; all in one line.
|
|
CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$')
|
|
# Some package, such as ring-0.13.5, has pattern '... running "cc"'.
|
|
|
|
# Rustc output of file location path pattern for a warning message.
|
|
WARNING_FILE_PAT = re.compile('^ *--> ([^:]*):[0-9]+')
|
|
|
|
# Rust package name with suffix -d1.d2.d3.
|
|
VERSION_SUFFIX_PAT = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+$')
|
|
|
|
|
|
def altered_name(name):
|
|
return RENAME_MAP[name] if (name in RENAME_MAP) else name
|
|
|
|
|
|
def altered_stem(name):
|
|
return RENAME_STEM_MAP[name] if (name in RENAME_STEM_MAP) else name
|
|
|
|
|
|
def altered_defaults(name):
|
|
return RENAME_DEFAULTS_MAP[name] if (name in RENAME_DEFAULTS_MAP) else name
|
|
|
|
|
|
def is_build_crate_name(name):
|
|
# We added special prefix to build script crate names.
|
|
return name.startswith('build_script_')
|
|
|
|
|
|
def is_dependent_file_path(path):
|
|
# Absolute or dependent '.../' paths are not main files of this crate.
|
|
return path.startswith('/') or path.startswith('.../')
|
|
|
|
|
|
def get_module_name(crate): # to sort crates in a list
|
|
return crate.module_name
|
|
|
|
|
|
def pkg2crate_name(s):
|
|
return s.replace('-', '_').replace('.', '_')
|
|
|
|
|
|
def file_base_name(path):
|
|
return os.path.splitext(os.path.basename(path))[0]
|
|
|
|
|
|
def test_base_name(path):
|
|
return pkg2crate_name(file_base_name(path))
|
|
|
|
|
|
def unquote(s): # remove quotes around str
|
|
if s and len(s) > 1 and s[0] == '"' and s[-1] == '"':
|
|
return s[1:-1]
|
|
return s
|
|
|
|
|
|
def remove_version_suffix(s): # remove -d1.d2.d3 suffix
|
|
if VERSION_SUFFIX_PAT.match(s):
|
|
return VERSION_SUFFIX_PAT.match(s).group(1)
|
|
return s
|
|
|
|
|
|
def short_out_name(pkg, s): # replace /.../pkg-*/out/* with .../out/*
|
|
return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s)
|
|
|
|
|
|
def escape_quotes(s): # replace '"' with '\\"'
|
|
return s.replace('"', '\\"')
|
|
|
|
|
|
class Crate(object):
|
|
"""Information of a Rust crate to collect/emit for an Android.bp module."""
|
|
|
|
def __init__(self, runner, outf_name):
|
|
# Remembered global runner and its members.
|
|
self.runner = runner
|
|
self.debug = runner.args.debug
|
|
self.cargo_dir = '' # directory of my Cargo.toml
|
|
self.outf_name = outf_name # path to Android.bp
|
|
self.outf = None # open file handle of outf_name during dump*
|
|
# Variants/results that could be merged from multiple rustc lines.
|
|
self.host_supported = False
|
|
self.device_supported = False
|
|
self.has_warning = False
|
|
# Android module properties derived from rustc parameters.
|
|
self.module_name = '' # unique in Android build system
|
|
self.module_type = '' # rust_{binary,library,test}[_host] etc.
|
|
self.defaults = '' # rust_defaults used by rust_test* modules
|
|
self.default_srcs = False # use 'srcs' defined in self.defaults
|
|
self.root_pkg = '' # parent package name of a sub/test packge, from -L
|
|
self.srcs = list() # main_src or merged multiple source files
|
|
self.stem = '' # real base name of output file
|
|
# Kept parsed status
|
|
self.errors = '' # all errors found during parsing
|
|
self.line_num = 1 # runner told input source line number
|
|
self.line = '' # original rustc command line parameters
|
|
# Parameters collected from rustc command line.
|
|
self.crate_name = '' # follows --crate-name
|
|
self.main_src = '' # follows crate_name parameter, shortened
|
|
self.crate_types = list() # follows --crate-type
|
|
self.cfgs = list() # follows --cfg, without feature= prefix
|
|
self.features = list() # follows --cfg, name in 'feature="..."'
|
|
self.codegens = list() # follows -C, some ignored
|
|
self.externs = list() # follows --extern
|
|
self.core_externs = list() # first part of self.externs elements
|
|
self.static_libs = list() # e.g. -l static=host_cpuid
|
|
self.shared_libs = list() # e.g. -l dylib=wayland-client, -l z
|
|
self.cap_lints = '' # follows --cap-lints
|
|
self.emit_list = '' # e.g., --emit=dep-info,metadata,link
|
|
self.edition = '2015' # rustc default, e.g., --edition=2018
|
|
self.target = '' # follows --target
|
|
|
|
def write(self, s):
|
|
# convenient way to output one line at a time with EOL.
|
|
self.outf.write(s + '\n')
|
|
|
|
def same_flags(self, other):
|
|
# host_supported, device_supported, has_warning are not compared but merged
|
|
# target is not compared, to merge different target/host modules
|
|
# externs is not compared; only core_externs is compared
|
|
return (not self.errors and not other.errors and
|
|
self.edition == other.edition and
|
|
self.cap_lints == other.cap_lints and
|
|
self.emit_list == other.emit_list and
|
|
self.core_externs == other.core_externs and
|
|
self.codegens == other.codegens and
|
|
self.features == other.features and
|
|
self.static_libs == other.static_libs and
|
|
self.shared_libs == other.shared_libs and self.cfgs == other.cfgs)
|
|
|
|
def merge_host_device(self, other):
|
|
"""Returns true if attributes are the same except host/device support."""
|
|
return (self.crate_name == other.crate_name and
|
|
self.crate_types == other.crate_types and
|
|
self.main_src == other.main_src and
|
|
# before merge, each test module has an unique module name and stem
|
|
(self.stem == other.stem or self.crate_types == ['test']) and
|
|
self.root_pkg == other.root_pkg and not self.skip_crate() and
|
|
self.same_flags(other))
|
|
|
|
def merge_test(self, other):
|
|
"""Returns true if self and other are tests of same root_pkg."""
|
|
# Before merger, each test has its own crate_name.
|
|
# A merged test uses its source file base name as output file name,
|
|
# so a test is mergeable only if its base name equals to its crate name.
|
|
return (self.crate_types == other.crate_types and
|
|
self.crate_types == ['test'] and self.root_pkg == other.root_pkg and
|
|
not self.skip_crate() and
|
|
other.crate_name == test_base_name(other.main_src) and
|
|
(len(self.srcs) > 1 or
|
|
(self.crate_name == test_base_name(self.main_src)) and
|
|
self.host_supported == other.host_supported and
|
|
self.device_supported == other.device_supported) and
|
|
self.same_flags(other))
|
|
|
|
def merge(self, other, outf_name):
|
|
"""Try to merge crate into self."""
|
|
# Cargo build --tests could recompile a library for tests.
|
|
# We need to merge such duplicated calls to rustc, with
|
|
# the algorithm in merge_host_device.
|
|
should_merge_host_device = self.merge_host_device(other)
|
|
should_merge_test = False
|
|
if not should_merge_host_device:
|
|
should_merge_test = self.merge_test(other)
|
|
if should_merge_host_device or should_merge_test:
|
|
self.runner.init_bp_file(outf_name)
|
|
with open(outf_name, 'a') as outf: # to write debug info
|
|
self.outf = outf
|
|
other.outf = outf
|
|
self.do_merge(other, should_merge_test)
|
|
return True
|
|
return False
|
|
|
|
def do_merge(self, other, should_merge_test):
|
|
"""Merge attributes of other to self."""
|
|
if self.debug:
|
|
self.write('\n// Before merge definition (1):')
|
|
self.dump_debug_info()
|
|
self.write('\n// Before merge definition (2):')
|
|
other.dump_debug_info()
|
|
# Merge properties of other to self.
|
|
self.has_warning = self.has_warning or other.has_warning
|
|
if not self.target: # okay to keep only the first target triple
|
|
self.target = other.target
|
|
# decide_module_type sets up default self.stem,
|
|
# which can be changed if self is a merged test module.
|
|
self.decide_module_type()
|
|
if should_merge_test:
|
|
self.srcs.append(other.main_src)
|
|
# use a short unique name as the merged module name.
|
|
prefix = self.root_pkg + '_tests'
|
|
self.module_name = self.runner.claim_module_name(prefix, self, 0)
|
|
self.stem = self.module_name
|
|
# This normalized root_pkg name although might be the same
|
|
# as other module's crate_name, it is not actually used for
|
|
# output file name. A merged test module always have multiple
|
|
# source files and each source file base name is used as
|
|
# its output file name.
|
|
self.crate_name = pkg2crate_name(self.root_pkg)
|
|
if self.debug:
|
|
self.write('\n// After merge definition (1):')
|
|
self.dump_debug_info()
|
|
|
|
def find_cargo_dir(self):
|
|
"""Deepest directory with Cargo.toml and contains the main_src."""
|
|
if not is_dependent_file_path(self.main_src):
|
|
dir_name = os.path.dirname(self.main_src)
|
|
while dir_name:
|
|
if os.path.exists(dir_name + '/Cargo.toml'):
|
|
self.cargo_dir = dir_name
|
|
return
|
|
dir_name = os.path.dirname(dir_name)
|
|
|
|
def add_codegens_flag(self, flag):
|
|
"""Ignore options not used in Android."""
|
|
# 'prefer-dynamic' does not work with common flag -C lto
|
|
# 'embed-bitcode' is ignored; we might control LTO with other .bp flag
|
|
# 'codegen-units' is set in Android global config or by default
|
|
if not (flag.startswith('codegen-units=') or
|
|
flag.startswith('debuginfo=') or
|
|
flag.startswith('embed-bitcode=') or
|
|
flag.startswith('extra-filename=') or
|
|
flag.startswith('incremental=') or
|
|
flag.startswith('metadata=') or
|
|
flag == 'prefer-dynamic'):
|
|
self.codegens.append(flag)
|
|
|
|
def parse(self, line_num, line):
|
|
"""Find important rustc arguments to convert to Android.bp properties."""
|
|
self.line_num = line_num
|
|
self.line = line
|
|
args = line.split() # Loop through every argument of rustc.
|
|
i = 0
|
|
while i < len(args):
|
|
arg = args[i]
|
|
if arg == '--crate-name':
|
|
i += 1
|
|
self.crate_name = args[i]
|
|
elif arg == '--crate-type':
|
|
i += 1
|
|
# cargo calls rustc with multiple --crate-type flags.
|
|
# rustc can accept:
|
|
# --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro]
|
|
self.crate_types.append(args[i])
|
|
elif arg == '--test':
|
|
self.crate_types.append('test')
|
|
elif arg == '--target':
|
|
i += 1
|
|
self.target = args[i]
|
|
elif arg == '--cfg':
|
|
i += 1
|
|
if args[i].startswith('\'feature='):
|
|
self.features.append(unquote(args[i].replace('\'feature=', '')[:-1]))
|
|
else:
|
|
self.cfgs.append(args[i])
|
|
elif arg == '--extern':
|
|
i += 1
|
|
extern_names = re.sub('=/[^ ]*/deps/', ' = ', args[i])
|
|
self.externs.append(extern_names)
|
|
self.core_externs.append(re.sub(' = .*', '', extern_names))
|
|
elif arg == '-C': # codegen options
|
|
i += 1
|
|
self.add_codegens_flag(args[i])
|
|
elif arg.startswith('-C'):
|
|
# cargo has been passing "-C <xyz>" flag to rustc,
|
|
# but newer cargo could pass '-Cembed-bitcode=no' to rustc.
|
|
self.add_codegens_flag(arg[2:])
|
|
elif arg == '--cap-lints':
|
|
i += 1
|
|
self.cap_lints = args[i]
|
|
elif arg == '-L':
|
|
i += 1
|
|
if args[i].startswith('dependency=') and args[i].endswith('/deps'):
|
|
if '/' + TARGET_TMP + '/' in args[i]:
|
|
self.root_pkg = re.sub(
|
|
'^.*/', '', re.sub('/' + TARGET_TMP + '/.*/deps$', '', args[i]))
|
|
else:
|
|
self.root_pkg = re.sub('^.*/', '',
|
|
re.sub('/[^/]+/[^/]+/deps$', '', args[i]))
|
|
self.root_pkg = remove_version_suffix(self.root_pkg)
|
|
elif arg == '-l':
|
|
i += 1
|
|
if args[i].startswith('static='):
|
|
self.static_libs.append(re.sub('static=', '', args[i]))
|
|
elif args[i].startswith('dylib='):
|
|
self.shared_libs.append(re.sub('dylib=', '', args[i]))
|
|
else:
|
|
self.shared_libs.append(args[i])
|
|
elif arg == '--out-dir' or arg == '--color': # ignored
|
|
i += 1
|
|
elif arg.startswith('--error-format=') or arg.startswith('--json='):
|
|
_ = arg # ignored
|
|
elif arg.startswith('--emit='):
|
|
self.emit_list = arg.replace('--emit=', '')
|
|
elif arg.startswith('--edition='):
|
|
self.edition = arg.replace('--edition=', '')
|
|
elif not arg.startswith('-'):
|
|
# shorten imported crate main source paths like $HOME/.cargo/
|
|
# registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/lib.rs
|
|
self.main_src = re.sub(r'^/[^ ]*/registry/src/', '.../', args[i])
|
|
self.main_src = re.sub(r'^\.\.\./github.com-[0-9a-f]*/', '.../',
|
|
self.main_src)
|
|
self.find_cargo_dir()
|
|
if self.cargo_dir: # for a subdirectory
|
|
if self.runner.args.no_subdir: # all .bp content to /dev/null
|
|
self.outf_name = '/dev/null'
|
|
elif not self.runner.args.onefile:
|
|
# Write to Android.bp in the subdirectory with Cargo.toml.
|
|
self.outf_name = self.cargo_dir + '/Android.bp'
|
|
self.main_src = self.main_src[len(self.cargo_dir) + 1:]
|
|
else:
|
|
self.errors += 'ERROR: unknown ' + arg + '\n'
|
|
i += 1
|
|
if not self.crate_name:
|
|
self.errors += 'ERROR: missing --crate-name\n'
|
|
if not self.main_src:
|
|
self.errors += 'ERROR: missing main source file\n'
|
|
else:
|
|
self.srcs.append(self.main_src)
|
|
if not self.crate_types:
|
|
# Treat "--cfg test" as "--test"
|
|
if 'test' in self.cfgs:
|
|
self.crate_types.append('test')
|
|
else:
|
|
self.errors += 'ERROR: missing --crate-type or --test\n'
|
|
elif len(self.crate_types) > 1:
|
|
if 'test' in self.crate_types:
|
|
self.errors += 'ERROR: cannot handle both --crate-type and --test\n'
|
|
if 'lib' in self.crate_types and 'rlib' in self.crate_types:
|
|
self.errors += 'ERROR: cannot generate both lib and rlib crate types\n'
|
|
if not self.root_pkg:
|
|
self.root_pkg = self.crate_name
|
|
self.device_supported = self.runner.args.device
|
|
self.host_supported = not self.runner.args.no_host
|
|
self.cfgs = sorted(set(self.cfgs))
|
|
self.features = sorted(set(self.features))
|
|
self.codegens = sorted(set(self.codegens))
|
|
self.externs = sorted(set(self.externs))
|
|
self.core_externs = sorted(set(self.core_externs))
|
|
self.static_libs = sorted(set(self.static_libs))
|
|
self.shared_libs = sorted(set(self.shared_libs))
|
|
self.crate_types = sorted(set(self.crate_types))
|
|
self.decide_module_type()
|
|
self.module_name = altered_name(self.stem)
|
|
return self
|
|
|
|
def dump_line(self):
|
|
self.write('\n// Line ' + str(self.line_num) + ' ' + self.line)
|
|
|
|
def feature_list(self):
|
|
"""Return a string of main_src + "feature_list"."""
|
|
pkg = self.main_src
|
|
if pkg.startswith('.../'): # keep only the main package name
|
|
pkg = re.sub('/.*', '', pkg[4:])
|
|
elif pkg.startswith('/'): # use relative path for a local package
|
|
pkg = os.path.relpath(pkg)
|
|
if not self.features:
|
|
return pkg
|
|
return pkg + ' "' + ','.join(self.features) + '"'
|
|
|
|
def dump_skip_crate(self, kind):
|
|
if self.debug:
|
|
self.write('\n// IGNORED: ' + kind + ' ' + self.main_src)
|
|
return self
|
|
|
|
def skip_crate(self):
|
|
"""Return crate_name or a message if this crate should be skipped."""
|
|
if (is_build_crate_name(self.crate_name) or
|
|
self.crate_name in EXCLUDED_CRATES):
|
|
return self.crate_name
|
|
if is_dependent_file_path(self.main_src):
|
|
return 'dependent crate'
|
|
return ''
|
|
|
|
def dump(self):
|
|
"""Dump all error/debug/module code to the output .bp file."""
|
|
self.runner.init_bp_file(self.outf_name)
|
|
with open(self.outf_name, 'a') as outf:
|
|
self.outf = outf
|
|
if self.errors:
|
|
self.dump_line()
|
|
self.write(self.errors)
|
|
elif self.skip_crate():
|
|
self.dump_skip_crate(self.skip_crate())
|
|
else:
|
|
if self.debug:
|
|
self.dump_debug_info()
|
|
self.dump_android_module()
|
|
|
|
def dump_debug_info(self):
|
|
"""Dump parsed data, when cargo2android is called with --debug."""
|
|
|
|
def dump(name, value):
|
|
self.write('//%12s = %s' % (name, value))
|
|
|
|
def opt_dump(name, value):
|
|
if value:
|
|
dump(name, value)
|
|
|
|
def dump_list(fmt, values):
|
|
for v in values:
|
|
self.write(fmt % v)
|
|
|
|
self.dump_line()
|
|
dump('module_name', self.module_name)
|
|
dump('crate_name', self.crate_name)
|
|
dump('crate_types', self.crate_types)
|
|
dump('main_src', self.main_src)
|
|
dump('has_warning', self.has_warning)
|
|
dump('for_host', self.host_supported)
|
|
dump('for_device', self.device_supported)
|
|
dump('module_type', self.module_type)
|
|
opt_dump('target', self.target)
|
|
opt_dump('edition', self.edition)
|
|
opt_dump('emit_list', self.emit_list)
|
|
opt_dump('cap_lints', self.cap_lints)
|
|
dump_list('// cfg = %s', self.cfgs)
|
|
dump_list('// cfg = \'feature "%s"\'', self.features)
|
|
# TODO(chh): escape quotes in self.features, but not in other dump_list
|
|
dump_list('// codegen = %s', self.codegens)
|
|
dump_list('// externs = %s', self.externs)
|
|
dump_list('// -l static = %s', self.static_libs)
|
|
dump_list('// -l (dylib) = %s', self.shared_libs)
|
|
|
|
def dump_android_module(self):
|
|
"""Dump one or more Android module definition, depending on crate_types."""
|
|
if len(self.crate_types) == 1:
|
|
self.dump_single_type_android_module()
|
|
return
|
|
if 'test' in self.crate_types:
|
|
self.write('\nERROR: multiple crate types cannot include test type')
|
|
return
|
|
# Dump one Android module per crate_type.
|
|
for crate_type in self.crate_types:
|
|
self.decide_one_module_type(crate_type)
|
|
self.dump_one_android_module(crate_type)
|
|
|
|
def build_default_name(self):
|
|
"""Return a short and readable name for the rust_defaults module."""
|
|
# Choices: (1) root_pkg + '_defaults',
|
|
# (2) root_pkg + '_defaults_' + crate_name
|
|
# (3) root_pkg + '_defaults_' + main_src_basename_path
|
|
# (4) root_pkg + '_defaults_' + a_positive_sequence_number
|
|
name1 = altered_defaults(self.root_pkg) + '_defaults'
|
|
if self.runner.try_claim_module_name(name1, self):
|
|
return name1
|
|
name2 = name1 + '_' + self.crate_name
|
|
if self.runner.try_claim_module_name(name2, self):
|
|
return name2
|
|
name3 = name1 + '_' + self.main_src_basename_path()
|
|
if self.runner.try_claim_module_name(name3, self):
|
|
return name3
|
|
return self.runner.claim_module_name(name1, self, 0)
|
|
|
|
def dump_srcs_list(self):
|
|
"""Dump the srcs list, for defaults or regular modules."""
|
|
if len(self.srcs) > 1:
|
|
srcs = sorted(set(self.srcs)) # make a copy and dedup
|
|
else:
|
|
srcs = [self.main_src]
|
|
copy_out = self.runner.copy_out_module_name()
|
|
if copy_out:
|
|
srcs.append(':' + copy_out)
|
|
self.dump_android_property_list('srcs', '"%s"', srcs)
|
|
|
|
def dump_defaults_module(self):
|
|
"""Dump a rust_defaults module to be shared by other modules."""
|
|
name = self.build_default_name()
|
|
self.defaults = name
|
|
self.write('\nrust_defaults {')
|
|
self.write(' name: "' + name + '",')
|
|
if self.runner.args.global_defaults:
|
|
self.write(' defaults: ["' + self.runner.args.global_defaults + '"],')
|
|
self.write(' crate_name: "' + self.crate_name + '",')
|
|
if len(self.srcs) == 1: # only one source file; share it in defaults
|
|
self.default_srcs = True
|
|
if self.has_warning and not self.cap_lints:
|
|
self.write(' // has rustc warnings')
|
|
self.dump_srcs_list()
|
|
if 'test' in self.crate_types:
|
|
self.write(' test_suites: ["general-tests"],')
|
|
self.write(' auto_gen_config: true,')
|
|
self.dump_edition_flags_libs()
|
|
self.write('}')
|
|
|
|
def dump_single_type_android_module(self):
|
|
"""Dump one simple Android module, which has only one crate_type."""
|
|
crate_type = self.crate_types[0]
|
|
if crate_type != 'test':
|
|
# do not change self.stem or self.module_name
|
|
self.dump_one_android_module(crate_type)
|
|
return
|
|
# Dump one test module per source file, and separate host and device tests.
|
|
# crate_type == 'test'
|
|
if (self.host_supported and self.device_supported) or len(self.srcs) > 1:
|
|
self.srcs = sorted(set(self.srcs))
|
|
self.dump_defaults_module()
|
|
saved_srcs = self.srcs
|
|
for src in saved_srcs:
|
|
self.srcs = [src]
|
|
saved_device_supported = self.device_supported
|
|
saved_host_supported = self.host_supported
|
|
saved_main_src = self.main_src
|
|
self.main_src = src
|
|
if saved_host_supported:
|
|
self.device_supported = False
|
|
self.host_supported = True
|
|
self.module_name = self.test_module_name()
|
|
self.decide_one_module_type(crate_type)
|
|
self.dump_one_android_module(crate_type)
|
|
if saved_device_supported:
|
|
self.device_supported = True
|
|
self.host_supported = False
|
|
self.module_name = self.test_module_name()
|
|
self.decide_one_module_type(crate_type)
|
|
self.dump_one_android_module(crate_type)
|
|
self.host_supported = saved_host_supported
|
|
self.device_supported = saved_device_supported
|
|
self.main_src = saved_main_src
|
|
self.srcs = saved_srcs
|
|
|
|
def dump_one_android_module(self, crate_type):
|
|
"""Dump one Android module definition."""
|
|
if not self.module_type:
|
|
self.write('\nERROR: unknown crate_type ' + crate_type)
|
|
return
|
|
self.write('\n' + self.module_type + ' {')
|
|
self.dump_android_core_properties()
|
|
if not self.defaults:
|
|
self.dump_edition_flags_libs()
|
|
if self.runner.args.host_first_multilib and self.host_supported and crate_type != 'test':
|
|
self.write(' compile_multilib: "first",')
|
|
self.write('}')
|
|
|
|
def dump_android_flags(self):
|
|
"""Dump Android module flags property."""
|
|
if not self.cfgs and not self.codegens and not self.cap_lints:
|
|
return
|
|
self.write(' flags: [')
|
|
if self.cap_lints:
|
|
self.write(' "--cap-lints ' + self.cap_lints + '",')
|
|
cfg_fmt = '"--cfg %s"'
|
|
codegens_fmt = '"-C %s"'
|
|
self.dump_android_property_list_items(cfg_fmt, self.cfgs)
|
|
self.dump_android_property_list_items(codegens_fmt, self.codegens)
|
|
self.write(' ],')
|
|
|
|
def dump_edition_flags_libs(self):
|
|
if self.edition:
|
|
self.write(' edition: "' + self.edition + '",')
|
|
self.dump_android_property_list('features', '"%s"', self.features)
|
|
self.dump_android_flags()
|
|
if self.externs:
|
|
self.dump_android_externs()
|
|
self.dump_android_property_list('static_libs', '"lib%s"', self.static_libs)
|
|
self.dump_android_property_list('shared_libs', '"lib%s"', self.shared_libs)
|
|
|
|
def main_src_basename_path(self):
|
|
return re.sub('/', '_', re.sub('.rs$', '', self.main_src))
|
|
|
|
def test_module_name(self):
|
|
"""Return a unique name for a test module."""
|
|
# root_pkg+(_host|_device) + '_test_'+source_file_name
|
|
suffix = self.main_src_basename_path()
|
|
host_device = '_host'
|
|
if self.device_supported:
|
|
host_device = '_device'
|
|
return self.root_pkg + host_device + '_test_' + suffix
|
|
|
|
def decide_module_type(self):
|
|
# Use the first crate type for the default/first module.
|
|
crate_type = self.crate_types[0] if self.crate_types else ''
|
|
self.decide_one_module_type(crate_type)
|
|
|
|
def decide_one_module_type(self, crate_type):
|
|
"""Decide which Android module type to use."""
|
|
host = '' if self.device_supported else '_host'
|
|
if crate_type == 'bin': # rust_binary[_host]
|
|
self.module_type = 'rust_binary' + host
|
|
# In rare cases like protobuf-codegen, the output binary name must
|
|
# be renamed to use as a plugin for protoc.
|
|
self.stem = altered_stem(self.crate_name)
|
|
self.module_name = altered_name(self.crate_name)
|
|
elif crate_type == 'lib': # rust_library[_host]
|
|
# TODO(chh): should this be rust_library[_host]?
|
|
# Assuming that Cargo.toml do not use both 'lib' and 'rlib',
|
|
# because we map them both to rlib.
|
|
self.module_type = 'rust_library' + host
|
|
self.stem = 'lib' + self.crate_name
|
|
self.module_name = altered_name(self.stem)
|
|
elif crate_type == 'rlib': # rust_library[_host]
|
|
self.module_type = 'rust_library' + host
|
|
self.stem = 'lib' + self.crate_name
|
|
self.module_name = altered_name(self.stem)
|
|
elif crate_type == 'dylib': # rust_library[_host]_dylib
|
|
self.module_type = 'rust_library' + host + '_dylib'
|
|
self.stem = 'lib' + self.crate_name
|
|
self.module_name = altered_name(self.stem) + '_dylib'
|
|
elif crate_type == 'cdylib': # rust_library[_host]_shared
|
|
self.module_type = 'rust_ffi' + host + '_shared'
|
|
self.stem = 'lib' + self.crate_name
|
|
self.module_name = altered_name(self.stem) + '_shared'
|
|
elif crate_type == 'staticlib': # rust_library[_host]_static
|
|
self.module_type = 'rust_ffi' + host + '_static'
|
|
self.stem = 'lib' + self.crate_name
|
|
self.module_name = altered_name(self.stem) + '_static'
|
|
elif crate_type == 'test': # rust_test[_host]
|
|
self.module_type = 'rust_test' + host
|
|
# Before do_merge, stem name is based on the --crate-name parameter.
|
|
# and test module name is based on stem.
|
|
self.stem = self.test_module_name()
|
|
# self.stem will be changed after merging with other tests.
|
|
# self.stem is NOT used for final test binary name.
|
|
# rust_test uses each source file base name as part of output file name.
|
|
# In do_merge, this function is called again, with a module_name.
|
|
# We make sure that the module name is unique in each package.
|
|
if self.module_name:
|
|
# Cargo uses "-C extra-filename=..." and "-C metadata=..." to add
|
|
# different suffixes and distinguish multiple tests of the same
|
|
# crate name. We ignore -C and use claim_module_name to get
|
|
# unique sequential suffix.
|
|
self.module_name = self.runner.claim_module_name(
|
|
self.module_name, self, 0)
|
|
# Now the module name is unique, stem should also match and unique.
|
|
self.stem = self.module_name
|
|
elif crate_type == 'proc-macro': # rust_proc_macro
|
|
self.module_type = 'rust_proc_macro'
|
|
self.stem = 'lib' + self.crate_name
|
|
self.module_name = altered_name(self.stem)
|
|
else: # unknown module type, rust_prebuilt_dylib? rust_library[_host]?
|
|
self.module_type = ''
|
|
self.stem = ''
|
|
|
|
def dump_android_property_list_items(self, fmt, values):
|
|
for v in values:
|
|
# fmt has quotes, so we need escape_quotes(v)
|
|
self.write(' ' + (fmt % escape_quotes(v)) + ',')
|
|
|
|
def dump_android_property_list(self, name, fmt, values):
|
|
if not values:
|
|
return
|
|
if len(values) > 1:
|
|
self.write(' ' + name + ': [')
|
|
self.dump_android_property_list_items(fmt, values)
|
|
self.write(' ],')
|
|
else:
|
|
self.write(' ' + name + ': [' +
|
|
(fmt % escape_quotes(values[0])) + '],')
|
|
|
|
def dump_android_core_properties(self):
|
|
"""Dump the module header, name, stem, etc."""
|
|
self.write(' name: "' + self.module_name + '",')
|
|
# see properties shared by dump_defaults_module
|
|
if self.defaults:
|
|
self.write(' defaults: ["' + self.defaults + '"],')
|
|
elif self.runner.args.global_defaults:
|
|
self.write(' defaults: ["' + self.runner.args.global_defaults + '"],')
|
|
if self.stem != self.module_name:
|
|
self.write(' stem: "' + self.stem + '",')
|
|
if self.has_warning and not self.cap_lints and not self.default_srcs:
|
|
self.write(' // has rustc warnings')
|
|
if self.host_supported and self.device_supported and self.module_type != 'rust_proc_macro':
|
|
self.write(' host_supported: true,')
|
|
if not self.defaults:
|
|
self.write(' crate_name: "' + self.crate_name + '",')
|
|
if not self.default_srcs:
|
|
self.dump_srcs_list()
|
|
if 'test' in self.crate_types and not self.defaults:
|
|
# self.root_pkg can have multiple test modules, with different *_tests[n]
|
|
# names, but their executables can all be installed under the same _tests
|
|
# directory. When built from Cargo.toml, all tests should have different
|
|
# file or crate names. So we used (root_pkg + '_tests') name as the
|
|
# relative_install_path.
|
|
# However, some package like 'slab' can have non-mergeable tests that
|
|
# must be separated by different module names. So, here we no longer
|
|
# emit relative_install_path.
|
|
# self.write(' relative_install_path: "' + self.root_pkg + '_tests",')
|
|
self.write(' test_suites: ["general-tests"],')
|
|
self.write(' auto_gen_config: true,')
|
|
if 'test' in self.crate_types and self.host_supported:
|
|
self.write(' test_options: {')
|
|
self.write(' unit_test: true,')
|
|
self.write(' },')
|
|
|
|
def dump_android_externs(self):
|
|
"""Dump the dependent rlibs and dylibs property."""
|
|
so_libs = list()
|
|
rust_libs = ''
|
|
deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$')
|
|
for lib in self.externs:
|
|
# normal value of lib: "libc = liblibc-*.rlib"
|
|
# strange case in rand crate: "getrandom_package = libgetrandom-*.rlib"
|
|
# we should use "libgetrandom", not "lib" + "getrandom_package"
|
|
groups = deps_libname.match(lib)
|
|
if groups is not None:
|
|
lib_name = groups.group(1)
|
|
else:
|
|
lib_name = re.sub(' .*$', '', lib)
|
|
if lib.endswith('.rlib') or lib.endswith('.rmeta'):
|
|
# On MacOS .rmeta is used when Linux uses .rlib or .rmeta.
|
|
rust_libs += ' "' + altered_name('lib' + lib_name) + '",\n'
|
|
elif lib.endswith('.so'):
|
|
so_libs.append(lib_name)
|
|
elif lib != 'proc_macro': # --extern proc_macro is special and ignored
|
|
rust_libs += ' // ERROR: unknown type of lib ' + lib + '\n'
|
|
if rust_libs:
|
|
self.write(' rustlibs: [\n' + rust_libs + ' ],')
|
|
# Are all dependent .so files proc_macros?
|
|
# TODO(chh): Separate proc_macros and dylib.
|
|
self.dump_android_property_list('proc_macros', '"lib%s"', so_libs)
|
|
|
|
|
|
class ARObject(object):
|
|
"""Information of an "ar" link command."""
|
|
|
|
def __init__(self, runner, outf_name):
|
|
# Remembered global runner and its members.
|
|
self.runner = runner
|
|
self.pkg = ''
|
|
self.outf_name = outf_name # path to Android.bp
|
|
# "ar" arguments
|
|
self.line_num = 1
|
|
self.line = ''
|
|
self.flags = '' # e.g. "crs"
|
|
self.lib = '' # e.g. "/.../out/lib*.a"
|
|
self.objs = list() # e.g. "/.../out/.../*.o"
|
|
|
|
def parse(self, pkg, line_num, args_line):
|
|
"""Collect ar obj/lib file names."""
|
|
self.pkg = pkg
|
|
self.line_num = line_num
|
|
self.line = args_line
|
|
args = args_line.split()
|
|
num_args = len(args)
|
|
if num_args < 3:
|
|
print('ERROR: "ar" command has too few arguments', args_line)
|
|
else:
|
|
self.flags = unquote(args[0])
|
|
self.lib = unquote(args[1])
|
|
self.objs = sorted(set(map(unquote, args[2:])))
|
|
return self
|
|
|
|
def write(self, s):
|
|
self.outf.write(s + '\n')
|
|
|
|
def dump_debug_info(self):
|
|
self.write('\n// Line ' + str(self.line_num) + ' "ar" ' + self.line)
|
|
self.write('// ar_object for %12s' % self.pkg)
|
|
self.write('// flags = %s' % self.flags)
|
|
self.write('// lib = %s' % short_out_name(self.pkg, self.lib))
|
|
for o in self.objs:
|
|
self.write('// obj = %s' % short_out_name(self.pkg, o))
|
|
|
|
def dump_android_lib(self):
|
|
"""Write cc_library_static into Android.bp."""
|
|
self.write('\ncc_library_static {')
|
|
self.write(' name: "' + file_base_name(self.lib) + '",')
|
|
self.write(' host_supported: true,')
|
|
if self.flags != 'crs':
|
|
self.write(' // ar flags = %s' % self.flags)
|
|
if self.pkg not in self.runner.pkg_obj2cc:
|
|
self.write(' ERROR: cannot find source files.\n}')
|
|
return
|
|
self.write(' srcs: [')
|
|
obj2cc = self.runner.pkg_obj2cc[self.pkg]
|
|
# Note: wflags are ignored.
|
|
dflags = list()
|
|
fflags = list()
|
|
for obj in self.objs:
|
|
self.write(' "' + short_out_name(self.pkg, obj2cc[obj].src) + '",')
|
|
# TODO(chh): union of dflags and flags of all obj
|
|
# Now, just a temporary hack that uses the last obj's flags
|
|
dflags = obj2cc[obj].dflags
|
|
fflags = obj2cc[obj].fflags
|
|
self.write(' ],')
|
|
self.write(' cflags: [')
|
|
self.write(' "-O3",') # TODO(chh): is this default correct?
|
|
self.write(' "-Wno-error",')
|
|
for x in fflags:
|
|
self.write(' "-f' + x + '",')
|
|
for x in dflags:
|
|
self.write(' "-D' + x + '",')
|
|
self.write(' ],')
|
|
self.write('}')
|
|
|
|
def dump(self):
|
|
"""Dump error/debug/module info to the output .bp file."""
|
|
self.runner.init_bp_file(self.outf_name)
|
|
with open(self.outf_name, 'a') as outf:
|
|
self.outf = outf
|
|
if self.runner.args.debug:
|
|
self.dump_debug_info()
|
|
self.dump_android_lib()
|
|
|
|
|
|
class CCObject(object):
|
|
"""Information of a "cc" compilation command."""
|
|
|
|
def __init__(self, runner, outf_name):
|
|
# Remembered global runner and its members.
|
|
self.runner = runner
|
|
self.pkg = ''
|
|
self.outf_name = outf_name # path to Android.bp
|
|
# "cc" arguments
|
|
self.line_num = 1
|
|
self.line = ''
|
|
self.src = ''
|
|
self.obj = ''
|
|
self.dflags = list() # -D flags
|
|
self.fflags = list() # -f flags
|
|
self.iflags = list() # -I flags
|
|
self.wflags = list() # -W flags
|
|
self.other_args = list()
|
|
|
|
def parse(self, pkg, line_num, args_line):
|
|
"""Collect cc compilation flags and src/out file names."""
|
|
self.pkg = pkg
|
|
self.line_num = line_num
|
|
self.line = args_line
|
|
args = args_line.split()
|
|
i = 0
|
|
while i < len(args):
|
|
arg = args[i]
|
|
if arg == '"-c"':
|
|
i += 1
|
|
if args[i].startswith('"-o'):
|
|
# ring-0.13.5 dumps: ... "-c" "-o/.../*.o" ".../*.c"
|
|
self.obj = unquote(args[i])[2:]
|
|
i += 1
|
|
self.src = unquote(args[i])
|
|
else:
|
|
self.src = unquote(args[i])
|
|
elif arg == '"-o"':
|
|
i += 1
|
|
self.obj = unquote(args[i])
|
|
elif arg == '"-I"':
|
|
i += 1
|
|
self.iflags.append(unquote(args[i]))
|
|
elif arg.startswith('"-D'):
|
|
self.dflags.append(unquote(args[i])[2:])
|
|
elif arg.startswith('"-f'):
|
|
self.fflags.append(unquote(args[i])[2:])
|
|
elif arg.startswith('"-W'):
|
|
self.wflags.append(unquote(args[i])[2:])
|
|
elif not (arg.startswith('"-O') or arg == '"-m64"' or arg == '"-g"' or
|
|
arg == '"-g3"'):
|
|
# ignore -O -m64 -g
|
|
self.other_args.append(unquote(args[i]))
|
|
i += 1
|
|
self.dflags = sorted(set(self.dflags))
|
|
self.fflags = sorted(set(self.fflags))
|
|
# self.wflags is not sorted because some are order sensitive
|
|
# and we ignore them anyway.
|
|
if self.pkg not in self.runner.pkg_obj2cc:
|
|
self.runner.pkg_obj2cc[self.pkg] = {}
|
|
self.runner.pkg_obj2cc[self.pkg][self.obj] = self
|
|
return self
|
|
|
|
def write(self, s):
|
|
self.outf.write(s + '\n')
|
|
|
|
def dump_debug_flags(self, name, flags):
|
|
self.write('// ' + name + ':')
|
|
for f in flags:
|
|
self.write('// %s' % f)
|
|
|
|
def dump(self):
|
|
"""Dump only error/debug info to the output .bp file."""
|
|
if not self.runner.args.debug:
|
|
return
|
|
self.runner.init_bp_file(self.outf_name)
|
|
with open(self.outf_name, 'a') as outf:
|
|
self.outf = outf
|
|
self.write('\n// Line ' + str(self.line_num) + ' "cc" ' + self.line)
|
|
self.write('// cc_object for %12s' % self.pkg)
|
|
self.write('// src = %s' % short_out_name(self.pkg, self.src))
|
|
self.write('// obj = %s' % short_out_name(self.pkg, self.obj))
|
|
self.dump_debug_flags('-I flags', self.iflags)
|
|
self.dump_debug_flags('-D flags', self.dflags)
|
|
self.dump_debug_flags('-f flags', self.fflags)
|
|
self.dump_debug_flags('-W flags', self.wflags)
|
|
if self.other_args:
|
|
self.dump_debug_flags('other args', self.other_args)
|
|
|
|
|
|
class Runner(object):
|
|
"""Main class to parse cargo -v output and print Android module definitions."""
|
|
|
|
def __init__(self, args):
|
|
self.bp_files = set() # Remember all output Android.bp files.
|
|
self.root_pkg = '' # name of package in ./Cargo.toml
|
|
# Saved flags, modes, and data.
|
|
self.args = args
|
|
self.dry_run = not args.run
|
|
self.skip_cargo = args.skipcargo
|
|
self.cargo_path = './cargo' # path to cargo, will be set later
|
|
self.checked_out_files = False # to check only once
|
|
self.build_out_files = [] # output files generated by build.rs
|
|
# All cc/ar objects, crates, dependencies, and warning files
|
|
self.cc_objects = list()
|
|
self.pkg_obj2cc = {}
|
|
# pkg_obj2cc[cc_object[i].pkg][cc_objects[i].obj] = cc_objects[i]
|
|
self.ar_objects = list()
|
|
self.crates = list()
|
|
self.dependencies = list() # dependent and build script crates
|
|
self.warning_files = set()
|
|
# Keep a unique mapping from (module name) to crate
|
|
self.name_owners = {}
|
|
# Save and dump all errors from cargo to Android.bp.
|
|
self.errors = ''
|
|
self.setup_cargo_path()
|
|
# Default action is cargo clean, followed by build or user given actions.
|
|
if args.cargo:
|
|
self.cargo = ['clean'] + args.cargo
|
|
else:
|
|
default_target = '--target x86_64-unknown-linux-gnu'
|
|
# Use the same target for both host and default device builds.
|
|
# Same target is used as default in host x86_64 Android compilation.
|
|
# Note: b/169872957, prebuilt cargo failed to build vsock
|
|
# on x86_64-unknown-linux-musl systems.
|
|
self.cargo = ['clean', 'build ' + default_target]
|
|
if args.tests:
|
|
self.cargo.append('build --tests ' + default_target)
|
|
|
|
def setup_cargo_path(self):
|
|
"""Find cargo in the --cargo_bin or prebuilt rust bin directory."""
|
|
if self.args.cargo_bin:
|
|
self.cargo_path = os.path.join(self.args.cargo_bin, 'cargo')
|
|
if not os.path.isfile(self.cargo_path):
|
|
sys.exit('ERROR: cannot find cargo in ' + self.args.cargo_bin)
|
|
print('WARNING: using cargo in ' + self.args.cargo_bin)
|
|
return
|
|
# We have only tested this on Linux.
|
|
if platform.system() != 'Linux':
|
|
sys.exit('ERROR: this script has only been tested on Linux with cargo.')
|
|
# Assuming that this script is in development/scripts.
|
|
my_dir = os.path.dirname(os.path.abspath(__file__))
|
|
linux_dir = os.path.join(my_dir, '..', '..',
|
|
'prebuilts', 'rust', 'linux-x86')
|
|
if not os.path.isdir(linux_dir):
|
|
sys.exit('ERROR: cannot find directory ' + linux_dir)
|
|
rust_version = self.find_rust_version(my_dir, linux_dir)
|
|
cargo_bin = os.path.join(linux_dir, rust_version, 'bin')
|
|
self.cargo_path = os.path.join(cargo_bin, 'cargo')
|
|
if not os.path.isfile(self.cargo_path):
|
|
sys.exit('ERROR: cannot find cargo in ' + cargo_bin
|
|
+ '; please try --cargo_bin= flag.')
|
|
return
|
|
|
|
def find_rust_version(self, my_dir, linux_dir):
|
|
"""Use my script directory, find prebuilt rust version."""
|
|
# First look up build/soong/rust/config/global.go.
|
|
path2global = os.path.join(my_dir, '..', '..',
|
|
'build', 'soong', 'rust', 'config', 'global.go')
|
|
if os.path.isfile(path2global):
|
|
# try to find: RustDefaultVersion = "1.44.0"
|
|
version_pat = re.compile(
|
|
r'\s*RustDefaultVersion\s*=\s*"([0-9]+\.[0-9]+\.[0-9]+)".*$')
|
|
with open(path2global, 'r') as inf:
|
|
for line in inf:
|
|
result = version_pat.match(line)
|
|
if result:
|
|
return result.group(1)
|
|
print('WARNING: cannot find RustDefaultVersion in ' + path2global)
|
|
# Otherwise, find the newest (largest) version number in linux_dir.
|
|
rust_version = (0, 0, 0) # the prebuilt version to use
|
|
version_pat = re.compile(r'([0-9]+)\.([0-9]+)\.([0-9]+)$')
|
|
for dir_name in os.listdir(linux_dir):
|
|
result = version_pat.match(dir_name)
|
|
if not result:
|
|
continue
|
|
version = (result.group(1), result.group(2), result.group(3))
|
|
if version > rust_version:
|
|
rust_version = version
|
|
return '.'.join(rust_version)
|
|
|
|
def find_out_files(self):
|
|
# list1 has build.rs output for normal crates
|
|
list1 = glob.glob(TARGET_TMP + '/*/*/build/' + self.root_pkg + '-*/out/*')
|
|
# list2 has build.rs output for proc-macro crates
|
|
list2 = glob.glob(TARGET_TMP + '/*/build/' + self.root_pkg + '-*/out/*')
|
|
return list1 + list2
|
|
|
|
def copy_out_files(self):
|
|
"""Copy build.rs output files to ./out and set up build_out_files."""
|
|
if self.checked_out_files:
|
|
return
|
|
self.checked_out_files = True
|
|
cargo_out_files = self.find_out_files()
|
|
out_files = set()
|
|
if cargo_out_files:
|
|
os.makedirs('out', exist_ok=True)
|
|
for path in cargo_out_files:
|
|
file_name = path.split('/')[-1]
|
|
out_files.add(file_name)
|
|
shutil.copy(path, 'out/' + file_name)
|
|
self.build_out_files = sorted(out_files)
|
|
|
|
def has_used_out_dir(self):
|
|
"""Returns true if env!("OUT_DIR") is found."""
|
|
return 0 == os.system('grep -rl --exclude build.rs --include \\*.rs' +
|
|
' \'env!("OUT_DIR")\' * > /dev/null')
|
|
|
|
def copy_out_module_name(self):
|
|
if self.args.copy_out and self.build_out_files:
|
|
return 'copy_' + self.root_pkg + '_build_out'
|
|
else:
|
|
return ''
|
|
|
|
def dump_copy_out_module(self, outf):
|
|
"""Output the genrule module to copy out/* to $(genDir)."""
|
|
copy_out = self.copy_out_module_name()
|
|
if not copy_out:
|
|
return
|
|
outf.write('\ngenrule {\n')
|
|
outf.write(' name: "' + copy_out + '",\n')
|
|
outf.write(' srcs: ["out/*"],\n')
|
|
outf.write(' cmd: "cp $(in) $(genDir)",\n')
|
|
if len(self.build_out_files) > 1:
|
|
outf.write(' out: [\n')
|
|
for f in self.build_out_files:
|
|
outf.write(' "' + f + '",\n')
|
|
outf.write(' ],\n')
|
|
else:
|
|
outf.write(' out: ["' + self.build_out_files[0] + '"],\n')
|
|
outf.write('}\n')
|
|
|
|
def init_bp_file(self, name):
|
|
# name could be Android.bp or sub_dir_path/Android.bp
|
|
if name not in self.bp_files:
|
|
self.bp_files.add(name)
|
|
with open(name, 'w') as outf:
|
|
outf.write(ANDROID_BP_HEADER.format(args=' '.join(sys.argv[1:])))
|
|
# at most one copy_out module per .bp file
|
|
self.dump_copy_out_module(outf)
|
|
|
|
def dump_test_mapping_files(self):
|
|
"""Dump all TEST_MAPPING files."""
|
|
if self.dry_run:
|
|
print('Dry-run skip dump of TEST_MAPPING')
|
|
else:
|
|
test_mapping = TestMapping()
|
|
for bp_file_name in self.bp_files:
|
|
test_mapping.create_test_mapping(os.path.dirname(bp_file_name))
|
|
return self
|
|
|
|
def try_claim_module_name(self, name, owner):
|
|
"""Reserve and return True if it has not been reserved yet."""
|
|
if name not in self.name_owners or owner == self.name_owners[name]:
|
|
self.name_owners[name] = owner
|
|
return True
|
|
return False
|
|
|
|
def claim_module_name(self, prefix, owner, counter):
|
|
"""Return prefix if not owned yet, otherwise, prefix+str(counter)."""
|
|
while True:
|
|
name = prefix
|
|
if counter > 0:
|
|
name += '_' + str(counter)
|
|
if self.try_claim_module_name(name, owner):
|
|
return name
|
|
counter += 1
|
|
|
|
def find_root_pkg(self):
|
|
"""Read name of [package] in ./Cargo.toml."""
|
|
if not os.path.exists('./Cargo.toml'):
|
|
return
|
|
with open('./Cargo.toml', 'r') as inf:
|
|
pkg_section = re.compile(r'^ *\[package\]')
|
|
name = re.compile('^ *name *= * "([^"]*)"')
|
|
in_pkg = False
|
|
for line in inf:
|
|
if in_pkg:
|
|
if name.match(line):
|
|
self.root_pkg = name.match(line).group(1)
|
|
break
|
|
else:
|
|
in_pkg = pkg_section.match(line) is not None
|
|
|
|
def run_cargo(self):
|
|
"""Calls cargo -v and save its output to ./cargo.out."""
|
|
if self.skip_cargo:
|
|
return self
|
|
cargo_toml = './Cargo.toml'
|
|
cargo_out = './cargo.out'
|
|
# Do not use Cargo.lock, because .bp rules are designed to
|
|
# run with "latest" crates avaialable on Android.
|
|
cargo_lock = './Cargo.lock'
|
|
cargo_lock_saved = './cargo.lock.saved'
|
|
had_cargo_lock = os.path.exists(cargo_lock)
|
|
if not os.access(cargo_toml, os.R_OK):
|
|
print('ERROR: Cannot find or read', cargo_toml)
|
|
return self
|
|
if not self.dry_run:
|
|
if os.path.exists(cargo_out):
|
|
os.remove(cargo_out)
|
|
if not self.args.use_cargo_lock and had_cargo_lock: # save it
|
|
os.rename(cargo_lock, cargo_lock_saved)
|
|
cmd_tail = ' --target-dir ' + TARGET_TMP + ' >> ' + cargo_out + ' 2>&1'
|
|
# set up search PATH for cargo to find the correct rustc
|
|
saved_path = os.environ['PATH']
|
|
os.environ['PATH'] = os.path.dirname(self.cargo_path) + ':' + saved_path
|
|
# Add [workspace] to Cargo.toml if it is not there.
|
|
added_workspace = False
|
|
if self.args.add_workspace:
|
|
with open(cargo_toml, 'r') as in_file:
|
|
cargo_toml_lines = in_file.readlines()
|
|
found_workspace = '[workspace]\n' in cargo_toml_lines
|
|
if found_workspace:
|
|
print('### WARNING: found [workspace] in Cargo.toml')
|
|
else:
|
|
with open(cargo_toml, 'a') as out_file:
|
|
out_file.write('[workspace]\n')
|
|
added_workspace = True
|
|
if self.args.verbose:
|
|
print('### INFO: added [workspace] to Cargo.toml')
|
|
for c in self.cargo:
|
|
features = ''
|
|
if c != 'clean':
|
|
if self.args.features is not None:
|
|
features = ' --no-default-features'
|
|
if self.args.features:
|
|
features += ' --features ' + self.args.features
|
|
cmd_v_flag = ' -vv ' if self.args.vv else ' -v '
|
|
cmd = self.cargo_path + cmd_v_flag
|
|
cmd += c + features + cmd_tail
|
|
if self.args.rustflags and c != 'clean':
|
|
cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cmd
|
|
if self.dry_run:
|
|
print('Dry-run skip:', cmd)
|
|
else:
|
|
if self.args.verbose:
|
|
print('Running:', cmd)
|
|
with open(cargo_out, 'a') as out_file:
|
|
out_file.write('### Running: ' + cmd + '\n')
|
|
os.system(cmd)
|
|
if added_workspace: # restore original Cargo.toml
|
|
with open(cargo_toml, 'w') as out_file:
|
|
out_file.writelines(cargo_toml_lines)
|
|
if self.args.verbose:
|
|
print('### INFO: restored original Cargo.toml')
|
|
os.environ['PATH'] = saved_path
|
|
if not self.dry_run:
|
|
if not had_cargo_lock: # restore to no Cargo.lock state
|
|
os.remove(cargo_lock)
|
|
elif not self.args.use_cargo_lock: # restore saved Cargo.lock
|
|
os.rename(cargo_lock_saved, cargo_lock)
|
|
return self
|
|
|
|
def dump_dependencies(self):
|
|
"""Append dependencies and their features to Android.bp."""
|
|
if not self.dependencies:
|
|
return
|
|
dependent_list = list()
|
|
for c in self.dependencies:
|
|
dependent_list.append(c.feature_list())
|
|
sorted_dependencies = sorted(set(dependent_list))
|
|
self.init_bp_file('Android.bp')
|
|
with open('Android.bp', 'a') as outf:
|
|
outf.write('\n// dependent_library ["feature_list"]\n')
|
|
for s in sorted_dependencies:
|
|
outf.write('// ' + s + '\n')
|
|
|
|
def dump_pkg_obj2cc(self):
|
|
"""Dump debug info of the pkg_obj2cc map."""
|
|
if not self.args.debug:
|
|
return
|
|
self.init_bp_file('Android.bp')
|
|
with open('Android.bp', 'a') as outf:
|
|
sorted_pkgs = sorted(self.pkg_obj2cc.keys())
|
|
for pkg in sorted_pkgs:
|
|
if not self.pkg_obj2cc[pkg]:
|
|
continue
|
|
outf.write('\n// obj => src for %s\n' % pkg)
|
|
obj2cc = self.pkg_obj2cc[pkg]
|
|
for obj in sorted(obj2cc.keys()):
|
|
outf.write('// ' + short_out_name(pkg, obj) + ' => ' +
|
|
short_out_name(pkg, obj2cc[obj].src) + '\n')
|
|
|
|
def apply_patch(self):
|
|
"""Apply local patch file if it is given."""
|
|
if self.args.patch:
|
|
if self.dry_run:
|
|
print('Dry-run skip patch file:', self.args.patch)
|
|
else:
|
|
if not os.path.exists(self.args.patch):
|
|
self.append_to_bp('ERROR cannot find patch file: ' + self.args.patch)
|
|
return self
|
|
if self.args.verbose:
|
|
print('### INFO: applying local patch file:', self.args.patch)
|
|
os.system('patch -s --no-backup-if-mismatch ./Android.bp ' +
|
|
self.args.patch)
|
|
return self
|
|
|
|
def gen_bp(self):
|
|
"""Parse cargo.out and generate Android.bp files."""
|
|
if self.dry_run:
|
|
print('Dry-run skip: read', CARGO_OUT, 'write Android.bp')
|
|
elif os.path.exists(CARGO_OUT):
|
|
self.find_root_pkg()
|
|
if self.args.copy_out:
|
|
self.copy_out_files()
|
|
elif self.find_out_files() and self.has_used_out_dir():
|
|
print('WARNING: ' + self.root_pkg + ' has cargo output files; ' +
|
|
'please rerun with the --copy-out flag.')
|
|
with open(CARGO_OUT, 'r') as cargo_out:
|
|
self.parse(cargo_out, 'Android.bp')
|
|
self.crates.sort(key=get_module_name)
|
|
for obj in self.cc_objects:
|
|
obj.dump()
|
|
self.dump_pkg_obj2cc()
|
|
for crate in self.crates:
|
|
crate.dump()
|
|
dumped_libs = set()
|
|
for lib in self.ar_objects:
|
|
if lib.pkg == self.root_pkg:
|
|
lib_name = file_base_name(lib.lib)
|
|
if lib_name not in dumped_libs:
|
|
dumped_libs.add(lib_name)
|
|
lib.dump()
|
|
if self.args.dependencies and self.dependencies:
|
|
self.dump_dependencies()
|
|
if self.errors:
|
|
self.append_to_bp('\nErrors in ' + CARGO_OUT + ':\n' + self.errors)
|
|
return self
|
|
|
|
def add_ar_object(self, obj):
|
|
self.ar_objects.append(obj)
|
|
|
|
def add_cc_object(self, obj):
|
|
self.cc_objects.append(obj)
|
|
|
|
def add_crate(self, crate):
|
|
"""Merge crate with someone in crates, or append to it. Return crates."""
|
|
if crate.skip_crate():
|
|
if self.args.debug: # include debug info of all crates
|
|
self.crates.append(crate)
|
|
if self.args.dependencies: # include only dependent crates
|
|
if (is_dependent_file_path(crate.main_src) and
|
|
not is_build_crate_name(crate.crate_name)):
|
|
self.dependencies.append(crate)
|
|
else:
|
|
for c in self.crates:
|
|
if c.merge(crate, 'Android.bp'):
|
|
return
|
|
# If not merged, decide module type and name now.
|
|
crate.decide_module_type()
|
|
self.crates.append(crate)
|
|
|
|
def find_warning_owners(self):
|
|
"""For each warning file, find its owner crate."""
|
|
missing_owner = False
|
|
for f in self.warning_files:
|
|
cargo_dir = '' # find lowest crate, with longest path
|
|
owner = None # owner crate of this warning
|
|
for c in self.crates:
|
|
if (f.startswith(c.cargo_dir + '/') and
|
|
len(cargo_dir) < len(c.cargo_dir)):
|
|
cargo_dir = c.cargo_dir
|
|
owner = c
|
|
if owner:
|
|
owner.has_warning = True
|
|
else:
|
|
missing_owner = True
|
|
if missing_owner and os.path.exists('Cargo.toml'):
|
|
# owner is the root cargo, with empty cargo_dir
|
|
for c in self.crates:
|
|
if not c.cargo_dir:
|
|
c.has_warning = True
|
|
|
|
def rustc_command(self, n, rustc_line, line, outf_name):
|
|
"""Process a rustc command line from cargo -vv output."""
|
|
# cargo build -vv output can have multiple lines for a rustc command
|
|
# due to '\n' in strings for environment variables.
|
|
# strip removes leading spaces and '\n' at the end
|
|
new_rustc = (rustc_line.strip() + line) if rustc_line else line
|
|
# Use an heuristic to detect the completions of a multi-line command.
|
|
# This might fail for some very rare case, but easy to fix manually.
|
|
if not line.endswith('`\n') or (new_rustc.count('`') % 2) != 0:
|
|
return new_rustc
|
|
if RUSTC_VV_CMD_ARGS.match(new_rustc):
|
|
args = RUSTC_VV_CMD_ARGS.match(new_rustc).group(1)
|
|
self.add_crate(Crate(self, outf_name).parse(n, args))
|
|
else:
|
|
self.assert_empty_vv_line(new_rustc)
|
|
return ''
|
|
|
|
def cc_ar_command(self, n, groups, outf_name):
|
|
pkg = groups.group(1)
|
|
line = groups.group(3)
|
|
if groups.group(2) == 'cc':
|
|
self.add_cc_object(CCObject(self, outf_name).parse(pkg, n, line))
|
|
else:
|
|
self.add_ar_object(ARObject(self, outf_name).parse(pkg, n, line))
|
|
|
|
def append_to_bp(self, line):
|
|
self.init_bp_file('Android.bp')
|
|
with open('Android.bp', 'a') as outf:
|
|
outf.write(line)
|
|
|
|
def assert_empty_vv_line(self, line):
|
|
if line: # report error if line is not empty
|
|
self.append_to_bp('ERROR -vv line: ' + line)
|
|
return ''
|
|
|
|
def parse(self, inf, outf_name):
|
|
"""Parse rustc and warning messages in inf, return a list of Crates."""
|
|
n = 0 # line number
|
|
prev_warning = False # true if the previous line was warning: ...
|
|
rustc_line = '' # previous line(s) matching RUSTC_VV_PAT
|
|
for line in inf:
|
|
n += 1
|
|
if line.startswith('warning: '):
|
|
prev_warning = True
|
|
rustc_line = self.assert_empty_vv_line(rustc_line)
|
|
continue
|
|
new_rustc = ''
|
|
if RUSTC_PAT.match(line):
|
|
args_line = RUSTC_PAT.match(line).group(1)
|
|
self.add_crate(Crate(self, outf_name).parse(n, args_line))
|
|
self.assert_empty_vv_line(rustc_line)
|
|
elif rustc_line or RUSTC_VV_PAT.match(line):
|
|
new_rustc = self.rustc_command(n, rustc_line, line, outf_name)
|
|
elif CC_AR_VV_PAT.match(line):
|
|
self.cc_ar_command(n, CC_AR_VV_PAT.match(line), outf_name)
|
|
elif prev_warning and WARNING_FILE_PAT.match(line):
|
|
self.assert_empty_vv_line(rustc_line)
|
|
fpath = WARNING_FILE_PAT.match(line).group(1)
|
|
if fpath[0] != '/': # ignore absolute path
|
|
self.warning_files.add(fpath)
|
|
elif line.startswith('error: ') or line.startswith('error[E'):
|
|
if not self.args.ignore_cargo_errors:
|
|
self.errors += line
|
|
prev_warning = False
|
|
rustc_line = new_rustc
|
|
self.find_warning_owners()
|
|
|
|
|
|
def parse_args():
|
|
"""Parse main arguments."""
|
|
parser = argparse.ArgumentParser('cargo2android')
|
|
parser.add_argument(
|
|
'--add_workspace',
|
|
action='store_true',
|
|
default=False,
|
|
help=('append [workspace] to Cargo.toml before calling cargo,' +
|
|
' to treat current directory as root of package source;' +
|
|
' otherwise the relative source file path in generated' +
|
|
' .bp file will be from the parent directory.'))
|
|
parser.add_argument(
|
|
'--cargo',
|
|
action='append',
|
|
metavar='args_string',
|
|
help=('extra cargo build -v args in a string, ' +
|
|
'each --cargo flag calls cargo build -v once'))
|
|
parser.add_argument(
|
|
'--cargo_bin',
|
|
type=str,
|
|
help='use cargo in the cargo_bin directory instead of the prebuilt one')
|
|
parser.add_argument(
|
|
'--copy-out',
|
|
action='store_true',
|
|
default=False,
|
|
help=('only for root directory, ' +
|
|
'copy build.rs output to ./out/* and add a genrule to copy ' +
|
|
'./out/* to genrule output; for crates with code pattern: ' +
|
|
'include!(concat!(env!("OUT_DIR"), "/<some_file>.rs"))'))
|
|
parser.add_argument(
|
|
'--debug',
|
|
action='store_true',
|
|
default=False,
|
|
help='dump debug info into Android.bp')
|
|
parser.add_argument(
|
|
'--dependencies',
|
|
action='store_true',
|
|
default=False,
|
|
help='dump debug info of dependent crates')
|
|
parser.add_argument(
|
|
'--device',
|
|
action='store_true',
|
|
default=False,
|
|
help='run cargo also for a default device target')
|
|
parser.add_argument(
|
|
'--features',
|
|
type=str,
|
|
help=('pass features to cargo build, ' +
|
|
'empty string means no default features'))
|
|
parser.add_argument(
|
|
'--global_defaults',
|
|
type=str,
|
|
help='add a defaults name to every module')
|
|
parser.add_argument(
|
|
'--host-first-multilib',
|
|
action='store_true',
|
|
default=False,
|
|
help=('add a compile_multilib:"first" property ' +
|
|
'to Android.bp host modules.'))
|
|
parser.add_argument(
|
|
'--ignore-cargo-errors',
|
|
action='store_true',
|
|
default=False,
|
|
help='do not append cargo/rustc error messages to Android.bp')
|
|
parser.add_argument(
|
|
'--no-host',
|
|
action='store_true',
|
|
default=False,
|
|
help='do not run cargo for the host; only for the device target')
|
|
parser.add_argument(
|
|
'--no-subdir',
|
|
action='store_true',
|
|
default=False,
|
|
help='do not output anything for sub-directories')
|
|
parser.add_argument(
|
|
'--onefile',
|
|
action='store_true',
|
|
default=False,
|
|
help=('output all into one ./Android.bp, default will generate ' +
|
|
'one Android.bp per Cargo.toml in subdirectories'))
|
|
parser.add_argument(
|
|
'--patch',
|
|
type=str,
|
|
help='apply the given patch file to generated ./Android.bp')
|
|
parser.add_argument(
|
|
'--run',
|
|
action='store_true',
|
|
default=False,
|
|
help='run it, default is dry-run')
|
|
parser.add_argument('--rustflags', type=str, help='passing flags to rustc')
|
|
parser.add_argument(
|
|
'--skipcargo',
|
|
action='store_true',
|
|
default=False,
|
|
help='skip cargo command, parse cargo.out, and generate Android.bp')
|
|
parser.add_argument(
|
|
'--tests',
|
|
action='store_true',
|
|
default=False,
|
|
help='run cargo build --tests after normal build')
|
|
parser.add_argument(
|
|
'--use-cargo-lock',
|
|
action='store_true',
|
|
default=False,
|
|
help=('run cargo build with existing Cargo.lock ' +
|
|
'(used when some latest dependent crates failed)'))
|
|
parser.add_argument(
|
|
'--verbose',
|
|
action='store_true',
|
|
default=False,
|
|
help='echo executed commands')
|
|
parser.add_argument(
|
|
'--vv',
|
|
action='store_true',
|
|
default=False,
|
|
help='run cargo with -vv instead of default -v')
|
|
return parser.parse_args()
|
|
|
|
|
|
def main():
|
|
args = parse_args()
|
|
if not args.run: # default is dry-run
|
|
print(DRY_RUN_NOTE)
|
|
Runner(args).run_cargo().gen_bp().apply_patch().dump_test_mapping_files()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|