From 77eb4ba473f56b984cae3291ea0fbcf784a6fceb Mon Sep 17 00:00:00 2001 From: Frederick Mayle Date: Sat, 19 Nov 2022 13:01:11 -0800 Subject: [PATCH] cargo_embargo: new cargo2android tool A fork of cargo2android, initially aimed at big imports like crosvm. * Faster run time when there are many packages (30 min -> 1 min for crosvm). * `--reuse-cargo-out` feature allows iterating on most of the config with sub second run time. * Most options can be specified per-package. Bug: 261486161 Test: ran on crosvm and checked diffs Change-Id: Iaf3f4732e828a6f8b281f0e63fe1e0e1e9e98631 --- tools/cargo_embargo/Android.bp | 33 ++ tools/cargo_embargo/OWNERS | 2 + tools/cargo_embargo/src/bp.rs | 186 +++++++++ tools/cargo_embargo/src/cargo_out.rs | 384 ++++++++++++++++++ tools/cargo_embargo/src/main.rs | 581 +++++++++++++++++++++++++++ 5 files changed, 1186 insertions(+) create mode 100644 tools/cargo_embargo/Android.bp create mode 100644 tools/cargo_embargo/OWNERS create mode 100644 tools/cargo_embargo/src/bp.rs create mode 100644 tools/cargo_embargo/src/cargo_out.rs create mode 100644 tools/cargo_embargo/src/main.rs diff --git a/tools/cargo_embargo/Android.bp b/tools/cargo_embargo/Android.bp new file mode 100644 index 000000000..07abe450d --- /dev/null +++ b/tools/cargo_embargo/Android.bp @@ -0,0 +1,33 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_binary_host { + name: "cargo_embargo", + srcs: ["src/main.rs"], + // Disable LTO for faster builds. Don't need the performance here. + flags: ["-C lto=off"], + rustlibs: [ + "libanyhow", + "libclap", + "libglob", + "libonce_cell", + "libregex", + "libserde", + "libserde_json", + ], +} diff --git a/tools/cargo_embargo/OWNERS b/tools/cargo_embargo/OWNERS new file mode 100644 index 000000000..204e0264f --- /dev/null +++ b/tools/cargo_embargo/OWNERS @@ -0,0 +1,2 @@ +fmayle@google.com +smoreland@google.com diff --git a/tools/cargo_embargo/src/bp.rs b/tools/cargo_embargo/src/bp.rs new file mode 100644 index 000000000..f660aed3f --- /dev/null +++ b/tools/cargo_embargo/src/bp.rs @@ -0,0 +1,186 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::Result; +use std::collections::BTreeMap; + +/// Build module. +pub struct BpModule { + module_type: String, + pub props: BpProperties, +} + +/// Properties of a build module, or of a nested object value. +#[derive(Clone, PartialEq, Eq)] +pub struct BpProperties { + map: BTreeMap, + /// A raw block of text to append after the last key-value pair, but before the closing brace. + /// For example, if you have the properties + /// + /// { + /// name: "foo", + /// srcs: ["main.rs"], + /// } + /// + /// and add `raw_block = "some random text"`, you'll get + /// + /// { + /// name: "foo", + /// srcs: ["main.rs"], + /// some random text + /// } + pub raw_block: Option, +} + +#[derive(Clone, PartialEq, Eq)] +pub enum BpValue { + Object(BpProperties), + Bool(bool), + String(String), + List(Vec), +} + +impl BpModule { + pub fn new(module_type: String) -> BpModule { + BpModule { module_type, props: BpProperties::new() } + } + + /// Serialize to Android.bp format. + pub fn write(&self, w: &mut impl std::fmt::Write) -> Result<()> { + w.write_str(&self.module_type)?; + w.write_str(" ")?; + self.props.write(w)?; + w.write_str("\n")?; + Ok(()) + } +} + +impl BpProperties { + pub fn new() -> Self { + BpProperties { map: BTreeMap::new(), raw_block: None } + } + + pub fn get_string(&self, k: &str) -> &str { + match self.map.get(k).unwrap() { + BpValue::String(s) => s, + _ => unreachable!(), + } + } + + pub fn set>(&mut self, k: &str, v: T) { + self.map.insert(k.to_string(), v.into()); + } + + pub fn object(&mut self, k: &str) -> &mut BpProperties { + let v = + self.map.entry(k.to_string()).or_insert_with(|| BpValue::Object(BpProperties::new())); + match v { + BpValue::Object(v) => v, + _ => panic!("key {k:?} already has non-object value"), + } + } + + /// Serialize to Android.bp format. + pub fn write(&self, w: &mut impl std::fmt::Write) -> Result<()> { + w.write_str("{\n")?; + // Sort stuff to match what cargo2android.py's output order. + let canonical_order = &[ + "name", + "defaults", + "stem", + "host_supported", + "prefer_rlib", + "crate_name", + "cargo_env_compat", + "cargo_pkg_version", + "srcs", + "test_suites", + "auto_gen_config", + "test_options", + "edition", + "features", + "rustlibs", + "proc_macros", + "static_libs", + "shared_libs", + "arch", + "target", + "ld_flags", + "apex_available", + ]; + let mut props: Vec<(&String, &BpValue)> = self.map.iter().collect(); + props.sort_by_key(|(k, _)| { + let i = canonical_order.iter().position(|x| k == x).unwrap_or(canonical_order.len()); + (i, (*k).clone()) + }); + for (k, v) in props { + w.write_str(k)?; + w.write_str(": ")?; + v.write(w)?; + w.write_str(",\n")?; + } + if let Some(raw_block) = &self.raw_block { + w.write_str(raw_block)?; + w.write_str(",\n")?; + } + w.write_str("}")?; + Ok(()) + } +} + +impl BpValue { + /// Serialize to Android.bp format. + pub fn write(&self, w: &mut impl std::fmt::Write) -> Result<()> { + match self { + BpValue::Object(p) => p.write(w)?, + BpValue::Bool(b) => write!(w, "{b}")?, + BpValue::String(s) => write!(w, "\"{s}\"")?, + BpValue::List(vs) => { + w.write_str("[")?; + for (i, v) in vs.iter().enumerate() { + v.write(w)?; + if i != vs.len() - 1 { + w.write_str(", ")?; + } + } + w.write_str("]")?; + } + } + Ok(()) + } +} + +impl From for BpValue { + fn from(x: bool) -> Self { + BpValue::Bool(x) + } +} + +impl From<&str> for BpValue { + fn from(x: &str) -> Self { + BpValue::String(x.to_string()) + } +} + +impl From for BpValue { + fn from(x: String) -> Self { + BpValue::String(x) + } +} + +impl> From> for BpValue { + fn from(x: Vec) -> Self { + BpValue::List(x.into_iter().map(|x| x.into()).collect()) + } +} diff --git a/tools/cargo_embargo/src/cargo_out.rs b/tools/cargo_embargo/src/cargo_out.rs new file mode 100644 index 000000000..2aa6ad462 --- /dev/null +++ b/tools/cargo_embargo/src/cargo_out.rs @@ -0,0 +1,384 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::anyhow; +use anyhow::bail; +use anyhow::Context; +use anyhow::Result; +use once_cell::sync::Lazy; +use regex::Regex; +use std::collections::BTreeMap; +use std::path::Path; +use std::path::PathBuf; + +/// Info extracted from `CargoOut` for a crate. +/// +/// Note that there is a 1-to-many relationship between a Cargo.toml file and these `Crate` +/// objects. For example, a Cargo.toml file might have a bin, a lib, and various tests. Each of +/// those will be a separate `Crate`. All of them will have the same `package_name`. +#[derive(Debug, Default)] +pub struct Crate { + pub name: String, + pub package_name: String, + pub version: Option, + // cargo calls rustc with multiple --crate-type flags. + // rustc can accept: + // --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro] + pub types: Vec, + pub test: bool, // --test + pub target: Option, // --target + pub features: Vec, // --cfg feature= + pub cfgs: Vec, // non-feature --cfg + pub externs: Vec<(String, Option)>, // name => rlib file + pub codegens: Vec, // -C + pub cap_lints: String, + pub static_libs: Vec, + pub shared_libs: Vec, + pub emit_list: String, + pub edition: String, + pub package_dir: PathBuf, // canonicalized + pub main_src: PathBuf, // relative to package_dir +} + +pub fn parse_cargo_out(cargo_out_path: &str, cargo_metadata_path: &str) -> Result> { + let metadata: WorkspaceMetadata = serde_json::from_str( + &std::fs::read_to_string(cargo_metadata_path).context("failed to read cargo.metadata")?, + ) + .context("failed to parse cargo.metadata")?; + + let cargo_out = CargoOut::parse( + &std::fs::read_to_string(cargo_out_path).context("failed to read cargo.out")?, + ) + .context("failed to parse cargo.out")?; + + assert!(cargo_out.cc_invocations.is_empty(), "cc not supported yet"); + assert!(cargo_out.ar_invocations.is_empty(), "ar not supported yet"); + + let mut crates = Vec::new(); + for rustc in cargo_out.rustc_invocations.iter() { + let c = Crate::from_rustc_invocation(rustc, &metadata) + .with_context(|| format!("failed to process rustc invocation: {rustc}"))?; + // Ignore build.rs crates. + if c.name.starts_with("build_script_") { + continue; + } + // Ignore crates outside the current directory. + let cwd = std::env::current_dir().unwrap().canonicalize().unwrap(); + if !c.package_dir.starts_with(cwd) { + continue; + } + crates.push(c); + } + Ok(crates) +} + +/// `cargo metadata` output. +#[derive(serde::Deserialize)] +struct WorkspaceMetadata { + packages: Vec, +} + +#[derive(serde::Deserialize)] +struct PackageMetadata { + name: String, + version: String, + edition: String, + manifest_path: String, +} + +/// Raw-ish data extracted from cargo.out file. +#[derive(Debug, Default)] +struct CargoOut { + rustc_invocations: Vec, + + // package name => cmd args + cc_invocations: BTreeMap, + ar_invocations: BTreeMap, + + // lines starting with "warning: ". + // line number => line + warning_lines: BTreeMap, + warning_files: Vec, + + errors: Vec, + test_errors: Vec, +} + +fn match1(regex: &Regex, s: &str) -> Option { + regex.captures(s).and_then(|x| x.get(1)).map(|x| x.as_str().to_string()) +} + +fn match3(regex: &Regex, s: &str) -> Option<(String, String, String)> { + regex.captures(s).and_then(|x| match (x.get(1), x.get(2), x.get(3)) { + (Some(a), Some(b), Some(c)) => { + Some((a.as_str().to_string(), b.as_str().to_string(), c.as_str().to_string())) + } + _ => None, + }) +} + +impl CargoOut { + /// Parse the output of a `cargo build -v` run. + fn parse(contents: &str) -> Result { + let mut result = CargoOut::default(); + let mut in_tests = false; + let mut lines_iter = contents.lines().enumerate(); + while let Some((n, line)) = lines_iter.next() { + if line.starts_with("warning: ") { + result.warning_lines.insert(n, line.to_string()); + continue; + } + + // Cargo -v output of a call to rustc. + static RUSTC_REGEX: Lazy = + Lazy::new(|| Regex::new(r"^ +Running `rustc (.*)`$").unwrap()); + if let Some(args) = match1(&RUSTC_REGEX, line) { + result.rustc_invocations.push(args); + continue; + } + // Cargo -vv output of a call to rustc could be split into multiple lines. + // Assume that the first line will contain some CARGO_* env definition. + static RUSTC_VV_REGEX: Lazy = + Lazy::new(|| Regex::new(r"^ +Running `.*CARGO_.*=.*$").unwrap()); + if RUSTC_VV_REGEX.is_match(line) { + // cargo build -vv output can have multiple lines for a rustc command due to + // '\n' in strings for environment variables. + let mut line = line.to_string(); + loop { + // Use an heuristic to detect the completions of a multi-line command. + if line.ends_with('`') && line.chars().filter(|c| *c == '`').count() % 2 == 0 { + break; + } + if let Some((_, next_line)) = lines_iter.next() { + line += next_line; + continue; + } + break; + } + // The combined -vv output rustc command line pattern. + static RUSTC_VV_CMD_ARGS: Lazy = + Lazy::new(|| Regex::new(r"^ *Running `.*CARGO_.*=.* rustc (.*)`$").unwrap()); + if let Some(args) = match1(&RUSTC_VV_CMD_ARGS, &line) { + result.rustc_invocations.push(args); + } else { + bail!("failed to parse cargo.out line: {}", line); + } + continue; + } + // Cargo -vv output of a "cc" or "ar" command; all in one line. + static CC_AR_VV_REGEX: Lazy = Lazy::new(|| { + Regex::new(r#"^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$"#).unwrap() + }); + if let Some((pkg, cmd, args)) = match3(&CC_AR_VV_REGEX, line) { + match cmd.as_str() { + "ar" => result.ar_invocations.insert(pkg, args), + "cc" => result.cc_invocations.insert(pkg, args), + _ => unreachable!(), + }; + continue; + } + // Rustc output of file location path pattern for a warning message. + static WARNING_FILE_REGEX: Lazy = + Lazy::new(|| Regex::new(r"^ *--> ([^:]*):[0-9]+").unwrap()); + if result.warning_lines.contains_key(&n.saturating_sub(1)) { + if let Some(fpath) = match1(&WARNING_FILE_REGEX, line) { + result.warning_files.push(fpath); + continue; + } + } + if line.starts_with("error: ") || line.starts_with("error[E") { + if in_tests { + result.test_errors.push(line.to_string()); + } else { + result.errors.push(line.to_string()); + } + continue; + } + static CARGO2ANDROID_RUNNING_REGEX: Lazy = + Lazy::new(|| Regex::new(r"^### Running: .*$").unwrap()); + if CARGO2ANDROID_RUNNING_REGEX.is_match(line) { + in_tests = line.contains("cargo test") && line.contains("--list"); + continue; + } + } + + // self.find_warning_owners() + + Ok(result) + } +} + +impl Crate { + fn from_rustc_invocation(rustc: &str, metadata: &WorkspaceMetadata) -> Result { + let mut out = Crate::default(); + + // split into args + let args: Vec<&str> = rustc.split_whitespace().collect(); + let mut arg_iter = args + .iter() + // Remove quotes from simple strings, panic for others. + .map(|arg| match (arg.chars().next(), arg.chars().skip(1).last()) { + (Some('"'), Some('"')) => &arg[1..arg.len() - 1], + (Some('\''), Some('\'')) => &arg[1..arg.len() - 1], + (Some('"'), _) => panic!("can't handle strings with whitespace"), + (Some('\''), _) => panic!("can't handle strings with whitespace"), + _ => arg, + }); + // process each arg + while let Some(arg) = arg_iter.next() { + match arg { + "--crate-name" => out.name = arg_iter.next().unwrap().to_string(), + "--crate-type" => out.types.push(arg_iter.next().unwrap().to_string()), + "--test" => out.test = true, + "--target" => out.target = Some(arg_iter.next().unwrap().to_string()), + "--cfg" => { + // example: feature=\"sink\" + let arg = arg_iter.next().unwrap(); + if let Some(feature) = + arg.strip_prefix("feature=\"").and_then(|s| s.strip_suffix('\"')) + { + out.features.push(feature.to_string()); + } else { + out.cfgs.push(arg.to_string()); + } + } + "--extern" => { + // example: proc_macro + // example: memoffset=/some/path/libmemoffset-2cfda327d156e680.rmeta + let arg = arg_iter.next().unwrap(); + if let Some((name, path)) = arg.split_once('=') { + out.externs.push(( + name.to_string(), + Some(path.split('/').last().unwrap().to_string()), + )); + } else { + out.externs.push((arg.to_string(), None)); + } + } + _ if arg.starts_with("-C") => { + // handle both "-Cfoo" and "-C foo" + let arg = if arg == "-C" { + arg_iter.next().unwrap() + } else { + arg.strip_prefix("-C").unwrap() + }; + // 'prefer-dynamic' does not work with common flag -C lto + // 'embed-bitcode' is ignored; we might control LTO with other .bp flag + // 'codegen-units' is set in Android global config or by default + // + // TODO: this is business logic. move it out of the parsing code + if !arg.starts_with("codegen-units=") + && !arg.starts_with("debuginfo=") + && !arg.starts_with("embed-bitcode=") + && !arg.starts_with("extra-filename=") + && !arg.starts_with("incremental=") + && !arg.starts_with("metadata=") + && arg != "prefer-dynamic" + { + out.codegens.push(arg.to_string()); + } + } + "--cap-lints" => out.cap_lints = arg_iter.next().unwrap().to_string(), + "-l" => { + let arg = arg_iter.next().unwrap(); + if let Some(lib) = arg.strip_prefix("static=") { + out.static_libs.push(lib.to_string()); + } else if let Some(lib) = arg.strip_prefix("dylib=") { + out.shared_libs.push(lib.to_string()); + } else { + out.shared_libs.push(arg.to_string()); + } + } + _ if arg.starts_with("--emit=") => { + out.emit_list = arg.strip_prefix("--emit=").unwrap().to_string(); + } + _ if !arg.starts_with('-') => { + let src_path = Path::new(arg); + // Canonicalize the path because: + // + // 1. We don't consistently get relative or absolute paths elsewhere. If we + // canonicalize everything, it becomes easy to compare paths. + // + // 2. We don't want to consider symlinks to code outside the cwd as part of the + // project (e.g. AOSP's import of crosvm has symlinks from crosvm's own 3p + // directory to the android 3p directories). + let src_path = src_path + .canonicalize() + .unwrap_or_else(|e| panic!("failed to canonicalize {src_path:?}: {}", e)); + out.package_dir = src_path.parent().unwrap().to_path_buf(); + while !out.package_dir.join("Cargo.toml").try_exists()? { + if let Some(parent) = out.package_dir.parent() { + out.package_dir = parent.to_path_buf(); + } else { + bail!("No Cargo.toml found in parents of {:?}", src_path); + } + } + out.main_src = src_path.strip_prefix(&out.package_dir).unwrap().to_path_buf(); + } + + // ignored flags + "-L" => { + arg_iter.next().unwrap(); + } + "--out-dir" => { + arg_iter.next().unwrap(); + } + "--color" => { + arg_iter.next().unwrap(); + } + _ if arg.starts_with("--error-format=") => {} + _ if arg.starts_with("--edition=") => {} + _ if arg.starts_with("--json=") => {} + _ if arg.starts_with("-Aclippy") => {} + _ if arg.starts_with("-Wclippy") => {} + "-W" => {} + "-D" => {} + + arg => bail!("unsupported rustc argument: {arg:?}"), + } + } + + if out.name.is_empty() { + bail!("missing --crate-name"); + } + if out.main_src.as_os_str().is_empty() { + bail!("missing main source file"); + } + if out.types.is_empty() != out.test { + bail!("expected exactly one of either --crate-type or --test"); + } + if out.types.iter().any(|x| x == "lib") && out.types.iter().any(|x| x == "rlib") { + bail!("cannot both have lib and rlib crate types"); + } + + // Find the metadata for the crates containing package by matching the manifest's path. + let manifest_path = out.package_dir.join("Cargo.toml"); + let package_metadata = metadata + .packages + .iter() + .find(|p| Path::new(&p.manifest_path).canonicalize().unwrap() == manifest_path) + .ok_or_else(|| { + anyhow!( + "can't find metadata for crate {:?} with manifest path {:?}", + out.name, + manifest_path, + ) + })?; + out.package_name = package_metadata.name.clone(); + out.version = Some(package_metadata.version.clone()); + out.edition = package_metadata.edition.clone(); + + Ok(out) + } +} diff --git a/tools/cargo_embargo/src/main.rs b/tools/cargo_embargo/src/main.rs new file mode 100644 index 000000000..c162e5a05 --- /dev/null +++ b/tools/cargo_embargo/src/main.rs @@ -0,0 +1,581 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Converts a cargo project to Soong. +//! +//! Forked from development/scripts/cargo2android.py. Missing many of its features. Adds various +//! features to make it easier to work with projects containing many crates. +//! +//! At a high level, this is done by +//! +//! 1. Running `cargo build -v` and saving the output to a "cargo.out" file. +//! 2. Parsing the "cargo.out" file to find invocations of compilers, e.g. `rustc` and `cc`. +//! 3. For each compiler invocation, generating a equivalent Soong module, e.g. a "rust_library". +//! +//! The last step often involves messy, project specific business logic, so many options are +//! available to tweak it via a config file. + +mod bp; +mod cargo_out; + +use anyhow::bail; +use anyhow::Context; +use anyhow::Result; +use bp::*; +use cargo_out::*; +use clap::Parser; +use once_cell::sync::Lazy; +use regex::Regex; +use std::collections::BTreeMap; +use std::collections::VecDeque; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; + +// Major TODOs +// * handle errors, esp. in cargo.out parsing. they should fail the program with an error code +// * handle warnings. put them in comments in the android.bp, some kind of report section + +#[derive(Parser, Debug)] +#[clap()] +struct Args { + /// Use the cargo binary in the `cargo_bin` directory. Defaults to cargo in $PATH. + /// + /// TODO: Should default to android prebuilts. + #[clap(long)] + cargo_bin: Option, + /// Config file. + #[clap(long)] + cfg: PathBuf, + /// Skip the `cargo build` commands and reuse the "cargo.out" file from a previous run if + /// available. + #[clap(long)] + reuse_cargo_out: bool, +} + +fn default_apex_available() -> Vec { + vec!["//apex_available:platform".to_string(), "//apex_available:anyapex".to_string()] +} + +/// Options that apply to everything. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct Config { + /// Whether to output "rust_test" modules. + tests: bool, + /// Set of features to enable. If non-empty, disables the default crate features. + #[serde(default)] + features: Vec, + /// Whether to build with --workspace. + #[serde(default)] + workspace: bool, + /// When workspace is enabled, list of --exclude crates. + #[serde(default)] + workspace_excludes: Vec, + /// Value to use for every generated module's "defaults" field. + global_defaults: Option, + /// Value to use for every generated library module's "apex_available" field. + #[serde(default = "default_apex_available")] + apex_available: Vec, + /// Map of renames for modules. For example, if a "libfoo" would be generated and there is an + /// entry ("libfoo", "libbar"), the generated module will be called "libbar" instead. + /// + /// Also, affects references to dependencies (e.g. in a "static_libs" list), even those outside + /// the project being processed. + #[serde(default)] + module_name_overrides: BTreeMap, + /// Package specific config options. + #[serde(default)] + package: BTreeMap, + /// Modules in this list will not be generated. + #[serde(default)] + module_blocklist: Vec, +} + +/// Options that apply to everything in a package (i.e. everything associated with a particular +/// Cargo.toml file). +#[derive(serde::Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct PackageConfig { + /// Whether to compile for device. Defaults to true. + #[serde(default)] + device_supported: Option, + /// Whether to compile for host. Defaults to true. + #[serde(default)] + host_supported: Option, + /// Generate "rust_library_rlib" instead of "rust_library". + #[serde(default)] + force_rlib: bool, + /// Whether to disable "unit_test" for "rust_test" modules. + // TODO: Should probably be a list of modules or crates. A package might have a mix of unit and + // integration tests. + #[serde(default)] + no_presubmit: bool, + /// File with content to append to the end of the generated Android.bp. + add_toplevel_block: Option, + /// File with content to append to the end of each generated module. + add_module_block: Option, + /// Modules in this list will not be added as dependencies of generated modules. + #[serde(default)] + dep_blocklist: Vec, + /// Patch file to apply after Android.bp is generated. + patch: Option, + /// Copy build.rs output to ./out/* and add a genrule to copy ./out/* to genrule output. + /// For crates with code pattern: + /// include!(concat!(env!("OUT_DIR"), "/.rs")) + #[serde(default)] + copy_out: bool, +} + +fn main() -> Result<()> { + let args = Args::parse(); + + let json_str = std::fs::read_to_string(&args.cfg) + .with_context(|| format!("failed to read file: {:?}", args.cfg))?; + // Add some basic support for comments to JSON. + let json_str: String = json_str.lines().filter(|l| !l.trim_start().starts_with("//")).collect(); + let cfg: Config = serde_json::from_str(&json_str).context("failed to parse config")?; + + if !Path::new("Cargo.toml").try_exists().context("when checking Cargo.toml")? { + bail!("Cargo.toml missing. Run in a directory with a Cargo.toml file."); + } + + // Add the custom cargo to PATH. + // NOTE: If the directory with cargo has more binaries, this could have some unpredictable side + // effects. That is partly intended though, because we want to use that cargo binary's + // associated rustc. + if let Some(cargo_bin) = args.cargo_bin { + let path = std::env::var_os("PATH").unwrap(); + let mut paths = std::env::split_paths(&path).collect::>(); + paths.push_front(cargo_bin); + let new_path = std::env::join_paths(paths)?; + std::env::set_var("PATH", &new_path); + } + + let cargo_out_path = "cargo.out"; + let cargo_metadata_path = "cargo.metadata"; + if !args.reuse_cargo_out || !Path::new(cargo_out_path).exists() { + generate_cargo_out(&cfg, cargo_out_path, cargo_metadata_path) + .context("generate_cargo_out failed")?; + } + + let crates = + parse_cargo_out(cargo_out_path, cargo_metadata_path).context("parse_cargo_out failed")?; + + // Find out files. + // Example: target.tmp/x86_64-unknown-linux-gnu/debug/build/metrics-d2dd799cebf1888d/out/event_details.rs + let mut package_out_files: BTreeMap> = BTreeMap::new(); + if cfg.package.iter().any(|(_, v)| v.copy_out) { + for entry in glob::glob("target.tmp/**/build/*/out/*")? { + match entry { + Ok(path) => { + let package_name = || -> Option<_> { + let dir_name = path.parent()?.parent()?.file_name()?.to_str()?; + Some(dir_name.rsplit_once('-')?.0) + }() + .unwrap_or_else(|| panic!("failed to parse out file path: {:?}", path)); + package_out_files + .entry(package_name.to_string()) + .or_default() + .push(path.clone()); + } + Err(e) => eprintln!("failed to check for out files: {}", e), + } + } + } + + // Group by package. + let mut module_by_package: BTreeMap> = BTreeMap::new(); + for c in crates { + module_by_package.entry(c.package_dir.clone()).or_default().push(c); + } + // Write an Android.bp file per package. + for (package_dir, crates) in module_by_package { + write_android_bp( + &cfg, + package_dir, + &crates, + package_out_files.get(&crates[0].package_name), + )?; + } + + Ok(()) +} + +fn run_cargo(cargo_out: &mut File, cmd: &mut Command) -> Result<()> { + use std::os::unix::io::OwnedFd; + use std::process::Stdio; + let fd: OwnedFd = cargo_out.try_clone()?.into(); + // eprintln!("Running: {:?}\n", cmd); + let output = cmd.stdout(Stdio::from(fd.try_clone()?)).stderr(Stdio::from(fd)).output()?; + if !output.status.success() { + bail!("cargo command failed with exit status: {:?}", output.status); + } + Ok(()) +} + +/// Run various cargo commands and save the output to `cargo_out_path`. +fn generate_cargo_out(cfg: &Config, cargo_out_path: &str, cargo_metadata_path: &str) -> Result<()> { + let mut cargo_out_file = std::fs::File::create(cargo_out_path)?; + let mut cargo_metadata_file = std::fs::File::create(cargo_metadata_path)?; + + let verbose_args = ["-v"]; + let target_dir_args = ["--target-dir", "target.tmp"]; + + // cargo clean + run_cargo(&mut cargo_out_file, Command::new("cargo").arg("clean").args(&target_dir_args))?; + + let default_target = "x86_64-unknown-linux-gnu"; + let feature_args = if cfg.features.is_empty() { + vec![] + } else { + vec!["--no-default-features".to_string(), "--features".to_string(), cfg.features.join(",")] + }; + + let workspace_args = if cfg.workspace { + let mut v = vec!["--workspace".to_string()]; + if !cfg.workspace_excludes.is_empty() { + for x in cfg.workspace_excludes.iter() { + v.push("--exclude".to_string()); + v.push(x.clone()); + } + } + v + } else { + vec![] + }; + + // cargo metadata + run_cargo( + &mut cargo_metadata_file, + Command::new("cargo") + .arg("metadata") + .arg("-q") // don't output warnings to stderr + .arg("--format-version") + .arg("1") + .args(&feature_args), + )?; + + // cargo build + run_cargo( + &mut cargo_out_file, + Command::new("cargo") + .args(["build", "--target", default_target]) + .args(&verbose_args) + .args(&target_dir_args) + .args(&workspace_args) + .args(&feature_args), + )?; + + if cfg.tests { + // cargo build --tests + run_cargo( + &mut cargo_out_file, + Command::new("cargo") + .args(["build", "--target", default_target, "--tests"]) + .args(&verbose_args) + .args(&target_dir_args) + .args(&workspace_args) + .args(&feature_args), + )?; + } + + Ok(()) +} + +/// Create the Android.bp file for `package_dir`. +fn write_android_bp( + cfg: &Config, + package_dir: PathBuf, + crates: &[Crate], + out_files: Option<&Vec>, +) -> Result<()> { + let bp_path = package_dir.join("Android.bp"); + + let package_name = crates[0].package_name.clone(); + let def = PackageConfig::default(); + let package_cfg = cfg.package.get(&package_name).unwrap_or(&def); + + // Keep the old license header. + let license_section = match std::fs::read_to_string(&bp_path) { + Ok(s) => s + .lines() + .skip_while(|l| l.starts_with("//")) + .take_while(|l| !l.starts_with("rust_") && !l.starts_with("genrule {")) + .collect::>() + .join("\n"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => "// TODO: Add license.\n".to_string(), + Err(e) => bail!("error when reading {bp_path:?}: {e}"), + }; + + let mut bp_contents = String::new(); + bp_contents += "// This file is generated by cargo_embargo.\n"; + bp_contents += "// Do not modify this file as changes will be overridden on upgrade.\n\n"; + bp_contents += license_section.trim(); + bp_contents += "\n"; + + let mut modules = Vec::new(); + + let extra_srcs = match (package_cfg.copy_out, out_files) { + (true, Some(out_files)) => { + let out_dir = package_dir.join("out"); + if !out_dir.exists() { + std::fs::create_dir(&out_dir).expect("failed to create out dir"); + } + + let mut outs: Vec = Vec::new(); + for f in out_files.iter() { + let dest = out_dir.join(f.file_name().unwrap()); + std::fs::copy(&f, &dest).expect("failed to copy out file"); + outs.push(f.file_name().unwrap().to_str().unwrap().to_string()); + } + + let mut m = BpModule::new("genrule".to_string()); + let module_name = format!("copy_{}_build_out", package_name); + m.props.set("name", module_name.clone()); + m.props.set("srcs", vec!["out/*"]); + m.props.set("cmd", "cp $(in) $(genDir)"); + m.props.set("out", outs); + modules.push(m); + + vec![":".to_string() + &module_name] + } + _ => vec![], + }; + + for c in crates { + modules.extend(crate_to_bp_modules(c, cfg, package_cfg, &extra_srcs)?); + } + if modules.is_empty() { + return Ok(()); + } + + modules.sort_by_key(|m| m.props.get_string("name").to_string()); + for m in modules { + m.write(&mut bp_contents)?; + bp_contents += "\n"; + } + if let Some(path) = &package_cfg.add_toplevel_block { + bp_contents += + &std::fs::read_to_string(path).with_context(|| format!("failed to read {path:?}"))?; + bp_contents += "\n"; + } + File::create(&bp_path)?.write_all(bp_contents.as_bytes())?; + + let bpfmt_output = Command::new("bpfmt").arg("-w").arg(&bp_path).output()?; + if !bpfmt_output.status.success() { + eprintln!( + "WARNING: bpfmt -w {:?} failed before patch: {}", + bp_path, + String::from_utf8_lossy(&bpfmt_output.stderr) + ); + } + + if let Some(patch_path) = &package_cfg.patch { + let patch_output = + Command::new("patch").arg("-s").arg(&bp_path).arg(patch_path).output()?; + if !patch_output.status.success() { + eprintln!("WARNING: failed to apply patch {:?}", patch_path); + } + // Re-run bpfmt after the patch so + let bpfmt_output = Command::new("bpfmt").arg("-w").arg(&bp_path).output()?; + if !bpfmt_output.status.success() { + eprintln!( + "WARNING: bpfmt -w {:?} failed after patch: {}", + bp_path, + String::from_utf8_lossy(&bpfmt_output.stderr) + ); + } + } + + Ok(()) +} + +/// Convert a `Crate` into `BpModule`s. +/// +/// If messy business logic is necessary, prefer putting it here. +fn crate_to_bp_modules( + crate_: &Crate, + cfg: &Config, + package_cfg: &PackageConfig, + extra_srcs: &[String], +) -> Result> { + let mut modules = Vec::new(); + let mut types = crate_.types.clone(); + if crate_.test { + types.push("test".to_string()); + } + for crate_type in types { + let host = if package_cfg.device_supported.unwrap_or(true) { "" } else { "_host" }; + let rlib = if package_cfg.force_rlib { "_rlib" } else { "" }; + let (module_type, module_name, stem) = match crate_type.as_str() { + "bin" => ("rust_binary".to_string() + host, crate_.name.clone(), crate_.name.clone()), + "lib" | "rlib" => { + let stem = "lib".to_string() + &crate_.name; + ("rust_library".to_string() + rlib + host, stem.clone(), stem) + } + "dylib" => { + let stem = "lib".to_string() + &crate_.name; + ("rust_library".to_string() + host + "_dylib", stem.clone() + "_dylib", stem) + } + "cdylib" => { + let stem = "lib".to_string() + &crate_.name; + ("rust_ffi".to_string() + host + "_shared", stem.clone() + "_shared", stem) + } + "staticlib" => { + let stem = "lib".to_string() + &crate_.name; + ("rust_ffi".to_string() + host + "_static", stem.clone() + "_static", stem) + } + "proc-macro" => { + let stem = "lib".to_string() + &crate_.name; + ("rust_proc_macro".to_string(), stem.clone(), stem) + } + "test" => { + let suffix = crate_.main_src.to_string_lossy().to_owned(); + let suffix = suffix.replace('/', "_").replace(".rs", ""); + let stem = crate_.package_name.clone() + "_test_" + &suffix; + ("rust_test".to_string() + host, stem.clone(), stem) + } + _ => panic!("unexpected crate type: {}", crate_type), + }; + + let mut m = BpModule::new(module_type.clone()); + let module_name = cfg.module_name_overrides.get(&module_name).unwrap_or(&module_name); + if cfg.module_blocklist.contains(module_name) { + continue; + } + m.props.set("name", module_name.clone()); + if &stem != module_name { + m.props.set("stem", stem); + } + + if let Some(defaults) = &cfg.global_defaults { + m.props.set("defaults", vec![defaults.clone()]); + } + + if package_cfg.host_supported.unwrap_or(true) + && package_cfg.device_supported.unwrap_or(true) + && module_type != "rust_proc_macro" + { + m.props.set("host_supported", true); + } + + m.props.set("crate_name", crate_.name.clone()); + m.props.set("cargo_env_compat", true); + + if let Some(version) = &crate_.version { + m.props.set("cargo_pkg_version", version.clone()); + } + + if crate_.test { + m.props.set("test_suites", vec!["general-tests"]); + m.props.set("auto_gen_config", true); + if package_cfg.host_supported.unwrap_or(true) { + m.props.object("test_options").set("unit_test", !package_cfg.no_presubmit); + } + } + + let mut srcs = vec![crate_.main_src.to_string_lossy().to_string()]; + srcs.extend(extra_srcs.iter().cloned()); + m.props.set("srcs", srcs); + + m.props.set("edition", crate_.edition.clone()); + if !crate_.features.is_empty() { + m.props.set("features", crate_.features.clone()); + } + if !crate_.cfgs.is_empty() { + m.props.set("cfgs", crate_.cfgs.clone()); + } + + let mut flags = Vec::new(); + if !crate_.cap_lints.is_empty() { + flags.push(crate_.cap_lints.clone()); + } + flags.extend(crate_.codegens.clone()); + if !flags.is_empty() { + m.props.set("flags", flags); + } + + let mut rust_libs = Vec::new(); + let mut proc_macro_libs = Vec::new(); + for (extern_name, filename) in &crate_.externs { + if extern_name == "proc_macro" { + continue; + } + let filename = + filename.as_ref().unwrap_or_else(|| panic!("no filename for {}", extern_name)); + // Example filename: "libgetrandom-fd8800939535fc59.rmeta" + static REGEX: Lazy = + Lazy::new(|| Regex::new(r"^lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$").unwrap()); + let lib_name = if let Some(x) = REGEX.captures(filename).and_then(|x| x.get(1)) { + x + } else { + bail!("bad filename for extern {}: {}", extern_name, filename); + }; + if filename.ends_with(".rlib") || filename.ends_with(".rmeta") { + rust_libs.push(lib_name.as_str().to_string()); + } else if filename.ends_with(".so") { + // Assume .so files are always proc_macros. May not always be right. + proc_macro_libs.push(lib_name.as_str().to_string()); + } else { + unreachable!(); + } + } + + // Add "lib" prefix and apply name overrides. + let process_lib_deps = |libs: Vec| -> Vec { + let mut result = Vec::new(); + for x in libs { + let module_name = "lib".to_string() + x.as_str(); + let module_name = + cfg.module_name_overrides.get(&module_name).unwrap_or(&module_name); + if package_cfg.dep_blocklist.contains(module_name) { + continue; + } + result.push(module_name.to_string()); + } + result.sort(); + result + }; + if !rust_libs.is_empty() { + m.props.set("rustlibs", process_lib_deps(rust_libs)); + } + if !proc_macro_libs.is_empty() { + m.props.set("proc_macros", process_lib_deps(proc_macro_libs)); + } + if !crate_.static_libs.is_empty() { + m.props.set("static_libs", process_lib_deps(crate_.static_libs.clone())); + } + if !crate_.shared_libs.is_empty() { + m.props.set("shared_libs", process_lib_deps(crate_.shared_libs.clone())); + } + + if !cfg.apex_available.is_empty() + && ["lib", "rlib", "dylib", "staticlib", "cdylib"].contains(&crate_type.as_str()) + { + m.props.set("apex_available", cfg.apex_available.clone()); + } + + if let Some(path) = &package_cfg.add_module_block { + let content = std::fs::read_to_string(path) + .with_context(|| format!("failed to read {path:?}"))?; + m.props.raw_block = Some(content); + } + + modules.push(m); + } + Ok(modules) +}