Merge "Update add3prf.py"

scripts/Android.bp

@@ -51,3 +51,14 @@ python_test_host {
     test_config: "symbol-tests.xml",
     test_suites: ["general-tests"],
 }
+
+python_test_host {
+    name: "add3prf_test",
+    srcs: [
+        "add3prf.py",
+        "add3prf_test.py",
+    ],
+    libs: [
+        "pyfakefs",
+    ]
+}

scripts/OWNERS

@@ -3,5 +3,5 @@ danalbert@google.com
 enh@google.com
 jmgao@google.com
 rprichard@google.com
-per-file add3prf.py,cargo2android.py,get_rust_pkg.py,update_crate_tests.py = ivanlozano@google.com,jeffv@google.com,jgalenson@google.com,mmaurer@google.com,srhines@google.com,tweek@google.com
+per-file add3prf.py,add3prf_test.py,cargo2android.py,get_rust_pkg.py,update_crate_tests.py = ivanlozano@google.com,jeffv@google.com,jgalenson@google.com,mmaurer@google.com,srhines@google.com,tweek@google.com
 per-file codegen = eugenesusla@google.com

scripts/add3prf.py

@@ -15,7 +15,9 @@
 # limitations under the License.
 """Add files to a Rust package for third party review."""
 
+import collections
 import datetime
+import enum
 import glob
 import json
 import os
@@ -47,6 +49,8 @@ MIT_PATTERN = r"^.*MIT License.*$"
 MIT_MATCHER = re.compile(MIT_PATTERN)
 BSD_PATTERN = r"^.*BSD .*License.*$"
 BSD_MATCHER = re.compile(BSD_PATTERN)
+MULTI_LICENSE_COMMENT = ("# Dual-licensed, using the least restrictive "
+        "per go/thirdpartylicenses#same.\n  ")
 
 # default owners added to OWNERS
 DEFAULT_OWNERS = "include platform/prebuilts/rust:/OWNERS\n"
@@ -55,23 +59,23 @@ DEFAULT_OWNERS = "include platform/prebuilts/rust:/OWNERS\n"
 # "license_type: NOTICE" might be optional,
 # but it is already used in most rust crate METADATA.
 # This line format should match the output of external_updater.
-METADATA_CONTENT = """name: "{}"
+METADATA_CONTENT = """name: "{name}"
-description: {}
+description: {description}
 third_party {{
   url {{
     type: HOMEPAGE
-    value: "https://crates.io/crates/{}"
+    value: "https://crates.io/crates/{name}"
   }}
   url {{
     type: ARCHIVE
-    value: "https://static.crates.io/crates/{}/{}-{}.crate"
+    value: "https://static.crates.io/crates/{name}/{name}-{version}.crate"
   }}
-  version: "{}"
+  version: "{version}"
-  license_type: NOTICE
+  {license_comment}license_type: NOTICE
   last_upgrade_date {{
-    year: {}
+    year: {year}
-    month: {}
+    month: {month}
-    day: {}
+    day: {day}
   }}
 }}
 """
@@ -104,17 +108,20 @@ def get_metadata_date():
   return today.year, today.month, today.day
 
 
-def add_metadata(name, version, description):
+def add_metadata(name, version, description, multi_license):
   """Update or add METADATA file."""
   if os.path.exists("METADATA"):
     print("### Updating METADATA")
   else:
     print("### Adding METADATA")
   year, month, day = get_metadata_date()
+  license_comment = ""
+  if multi_license:
+    license_comment = MULTI_LICENSE_COMMENT
   with open("METADATA", "w") as outf:
     outf.write(METADATA_CONTENT.format(
-        name, description, name, name, name,
+        name=name, description=description, version=version,
-        version, version, year, month, day))
+        license_comment=license_comment, year=year, month=month, day=day))
 
 
 def grep_license_keyword(license_file):
@@ -122,48 +129,63 @@ def grep_license_keyword(license_file):
   with open(license_file, "r") as input_file:
     for line in input_file:
       if APACHE_MATCHER.match(line):
-        return "APACHE2", license_file
+        return License(LicenseType.APACHE2, license_file)
       if MIT_MATCHER.match(line):
-        return "MIT", license_file
+        return License(LicenseType.MIT, license_file)
       if BSD_MATCHER.match(line):
-        return "BSD_LIKE", license_file
+        return License(LicenseType.BSD_LIKE, license_file)
   print("ERROR: cannot decide license type in", license_file,
         "assume BSD_LIKE")
-  return "BSD_LIKE", license_file
+  return License(LicenseType.BSD_LIKE, license_file)
+
+
+class LicenseType(enum.IntEnum):
+  """A type of license.
+
+  An IntEnum is used to be able to sort by preference. This is mainly the case
+  for dual-licensed Apache/MIT code, for which we prefer the Apache license.
+  The enum name is used to generate the corresponding MODULE_LICENSE_* file.
+  """
+  APACHE2 = 1
+  MIT = 2
+  BSD_LIKE = 3
+  ISC = 4
+
+
+License = collections.namedtuple('License', ['type', 'filename'])
 
 
 def decide_license_type(cargo_license):
-  """Check LICENSE* files to determine the license type."""
+  """Check LICENSE* files to determine the license type.
+
+  Returns: A list of Licenses. The first element is the license we prefer.
+  """
   # Most crates.io packages have both APACHE and MIT.
   # Some crate like time-macros-impl uses lower case names like LICENSE-Apache.
-  targets = {}
+  licenses = []
-  license_file = "unknown-file"
+  license_file = None
-  for license_file in glob.glob("./LICENSE*"):
+  for license_file in glob.glob("LICENSE*"):
-    license_file = license_file[2:]
     lowered_name = license_file.lower()
     if lowered_name == "license-apache":
-      targets["APACHE2"] = license_file
+      licenses.append(License(LicenseType.APACHE2, license_file))
     elif lowered_name == "license-mit":
-      targets["MIT"] = license_file
+      licenses.append(License(LicenseType.MIT, license_file))
-  # Prefer APACHE2 over MIT license type.
+  if licenses:
-  for license_type in ["APACHE2", "MIT"]:
+    licenses.sort(key=lambda l: l.type)
-    if license_type in targets:
+    return licenses
-      return license_type, targets[license_type]
+  if not license_file:
-  # Use cargo_license found in Cargo.toml.
+    raise FileNotFoundError("No license file has been found.")
+  # There is a LICENSE or LICENSE.txt file, use cargo_license found in
+  # Cargo.toml.
   if "Apache" in cargo_license:
-    return "APACHE2", license_file
+    return [License(LicenseType.APACHE2, license_file)]
   if "MIT" in cargo_license:
-    return "MIT", license_file
+    return [License(LicenseType.MIT, license_file)]
   if "BSD" in cargo_license:
-    return "BSD_LIKE", license_file
+    return [License(LicenseType.BSD_LIKE, license_file)]
   if "ISC" in cargo_license:
-    return "ISC", license_file
+    return [License(LicenseType.ISC, license_file)]
-  # Try to find key words in LICENSE* files.
+  return [grep_license_keyword(license_file)]
-  for license_file in ["LICENSE", "LICENSE.txt"]:
-    if os.path.exists(license_file):
-      return grep_license_keyword(license_file)
-  print("ERROR: missing LICENSE-{APACHE,MIT}; assume BSD_LIKE")
-  return "BSD_LIKE", "unknown-file"
 
 
 def add_notice():
@@ -192,7 +214,7 @@ def add_license(target):
     if os.path.islink("LICENSE"):
       check_license_link(target)
     else:
-      print("NOTE: found LICENSE and it is not a link!")
+      print("NOTE: found LICENSE and it is not a link.")
     return
   print("### Creating LICENSE link to", target)
   os.symlink(target, "LICENSE")
@@ -204,10 +226,10 @@ def add_module_license(license_type):
   for suffix in ["MIT", "APACHE", "APACHE2", "BSD_LIKE"]:
     module_file = "MODULE_LICENSE_" + suffix
     if os.path.exists(module_file):
-      if license_type != suffix:
+      if license_type.name != suffix:
-        print("ERROR: found unexpected", module_file)
+        raise Exception("Found unexpected license " + module_file)
       return
-  module_file = "MODULE_LICENSE_" + license_type
+  module_file = "MODULE_LICENSE_" + license_type.name.upper()
   pathlib.Path(module_file).touch()
   print("### Touched", module_file)
 
@@ -285,11 +307,12 @@ def main():
     print("ERROR: Cannot find name, version, or description in", cargo)
     return
   print("### Cargo.toml license:", cargo_license)
-  add_metadata(name, version, description)
+  licenses = decide_license_type(cargo_license)
+  preferred_license = licenses[0]
+  add_metadata(name, version, description, len(licenses) > 1)
   add_owners()
-  license_type, file_name = decide_license_type(cargo_license)
+  add_license(preferred_license.filename)
-  add_license(file_name)
+  add_module_license(preferred_license.type)
-  add_module_license(license_type)
   # It is unclear yet if a NOTICE file is required.
   # add_notice()
 

scripts/add3prf_test.py

@@ -0,0 +1,76 @@
+# Copyright 2021 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os.path
+import unittest
+
+from pyfakefs import fake_filesystem_unittest
+
+import add3prf
+
+class LicenseDetectionTestCase(fake_filesystem_unittest.TestCase):
+
+  def setUp(self):
+    self.setUpPyfakefs()
+
+  def test_dual_license(self):
+    self.fs.create_file("LICENSE-APACHE")
+    self.fs.create_file("LICENSE-MIT")
+    licenses = add3prf.decide_license_type("MIT OR Apache-2.0")
+    self.assertEqual(len(licenses), 2)
+    preferred_license = licenses[0]
+    self.assertEqual(preferred_license.type, add3prf.LicenseType.APACHE2)
+    self.assertEqual(preferred_license.filename, "LICENSE-APACHE")
+
+  def test_mit_license(self):
+    self.fs.create_file("LICENSE")
+    licenses = add3prf.decide_license_type("MIT")
+    self.assertEqual(len(licenses), 1)
+    preferred_license = licenses[0]
+    self.assertEqual(preferred_license.type, add3prf.LicenseType.MIT)
+    self.assertEqual(preferred_license.filename, "LICENSE")
+
+  def test_misc_license(self):
+    self.fs.create_file("LICENSE.txt")
+    licenses = add3prf.decide_license_type("")
+    self.assertEqual(len(licenses), 1)
+    preferred_license = licenses[0]
+    self.assertEqual(preferred_license.type, add3prf.LicenseType.BSD_LIKE)
+    self.assertEqual(preferred_license.filename, "LICENSE.txt")
+
+  def test_missing_license_file(self):
+    with self.assertRaises(FileNotFoundError):
+      add3prf.decide_license_type("MIT OR Apache-2.0")
+
+
+class AddModuleLicenseTestCase(fake_filesystem_unittest.TestCase):
+
+  def setUp(self):
+    self.setUpPyfakefs()
+
+  def test_no_file(self):
+    add3prf.add_module_license(add3prf.LicenseType.APACHE2)
+    self.assertTrue(os.path.exists("MODULE_LICENSE_APACHE2"))
+
+  def test_already_exists(self):
+    self.fs.create_file("MODULE_LICENSE_APACHE2")
+    add3prf.add_module_license(add3prf.LicenseType.APACHE2)
+
+  def test_mit_apache(self):
+    self.fs.create_file("MODULE_LICENSE_MIT")
+    with self.assertRaises(Exception):
+      add3prf.add_module_license(add3prf.LicenseType.APACHE2)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)