Optional modules need a module specific soong config variable

Optional modules, i.e. those modules which may be provided by Google or
vendors depending on the vendor, need to have its own Soong config
variable that controls whether prebuilts are used or not. Without that
the build will always attempt to use the Google prebuilt module instead
of the vendor provided module.

This change:
1. Adds support for specifying which modules are optional and will
   generate a module specific soong_config_module_type that uses a
   module specific Soong config variable.

2. Generates the soong_config_module_type for optional modules inline
   in the snapshot Android.bp file (instead of importing from a
   manually curated definitions files). That simplifies the cost of
   adding optional modules.

3. Adds some extra tests to ensure that S and Tiramisu behave the
   same way.

Bug: 233965247
Test: atest mainline_modules_sdks_test
      packages/modules/common/build/mainline_modules_sdks.sh
      # Check the output to ensure that wifi uses the wifi specific
      # Soong config but ipsec (as a non-optional module) does not.
      # Unpack the wifi snapshot into prebuilts/module_sdk/Wifi
Change-Id: I6a85b6f9877fc251010ff2bbee75fe8fa99db9b4
(cherry picked from commit d20edd6c69)
Merged-In: I6a85b6f9877fc251010ff2bbee75fe8fa99db9b4

build/mainline_modules_sdks.py

@@ -20,6 +20,7 @@ the APEXes in it are built, otherwise all configured SDKs are built.
 """
 import argparse
 import dataclasses
+import functools
 import io
 import os
 import re
@@ -161,24 +162,28 @@ class SoongConfigBoilerplateInserter(FileTransformation):
         }},
     }},""")
 
+                # Add the module type to the list of module types that need to
+                # have corresponding config module types.
+                config_module_types.add(module_type)
+
                 # Change the module type to the corresponding soong config
                 # module type by adding the prefix.
                 module_type = self.configModuleTypePrefix + module_type
-                # Add the module type to the list of module types that need to
-                # be imported into the bp file.
-                config_module_types.add(module_type)
 
             # Generate the module, possibly with the new module type and
-            # containing the
+            # containing the soong config variables entry.
             content_lines.append(module_type + " {")
             content_lines.extend(module_content)
             content_lines.append("}")
 
-        # Add the soong_config_module_type_import module definition that imports
+        if self.configBpDefFile:
-        # the soong config module types into this bp file to the header lines so
+            # Add the soong_config_module_type_import module definition that
-        # that they appear before any uses.
+            # imports the soong config module types into this bp file to the
-        module_types = "\n".join(
+            # header lines so that they appear before any uses.
-            [f'        "{mt}",' for mt in sorted(config_module_types)])
+            module_types = "\n".join([
+                f'        "{self.configModuleTypePrefix}{mt}",'
+                for mt in sorted(config_module_types)
+            ])
             header_lines.append(f"""
 // Soong config variable stanza added by {producer.script}.
 soong_config_module_type_import {{
@@ -188,6 +193,24 @@ soong_config_module_type_import {{
     ],
 }}
 """)
+        else:
+            # Add the soong_config_module_type module definitions to the header
+            # lines so that they appear before any uses.
+            header_lines.append("")
+            for module_type in sorted(config_module_types):
+                # Create the corresponding soong config module type name by
+                # adding the prefix.
+                config_module_type = self.configModuleTypePrefix + module_type
+                header_lines.append(f"""
+// Soong config variable module type added by {producer.script}.
+soong_config_module_type {{
+    name: "{config_module_type}",
+    module_type: "{module_type}",
+    config_namespace: "{self.configVar.namespace}",
+    bool_variables: ["{self.configVar.name}"],
+    properties: ["prefer"],
+}}
+""".lstrip())
 
         # Overwrite the file with the updated contents.
         file.seek(0)
@@ -396,6 +419,7 @@ ALL_BUILD_RELEASES = []
 
 
 @dataclasses.dataclass(frozen=True)
+@functools.total_ordering
 class BuildRelease:
     """Represents a build release"""
 
@@ -453,6 +477,9 @@ class BuildRelease:
                     "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": self.name,
                 })
 
+    def __eq__(self, other):
+        return self.ordinal == other.ordinal
+
     def __le__(self, other):
         return self.ordinal <= other.ordinal
 
@@ -628,6 +655,30 @@ class MainlineModule:
 
     for_r_build: typing.Optional[ForRBuild] = None
 
+    # The last release on which this module was optional.
+    #
+    # Some modules are optional when they are first released, usually because
+    # some vendors of Android devices have their own customizations of the
+    # module that they would like to preserve and which cannot yet be achieved
+    # through the existing APIs. Once those issues have been resolved then they
+    # will become mandatory.
+    #
+    # This field records the last build release in which they are optional. It
+    # defaults to None which indicates that the module was never optional.
+    last_optional_release: typing.Optional[BuildRelease] = None
+
+    # The short name for the module.
+    #
+    # Defaults to the last part of the apex name.
+    short_name: str = ""
+
+    def __post_init__(self):
+        # If short_name is not set then set it to the last component of the apex
+        # name.
+        if not self.short_name:
+            short_name = self.apex.rsplit(".", 1)[-1]
+            object.__setattr__(self, "short_name", short_name)
+
     def is_bundled(self):
         """Returns true for bundled modules. See BundledMainlineModule."""
         return False
@@ -636,11 +687,29 @@ class MainlineModule:
         """Returns the transformations to apply to this module's snapshot(s)."""
         transformations = []
         if build_release.supports_soong_config_boilerplate:
+
+            config_var = self.configVar
+            config_module_type_prefix = self.configModuleTypePrefix
+            config_bp_def_file = self.configBpDefFile
+
+            # If the module is optional then it needs its own Soong config
+            # variable to allow it to be managed separately from other modules.
+            if (self.last_optional_release and
+                    self.last_optional_release > build_release):
+                config_var = ConfigVar(
+                    namespace=f"{self.short_name}_module",
+                    name="source_build",
+                )
+                config_module_type_prefix = f"{self.short_name}_prebuilt_"
+                # Optional modules don't have their own config_bp_def_file so
+                # they have to generate the soong_config_module_types inline.
+                config_bp_def_file = ""
+
             inserter = SoongConfigBoilerplateInserter(
                 "Android.bp",
-                configVar=self.configVar,
+                configVar=config_var,
-                configModuleTypePrefix=self.configModuleTypePrefix,
+                configModuleTypePrefix=config_module_type_prefix,
-                configBpDefFile=self.configBpDefFile)
+                configBpDefFile=config_bp_def_file)
             transformations.append(inserter)
         return transformations
 
@@ -800,6 +869,8 @@ MAINLINE_MODULES = [
         for_r_build=ForRBuild(sdk_libraries=[
             SdkLibrary(name="framework-wifi"),
         ]),
+        # Wifi has always been and is still optional.
+        last_optional_release=LATEST,
     ),
 ]
 

build/mainline_modules_sdks_test.py

@@ -413,6 +413,36 @@ class TestSoongConfigBoilerplateInserter(unittest.TestCase):
 
         self.apply_transformations(src, transformations, expected)
 
+        # Check that Tiramisu provides the same transformations as S.
+        tiramisu_transformations = module.transformations(mm.Tiramisu)
+        self.assertEqual(
+            transformations,
+            tiramisu_transformations,
+            msg="Tiramisu must use the same transformations as S")
+
+    def test_optional_mainline_module(self):
+        """Tests the transformations applied to an optional mainline module.
+
+        This uses wifi as an example of a optional mainline module. This checks
+        that the module specific Soong config module types and variables are
+        used.
+        """
+        src = read_test_data("wifi_Android.bp.input")
+
+        expected = read_test_data("wifi_Android.bp.expected")
+
+        module = MAINLINE_MODULES_BY_APEX["com.android.wifi"]
+        transformations = module.transformations(mm.S)
+
+        self.apply_transformations(src, transformations, expected)
+
+        # Check that Tiramisu provides the same transformations as S.
+        tiramisu_transformations = module.transformations(mm.Tiramisu)
+        self.assertEqual(
+            transformations,
+            tiramisu_transformations,
+            msg="Tiramisu must use the same transformations as S")
+
     def test_art(self):
         """Tests the transformations applied to a the ART mainline module.
 

build/mainline_modules_sdks_test_data/wifi_Android.bp.expected

@@ -0,0 +1,168 @@
+// This is auto-generated. DO NOT EDIT.
+
+// Soong config variable module type added by test_optional_mainline_module.
+soong_config_module_type {
+    name: "wifi_prebuilt_java_import",
+    module_type: "java_import",
+    config_namespace: "wifi_module",
+    bool_variables: ["source_build"],
+    properties: ["prefer"],
+}
+
+// Soong config variable module type added by test_optional_mainline_module.
+soong_config_module_type {
+    name: "wifi_prebuilt_java_sdk_library_import",
+    module_type: "java_sdk_library_import",
+    config_namespace: "wifi_module",
+    bool_variables: ["source_build"],
+    properties: ["prefer"],
+}
+
+// Soong config variable module type added by test_optional_mainline_module.
+soong_config_module_type {
+    name: "wifi_prebuilt_prebuilt_bootclasspath_fragment",
+    module_type: "prebuilt_bootclasspath_fragment",
+    config_namespace: "wifi_module",
+    bool_variables: ["source_build"],
+    properties: ["prefer"],
+}
+
+// Soong config variable module type added by test_optional_mainline_module.
+soong_config_module_type {
+    name: "wifi_prebuilt_prebuilt_systemserverclasspath_fragment",
+    module_type: "prebuilt_systemserverclasspath_fragment",
+    config_namespace: "wifi_module",
+    bool_variables: ["source_build"],
+    properties: ["prefer"],
+}
+
+package {
+    // A default list here prevents the license LSC from adding its own list which would
+    // be unnecessary as every module in the sdk already has its own licenses property.
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+wifi_prebuilt_prebuilt_bootclasspath_fragment {
+    name: "com.android.wifi-bootclasspath-fragment",
+    // Do not prefer prebuilt if the Soong config variable "source_build" in namespace "wifi_module" is true.
+    prefer: true,
+    soong_config_variables: {
+        source_build: {
+            prefer: false,
+        },
+    },
+    visibility: ["//visibility:public"],
+    apex_available: ["com.android.wifi"],
+    licenses: ["wifi-module-sdk_Android-Apache-2.0"],
+    contents: ["framework-wifi"],
+    fragments: [
+        {
+            apex: "com.android.art",
+            module: "art-bootclasspath-fragment",
+        },
+    ],
+    hidden_api: {
+        unsupported: ["hiddenapi/hiddenapi-unsupported.txt"],
+        max_target_r_low_priority: ["hiddenapi/hiddenapi-max-target-r-low-priority.txt"],
+        max_target_o_low_priority: ["hiddenapi/hiddenapi-max-target-o-low-priority.txt"],
+        annotation_flags: "hiddenapi/annotation-flags.csv",
+        metadata: "hiddenapi/metadata.csv",
+        index: "hiddenapi/index.csv",
+        signature_patterns: "hiddenapi/signature-patterns.csv",
+        filtered_stub_flags: "hiddenapi/filtered-stub-flags.csv",
+        filtered_flags: "hiddenapi/filtered-flags.csv",
+    },
+}
+
+wifi_prebuilt_java_sdk_library_import {
+    name: "framework-wifi",
+    // Do not prefer prebuilt if the Soong config variable "source_build" in namespace "wifi_module" is true.
+    prefer: true,
+    soong_config_variables: {
+        source_build: {
+            prefer: false,
+        },
+    },
+    visibility: ["//visibility:public"],
+    apex_available: [
+        "com.android.wifi",
+        "test_com.android.wifi",
+    ],
+    licenses: ["wifi-module-sdk_Android-Apache-2.0"],
+    shared_library: false,
+    permitted_packages: [
+        "android.hardware.wifi",
+        "android.net.wifi",
+        "com.android.wifi.x",
+    ],
+    public: {
+        jars: ["sdk_library/public/framework-wifi-stubs.jar"],
+        stub_srcs: ["sdk_library/public/framework-wifi_stub_sources"],
+        current_api: "sdk_library/public/framework-wifi.txt",
+        removed_api: "sdk_library/public/framework-wifi-removed.txt",
+        annotations: "sdk_library/public/framework-wifi_annotations.zip",
+        sdk_version: "module_current",
+    },
+    system: {
+        jars: ["sdk_library/system/framework-wifi-stubs.jar"],
+        stub_srcs: ["sdk_library/system/framework-wifi_stub_sources"],
+        current_api: "sdk_library/system/framework-wifi.txt",
+        removed_api: "sdk_library/system/framework-wifi-removed.txt",
+        annotations: "sdk_library/system/framework-wifi_annotations.zip",
+        sdk_version: "module_current",
+    },
+    module_lib: {
+        jars: ["sdk_library/module-lib/framework-wifi-stubs.jar"],
+        stub_srcs: ["sdk_library/module-lib/framework-wifi_stub_sources"],
+        current_api: "sdk_library/module-lib/framework-wifi.txt",
+        removed_api: "sdk_library/module-lib/framework-wifi-removed.txt",
+        annotations: "sdk_library/module-lib/framework-wifi_annotations.zip",
+        sdk_version: "module_current",
+    },
+}
+
+wifi_prebuilt_java_import {
+    name: "service-wifi",
+    // Do not prefer prebuilt if the Soong config variable "source_build" in namespace "wifi_module" is true.
+    prefer: true,
+    soong_config_variables: {
+        source_build: {
+            prefer: false,
+        },
+    },
+    visibility: [
+        "//frameworks/opt/net/wifi/service/apex",
+        "//frameworks/opt/net/wifi/tests/wifitests/apex",
+        "//packages/modules/Wifi/apex",
+        "//packages/modules/Wifi/service",
+        "//packages/modules/Wifi/service/tests/wifitests/apex",
+    ],
+    apex_available: [
+        "com.android.wifi",
+        "test_com.android.wifi",
+    ],
+    licenses: ["wifi-module-sdk_Android-Apache-2.0"],
+    jars: ["java_systemserver_libs/snapshot/jars/are/invalid/service-wifi.jar"],
+}
+
+license {
+    name: "wifi-module-sdk_Android-Apache-2.0",
+    visibility: ["//visibility:private"],
+    license_kinds: ["SPDX-license-identifier-Apache-2.0"],
+    license_text: ["licenses/build/soong/licenses/LICENSE"],
+}
+
+wifi_prebuilt_prebuilt_systemserverclasspath_fragment {
+    name: "com.android.wifi-systemserverclasspath-fragment",
+    // Do not prefer prebuilt if the Soong config variable "source_build" in namespace "wifi_module" is true.
+    prefer: true,
+    soong_config_variables: {
+        source_build: {
+            prefer: false,
+        },
+    },
+    visibility: ["//visibility:public"],
+    apex_available: ["com.android.wifi"],
+    licenses: ["wifi-module-sdk_Android-Apache-2.0"],
+    standalone_contents: ["service-wifi"],
+}

build/mainline_modules_sdks_test_data/wifi_Android.bp.input

@@ -0,0 +1,108 @@
+// This is auto-generated. DO NOT EDIT.
+
+package {
+    // A default list here prevents the license LSC from adding its own list which would
+    // be unnecessary as every module in the sdk already has its own licenses property.
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+prebuilt_bootclasspath_fragment {
+    name: "com.android.wifi-bootclasspath-fragment",
+    prefer: false,
+    visibility: ["//visibility:public"],
+    apex_available: ["com.android.wifi"],
+    licenses: ["wifi-module-sdk_Android-Apache-2.0"],
+    contents: ["framework-wifi"],
+    fragments: [
+        {
+            apex: "com.android.art",
+            module: "art-bootclasspath-fragment",
+        },
+    ],
+    hidden_api: {
+        unsupported: ["hiddenapi/hiddenapi-unsupported.txt"],
+        max_target_r_low_priority: ["hiddenapi/hiddenapi-max-target-r-low-priority.txt"],
+        max_target_o_low_priority: ["hiddenapi/hiddenapi-max-target-o-low-priority.txt"],
+        annotation_flags: "hiddenapi/annotation-flags.csv",
+        metadata: "hiddenapi/metadata.csv",
+        index: "hiddenapi/index.csv",
+        signature_patterns: "hiddenapi/signature-patterns.csv",
+        filtered_stub_flags: "hiddenapi/filtered-stub-flags.csv",
+        filtered_flags: "hiddenapi/filtered-flags.csv",
+    },
+}
+
+java_sdk_library_import {
+    name: "framework-wifi",
+    prefer: false,
+    visibility: ["//visibility:public"],
+    apex_available: [
+        "com.android.wifi",
+        "test_com.android.wifi",
+    ],
+    licenses: ["wifi-module-sdk_Android-Apache-2.0"],
+    shared_library: false,
+    permitted_packages: [
+        "android.hardware.wifi",
+        "android.net.wifi",
+        "com.android.wifi.x",
+    ],
+    public: {
+        jars: ["sdk_library/public/framework-wifi-stubs.jar"],
+        stub_srcs: ["sdk_library/public/framework-wifi_stub_sources"],
+        current_api: "sdk_library/public/framework-wifi.txt",
+        removed_api: "sdk_library/public/framework-wifi-removed.txt",
+        annotations: "sdk_library/public/framework-wifi_annotations.zip",
+        sdk_version: "module_current",
+    },
+    system: {
+        jars: ["sdk_library/system/framework-wifi-stubs.jar"],
+        stub_srcs: ["sdk_library/system/framework-wifi_stub_sources"],
+        current_api: "sdk_library/system/framework-wifi.txt",
+        removed_api: "sdk_library/system/framework-wifi-removed.txt",
+        annotations: "sdk_library/system/framework-wifi_annotations.zip",
+        sdk_version: "module_current",
+    },
+    module_lib: {
+        jars: ["sdk_library/module-lib/framework-wifi-stubs.jar"],
+        stub_srcs: ["sdk_library/module-lib/framework-wifi_stub_sources"],
+        current_api: "sdk_library/module-lib/framework-wifi.txt",
+        removed_api: "sdk_library/module-lib/framework-wifi-removed.txt",
+        annotations: "sdk_library/module-lib/framework-wifi_annotations.zip",
+        sdk_version: "module_current",
+    },
+}
+
+java_import {
+    name: "service-wifi",
+    prefer: false,
+    visibility: [
+        "//frameworks/opt/net/wifi/service/apex",
+        "//frameworks/opt/net/wifi/tests/wifitests/apex",
+        "//packages/modules/Wifi/apex",
+        "//packages/modules/Wifi/service",
+        "//packages/modules/Wifi/service/tests/wifitests/apex",
+    ],
+    apex_available: [
+        "com.android.wifi",
+        "test_com.android.wifi",
+    ],
+    licenses: ["wifi-module-sdk_Android-Apache-2.0"],
+    jars: ["java_systemserver_libs/snapshot/jars/are/invalid/service-wifi.jar"],
+}
+
+license {
+    name: "wifi-module-sdk_Android-Apache-2.0",
+    visibility: ["//visibility:private"],
+    license_kinds: ["SPDX-license-identifier-Apache-2.0"],
+    license_text: ["licenses/build/soong/licenses/LICENSE"],
+}
+
+prebuilt_systemserverclasspath_fragment {
+    name: "com.android.wifi-systemserverclasspath-fragment",
+    prefer: false,
+    visibility: ["//visibility:public"],
+    apex_available: ["com.android.wifi"],
+    licenses: ["wifi-module-sdk_Android-Apache-2.0"],
+    standalone_contents: ["service-wifi"],
+}