Merge changes I524df983,I23578fd5 into main

* changes:
  compare_cts_reports: allow abi selection while parsing reports
  compare_cts_reports: support `ignore_abi` for csv report

tools/compare_cts_reports/README.md

@@ -10,17 +10,19 @@ A report can be either a standard CTS result zip file, or a test_result.xml file
 ## parse_cts_report.py
 ### usage
 ```
-./parse_cts_report.py -r REPORT -d OUTPUT_DIR
+./parse_cts_report.py -r REPORT -d OUTPUT_DIR [--abi [{armeabi-v7a,arm64-v8a,x86,x86_64} ...]]
 ```
 
 The `-r` flag must be followed by exactly one report.
 
 The `-d` flag specifies the directory in which the information files will be stored.
 
+The `--abi` flag can be used to select one or more test ABIs to be parsed.
+
 ## aggregate_cts_reports.py
 ### usage
 ```
-./aggregate_cts_reports.py -r REPORT [REPORT ...] -d OUTPUT_DIR [--ignore-abi]
+./aggregate_cts_reports.py -r REPORT [REPORT ...] -d OUTPUT_DIR [--ignore-abi] [--abi [{armeabi-v7a,arm64-v8a,x86,x86_64} ...]]
 ```
 
 The `-r` flag can be followed by one or more reports.
@@ -29,6 +31,8 @@ The `-d` flag has the same behavior as `parse_cts_report.py`.
 
 `--ignore-abi` is a boolean flag. If users specify this flag, tests ABI would be ignored while doing the aggregation. It means that two tests would be considered as the same one as long as they have the same module_name#class_name.test_name.
 
+The `--abi` flag can be used to select one or more test ABIs to be aggregated.
+
 ## compare_cts_reports.py
 ### usage
 ```

tools/compare_cts_reports/aggregate_cts_reports.py

@@ -26,10 +26,13 @@ import os
 import tempfile
 import zipfile
 
+import constant
 import parse_cts_report
 
 
-def aggregate_cts_reports(report_files, ignore_abi=False):
+def aggregate_cts_reports(report_files,
+                          selected_abis=constant.ALL_TEST_ABIS,
+                          ignore_abi=False):
   """Aggregate all report files and produce information files to output_dir.
 
   If the results of the same test are different in two reports, choose the one
@@ -51,7 +54,8 @@ def aggregate_cts_reports(report_files, ignore_abi=False):
 
   first_report_file = report_files[0]
 
-  report = parse_cts_report.parse_report_file(first_report_file, ignore_abi)
+  report = parse_cts_report.parse_report_file(
+      first_report_file, selected_abis, ignore_abi)
 
   with tempfile.TemporaryDirectory() as temp_dir:
 
@@ -82,6 +86,9 @@ def main():
                       help=('Path to the directory to store output files.'))
   parser.add_argument('--ignore-abi', action='store_true',
                       help='Ignore the tests ABI while aggregating reports.')
+  parser.add_argument('--abi', choices=constant.ALL_TEST_ABIS, nargs='*',
+                      default=constant.ALL_TEST_ABIS,
+                      help='Selected test ABIs to be aggregated.')
 
   args = parser.parse_args()
 
@@ -91,7 +98,7 @@ def main():
   if not os.path.exists(output_dir):
     raise FileNotFoundError(f'Output directory {output_dir} does not exist.')
 
-  report = aggregate_cts_reports(report_files, args.ignore_abi)
+  report = aggregate_cts_reports(report_files, args.abi, args.ignore_abi)
   report.output_files(output_dir)
 
 

tools/compare_cts_reports/compare_cts_reports.py

@@ -227,7 +227,7 @@ def n_way_compare(reports, diff_csv):
         diff_writer.writerow([module_with_abi, item] + row)
 
 
-def load_parsed_report(report_dir):
+def load_parsed_report(report_dir, ignore_abi=False):
   """Load CtsReport() from a directory that stores a parsed report."""
 
   if not os.path.isdir(report_dir):
@@ -246,7 +246,7 @@ def load_parsed_report(report_dir):
   report = parse_cts_report.CtsReport(info)
 
   with open(result_path, 'r') as result_csvfile:
-    report.load_from_csv(result_csvfile)
+    report.load_from_csv(result_csvfile, ignore_abi)
 
   return report
 
@@ -291,12 +291,14 @@ def main():
   ctsreports = []
   ignore_abi = args.ignore_abi
   for i, report_path in enumerate(reports):
+    # path(s) from the `--report` flag is a list
     is_report_files = isinstance(report_path, list)
-    report = (
-        aggregate_cts_reports.aggregate_cts_reports(report_path, ignore_abi)
-        if is_report_files  # path(s) come from --report flag
-        else load_parsed_report(report_path)
-    )
+
+    if is_report_files:
+      report = aggregate_cts_reports.aggregate_cts_reports(
+          report_path, constant.ALL_TEST_ABIS, ignore_abi)
+    else:
+      report = load_parsed_report(report_path, ignore_abi)
 
     if is_report_files and args.output_files:
       device_name = report.info['build_device']

tools/compare_cts_reports/constant.py

@@ -24,3 +24,8 @@ TESTED_ITEMS = 'tested_items'
 PASS_RATE = 'pass_rate'
 
 ABI_IGNORED = 'abi-ignored'
+ABI_ARM_V7A = 'armeabi-v7a'
+ABI_ARM_V8A = 'arm64-v8a'
+ABI_X86 = 'x86'
+ABI_X86_64 = 'x86_64'
+ALL_TEST_ABIS = [ABI_ARM_V7A, ABI_ARM_V8A, ABI_X86, ABI_X86_64]

tools/compare_cts_reports/parse_cts_report.py

@@ -48,8 +48,9 @@ class CtsReport:
 
   FAIL_INDEX = STATUS_ORDER.index('fail')
 
-  def __init__(self, info):
+  def __init__(self, info, selected_abis=constant.ALL_TEST_ABIS):
     self.info = info
+    self.selected_abis = selected_abis
     self.result_tree = {}
     self.module_summaries = {}
 
@@ -134,7 +135,11 @@ class CtsReport:
 
     for module in root.iter('Module'):
       module_name = module.attrib['name']
-      abi = constant.ABI_IGNORED if ignore_abi else module.attrib['abi']
+      abi = module.attrib['abi']
+      if abi not in self.selected_abis:
+        continue
+      if ignore_abi:
+        abi = constant.ABI_IGNORED
 
       for testcase in module.iter('TestCase'):
         class_name = testcase.attrib['name']
@@ -144,11 +149,12 @@ class CtsReport:
           result = test.attrib['result']
           self.set_test_status(module_name, abi, class_name, test_name, result)
 
-  def load_from_csv(self, result_csvfile):
+  def load_from_csv(self, result_csvfile, ignore_abi=False):
     """Read the information of the report from the csv files.
 
     Args:
       result_csvfile: path to result.csv
+      ignore_abi: if specified, load the test ABI name as constant.ABI_IGNORED
     """
 
     result_reader = csv.reader(result_csvfile)
@@ -161,6 +167,10 @@ class CtsReport:
 
     for row in result_reader:
       module_name, abi, class_name, test_name, result = row
+      if abi not in self.selected_abis:
+        continue
+      if ignore_abi:
+        abi = constant.ABI_IGNORED
       self.set_test_status(module_name, abi, class_name, test_name, result)
 
   def write_to_csv(self, result_csvfile, summary_csvfile):
@@ -341,7 +351,9 @@ def extract_test_result_from_zip(zip_file_path, dest_dir):
   return extracted
 
 
-def parse_report_file(report_file, ignore_abi=False):
+def parse_report_file(report_file,
+                      selected_abis=constant.ALL_TEST_ABIS,
+                      ignore_abi=False):
   """Turn one cts report into a CtsReport object."""
 
   with tempfile.TemporaryDirectory() as temp_dir:
@@ -352,9 +364,10 @@ def parse_report_file(report_file, ignore_abi=False):
     )
 
     test_info = get_test_info_xml(xml_path)
+    print(f'Parsing {selected_abis} test results from: ')
     print_test_info(test_info)
 
-    report = CtsReport(test_info)
+    report = CtsReport(test_info, selected_abis)
     report.read_test_result_xml(xml_path, ignore_abi)
 
   return report
@@ -378,6 +391,13 @@ def main():
       required=True,
       help='Path to the directory to store output files.',
   )
+  parser.add_argument(
+      '--abi',
+      choices=constant.ALL_TEST_ABIS,
+      nargs='*',
+      default=constant.ALL_TEST_ABIS,
+      help='Selected test ABIs to be parsed.',
+  )
 
   args = parser.parse_args()
 
@@ -387,7 +407,7 @@ def main():
   if not os.path.exists(output_dir):
     raise FileNotFoundError(f'Output directory {output_dir} does not exist.')
 
-  report = parse_report_file(report_file)
+  report = parse_report_file(report_file, args.abi)
 
   report.output_files(output_dir)
 

tools/compare_cts_reports/test_parse_cts_report.py

@@ -14,10 +14,13 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 #
+
 import filecmp
 import os
 import tempfile
 import unittest
+
+import constant
 import parse_cts_report
 
 
@@ -37,6 +40,20 @@ class TestParse(unittest.TestCase):
 
     self.assertEqual(report.get_test_status(*test_item), 'pass')
 
+  def test_select_abi(self):
+    report_file = 'testdata/test_result_multiple_abis.xml'
+    report_all_abi = parse_cts_report.parse_report_file(report_file)
+    report_selected_abi = parse_cts_report.parse_report_file(report_file,
+                                                             ['arm64-v8a'])
+    test_item_arm = ('module', 'armeabi-v7a', 'class', 'test')
+    test_item_arm64 = ('module', 'arm64-v8a', 'class', 'test')
+    self.assertEqual(report_all_abi.get_test_status(*test_item_arm), 'pass')
+    self.assertEqual(report_all_abi.get_test_status(*test_item_arm64), 'pass')
+    self.assertEqual(report_selected_abi.get_test_status(*test_item_arm),
+                     constant.NO_DATA)
+    self.assertEqual(report_selected_abi.get_test_status(*test_item_arm64),
+                     'pass')
+
   def test_parse_xml(self):
     report_file = 'testdata/test_result_1.xml'
     report = parse_cts_report.parse_report_file(report_file)

tools/compare_cts_reports/testdata/test_result_multiple_abis.xml

@@ -0,0 +1,15 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<Result suite_name="CTS" suite_plan="cts" suite_build_number="" start_display="" suite_version="">
+    <Build build_model="model" build_device="device" build_id="123456" build_version_sdk="34" build_version_security_patch="2023-06-05" build_board="board" build_type="userdebug" build_version_release="14" build_fingerprint="fingerprint" />
+    <Summary pass="2" failed="0" modules_done="2" modules_total="2" />
+    <Module name="module" abi="armeabi-v7a" done="true" pass="1" total_tests="1">
+        <TestCase name="class">
+            <Test result="pass" name="test"></Test>
+        </TestCase>
+    </Module>
+    <Module name="module" abi="arm64-v8a" done="true" pass="1" total_tests="1">
+        <TestCase name="class">
+            <Test result="pass" name="test"></Test>
+        </TestCase>
+    </Module>
+</Result>