Merge "compare_cts_reports: add the --ignore-abi option" into main

tools/compare_cts_reports/README.md

@@ -20,17 +20,19 @@ The `-d` flag specifies the directory in which the information files will be sto
 ## aggregate_cts_reports.py
 ### usage
 ```
-./aggregate_cts_reports.py -r REPORT [REPORT ...] -d OUTPUT_DIR
+./aggregate_cts_reports.py -r REPORT [REPORT ...] -d OUTPUT_DIR [--ignore-abi]
 ```
 
 The `-r` flag can be followed by one or more reports.
 
 The `-d` flag has the same behavior as `parse_cts_report.py`.
 
+`--ignore-abi` is a boolean flag. If users specify this flag, tests ABI would be ignored while doing the aggregation. It means that two tests would be considered as the same one as long as they have the same module_name#class_name.test_name.
+
 ## compare_cts_reports.py
 ### usage
 ```
-./compare_cts_reports.py [-h] [-r CTS_REPORTS [CTS_REPORTS ...]] [-f CTS_REPORTS] --mode {1,2,n} --output-dir OUTPUT_DIR [--csv CSV] [--output-files]
+./compare_cts_reports.py [-h] [-r CTS_REPORTS [CTS_REPORTS ...]] [-f CTS_REPORTS] --mode {1,2,n} --output-dir OUTPUT_DIR [--csv CSV] [--output-files] [--ignore-abi]
 ```
 
 One `-r` flag is followed by a group of report files that you want to aggregate.
@@ -41,9 +43,11 @@ The `-m` flag has to be specified: `1` for one-way mode, `2` for two-way mode, a
 
 The `-d` flag has to be specified. Behavior same as `parse_cts_report.py`.
 
+`--csv` allows users to specify the file name of the comparison result. The default value is `diff.csv`.
+
 `--output-files/-o` is a boolean flag. If users specify this flag, the parsed results of reports after flags `-r` will be outputed into the information files.
 
-`--csv` allows users to specify the file name of the comparison result. The default value is `diff.csv`.
+`--ignore-abi` is a boolean flag, which has the same behavior as `aggregate_cts_reports.py`.
 
 ### modes
 #### One-way Comparison
@@ -64,4 +68,4 @@ N-way comparison mode is used to show a summary of several test reports.
 ./test_parse_cts_report.py
 ./test_aggregate_cts_reports.py
 ./test_compare_cts_reports.py
-```
+```

tools/compare_cts_reports/aggregate_cts_reports.py

@@ -29,7 +29,7 @@ import zipfile
 import parse_cts_report
 
 
-def aggregate_cts_reports(report_files):
+def aggregate_cts_reports(report_files, ignore_abi=False):
   """Aggregate all report files and produce information files to output_dir.
 
   If the results of the same test are different in two reports, choose the one
@@ -38,6 +38,9 @@ def aggregate_cts_reports(report_files):
 
   Args:
     report_files: A list of paths to cts reports.
+    ignore_abi: Ignore the tests ABI when doing the aggregation, which means
+                that tests would be considered as the same one as long as they
+                have the same module_name#class_name.test_name.
 
   Raises:
     UserWarning: Report files not compatible.
@@ -48,7 +51,7 @@ def aggregate_cts_reports(report_files):
 
   first_report_file = report_files[0]
 
-  report = parse_cts_report.parse_report_file(first_report_file)
+  report = parse_cts_report.parse_report_file(first_report_file, ignore_abi)
 
   with tempfile.TemporaryDirectory() as temp_dir:
 
@@ -64,7 +67,7 @@ def aggregate_cts_reports(report_files):
         msg = (f'{report_file} is incompatible to {first_report_file}.')
         raise UserWarning(msg)
 
-      report.read_test_result_xml(xml_path)
+      report.read_test_result_xml(xml_path, ignore_abi)
 
   return report
 
@@ -77,6 +80,8 @@ def main():
                             'be a zip archive or a xml file.'))
   parser.add_argument('-d', '--output-dir', required=True,
                       help=('Path to the directory to store output files.'))
+  parser.add_argument('--ignore-abi', action='store_true',
+                      help='Ignore the tests ABI while aggregating reports.')
 
   args = parser.parse_args()
 
@@ -86,7 +91,7 @@ def main():
   if not os.path.exists(output_dir):
     raise FileNotFoundError(f'Output directory {output_dir} does not exist.')
 
-  report = aggregate_cts_reports(report_files)
+  report = aggregate_cts_reports(report_files, args.ignore_abi)
   report.output_files(output_dir)
 
 

tools/compare_cts_reports/compare_cts_reports.py

@@ -270,6 +270,8 @@ def main():
   parser.add_argument('--csv', default='diff.csv', help='Path to csv output.')
   parser.add_argument('--output-files', '-o', action='store_true',
                       help='Output parsed csv files.')
+  parser.add_argument('--ignore-abi', action='store_true',
+                      help='Ignore the tests ABI while comparing.')
 
   args = parser.parse_args()
 
@@ -287,10 +289,11 @@ def main():
   diff_csv = os.path.join(output_dir, args.csv)
 
   ctsreports = []
+  ignore_abi = args.ignore_abi
   for i, report_path in enumerate(reports):
     is_report_files = isinstance(report_path, list)
     report = (
-        aggregate_cts_reports.aggregate_cts_reports(report_path)
+        aggregate_cts_reports.aggregate_cts_reports(report_path, ignore_abi)
         if is_report_files  # path(s) come from --report flag
         else load_parsed_report(report_path)
     )

tools/compare_cts_reports/parse_cts_report.py

@@ -126,7 +126,7 @@ class CtsReport:
       summary.counter[previous] -= 1
       summary.counter[test_status] += 1
 
-  def read_test_result_xml(self, test_result_path):
+  def read_test_result_xml(self, test_result_path, ignore_abi=False):
     """Read the result from test_result.xml into a CtsReport object."""
 
     tree = ET.parse(test_result_path)
@@ -134,7 +134,7 @@ class CtsReport:
 
     for module in root.iter('Module'):
       module_name = module.attrib['name']
-      abi = module.attrib['abi']
+      abi = 'ignored' if ignore_abi else module.attrib['abi']
 
       for testcase in module.iter('TestCase'):
         class_name = testcase.attrib['name']
@@ -341,7 +341,7 @@ def extract_test_result_from_zip(zip_file_path, dest_dir):
   return extracted
 
 
-def parse_report_file(report_file):
+def parse_report_file(report_file, ignore_abi=False):
   """Turn one cts report into a CtsReport object."""
 
   with tempfile.TemporaryDirectory() as temp_dir:
@@ -355,7 +355,7 @@ def parse_report_file(report_file):
     print_test_info(test_info)
 
     report = CtsReport(test_info)
-    report.read_test_result_xml(xml_path)
+    report.read_test_result_xml(xml_path, ignore_abi)
 
   return report