diff --git a/tools/compare_cts_reports/README.md b/tools/compare_cts_reports/README.md index f7c5cd2f9..c878a1b41 100644 --- a/tools/compare_cts_reports/README.md +++ b/tools/compare_cts_reports/README.md @@ -10,17 +10,19 @@ A report can be either a standard CTS result zip file, or a test_result.xml file ## parse_cts_report.py ### usage ``` -./parse_cts_report.py -r REPORT -d OUTPUT_DIR +./parse_cts_report.py -r REPORT -d OUTPUT_DIR [--abi [{armeabi-v7a,arm64-v8a,x86,x86_64} ...]] ``` The `-r` flag must be followed by exactly one report. The `-d` flag specifies the directory in which the information files will be stored. +The `--abi` flag can be used to select one or more test ABIs to be parsed. + ## aggregate_cts_reports.py ### usage ``` -./aggregate_cts_reports.py -r REPORT [REPORT ...] -d OUTPUT_DIR [--ignore-abi] +./aggregate_cts_reports.py -r REPORT [REPORT ...] -d OUTPUT_DIR [--ignore-abi] [--abi [{armeabi-v7a,arm64-v8a,x86,x86_64} ...]] ``` The `-r` flag can be followed by one or more reports. @@ -29,6 +31,8 @@ The `-d` flag has the same behavior as `parse_cts_report.py`. `--ignore-abi` is a boolean flag. If users specify this flag, tests ABI would be ignored while doing the aggregation. It means that two tests would be considered as the same one as long as they have the same module_name#class_name.test_name. +The `--abi` flag can be used to select one or more test ABIs to be aggregated. + ## compare_cts_reports.py ### usage ``` diff --git a/tools/compare_cts_reports/aggregate_cts_reports.py b/tools/compare_cts_reports/aggregate_cts_reports.py index f2db373e2..acb9ff820 100755 --- a/tools/compare_cts_reports/aggregate_cts_reports.py +++ b/tools/compare_cts_reports/aggregate_cts_reports.py @@ -26,10 +26,13 @@ import os import tempfile import zipfile +import constant import parse_cts_report -def aggregate_cts_reports(report_files, ignore_abi=False): +def aggregate_cts_reports(report_files, + selected_abis=constant.ALL_TEST_ABIS, + ignore_abi=False): """Aggregate all report files and produce information files to output_dir. If the results of the same test are different in two reports, choose the one @@ -51,7 +54,8 @@ def aggregate_cts_reports(report_files, ignore_abi=False): first_report_file = report_files[0] - report = parse_cts_report.parse_report_file(first_report_file, ignore_abi) + report = parse_cts_report.parse_report_file( + first_report_file, selected_abis, ignore_abi) with tempfile.TemporaryDirectory() as temp_dir: @@ -82,6 +86,9 @@ def main(): help=('Path to the directory to store output files.')) parser.add_argument('--ignore-abi', action='store_true', help='Ignore the tests ABI while aggregating reports.') + parser.add_argument('--abi', choices=constant.ALL_TEST_ABIS, nargs='*', + default=constant.ALL_TEST_ABIS, + help='Selected test ABIs to be aggregated.') args = parser.parse_args() @@ -91,7 +98,7 @@ def main(): if not os.path.exists(output_dir): raise FileNotFoundError(f'Output directory {output_dir} does not exist.') - report = aggregate_cts_reports(report_files, args.ignore_abi) + report = aggregate_cts_reports(report_files, args.abi, args.ignore_abi) report.output_files(output_dir) diff --git a/tools/compare_cts_reports/compare_cts_reports.py b/tools/compare_cts_reports/compare_cts_reports.py index 168af2058..875ebd79f 100755 --- a/tools/compare_cts_reports/compare_cts_reports.py +++ b/tools/compare_cts_reports/compare_cts_reports.py @@ -227,7 +227,7 @@ def n_way_compare(reports, diff_csv): diff_writer.writerow([module_with_abi, item] + row) -def load_parsed_report(report_dir): +def load_parsed_report(report_dir, ignore_abi=False): """Load CtsReport() from a directory that stores a parsed report.""" if not os.path.isdir(report_dir): @@ -246,7 +246,7 @@ def load_parsed_report(report_dir): report = parse_cts_report.CtsReport(info) with open(result_path, 'r') as result_csvfile: - report.load_from_csv(result_csvfile) + report.load_from_csv(result_csvfile, ignore_abi) return report @@ -291,12 +291,14 @@ def main(): ctsreports = [] ignore_abi = args.ignore_abi for i, report_path in enumerate(reports): + # path(s) from the `--report` flag is a list is_report_files = isinstance(report_path, list) - report = ( - aggregate_cts_reports.aggregate_cts_reports(report_path, ignore_abi) - if is_report_files # path(s) come from --report flag - else load_parsed_report(report_path) - ) + + if is_report_files: + report = aggregate_cts_reports.aggregate_cts_reports( + report_path, constant.ALL_TEST_ABIS, ignore_abi) + else: + report = load_parsed_report(report_path, ignore_abi) if is_report_files and args.output_files: device_name = report.info['build_device'] diff --git a/tools/compare_cts_reports/constant.py b/tools/compare_cts_reports/constant.py index 8c5dc72fa..99a2331d6 100755 --- a/tools/compare_cts_reports/constant.py +++ b/tools/compare_cts_reports/constant.py @@ -24,3 +24,8 @@ TESTED_ITEMS = 'tested_items' PASS_RATE = 'pass_rate' ABI_IGNORED = 'abi-ignored' +ABI_ARM_V7A = 'armeabi-v7a' +ABI_ARM_V8A = 'arm64-v8a' +ABI_X86 = 'x86' +ABI_X86_64 = 'x86_64' +ALL_TEST_ABIS = [ABI_ARM_V7A, ABI_ARM_V8A, ABI_X86, ABI_X86_64] diff --git a/tools/compare_cts_reports/parse_cts_report.py b/tools/compare_cts_reports/parse_cts_report.py index 8ed7111e3..8286d3505 100755 --- a/tools/compare_cts_reports/parse_cts_report.py +++ b/tools/compare_cts_reports/parse_cts_report.py @@ -48,8 +48,9 @@ class CtsReport: FAIL_INDEX = STATUS_ORDER.index('fail') - def __init__(self, info): + def __init__(self, info, selected_abis=constant.ALL_TEST_ABIS): self.info = info + self.selected_abis = selected_abis self.result_tree = {} self.module_summaries = {} @@ -134,7 +135,11 @@ class CtsReport: for module in root.iter('Module'): module_name = module.attrib['name'] - abi = constant.ABI_IGNORED if ignore_abi else module.attrib['abi'] + abi = module.attrib['abi'] + if abi not in self.selected_abis: + continue + if ignore_abi: + abi = constant.ABI_IGNORED for testcase in module.iter('TestCase'): class_name = testcase.attrib['name'] @@ -144,11 +149,12 @@ class CtsReport: result = test.attrib['result'] self.set_test_status(module_name, abi, class_name, test_name, result) - def load_from_csv(self, result_csvfile): + def load_from_csv(self, result_csvfile, ignore_abi=False): """Read the information of the report from the csv files. Args: result_csvfile: path to result.csv + ignore_abi: if specified, load the test ABI name as constant.ABI_IGNORED """ result_reader = csv.reader(result_csvfile) @@ -161,6 +167,10 @@ class CtsReport: for row in result_reader: module_name, abi, class_name, test_name, result = row + if abi not in self.selected_abis: + continue + if ignore_abi: + abi = constant.ABI_IGNORED self.set_test_status(module_name, abi, class_name, test_name, result) def write_to_csv(self, result_csvfile, summary_csvfile): @@ -341,7 +351,9 @@ def extract_test_result_from_zip(zip_file_path, dest_dir): return extracted -def parse_report_file(report_file, ignore_abi=False): +def parse_report_file(report_file, + selected_abis=constant.ALL_TEST_ABIS, + ignore_abi=False): """Turn one cts report into a CtsReport object.""" with tempfile.TemporaryDirectory() as temp_dir: @@ -352,9 +364,10 @@ def parse_report_file(report_file, ignore_abi=False): ) test_info = get_test_info_xml(xml_path) + print(f'Parsing {selected_abis} test results from: ') print_test_info(test_info) - report = CtsReport(test_info) + report = CtsReport(test_info, selected_abis) report.read_test_result_xml(xml_path, ignore_abi) return report @@ -378,6 +391,13 @@ def main(): required=True, help='Path to the directory to store output files.', ) + parser.add_argument( + '--abi', + choices=constant.ALL_TEST_ABIS, + nargs='*', + default=constant.ALL_TEST_ABIS, + help='Selected test ABIs to be parsed.', + ) args = parser.parse_args() @@ -387,7 +407,7 @@ def main(): if not os.path.exists(output_dir): raise FileNotFoundError(f'Output directory {output_dir} does not exist.') - report = parse_report_file(report_file) + report = parse_report_file(report_file, args.abi) report.output_files(output_dir) diff --git a/tools/compare_cts_reports/test_parse_cts_report.py b/tools/compare_cts_reports/test_parse_cts_report.py index 29ad1f7e7..09ff90d50 100755 --- a/tools/compare_cts_reports/test_parse_cts_report.py +++ b/tools/compare_cts_reports/test_parse_cts_report.py @@ -14,10 +14,13 @@ # License for the specific language governing permissions and limitations under # the License. # + import filecmp import os import tempfile import unittest + +import constant import parse_cts_report @@ -37,6 +40,20 @@ class TestParse(unittest.TestCase): self.assertEqual(report.get_test_status(*test_item), 'pass') + def test_select_abi(self): + report_file = 'testdata/test_result_multiple_abis.xml' + report_all_abi = parse_cts_report.parse_report_file(report_file) + report_selected_abi = parse_cts_report.parse_report_file(report_file, + ['arm64-v8a']) + test_item_arm = ('module', 'armeabi-v7a', 'class', 'test') + test_item_arm64 = ('module', 'arm64-v8a', 'class', 'test') + self.assertEqual(report_all_abi.get_test_status(*test_item_arm), 'pass') + self.assertEqual(report_all_abi.get_test_status(*test_item_arm64), 'pass') + self.assertEqual(report_selected_abi.get_test_status(*test_item_arm), + constant.NO_DATA) + self.assertEqual(report_selected_abi.get_test_status(*test_item_arm64), + 'pass') + def test_parse_xml(self): report_file = 'testdata/test_result_1.xml' report = parse_cts_report.parse_report_file(report_file) diff --git a/tools/compare_cts_reports/testdata/test_result_multiple_abis.xml b/tools/compare_cts_reports/testdata/test_result_multiple_abis.xml new file mode 100644 index 000000000..8b26e4870 --- /dev/null +++ b/tools/compare_cts_reports/testdata/test_result_multiple_abis.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + +