diff --git a/tools/compare_cts_reports/test_aggregate_cts_reports.py b/tools/compare_cts_reports/test_aggregate_cts_reports.py new file mode 100755 index 000000000..0d17868b8 --- /dev/null +++ b/tools/compare_cts_reports/test_aggregate_cts_reports.py @@ -0,0 +1,121 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +import unittest +import aggregate_cts_reports + + +class TestParse(unittest.TestCase): + + def test_aggregate(self): + report_files = ['testdata/test_result_1.xml', 'testdata/test_result_2.xml'] + report = aggregate_cts_reports.aggregate_cts_reports(report_files) + + self.check_ctsreport(report) + + def check_ctsreport(self, report): + self.assertEqual( + report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_1'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_2'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_2', 'test_3'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_4'), + 'ASSUMPTION_FAILURE', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_5'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_6'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_7'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_8'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_3', 'arm64-v8a', 'testcase_5', 'test_9'), + 'pass', + ) + self.assertEqual( + report.get_test_status( + 'module_3', 'arm64-v8a', 'testcase_5', 'test_10' + ), + 'TEST_ERROR', + ) + self.assertEqual( + report.get_test_status( + 'module_3', 'arm64-v8a', 'testcase_5', 'test_11' + ), + 'pass', + ) + self.assertEqual( + report.get_test_status( + 'module_4', 'arm64-v8a', 'testcase_6', 'test_12' + ), + 'fail', + ) + + self.assertEqual(report.info['build_model'], 'this_model') + self.assertEqual(report.info['build_id'], '1412') + self.assertEqual(report.info['build_fingerprint'], 'this_build_fingerprint') + self.assertEqual(report.info['build_device'], 'this_device') + self.assertEqual(report.info['build_version_sdk'], '34') + self.assertEqual(report.info['build_version_security_patch'], '2023-06-05') + self.assertEqual(report.info['build_board'], 'this_board') + self.assertEqual(report.info['build_type'], 'userdebug') + self.assertEqual(report.info['build_version_release'], '14') + self.assertEqual(report.info['suite_name'], 'CTS') + self.assertEqual(report.info['suite_version'], '14_r1') + self.assertEqual(report.info['suite_plan'], 'cts') + self.assertEqual(report.info['suite_build_number'], '1234567') + + module_summaries = report.module_summaries + summary = module_summaries['module_1']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 1) + self.assertEqual(summary.counter['fail'], 1) + + summary = module_summaries['module_2']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 3) + self.assertEqual( + summary.counter['ASSUMPTION_FAILURE'], + 1, + ) + self.assertEqual(summary.counter['fail'], 2) + + summary = module_summaries['module_3']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 2) + self.assertEqual(summary.counter['TEST_ERROR'], 1) + + summary = module_summaries['module_4']['arm64-v8a'] + self.assertEqual(summary.counter['fail'], 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/compare_cts_reports/test_parse_cts_report.py b/tools/compare_cts_reports/test_parse_cts_report.py new file mode 100755 index 000000000..efa60ab08 --- /dev/null +++ b/tools/compare_cts_reports/test_parse_cts_report.py @@ -0,0 +1,155 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +import filecmp +import os +import tempfile +import unittest +import parse_cts_report + + +class TestParse(unittest.TestCase): + + def test_set(self): + info = {} + report = parse_cts_report.CtsReport(info) + test_item = ('module', 'abi', 'class', 'test') + + report.set_test_status(*test_item, 'IGNORED') + report.set_test_status(*test_item, 'fail') + + self.assertEqual(report.get_test_status(*test_item), 'IGNORED') + + report.set_test_status(*test_item, 'pass') + + self.assertEqual(report.get_test_status(*test_item), 'pass') + + def test_parse_xml(self): + report_file = 'testdata/test_result_1.xml' + report = parse_cts_report.parse_report_file(report_file) + + self.check_ctsreport(report) + + def test_parse_zip(self): + report_file = 'testdata/report.zip' + report = parse_cts_report.parse_report_file(report_file) + + self.check_ctsreport(report) + + def check_ctsreport(self, report): + self.assertEqual( + report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_1'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_2'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_2', 'test_3'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_4'), + 'ASSUMPTION_FAILURE', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_5'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_6'), + 'IGNORED', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_7'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_8'), + 'TEST_STATUS_UNSPECIFIED', + ) + self.assertEqual( + report.get_test_status('module_3', 'arm64-v8a', 'testcase_5', 'test_9'), + 'pass', + ) + self.assertEqual( + report.get_test_status( + 'module_3', 'arm64-v8a', 'testcase_5', 'test_10' + ), + 'TEST_ERROR', + ) + + self.assertEqual(report.info['build_model'], 'this_model') + self.assertEqual(report.info['build_id'], '1412') + self.assertEqual(report.info['build_fingerprint'], 'this_build_fingerprint') + self.assertEqual(report.info['build_device'], 'this_device') + self.assertEqual(report.info['build_version_sdk'], '34') + self.assertEqual(report.info['build_version_security_patch'], '2023-06-05') + self.assertEqual(report.info['build_board'], 'this_board') + self.assertEqual(report.info['build_type'], 'userdebug') + self.assertEqual(report.info['build_version_release'], '14') + self.assertEqual(report.info['suite_name'], 'CTS') + self.assertEqual(report.info['suite_version'], '14_r1') + self.assertEqual(report.info['suite_plan'], 'cts') + self.assertEqual(report.info['suite_build_number'], '1234567') + + module_summaries = report.module_summaries + summary = module_summaries['module_1']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 1) + self.assertEqual(summary.counter['fail'], 1) + + summary = module_summaries['module_2']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 1) + self.assertEqual(summary.counter['IGNORED'], 1) + self.assertEqual( + summary.counter['ASSUMPTION_FAILURE'], + 1, + ) + self.assertEqual(summary.counter['fail'], 2) + self.assertEqual( + summary.counter['TEST_STATUS_UNSPECIFIED'], + 1, + ) + + summary = module_summaries['module_3']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 1) + self.assertEqual(summary.counter['TEST_ERROR'], 1) + + def test_output(self): + report_file = 'testdata/test_result_1.xml' + report = parse_cts_report.parse_report_file(report_file) + + with tempfile.TemporaryDirectory() as temp_dir: + report.output_files(temp_dir) + + parsed_info_path = os.path.join(temp_dir, 'info.json') + parsed_result_path = os.path.join(temp_dir, 'result.csv') + parsed_summary_path = os.path.join(temp_dir, 'summary.csv') + + files = [parsed_info_path, parsed_result_path, parsed_summary_path] + + match, mismatch, errors = filecmp.cmpfiles( + 'testdata/output', temp_dir, files + ) + + self.assertEqual(match, files) + self.assertEqual(mismatch, []) + self.assertEqual(errors, []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/compare_cts_reports/testdata/output/info_1.json b/tools/compare_cts_reports/testdata/output/info_1.json new file mode 100644 index 000000000..b70901b02 --- /dev/null +++ b/tools/compare_cts_reports/testdata/output/info_1.json @@ -0,0 +1,16 @@ +{ + "source_path": "testdata/test_result_1.xml", + "build_model": "this_model", + "build_id": "1412", + "build_fingerprint": "this_build_fingerprint", + "build_device": "this_device", + "build_version_sdk": "34", + "build_version_security_patch": "2023-06-05", + "build_board": "this_board", + "build_type": "userdebug", + "build_version_release": "14", + "suite_name": "CTS", + "suite_version": "14_r1", + "suite_plan": "cts", + "suite_build_number": "1234567" +} \ No newline at end of file diff --git a/tools/compare_cts_reports/testdata/output/result_1.csv b/tools/compare_cts_reports/testdata/output/result_1.csv new file mode 100644 index 000000000..b8e6e6703 --- /dev/null +++ b/tools/compare_cts_reports/testdata/output/result_1.csv @@ -0,0 +1,11 @@ +module_name,abi,class_name,test_name,result +module_1,arm64-v8a,testcase_1,test_1,pass +module_1,arm64-v8a,testcase_1,test_2,fail +module_2,arm64-v8a,testcase_2,test_3,pass +module_2,arm64-v8a,testcase_3,test_4,ASSUMPTION_FAILURE +module_2,arm64-v8a,testcase_3,test_5,fail +module_2,arm64-v8a,testcase_4,test_6,IGNORED +module_2,arm64-v8a,testcase_4,test_7,fail +module_2,arm64-v8a,testcase_4,test_8,TEST_STATUS_UNSPECIFIED +module_3,arm64-v8a,testcase_5,test_9,pass +module_3,arm64-v8a,testcase_5,test_10,TEST_ERROR diff --git a/tools/compare_cts_reports/testdata/output/summary_1.csv b/tools/compare_cts_reports/testdata/output/summary_1.csv new file mode 100644 index 000000000..e794f4f88 --- /dev/null +++ b/tools/compare_cts_reports/testdata/output/summary_1.csv @@ -0,0 +1,4 @@ +module_name,abi,pass,IGNORED,ASSUMPTION_FAILURE,fail,TEST_ERROR,TEST_STATUS_UNSPECIFIED +module_1,arm64-v8a,1,0,0,1,0,0 +module_2,arm64-v8a,1,1,1,2,0,1 +module_3,arm64-v8a,1,0,0,0,1,0 diff --git a/tools/compare_cts_reports/testdata/report.zip b/tools/compare_cts_reports/testdata/report.zip new file mode 100644 index 000000000..00f665992 Binary files /dev/null and b/tools/compare_cts_reports/testdata/report.zip differ diff --git a/tools/compare_cts_reports/testdata/test_result_1.xml b/tools/compare_cts_reports/testdata/test_result_1.xml new file mode 100644 index 000000000..e75e1d6e3 --- /dev/null +++ b/tools/compare_cts_reports/testdata/test_result_1.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/compare_cts_reports/testdata/test_result_2.xml b/tools/compare_cts_reports/testdata/test_result_2.xml new file mode 100644 index 000000000..416fbfdd4 --- /dev/null +++ b/tools/compare_cts_reports/testdata/test_result_2.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +