From 83445923e50d4e51f06bfca62c3eb1b648d9e998 Mon Sep 17 00:00:00 2001 From: Zoe Tsou Date: Wed, 13 Sep 2023 03:41:33 +0000 Subject: [PATCH] Add unit tests for tools/compare_cts_reports. Bug: 293809772 Test: ./test_parse_cts_report.py Test: ./test_aggregate_cts_reports.py Change-Id: I813482cfdfdcc9a5f2384f95fd289fb7ff89619f --- .../test_aggregate_cts_reports.py | 121 ++++++++++++++ .../test_parse_cts_report.py | 155 ++++++++++++++++++ .../testdata/output/info_1.json | 16 ++ .../testdata/output/result_1.csv | 11 ++ .../testdata/output/summary_1.csv | 4 + tools/compare_cts_reports/testdata/report.zip | Bin 0 -> 844 bytes .../testdata/test_result_1.xml | 41 +++++ .../testdata/test_result_2.xml | 45 +++++ 8 files changed, 393 insertions(+) create mode 100755 tools/compare_cts_reports/test_aggregate_cts_reports.py create mode 100755 tools/compare_cts_reports/test_parse_cts_report.py create mode 100644 tools/compare_cts_reports/testdata/output/info_1.json create mode 100644 tools/compare_cts_reports/testdata/output/result_1.csv create mode 100644 tools/compare_cts_reports/testdata/output/summary_1.csv create mode 100644 tools/compare_cts_reports/testdata/report.zip create mode 100644 tools/compare_cts_reports/testdata/test_result_1.xml create mode 100644 tools/compare_cts_reports/testdata/test_result_2.xml diff --git a/tools/compare_cts_reports/test_aggregate_cts_reports.py b/tools/compare_cts_reports/test_aggregate_cts_reports.py new file mode 100755 index 000000000..0d17868b8 --- /dev/null +++ b/tools/compare_cts_reports/test_aggregate_cts_reports.py @@ -0,0 +1,121 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +import unittest +import aggregate_cts_reports + + +class TestParse(unittest.TestCase): + + def test_aggregate(self): + report_files = ['testdata/test_result_1.xml', 'testdata/test_result_2.xml'] + report = aggregate_cts_reports.aggregate_cts_reports(report_files) + + self.check_ctsreport(report) + + def check_ctsreport(self, report): + self.assertEqual( + report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_1'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_2'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_2', 'test_3'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_4'), + 'ASSUMPTION_FAILURE', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_5'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_6'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_7'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_8'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_3', 'arm64-v8a', 'testcase_5', 'test_9'), + 'pass', + ) + self.assertEqual( + report.get_test_status( + 'module_3', 'arm64-v8a', 'testcase_5', 'test_10' + ), + 'TEST_ERROR', + ) + self.assertEqual( + report.get_test_status( + 'module_3', 'arm64-v8a', 'testcase_5', 'test_11' + ), + 'pass', + ) + self.assertEqual( + report.get_test_status( + 'module_4', 'arm64-v8a', 'testcase_6', 'test_12' + ), + 'fail', + ) + + self.assertEqual(report.info['build_model'], 'this_model') + self.assertEqual(report.info['build_id'], '1412') + self.assertEqual(report.info['build_fingerprint'], 'this_build_fingerprint') + self.assertEqual(report.info['build_device'], 'this_device') + self.assertEqual(report.info['build_version_sdk'], '34') + self.assertEqual(report.info['build_version_security_patch'], '2023-06-05') + self.assertEqual(report.info['build_board'], 'this_board') + self.assertEqual(report.info['build_type'], 'userdebug') + self.assertEqual(report.info['build_version_release'], '14') + self.assertEqual(report.info['suite_name'], 'CTS') + self.assertEqual(report.info['suite_version'], '14_r1') + self.assertEqual(report.info['suite_plan'], 'cts') + self.assertEqual(report.info['suite_build_number'], '1234567') + + module_summaries = report.module_summaries + summary = module_summaries['module_1']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 1) + self.assertEqual(summary.counter['fail'], 1) + + summary = module_summaries['module_2']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 3) + self.assertEqual( + summary.counter['ASSUMPTION_FAILURE'], + 1, + ) + self.assertEqual(summary.counter['fail'], 2) + + summary = module_summaries['module_3']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 2) + self.assertEqual(summary.counter['TEST_ERROR'], 1) + + summary = module_summaries['module_4']['arm64-v8a'] + self.assertEqual(summary.counter['fail'], 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/compare_cts_reports/test_parse_cts_report.py b/tools/compare_cts_reports/test_parse_cts_report.py new file mode 100755 index 000000000..efa60ab08 --- /dev/null +++ b/tools/compare_cts_reports/test_parse_cts_report.py @@ -0,0 +1,155 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +import filecmp +import os +import tempfile +import unittest +import parse_cts_report + + +class TestParse(unittest.TestCase): + + def test_set(self): + info = {} + report = parse_cts_report.CtsReport(info) + test_item = ('module', 'abi', 'class', 'test') + + report.set_test_status(*test_item, 'IGNORED') + report.set_test_status(*test_item, 'fail') + + self.assertEqual(report.get_test_status(*test_item), 'IGNORED') + + report.set_test_status(*test_item, 'pass') + + self.assertEqual(report.get_test_status(*test_item), 'pass') + + def test_parse_xml(self): + report_file = 'testdata/test_result_1.xml' + report = parse_cts_report.parse_report_file(report_file) + + self.check_ctsreport(report) + + def test_parse_zip(self): + report_file = 'testdata/report.zip' + report = parse_cts_report.parse_report_file(report_file) + + self.check_ctsreport(report) + + def check_ctsreport(self, report): + self.assertEqual( + report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_1'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_2'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_2', 'test_3'), + 'pass', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_4'), + 'ASSUMPTION_FAILURE', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_5'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_6'), + 'IGNORED', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_7'), + 'fail', + ) + self.assertEqual( + report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_8'), + 'TEST_STATUS_UNSPECIFIED', + ) + self.assertEqual( + report.get_test_status('module_3', 'arm64-v8a', 'testcase_5', 'test_9'), + 'pass', + ) + self.assertEqual( + report.get_test_status( + 'module_3', 'arm64-v8a', 'testcase_5', 'test_10' + ), + 'TEST_ERROR', + ) + + self.assertEqual(report.info['build_model'], 'this_model') + self.assertEqual(report.info['build_id'], '1412') + self.assertEqual(report.info['build_fingerprint'], 'this_build_fingerprint') + self.assertEqual(report.info['build_device'], 'this_device') + self.assertEqual(report.info['build_version_sdk'], '34') + self.assertEqual(report.info['build_version_security_patch'], '2023-06-05') + self.assertEqual(report.info['build_board'], 'this_board') + self.assertEqual(report.info['build_type'], 'userdebug') + self.assertEqual(report.info['build_version_release'], '14') + self.assertEqual(report.info['suite_name'], 'CTS') + self.assertEqual(report.info['suite_version'], '14_r1') + self.assertEqual(report.info['suite_plan'], 'cts') + self.assertEqual(report.info['suite_build_number'], '1234567') + + module_summaries = report.module_summaries + summary = module_summaries['module_1']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 1) + self.assertEqual(summary.counter['fail'], 1) + + summary = module_summaries['module_2']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 1) + self.assertEqual(summary.counter['IGNORED'], 1) + self.assertEqual( + summary.counter['ASSUMPTION_FAILURE'], + 1, + ) + self.assertEqual(summary.counter['fail'], 2) + self.assertEqual( + summary.counter['TEST_STATUS_UNSPECIFIED'], + 1, + ) + + summary = module_summaries['module_3']['arm64-v8a'] + self.assertEqual(summary.counter['pass'], 1) + self.assertEqual(summary.counter['TEST_ERROR'], 1) + + def test_output(self): + report_file = 'testdata/test_result_1.xml' + report = parse_cts_report.parse_report_file(report_file) + + with tempfile.TemporaryDirectory() as temp_dir: + report.output_files(temp_dir) + + parsed_info_path = os.path.join(temp_dir, 'info.json') + parsed_result_path = os.path.join(temp_dir, 'result.csv') + parsed_summary_path = os.path.join(temp_dir, 'summary.csv') + + files = [parsed_info_path, parsed_result_path, parsed_summary_path] + + match, mismatch, errors = filecmp.cmpfiles( + 'testdata/output', temp_dir, files + ) + + self.assertEqual(match, files) + self.assertEqual(mismatch, []) + self.assertEqual(errors, []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/compare_cts_reports/testdata/output/info_1.json b/tools/compare_cts_reports/testdata/output/info_1.json new file mode 100644 index 000000000..b70901b02 --- /dev/null +++ b/tools/compare_cts_reports/testdata/output/info_1.json @@ -0,0 +1,16 @@ +{ + "source_path": "testdata/test_result_1.xml", + "build_model": "this_model", + "build_id": "1412", + "build_fingerprint": "this_build_fingerprint", + "build_device": "this_device", + "build_version_sdk": "34", + "build_version_security_patch": "2023-06-05", + "build_board": "this_board", + "build_type": "userdebug", + "build_version_release": "14", + "suite_name": "CTS", + "suite_version": "14_r1", + "suite_plan": "cts", + "suite_build_number": "1234567" +} \ No newline at end of file diff --git a/tools/compare_cts_reports/testdata/output/result_1.csv b/tools/compare_cts_reports/testdata/output/result_1.csv new file mode 100644 index 000000000..b8e6e6703 --- /dev/null +++ b/tools/compare_cts_reports/testdata/output/result_1.csv @@ -0,0 +1,11 @@ +module_name,abi,class_name,test_name,result +module_1,arm64-v8a,testcase_1,test_1,pass +module_1,arm64-v8a,testcase_1,test_2,fail +module_2,arm64-v8a,testcase_2,test_3,pass +module_2,arm64-v8a,testcase_3,test_4,ASSUMPTION_FAILURE +module_2,arm64-v8a,testcase_3,test_5,fail +module_2,arm64-v8a,testcase_4,test_6,IGNORED +module_2,arm64-v8a,testcase_4,test_7,fail +module_2,arm64-v8a,testcase_4,test_8,TEST_STATUS_UNSPECIFIED +module_3,arm64-v8a,testcase_5,test_9,pass +module_3,arm64-v8a,testcase_5,test_10,TEST_ERROR diff --git a/tools/compare_cts_reports/testdata/output/summary_1.csv b/tools/compare_cts_reports/testdata/output/summary_1.csv new file mode 100644 index 000000000..e794f4f88 --- /dev/null +++ b/tools/compare_cts_reports/testdata/output/summary_1.csv @@ -0,0 +1,4 @@ +module_name,abi,pass,IGNORED,ASSUMPTION_FAILURE,fail,TEST_ERROR,TEST_STATUS_UNSPECIFIED +module_1,arm64-v8a,1,0,0,1,0,0 +module_2,arm64-v8a,1,1,1,2,0,1 +module_3,arm64-v8a,1,0,0,0,1,0 diff --git a/tools/compare_cts_reports/testdata/report.zip b/tools/compare_cts_reports/testdata/report.zip new file mode 100644 index 0000000000000000000000000000000000000000..00f665992d66502cb046711736f0e796bd4abd9a GIT binary patch literal 844 zcmWIWW@h1H00B2O-Ec4iO0YA?FchU0cqsF|+Mkqzj>d~t7)?`NF>hy1&J8uAvIm;k#=V!9R-|2w zS^xRApiA0)fiKPWhjgwstyaDnJk?G)<0kK%j|U`6r{3=BS|b~JJJw0|dt1{qpOl={ z<vE3M^D{5j zc{RWL&$YrUaPztaM=MqwUSN5CS=$bQ3%c$Tx{_pg_=LF}nHRU1Y3Cf9xk#!@Lv89u z+iC-IMYDT%G_nM9jZF1co?m;GIWS(8MOWz0+E3|MFHDQg*udm3wdZr&u}wT{b|osm zS6z6ox}`5({8g;#mHMgQPrvx5`Xy3Eoxi_gYvkmoIbTgX#nd-9ZPzRP;#YJ~wP>~e zYu_zDYK#*8gt=|6&pgmAFS$n-_tDnBgdGTcax~~87_l~?+wffUXskhx4 zw_KL{pWGRgZ;?~?E%QBlfHxzP95b%uC;?5F0t_!4K}>k!WQ8P7w1fmP3|G>G7zRwK k3=0}Tra=wENT@)wu_V#}Z&qLeXJB9g!j(X}mkGoJ038E2zyJUM literal 0 HcmV?d00001 diff --git a/tools/compare_cts_reports/testdata/test_result_1.xml b/tools/compare_cts_reports/testdata/test_result_1.xml new file mode 100644 index 000000000..e75e1d6e3 --- /dev/null +++ b/tools/compare_cts_reports/testdata/test_result_1.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/compare_cts_reports/testdata/test_result_2.xml b/tools/compare_cts_reports/testdata/test_result_2.xml new file mode 100644 index 000000000..416fbfdd4 --- /dev/null +++ b/tools/compare_cts_reports/testdata/test_result_2.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +