diff --git a/tools/compare_cts_reports/test_compare_cts_reports.py b/tools/compare_cts_reports/test_compare_cts_reports.py new file mode 100755 index 000000000..3a4d63d3e --- /dev/null +++ b/tools/compare_cts_reports/test_compare_cts_reports.py @@ -0,0 +1,63 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +import filecmp +import os +import tempfile +import unittest + +import compare_cts_reports +import parse_cts_report + + +class TestParse(unittest.TestCase): + + def test_one_way(self): + ctsreports = [ + parse_cts_report.parse_report_file('testdata/test_result_1.xml'), + parse_cts_report.parse_report_file('testdata/test_result_2.xml'), + ] + with tempfile.TemporaryDirectory() as temp_dir: + csvfile = os.path.join(temp_dir, 'one_way_diff.csv') + compare_cts_reports.one_way_compare(ctsreports, csvfile) + + self.assertTrue(filecmp.cmp('testdata/compare/one_way_diff.csv', csvfile)) + + def test_two_way(self): + ctsreports = [ + parse_cts_report.parse_report_file('testdata/test_result_1.xml'), + parse_cts_report.parse_report_file('testdata/test_result_2.xml'), + ] + with tempfile.TemporaryDirectory() as temp_dir: + csvfile = os.path.join(temp_dir, 'two_way_diff.csv') + compare_cts_reports.two_way_compare(ctsreports, csvfile) + + self.assertTrue(filecmp.cmp('testdata/compare/two_way_diff.csv', csvfile)) + + def test_n_way(self): + ctsreports = [ + parse_cts_report.parse_report_file('testdata/test_result_1.xml'), + parse_cts_report.parse_report_file('testdata/test_result_2.xml'), + ] + with tempfile.TemporaryDirectory() as temp_dir: + csvfile = os.path.join(temp_dir, 'n_way_diff.csv') + compare_cts_reports.n_way_compare(ctsreports, csvfile) + + self.assertTrue(filecmp.cmp('testdata/compare/n_way_diff.csv', csvfile)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/compare_cts_reports/test_parse_cts_report.py b/tools/compare_cts_reports/test_parse_cts_report.py index efa60ab08..29ad1f7e7 100755 --- a/tools/compare_cts_reports/test_parse_cts_report.py +++ b/tools/compare_cts_reports/test_parse_cts_report.py @@ -140,15 +140,15 @@ class TestParse(unittest.TestCase): parsed_result_path = os.path.join(temp_dir, 'result.csv') parsed_summary_path = os.path.join(temp_dir, 'summary.csv') - files = [parsed_info_path, parsed_result_path, parsed_summary_path] - - match, mismatch, errors = filecmp.cmpfiles( - 'testdata/output', temp_dir, files + self.assertTrue( + filecmp.cmp('testdata/output/info_1.json', parsed_info_path) + ) + self.assertTrue( + filecmp.cmp('testdata/output/result_1.csv', parsed_result_path) + ) + self.assertTrue( + filecmp.cmp('testdata/output/summary_1.csv', parsed_summary_path) ) - - self.assertEqual(match, files) - self.assertEqual(mismatch, []) - self.assertEqual(errors, []) if __name__ == '__main__': diff --git a/tools/compare_cts_reports/testdata/compare/n_way_diff.csv b/tools/compare_cts_reports/testdata/compare/n_way_diff.csv new file mode 100644 index 000000000..b2b05a301 --- /dev/null +++ b/tools/compare_cts_reports/testdata/compare/n_way_diff.csv @@ -0,0 +1,33 @@ +module_with_abi,item,0_device_this_device,1_device_that_device +module_4[arm64-v8a],pass,0,0 +module_4[arm64-v8a],IGNORED,0,0 +module_4[arm64-v8a],ASSUMPTION_FAILURE,0,0 +module_4[arm64-v8a],fail,0,1 +module_4[arm64-v8a],TEST_ERROR,0,0 +module_4[arm64-v8a],TEST_STATUS_UNSPECIFIED,0,0 +module_4[arm64-v8a],tested_items,0,1 +module_4[arm64-v8a],pass_rate,0.0,0.0 +module_1[arm64-v8a],pass,1,1 +module_1[arm64-v8a],IGNORED,0,0 +module_1[arm64-v8a],ASSUMPTION_FAILURE,0,0 +module_1[arm64-v8a],fail,1,1 +module_1[arm64-v8a],TEST_ERROR,0,0 +module_1[arm64-v8a],TEST_STATUS_UNSPECIFIED,0,0 +module_1[arm64-v8a],tested_items,2,2 +module_1[arm64-v8a],pass_rate,0.5,0.5 +module_2[arm64-v8a],pass,1,2 +module_2[arm64-v8a],IGNORED,1,1 +module_2[arm64-v8a],ASSUMPTION_FAILURE,1,0 +module_2[arm64-v8a],fail,2,2 +module_2[arm64-v8a],TEST_ERROR,0,0 +module_2[arm64-v8a],TEST_STATUS_UNSPECIFIED,1,1 +module_2[arm64-v8a],tested_items,6,6 +module_2[arm64-v8a],pass_rate,0.5,0.5 +module_3[arm64-v8a],pass,1,1 +module_3[arm64-v8a],IGNORED,0,0 +module_3[arm64-v8a],ASSUMPTION_FAILURE,0,0 +module_3[arm64-v8a],fail,0,0 +module_3[arm64-v8a],TEST_ERROR,1,0 +module_3[arm64-v8a],TEST_STATUS_UNSPECIFIED,0,0 +module_3[arm64-v8a],tested_items,2,1 +module_3[arm64-v8a],pass_rate,0.5,1.0 diff --git a/tools/compare_cts_reports/testdata/compare/one_way_diff.csv b/tools/compare_cts_reports/testdata/compare/one_way_diff.csv new file mode 100644 index 000000000..9fc36fcf7 --- /dev/null +++ b/tools/compare_cts_reports/testdata/compare/one_way_diff.csv @@ -0,0 +1,6 @@ +module_name,abi,class_name,test_name,result in A,result in B +module_1,arm64-v8a,testcase_1,test_2,fail,fail +module_2,arm64-v8a,testcase_3,test_5,fail,TEST_STATUS_UNSPECIFIED +module_2,arm64-v8a,testcase_4,test_7,fail,pass +module_2,arm64-v8a,testcase_4,test_8,TEST_STATUS_UNSPECIFIED,fail +module_3,arm64-v8a,testcase_5,test_10,TEST_ERROR,null diff --git a/tools/compare_cts_reports/testdata/compare/two_way_diff.csv b/tools/compare_cts_reports/testdata/compare/two_way_diff.csv new file mode 100644 index 000000000..a3ac77487 --- /dev/null +++ b/tools/compare_cts_reports/testdata/compare/two_way_diff.csv @@ -0,0 +1,11 @@ +module_name,abi,class_name,test_name,result in A,result in B +module_2,arm64-v8a,testcase_2,test_3,pass,IGNORED +module_2,arm64-v8a,testcase_3,test_4,ASSUMPTION_FAILURE,fail +module_2,arm64-v8a,testcase_3,test_5,fail,TEST_STATUS_UNSPECIFIED +module_2,arm64-v8a,testcase_4,test_6,IGNORED,pass +module_2,arm64-v8a,testcase_4,test_7,fail,pass +module_2,arm64-v8a,testcase_4,test_8,TEST_STATUS_UNSPECIFIED,fail +module_3,arm64-v8a,testcase_5,test_9,pass,null +module_3,arm64-v8a,testcase_5,test_10,TEST_ERROR,null +module_3,arm64-v8a,testcase_5,test_11,null,pass +module_4,arm64-v8a,testcase_6,test_12,null,fail diff --git a/tools/compare_cts_reports/testdata/output/info_1.json b/tools/compare_cts_reports/testdata/output/info_1.json index b70901b02..851b78c2b 100644 --- a/tools/compare_cts_reports/testdata/output/info_1.json +++ b/tools/compare_cts_reports/testdata/output/info_1.json @@ -1,4 +1,5 @@ { + "tool_version": "1.0", "source_path": "testdata/test_result_1.xml", "build_model": "this_model", "build_id": "1412",