Merge "Add unit tests for tools/compare_cts_reports." into main
This commit is contained in:
Zoe Tsou
121
tools/compare_cts_reports/test_aggregate_cts_reports.py
Executable file
121
tools/compare_cts_reports/test_aggregate_cts_reports.py
Executable file
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# Copyright (C) 2023 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy of
|
||||
# the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations under
|
||||
# the License.
|
||||
#
|
||||
import unittest
|
||||
import aggregate_cts_reports
|
||||
|
||||
|
||||
class TestParse(unittest.TestCase):
|
||||
|
||||
def test_aggregate(self):
|
||||
report_files = ['testdata/test_result_1.xml', 'testdata/test_result_2.xml']
|
||||
report = aggregate_cts_reports.aggregate_cts_reports(report_files)
|
||||
|
||||
self.check_ctsreport(report)
|
||||
|
||||
def check_ctsreport(self, report):
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_1'),
|
||||
'pass',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_2'),
|
||||
'fail',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_2', 'test_3'),
|
||||
'pass',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_4'),
|
||||
'ASSUMPTION_FAILURE',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_5'),
|
||||
'fail',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_6'),
|
||||
'pass',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_7'),
|
||||
'pass',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_8'),
|
||||
'fail',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_3', 'arm64-v8a', 'testcase_5', 'test_9'),
|
||||
'pass',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status(
|
||||
'module_3', 'arm64-v8a', 'testcase_5', 'test_10'
|
||||
),
|
||||
'TEST_ERROR',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status(
|
||||
'module_3', 'arm64-v8a', 'testcase_5', 'test_11'
|
||||
),
|
||||
'pass',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status(
|
||||
'module_4', 'arm64-v8a', 'testcase_6', 'test_12'
|
||||
),
|
||||
'fail',
|
||||
)
|
||||
|
||||
self.assertEqual(report.info['build_model'], 'this_model')
|
||||
self.assertEqual(report.info['build_id'], '1412')
|
||||
self.assertEqual(report.info['build_fingerprint'], 'this_build_fingerprint')
|
||||
self.assertEqual(report.info['build_device'], 'this_device')
|
||||
self.assertEqual(report.info['build_version_sdk'], '34')
|
||||
self.assertEqual(report.info['build_version_security_patch'], '2023-06-05')
|
||||
self.assertEqual(report.info['build_board'], 'this_board')
|
||||
self.assertEqual(report.info['build_type'], 'userdebug')
|
||||
self.assertEqual(report.info['build_version_release'], '14')
|
||||
self.assertEqual(report.info['suite_name'], 'CTS')
|
||||
self.assertEqual(report.info['suite_version'], '14_r1')
|
||||
self.assertEqual(report.info['suite_plan'], 'cts')
|
||||
self.assertEqual(report.info['suite_build_number'], '1234567')
|
||||
|
||||
module_summaries = report.module_summaries
|
||||
summary = module_summaries['module_1']['arm64-v8a']
|
||||
self.assertEqual(summary.counter['pass'], 1)
|
||||
self.assertEqual(summary.counter['fail'], 1)
|
||||
|
||||
summary = module_summaries['module_2']['arm64-v8a']
|
||||
self.assertEqual(summary.counter['pass'], 3)
|
||||
self.assertEqual(
|
||||
summary.counter['ASSUMPTION_FAILURE'],
|
||||
1,
|
||||
)
|
||||
self.assertEqual(summary.counter['fail'], 2)
|
||||
|
||||
summary = module_summaries['module_3']['arm64-v8a']
|
||||
self.assertEqual(summary.counter['pass'], 2)
|
||||
self.assertEqual(summary.counter['TEST_ERROR'], 1)
|
||||
|
||||
summary = module_summaries['module_4']['arm64-v8a']
|
||||
self.assertEqual(summary.counter['fail'], 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
155
tools/compare_cts_reports/test_parse_cts_report.py
Executable file
155
tools/compare_cts_reports/test_parse_cts_report.py
Executable file
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# Copyright (C) 2023 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy of
|
||||
# the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations under
|
||||
# the License.
|
||||
#
|
||||
import filecmp
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import parse_cts_report
|
||||
|
||||
|
||||
class TestParse(unittest.TestCase):
|
||||
|
||||
def test_set(self):
|
||||
info = {}
|
||||
report = parse_cts_report.CtsReport(info)
|
||||
test_item = ('module', 'abi', 'class', 'test')
|
||||
|
||||
report.set_test_status(*test_item, 'IGNORED')
|
||||
report.set_test_status(*test_item, 'fail')
|
||||
|
||||
self.assertEqual(report.get_test_status(*test_item), 'IGNORED')
|
||||
|
||||
report.set_test_status(*test_item, 'pass')
|
||||
|
||||
self.assertEqual(report.get_test_status(*test_item), 'pass')
|
||||
|
||||
def test_parse_xml(self):
|
||||
report_file = 'testdata/test_result_1.xml'
|
||||
report = parse_cts_report.parse_report_file(report_file)
|
||||
|
||||
self.check_ctsreport(report)
|
||||
|
||||
def test_parse_zip(self):
|
||||
report_file = 'testdata/report.zip'
|
||||
report = parse_cts_report.parse_report_file(report_file)
|
||||
|
||||
self.check_ctsreport(report)
|
||||
|
||||
def check_ctsreport(self, report):
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_1'),
|
||||
'pass',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_1', 'arm64-v8a', 'testcase_1', 'test_2'),
|
||||
'fail',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_2', 'test_3'),
|
||||
'pass',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_4'),
|
||||
'ASSUMPTION_FAILURE',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_3', 'test_5'),
|
||||
'fail',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_6'),
|
||||
'IGNORED',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_7'),
|
||||
'fail',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_2', 'arm64-v8a', 'testcase_4', 'test_8'),
|
||||
'TEST_STATUS_UNSPECIFIED',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status('module_3', 'arm64-v8a', 'testcase_5', 'test_9'),
|
||||
'pass',
|
||||
)
|
||||
self.assertEqual(
|
||||
report.get_test_status(
|
||||
'module_3', 'arm64-v8a', 'testcase_5', 'test_10'
|
||||
),
|
||||
'TEST_ERROR',
|
||||
)
|
||||
|
||||
self.assertEqual(report.info['build_model'], 'this_model')
|
||||
self.assertEqual(report.info['build_id'], '1412')
|
||||
self.assertEqual(report.info['build_fingerprint'], 'this_build_fingerprint')
|
||||
self.assertEqual(report.info['build_device'], 'this_device')
|
||||
self.assertEqual(report.info['build_version_sdk'], '34')
|
||||
self.assertEqual(report.info['build_version_security_patch'], '2023-06-05')
|
||||
self.assertEqual(report.info['build_board'], 'this_board')
|
||||
self.assertEqual(report.info['build_type'], 'userdebug')
|
||||
self.assertEqual(report.info['build_version_release'], '14')
|
||||
self.assertEqual(report.info['suite_name'], 'CTS')
|
||||
self.assertEqual(report.info['suite_version'], '14_r1')
|
||||
self.assertEqual(report.info['suite_plan'], 'cts')
|
||||
self.assertEqual(report.info['suite_build_number'], '1234567')
|
||||
|
||||
module_summaries = report.module_summaries
|
||||
summary = module_summaries['module_1']['arm64-v8a']
|
||||
self.assertEqual(summary.counter['pass'], 1)
|
||||
self.assertEqual(summary.counter['fail'], 1)
|
||||
|
||||
summary = module_summaries['module_2']['arm64-v8a']
|
||||
self.assertEqual(summary.counter['pass'], 1)
|
||||
self.assertEqual(summary.counter['IGNORED'], 1)
|
||||
self.assertEqual(
|
||||
summary.counter['ASSUMPTION_FAILURE'],
|
||||
1,
|
||||
)
|
||||
self.assertEqual(summary.counter['fail'], 2)
|
||||
self.assertEqual(
|
||||
summary.counter['TEST_STATUS_UNSPECIFIED'],
|
||||
1,
|
||||
)
|
||||
|
||||
summary = module_summaries['module_3']['arm64-v8a']
|
||||
self.assertEqual(summary.counter['pass'], 1)
|
||||
self.assertEqual(summary.counter['TEST_ERROR'], 1)
|
||||
|
||||
def test_output(self):
|
||||
report_file = 'testdata/test_result_1.xml'
|
||||
report = parse_cts_report.parse_report_file(report_file)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
report.output_files(temp_dir)
|
||||
|
||||
parsed_info_path = os.path.join(temp_dir, 'info.json')
|
||||
parsed_result_path = os.path.join(temp_dir, 'result.csv')
|
||||
parsed_summary_path = os.path.join(temp_dir, 'summary.csv')
|
||||
|
||||
files = [parsed_info_path, parsed_result_path, parsed_summary_path]
|
||||
|
||||
match, mismatch, errors = filecmp.cmpfiles(
|
||||
'testdata/output', temp_dir, files
|
||||
)
|
||||
|
||||
self.assertEqual(match, files)
|
||||
self.assertEqual(mismatch, [])
|
||||
self.assertEqual(errors, [])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
16
tools/compare_cts_reports/testdata/output/info_1.json
vendored
Normal file
16
tools/compare_cts_reports/testdata/output/info_1.json
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"source_path": "testdata/test_result_1.xml",
|
||||
"build_model": "this_model",
|
||||
"build_id": "1412",
|
||||
"build_fingerprint": "this_build_fingerprint",
|
||||
"build_device": "this_device",
|
||||
"build_version_sdk": "34",
|
||||
"build_version_security_patch": "2023-06-05",
|
||||
"build_board": "this_board",
|
||||
"build_type": "userdebug",
|
||||
"build_version_release": "14",
|
||||
"suite_name": "CTS",
|
||||
"suite_version": "14_r1",
|
||||
"suite_plan": "cts",
|
||||
"suite_build_number": "1234567"
|
||||
}
|
||||
11
tools/compare_cts_reports/testdata/output/result_1.csv
vendored
Normal file
11
tools/compare_cts_reports/testdata/output/result_1.csv
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
module_name,abi,class_name,test_name,result
|
||||
module_1,arm64-v8a,testcase_1,test_1,pass
|
||||
module_1,arm64-v8a,testcase_1,test_2,fail
|
||||
module_2,arm64-v8a,testcase_2,test_3,pass
|
||||
module_2,arm64-v8a,testcase_3,test_4,ASSUMPTION_FAILURE
|
||||
module_2,arm64-v8a,testcase_3,test_5,fail
|
||||
module_2,arm64-v8a,testcase_4,test_6,IGNORED
|
||||
module_2,arm64-v8a,testcase_4,test_7,fail
|
||||
module_2,arm64-v8a,testcase_4,test_8,TEST_STATUS_UNSPECIFIED
|
||||
module_3,arm64-v8a,testcase_5,test_9,pass
|
||||
module_3,arm64-v8a,testcase_5,test_10,TEST_ERROR
|
||||
|
4
tools/compare_cts_reports/testdata/output/summary_1.csv
vendored
Normal file
4
tools/compare_cts_reports/testdata/output/summary_1.csv
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
module_name,abi,pass,IGNORED,ASSUMPTION_FAILURE,fail,TEST_ERROR,TEST_STATUS_UNSPECIFIED
|
||||
module_1,arm64-v8a,1,0,0,1,0,0
|
||||
module_2,arm64-v8a,1,1,1,2,0,1
|
||||
module_3,arm64-v8a,1,0,0,0,1,0
|
||||
|
BIN
tools/compare_cts_reports/testdata/report.zip
vendored
Normal file
BIN
tools/compare_cts_reports/testdata/report.zip
vendored
Normal file
Binary file not shown.
41
tools/compare_cts_reports/testdata/test_result_1.xml
vendored
Normal file
41
tools/compare_cts_reports/testdata/test_result_1.xml
vendored
Normal file
@@ -0,0 +1,41 @@
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<Result suite_name="CTS" suite_plan="cts" suite_build_number="1234567" start_display="Tue Aug 01 08:34:01 CST 2023" suite_version="14_r1">
|
||||
<Build build_model="this_model" build_device="this_device" build_id="1412" build_version_sdk="34" build_version_security_patch="2023-06-05" build_board="this_board" build_type="userdebug" build_version_release="14" build_fingerprint="this_build_fingerprint" />
|
||||
<Summary pass="3" failed="3" modules_done="3" modules_total="4" />
|
||||
<Module name="module_1" abi="arm64-v8a" done="true" pass="1" total_tests="2">
|
||||
<TestCase name="testcase_1">
|
||||
<Test result="pass" name="test_1">
|
||||
</Test>
|
||||
<Test result="fail" name="test_2">
|
||||
</Test>
|
||||
</TestCase>
|
||||
</Module>
|
||||
<Module name="module_2" abi="arm64-v8a" done="true" pass="1" total_tests="6">
|
||||
<TestCase name="testcase_2">
|
||||
<Test result="pass" name="test_3">
|
||||
</Test>
|
||||
</TestCase>
|
||||
<TestCase name="testcase_3">
|
||||
<Test result="ASSUMPTION_FAILURE" name="test_4">
|
||||
</Test>
|
||||
<Test result="fail" name="test_5">
|
||||
</Test>
|
||||
</TestCase>
|
||||
<TestCase name="testcase_4">
|
||||
<Test result="IGNORED" name="test_6">
|
||||
</Test>
|
||||
<Test result="fail" name="test_7">
|
||||
</Test>
|
||||
<Test result="TEST_STATUS_UNSPECIFIED" name="test_8">
|
||||
</Test>
|
||||
</TestCase>
|
||||
</Module>
|
||||
<Module name="module_3" abi="arm64-v8a" done="true" pass="1" total_tests="2">
|
||||
<TestCase name="testcase_5">
|
||||
<Test result="pass" name="test_9">
|
||||
</Test>
|
||||
<Test result="TEST_ERROR" name="test_10">
|
||||
</Test>
|
||||
</TestCase>
|
||||
</Module>
|
||||
</Result>
|
||||
45
tools/compare_cts_reports/testdata/test_result_2.xml
vendored
Normal file
45
tools/compare_cts_reports/testdata/test_result_2.xml
vendored
Normal file
@@ -0,0 +1,45 @@
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<Result suite_name="CTS" suite_plan="cts" suite_build_number="7654321" start_display="Tue Aug 01 08:34:01 CST 2023" suite_version="14_r1">
|
||||
<Build build_model="that_model" build_device="that_device" build_id="0621" build_version_sdk="34" build_version_security_patch="2023-06-05" build_board="that_board" build_type="userdebug" build_version_release="14" build_fingerprint="this_build_fingerprint" />
|
||||
<Summary pass="4" failed="4" modules_done="4" modules_total="4" />
|
||||
<Module name="module_1" abi="arm64-v8a" done="true" pass="1" total_tests="2">
|
||||
<TestCase name="testcase_1">
|
||||
<Test result="pass" name="test_1">
|
||||
</Test>
|
||||
<Test result="fail" name="test_2">
|
||||
</Test>
|
||||
</TestCase>
|
||||
</Module>
|
||||
<Module name="module_2" abi="arm64-v8a" done="true" pass="2" total_tests="6">
|
||||
<TestCase name="testcase_2">
|
||||
<Test result="IGNORED" name="test_3">
|
||||
</Test>
|
||||
</TestCase>
|
||||
<TestCase name="testcase_3">
|
||||
<Test result="fail" name="test_4">
|
||||
</Test>
|
||||
<Test result="TEST_STATUS_UNSPECIFIED" name="test_5">
|
||||
</Test>
|
||||
</TestCase>
|
||||
<TestCase name="testcase_4">
|
||||
<Test result="pass" name="test_6">
|
||||
</Test>
|
||||
<Test result="pass" name="test_7">
|
||||
</Test>
|
||||
<Test result="fail" name="test_8">
|
||||
</Test>
|
||||
</TestCase>
|
||||
</Module>
|
||||
<Module name="module_3" abi="arm64-v8a" done="true" pass="1" total_tests="1">
|
||||
<TestCase name="testcase_5">
|
||||
<Test result="pass" name="test_11">
|
||||
</Test>
|
||||
</TestCase>
|
||||
</Module>
|
||||
<Module name="module_4" abi="arm64-v8a" done="true" pass="0" total_tests="1">
|
||||
<TestCase name="testcase_6">
|
||||
<Test result="fail" name="test_12">
|
||||
</Test>
|
||||
</TestCase>
|
||||
</Module>
|
||||
</Result>
|
||||
Reference in New Issue
Block a user