Update Google Benchmark library
git-svn-id: https://llvm.org/svn/llvm-project/libcxx/trunk@322812 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Eric Fiselier
312
utils/google-benchmark/tools/compare.py
Executable file
312
utils/google-benchmark/tools/compare.py
Executable file
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
compare.py - versatile benchmark output compare tool
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from argparse import ArgumentParser
|
||||
import sys
|
||||
import gbench
|
||||
from gbench import util, report
|
||||
from gbench.util import *
|
||||
|
||||
|
||||
def check_inputs(in1, in2, flags):
|
||||
"""
|
||||
Perform checking on the user provided inputs and diagnose any abnormalities
|
||||
"""
|
||||
in1_kind, in1_err = classify_input_file(in1)
|
||||
in2_kind, in2_err = classify_input_file(in2)
|
||||
output_file = find_benchmark_flag('--benchmark_out=', flags)
|
||||
output_type = find_benchmark_flag('--benchmark_out_format=', flags)
|
||||
if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file:
|
||||
print(("WARNING: '--benchmark_out=%s' will be passed to both "
|
||||
"benchmarks causing it to be overwritten") % output_file)
|
||||
if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0:
|
||||
print("WARNING: passing optional flags has no effect since both "
|
||||
"inputs are JSON")
|
||||
if output_type is not None and output_type != 'json':
|
||||
print(("ERROR: passing '--benchmark_out_format=%s' to 'compare.py`"
|
||||
" is not supported.") % output_type)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def create_parser():
|
||||
parser = ArgumentParser(
|
||||
description='versatile benchmark output compare tool')
|
||||
subparsers = parser.add_subparsers(
|
||||
help='This tool has multiple modes of operation:',
|
||||
dest='mode')
|
||||
|
||||
parser_a = subparsers.add_parser(
|
||||
'benchmarks',
|
||||
help='The most simple use-case, compare all the output of these two benchmarks')
|
||||
baseline = parser_a.add_argument_group(
|
||||
'baseline', 'The benchmark baseline')
|
||||
baseline.add_argument(
|
||||
'test_baseline',
|
||||
metavar='test_baseline',
|
||||
type=argparse.FileType('r'),
|
||||
nargs=1,
|
||||
help='A benchmark executable or JSON output file')
|
||||
contender = parser_a.add_argument_group(
|
||||
'contender', 'The benchmark that will be compared against the baseline')
|
||||
contender.add_argument(
|
||||
'test_contender',
|
||||
metavar='test_contender',
|
||||
type=argparse.FileType('r'),
|
||||
nargs=1,
|
||||
help='A benchmark executable or JSON output file')
|
||||
parser_a.add_argument(
|
||||
'benchmark_options',
|
||||
metavar='benchmark_options',
|
||||
nargs=argparse.REMAINDER,
|
||||
help='Arguments to pass when running benchmark executables')
|
||||
|
||||
parser_b = subparsers.add_parser(
|
||||
'filters', help='Compare filter one with the filter two of benchmark')
|
||||
baseline = parser_b.add_argument_group(
|
||||
'baseline', 'The benchmark baseline')
|
||||
baseline.add_argument(
|
||||
'test',
|
||||
metavar='test',
|
||||
type=argparse.FileType('r'),
|
||||
nargs=1,
|
||||
help='A benchmark executable or JSON output file')
|
||||
baseline.add_argument(
|
||||
'filter_baseline',
|
||||
metavar='filter_baseline',
|
||||
type=str,
|
||||
nargs=1,
|
||||
help='The first filter, that will be used as baseline')
|
||||
contender = parser_b.add_argument_group(
|
||||
'contender', 'The benchmark that will be compared against the baseline')
|
||||
contender.add_argument(
|
||||
'filter_contender',
|
||||
metavar='filter_contender',
|
||||
type=str,
|
||||
nargs=1,
|
||||
help='The second filter, that will be compared against the baseline')
|
||||
parser_b.add_argument(
|
||||
'benchmark_options',
|
||||
metavar='benchmark_options',
|
||||
nargs=argparse.REMAINDER,
|
||||
help='Arguments to pass when running benchmark executables')
|
||||
|
||||
parser_c = subparsers.add_parser(
|
||||
'benchmarksfiltered',
|
||||
help='Compare filter one of first benchmark with filter two of the second benchmark')
|
||||
baseline = parser_c.add_argument_group(
|
||||
'baseline', 'The benchmark baseline')
|
||||
baseline.add_argument(
|
||||
'test_baseline',
|
||||
metavar='test_baseline',
|
||||
type=argparse.FileType('r'),
|
||||
nargs=1,
|
||||
help='A benchmark executable or JSON output file')
|
||||
baseline.add_argument(
|
||||
'filter_baseline',
|
||||
metavar='filter_baseline',
|
||||
type=str,
|
||||
nargs=1,
|
||||
help='The first filter, that will be used as baseline')
|
||||
contender = parser_c.add_argument_group(
|
||||
'contender', 'The benchmark that will be compared against the baseline')
|
||||
contender.add_argument(
|
||||
'test_contender',
|
||||
metavar='test_contender',
|
||||
type=argparse.FileType('r'),
|
||||
nargs=1,
|
||||
help='The second benchmark executable or JSON output file, that will be compared against the baseline')
|
||||
contender.add_argument(
|
||||
'filter_contender',
|
||||
metavar='filter_contender',
|
||||
type=str,
|
||||
nargs=1,
|
||||
help='The second filter, that will be compared against the baseline')
|
||||
parser_c.add_argument(
|
||||
'benchmark_options',
|
||||
metavar='benchmark_options',
|
||||
nargs=argparse.REMAINDER,
|
||||
help='Arguments to pass when running benchmark executables')
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
# Parse the command line flags
|
||||
parser = create_parser()
|
||||
args, unknown_args = parser.parse_known_args()
|
||||
assert not unknown_args
|
||||
benchmark_options = args.benchmark_options
|
||||
|
||||
if args.mode == 'benchmarks':
|
||||
test_baseline = args.test_baseline[0].name
|
||||
test_contender = args.test_contender[0].name
|
||||
filter_baseline = ''
|
||||
filter_contender = ''
|
||||
|
||||
# NOTE: if test_baseline == test_contender, you are analyzing the stdev
|
||||
|
||||
description = 'Comparing %s to %s' % (test_baseline, test_contender)
|
||||
elif args.mode == 'filters':
|
||||
test_baseline = args.test[0].name
|
||||
test_contender = args.test[0].name
|
||||
filter_baseline = args.filter_baseline[0]
|
||||
filter_contender = args.filter_contender[0]
|
||||
|
||||
# NOTE: if filter_baseline == filter_contender, you are analyzing the
|
||||
# stdev
|
||||
|
||||
description = 'Comparing %s to %s (from %s)' % (
|
||||
filter_baseline, filter_contender, args.test[0].name)
|
||||
elif args.mode == 'benchmarksfiltered':
|
||||
test_baseline = args.test_baseline[0].name
|
||||
test_contender = args.test_contender[0].name
|
||||
filter_baseline = args.filter_baseline[0]
|
||||
filter_contender = args.filter_contender[0]
|
||||
|
||||
# NOTE: if test_baseline == test_contender and
|
||||
# filter_baseline == filter_contender, you are analyzing the stdev
|
||||
|
||||
description = 'Comparing %s (from %s) to %s (from %s)' % (
|
||||
filter_baseline, test_baseline, filter_contender, test_contender)
|
||||
else:
|
||||
# should never happen
|
||||
print("Unrecognized mode of operation: '%s'" % args.mode)
|
||||
exit(1)
|
||||
|
||||
check_inputs(test_baseline, test_contender, benchmark_options)
|
||||
|
||||
options_baseline = []
|
||||
options_contender = []
|
||||
|
||||
if filter_baseline and filter_contender:
|
||||
options_baseline = ['--benchmark_filter=%s' % filter_baseline]
|
||||
options_contender = ['--benchmark_filter=%s' % filter_contender]
|
||||
|
||||
# Run the benchmarks and report the results
|
||||
json1 = json1_orig = gbench.util.run_or_load_benchmark(
|
||||
test_baseline, benchmark_options + options_baseline)
|
||||
json2 = json2_orig = gbench.util.run_or_load_benchmark(
|
||||
test_contender, benchmark_options + options_contender)
|
||||
|
||||
# Now, filter the benchmarks so that the difference report can work
|
||||
if filter_baseline and filter_contender:
|
||||
replacement = '[%s vs. %s]' % (filter_baseline, filter_contender)
|
||||
json1 = gbench.report.filter_benchmark(
|
||||
json1_orig, filter_baseline, replacement)
|
||||
json2 = gbench.report.filter_benchmark(
|
||||
json2_orig, filter_contender, replacement)
|
||||
|
||||
# Diff and output
|
||||
output_lines = gbench.report.generate_difference_report(json1, json2)
|
||||
print(description)
|
||||
for ln in output_lines:
|
||||
print(ln)
|
||||
|
||||
|
||||
import unittest
|
||||
|
||||
|
||||
class TestParser(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.parser = create_parser()
|
||||
testInputs = os.path.join(
|
||||
os.path.dirname(
|
||||
os.path.realpath(__file__)),
|
||||
'gbench',
|
||||
'Inputs')
|
||||
self.testInput0 = os.path.join(testInputs, 'test_baseline_run1.json')
|
||||
self.testInput1 = os.path.join(testInputs, 'test_baseline_run2.json')
|
||||
|
||||
def test_benchmarks_basic(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarks', self.testInput0, self.testInput1])
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
||||
self.assertFalse(parsed.benchmark_options)
|
||||
|
||||
def test_benchmarks_with_remainder(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarks', self.testInput0, self.testInput1, 'd'])
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
||||
self.assertEqual(parsed.benchmark_options, ['d'])
|
||||
|
||||
def test_benchmarks_with_remainder_after_doubleminus(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarks', self.testInput0, self.testInput1, '--', 'e'])
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
||||
self.assertEqual(parsed.benchmark_options, ['e'])
|
||||
|
||||
def test_filters_basic(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['filters', self.testInput0, 'c', 'd'])
|
||||
self.assertEqual(parsed.mode, 'filters')
|
||||
self.assertEqual(parsed.test[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.filter_baseline[0], 'c')
|
||||
self.assertEqual(parsed.filter_contender[0], 'd')
|
||||
self.assertFalse(parsed.benchmark_options)
|
||||
|
||||
def test_filters_with_remainder(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['filters', self.testInput0, 'c', 'd', 'e'])
|
||||
self.assertEqual(parsed.mode, 'filters')
|
||||
self.assertEqual(parsed.test[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.filter_baseline[0], 'c')
|
||||
self.assertEqual(parsed.filter_contender[0], 'd')
|
||||
self.assertEqual(parsed.benchmark_options, ['e'])
|
||||
|
||||
def test_filters_with_remainder_after_doubleminus(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['filters', self.testInput0, 'c', 'd', '--', 'f'])
|
||||
self.assertEqual(parsed.mode, 'filters')
|
||||
self.assertEqual(parsed.test[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.filter_baseline[0], 'c')
|
||||
self.assertEqual(parsed.filter_contender[0], 'd')
|
||||
self.assertEqual(parsed.benchmark_options, ['f'])
|
||||
|
||||
def test_benchmarksfiltered_basic(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e'])
|
||||
self.assertEqual(parsed.mode, 'benchmarksfiltered')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.filter_baseline[0], 'c')
|
||||
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
||||
self.assertEqual(parsed.filter_contender[0], 'e')
|
||||
self.assertFalse(parsed.benchmark_options)
|
||||
|
||||
def test_benchmarksfiltered_with_remainder(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f'])
|
||||
self.assertEqual(parsed.mode, 'benchmarksfiltered')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.filter_baseline[0], 'c')
|
||||
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
||||
self.assertEqual(parsed.filter_contender[0], 'e')
|
||||
self.assertEqual(parsed.benchmark_options[0], 'f')
|
||||
|
||||
def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g'])
|
||||
self.assertEqual(parsed.mode, 'benchmarksfiltered')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.filter_baseline[0], 'c')
|
||||
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
||||
self.assertEqual(parsed.filter_contender[0], 'e')
|
||||
self.assertEqual(parsed.benchmark_options[0], 'g')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# unittest.main()
|
||||
main()
|
||||
|
||||
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
|
||||
# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
|
||||
# kate: indent-mode python; remove-trailing-spaces modified;
|
||||
Reference in New Issue
Block a user