6686257924
New test tools are adding --no-hidden-api-checks when tests have hidden-api-checks=false in their AndroidTest.xml, however this is not supported by runtest. As only runtest is available in AOSP, this leaves no option to make tests of internal classes pass. Add an option to disable the check in runtest so we can still run tests in AOSP. Test: runtest frameworks-net still fails Test: runtest --no-hidden-api-checks frameworks-net now passes Change-Id: I9be9674d0bf2588ff509121273379f08d4cb3aea
363 lines
12 KiB
Python
363 lines
12 KiB
Python
#!/usr/bin/python2.4
|
|
#
|
|
#
|
|
# Copyright 2008, The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""TestSuite definition for Android instrumentation tests."""
|
|
|
|
import os
|
|
import re
|
|
|
|
# local imports
|
|
import android_manifest
|
|
from coverage import coverage
|
|
import errors
|
|
import logger
|
|
import test_suite
|
|
|
|
|
|
class InstrumentationTestSuite(test_suite.AbstractTestSuite):
|
|
"""Represents a java instrumentation test suite definition run on device."""
|
|
|
|
DEFAULT_RUNNER = "android.test.InstrumentationTestRunner"
|
|
|
|
def __init__(self):
|
|
test_suite.AbstractTestSuite.__init__(self)
|
|
self._package_name = None
|
|
self._runner_name = self.DEFAULT_RUNNER
|
|
self._class_name = None
|
|
self._target_name = None
|
|
self._java_package = None
|
|
|
|
def GetPackageName(self):
|
|
return self._package_name
|
|
|
|
def SetPackageName(self, package_name):
|
|
self._package_name = package_name
|
|
return self
|
|
|
|
def GetRunnerName(self):
|
|
return self._runner_name
|
|
|
|
def SetRunnerName(self, runner_name):
|
|
self._runner_name = runner_name
|
|
return self
|
|
|
|
def GetClassName(self):
|
|
return self._class_name
|
|
|
|
def SetClassName(self, class_name):
|
|
self._class_name = class_name
|
|
return self
|
|
|
|
def GetJavaPackageFilter(self):
|
|
return self._java_package
|
|
|
|
def SetJavaPackageFilter(self, java_package_name):
|
|
"""Configure the suite to only run tests in given java package."""
|
|
self._java_package = java_package_name
|
|
return self
|
|
|
|
def GetTargetName(self):
|
|
"""Retrieve module that this test is targeting.
|
|
|
|
Used for generating code coverage metrics.
|
|
Returns:
|
|
the module target name
|
|
"""
|
|
return self._target_name
|
|
|
|
def SetTargetName(self, target_name):
|
|
self._target_name = target_name
|
|
return self
|
|
|
|
def GetBuildDependencies(self, options):
|
|
if options.coverage_target_path:
|
|
return [options.coverage_target_path]
|
|
return []
|
|
|
|
def Run(self, options, adb):
|
|
"""Run the provided test suite.
|
|
|
|
Builds up an adb instrument command using provided input arguments.
|
|
|
|
Args:
|
|
options: command line options to provide to test run
|
|
adb: adb_interface to device under test
|
|
|
|
Raises:
|
|
errors.AbortError: if fatal error occurs
|
|
"""
|
|
|
|
test_class = self.GetClassName()
|
|
if options.test_class is not None:
|
|
test_class = options.test_class.lstrip()
|
|
if test_class.startswith("."):
|
|
test_class = self.GetPackageName() + test_class
|
|
if options.test_method is not None:
|
|
test_class = "%s#%s" % (test_class, options.test_method)
|
|
|
|
test_package = self.GetJavaPackageFilter()
|
|
if options.test_package:
|
|
test_package = options.test_package
|
|
|
|
if test_class and test_package:
|
|
logger.Log('Error: both class and java package options are specified')
|
|
|
|
instrumentation_args = {}
|
|
if test_class is not None:
|
|
instrumentation_args["class"] = test_class
|
|
if test_package:
|
|
instrumentation_args["package"] = test_package
|
|
if options.test_size:
|
|
instrumentation_args["size"] = options.test_size
|
|
if options.wait_for_debugger:
|
|
instrumentation_args["debug"] = "true"
|
|
if options.suite_assign_mode:
|
|
instrumentation_args["suiteAssignment"] = "true"
|
|
if options.coverage:
|
|
instrumentation_args["coverage"] = "true"
|
|
if options.test_annotation:
|
|
instrumentation_args["annotation"] = options.test_annotation
|
|
if options.test_not_annotation:
|
|
instrumentation_args["notAnnotation"] = options.test_not_annotation
|
|
if options.preview:
|
|
adb_cmd = adb.PreviewInstrumentationCommand(
|
|
package_name=self.GetPackageName(),
|
|
runner_name=self.GetRunnerName(),
|
|
raw_mode=options.raw_mode,
|
|
instrumentation_args=instrumentation_args)
|
|
logger.Log(adb_cmd)
|
|
elif options.coverage:
|
|
coverage_gen = coverage.CoverageGenerator(adb)
|
|
if options.coverage_target_path:
|
|
coverage_target = coverage_gen.GetCoverageTargetForPath(options.coverage_target_path)
|
|
elif self.GetTargetName():
|
|
coverage_target = coverage_gen.GetCoverageTarget(self.GetTargetName())
|
|
self._CheckInstrumentationInstalled(adb)
|
|
# need to parse test output to determine path to coverage file
|
|
logger.Log("Running in coverage mode, suppressing test output")
|
|
try:
|
|
(test_results, status_map) = adb.StartInstrumentationForPackage(
|
|
package_name=self.GetPackageName(),
|
|
runner_name=self.GetRunnerName(),
|
|
timeout_time=60*60,
|
|
instrumentation_args=instrumentation_args,
|
|
user=options.user,
|
|
no_hidden_api_checks=options.no_hidden_api_checks)
|
|
except errors.InstrumentationError, errors.DeviceUnresponsiveError:
|
|
return
|
|
self._PrintTestResults(test_results)
|
|
device_coverage_path = status_map.get("coverageFilePath", None)
|
|
if device_coverage_path is None:
|
|
logger.Log("Error: could not find coverage data on device")
|
|
return
|
|
|
|
coverage_file = coverage_gen.ExtractReport(
|
|
self.GetName(), coverage_target, device_coverage_path,
|
|
test_qualifier=options.test_size)
|
|
if coverage_file is not None:
|
|
logger.Log("Coverage report generated at %s" % coverage_file)
|
|
|
|
else:
|
|
self._CheckInstrumentationInstalled(adb)
|
|
adb.StartInstrumentationNoResults(
|
|
package_name=self.GetPackageName(),
|
|
runner_name=self.GetRunnerName(),
|
|
raw_mode=options.raw_mode,
|
|
instrumentation_args=instrumentation_args,
|
|
user=options.user,
|
|
no_hidden_api_checks=options.no_hidden_api_checks)
|
|
|
|
def _CheckInstrumentationInstalled(self, adb):
|
|
if not adb.IsInstrumentationInstalled(self.GetPackageName(),
|
|
self.GetRunnerName()):
|
|
msg=("Could not find instrumentation %s/%s on device. Try forcing a "
|
|
"rebuild by updating a source file, and re-executing runtest." %
|
|
(self.GetPackageName(), self.GetRunnerName()))
|
|
raise errors.AbortError(msg=msg)
|
|
|
|
def _PrintTestResults(self, test_results):
|
|
"""Prints a summary of test result data to stdout.
|
|
|
|
Args:
|
|
test_results: a list of am_instrument_parser.TestResult
|
|
"""
|
|
total_count = 0
|
|
error_count = 0
|
|
fail_count = 0
|
|
for test_result in test_results:
|
|
if test_result.GetStatusCode() == -1: # error
|
|
logger.Log("Error in %s: %s" % (test_result.GetTestName(),
|
|
test_result.GetFailureReason()))
|
|
error_count+=1
|
|
elif test_result.GetStatusCode() == -2: # failure
|
|
logger.Log("Failure in %s: %s" % (test_result.GetTestName(),
|
|
test_result.GetFailureReason()))
|
|
fail_count+=1
|
|
total_count+=1
|
|
logger.Log("Tests run: %d, Failures: %d, Errors: %d" %
|
|
(total_count, fail_count, error_count))
|
|
|
|
def HasInstrumentationTest(path):
|
|
"""Determine if given path defines an instrumentation test.
|
|
|
|
Args:
|
|
path: file system path to instrumentation test.
|
|
"""
|
|
manifest_parser = android_manifest.CreateAndroidManifest(path)
|
|
if manifest_parser:
|
|
return manifest_parser.GetInstrumentationNames()
|
|
return False
|
|
|
|
class InstrumentationTestFactory(test_suite.AbstractTestFactory):
|
|
"""A factory for creating InstrumentationTestSuites"""
|
|
|
|
def __init__(self, test_root_path, build_path):
|
|
test_suite.AbstractTestFactory.__init__(self, test_root_path,
|
|
build_path)
|
|
|
|
def CreateTests(self, sub_tests_path=None):
|
|
"""Create tests found in test_path.
|
|
|
|
Will create a single InstrumentationTestSuite based on info found in
|
|
AndroidManifest.xml found at build_path. Will set additional filters if
|
|
test_path refers to a java package or java class.
|
|
"""
|
|
tests = []
|
|
class_name_arg = None
|
|
java_package_name = None
|
|
if sub_tests_path:
|
|
# if path is java file, populate class name
|
|
if self._IsJavaFile(sub_tests_path):
|
|
class_name_arg = self._GetClassNameFromFile(sub_tests_path)
|
|
logger.SilentLog('Using java test class %s' % class_name_arg)
|
|
elif self._IsJavaPackage(sub_tests_path):
|
|
java_package_name = self._GetPackageNameFromDir(sub_tests_path)
|
|
logger.SilentLog('Using java package %s' % java_package_name)
|
|
try:
|
|
manifest_parser = android_manifest.AndroidManifest(app_path=
|
|
self.GetTestsRootPath())
|
|
instrs = manifest_parser.GetInstrumentationNames()
|
|
if not instrs:
|
|
logger.Log('Could not find instrumentation declarations in %s at %s' %
|
|
(android_manifest.AndroidManifest.FILENAME,
|
|
self.GetBuildPath()))
|
|
return tests
|
|
elif len(instrs) > 1:
|
|
logger.Log("Found multiple instrumentation declarations in %s/%s. "
|
|
"Only using first declared." %
|
|
(self.GetBuildPath(),
|
|
android_manifest.AndroidManifest.FILENAME))
|
|
instr_name = manifest_parser.GetInstrumentationNames()[0]
|
|
# escape inner class names
|
|
instr_name = instr_name.replace('$', '\$')
|
|
pkg_name = manifest_parser.GetPackageName()
|
|
if instr_name.find(".") < 0:
|
|
instr_name = "." + instr_name
|
|
logger.SilentLog('Found instrumentation %s/%s' % (pkg_name, instr_name))
|
|
suite = InstrumentationTestSuite()
|
|
suite.SetPackageName(pkg_name)
|
|
suite.SetBuildPath(self.GetBuildPath())
|
|
suite.SetRunnerName(instr_name)
|
|
suite.SetName(pkg_name)
|
|
suite.SetClassName(class_name_arg)
|
|
suite.SetJavaPackageFilter(java_package_name)
|
|
tests.append(suite)
|
|
return tests
|
|
|
|
except:
|
|
logger.Log('Could not find or parse %s at %s' %
|
|
(android_manifest.AndroidManifest.FILENAME,
|
|
self.GetBuildPath()))
|
|
return tests
|
|
|
|
def _IsJavaFile(self, path):
|
|
"""Returns true if given file system path is a java file."""
|
|
return os.path.isfile(path) and self._IsJavaFileName(path)
|
|
|
|
def _IsJavaFileName(self, filename):
|
|
"""Returns true if given file name is a java file name."""
|
|
return os.path.splitext(filename)[1] == '.java'
|
|
|
|
def _IsJavaPackage(self, path):
|
|
"""Returns true if given file path is a java package.
|
|
|
|
Currently assumes if any java file exists in this directory, than it
|
|
represents a java package.
|
|
|
|
Args:
|
|
path: file system path of directory to check
|
|
|
|
Returns:
|
|
True if path is a java package
|
|
"""
|
|
if not os.path.isdir(path):
|
|
return False
|
|
for file_name in os.listdir(path):
|
|
if self._IsJavaFileName(file_name):
|
|
return True
|
|
return False
|
|
|
|
def _GetClassNameFromFile(self, java_file_path):
|
|
"""Gets the fully qualified java class name from path.
|
|
|
|
Args:
|
|
java_file_path: file system path of java file
|
|
|
|
Returns:
|
|
fully qualified java class name or None.
|
|
"""
|
|
package_name = self._GetPackageNameFromFile(java_file_path)
|
|
if package_name:
|
|
filename = os.path.basename(java_file_path)
|
|
class_name = os.path.splitext(filename)[0]
|
|
return '%s.%s' % (package_name, class_name)
|
|
return None
|
|
|
|
def _GetPackageNameFromDir(self, path):
|
|
"""Gets the java package name associated with given directory path.
|
|
|
|
Caveat: currently just parses defined java package name from first java
|
|
file found in directory.
|
|
|
|
Args:
|
|
path: file system path of directory
|
|
|
|
Returns:
|
|
the java package name or None
|
|
"""
|
|
for filename in os.listdir(path):
|
|
if self._IsJavaFileName(filename):
|
|
return self._GetPackageNameFromFile(os.path.join(path, filename))
|
|
|
|
def _GetPackageNameFromFile(self, java_file_path):
|
|
"""Gets the java package name associated with given java file path.
|
|
|
|
Args:
|
|
java_file_path: file system path of java file
|
|
|
|
Returns:
|
|
the java package name or None
|
|
"""
|
|
logger.SilentLog('Looking for java package name in %s' % java_file_path)
|
|
re_package = re.compile(r'package\s+(.*);')
|
|
file_handle = open(java_file_path, 'r')
|
|
for line in file_handle:
|
|
match = re_package.match(line)
|
|
if match:
|
|
return match.group(1)
|
|
return None
|