287 lines
10 KiB
Python
Executable File
287 lines
10 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
"""Run layout tests using Android emulator and instrumentation.
|
|
|
|
First, you need to get an SD card or sdcard image that has layout tests on it.
|
|
Layout tests are in following directory:
|
|
/sdcard/android/layout_tests
|
|
For example, /sdcard/android/layout_tests/fast
|
|
|
|
Usage:
|
|
Run all tests under fast/ directory:
|
|
run_layout_tests.py, or
|
|
run_layout_tests.py fast
|
|
|
|
Run all tests under a sub directory:
|
|
run_layout_tests.py fast/dom
|
|
|
|
Run a single test:
|
|
run_layout_tests.py fast/dom/
|
|
|
|
After a merge, if there are changes of layout tests in SD card, you need to
|
|
use --refresh-test-list option *once* to re-generate test list on the card.
|
|
|
|
Some other options are:
|
|
--rebaseline generates expected layout tests results under /sdcard/android/expected_result/
|
|
--time-out-ms (default is 8000 millis) for each test
|
|
--adb-options="-e" passes option string to adb
|
|
--results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
|
|
"""
|
|
|
|
import logging
|
|
import optparse
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
def CountLineNumber(filename):
|
|
"""Compute the number of lines in a given file.
|
|
|
|
Args:
|
|
filename: a file name related to the current directory.
|
|
"""
|
|
|
|
fp = open(os.path.abspath(filename), "r");
|
|
lines = 0
|
|
for line in fp.readlines():
|
|
lines = lines + 1
|
|
fp.close()
|
|
return lines
|
|
|
|
def DumpRenderTreeFinished(adb_cmd):
|
|
""" Check if DumpRenderTree finished running tests
|
|
|
|
Args:
|
|
output: adb_cmd string
|
|
"""
|
|
|
|
# pull /sdcard/android/running_test.txt, if the content is "#DONE", it's done
|
|
shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
|
|
adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
|
|
return adb_output.strip() == "#DONE"
|
|
|
|
def DiffResults(marker, new_results, old_results, diff_results, strip_reason):
|
|
""" Given two result files, generate diff and
|
|
write to diff_results file. All arguments are absolute paths
|
|
to files.
|
|
"""
|
|
old_file = open(old_results, "r")
|
|
new_file = open(new_results, "r")
|
|
diff_file = open(diff_results, "a")
|
|
|
|
# Read lines from each file
|
|
ndict = new_file.readlines()
|
|
cdict = old_file.readlines()
|
|
|
|
# Write marker to diff file
|
|
diff_file.writelines(marker + "\n")
|
|
diff_file.writelines("###############\n")
|
|
|
|
# Strip reason from result lines
|
|
if strip_reason is True:
|
|
for i in range(0, len(ndict)):
|
|
ndict[i] = ndict[i].split(' ')[0] + "\n"
|
|
for i in range(0, len(cdict)):
|
|
cdict[i] = cdict[i].split(' ')[0] + "\n"
|
|
|
|
# Find results in new_results missing in old_results
|
|
new_count=0
|
|
for line in ndict:
|
|
if line not in cdict:
|
|
diff_file.writelines("+ " + line)
|
|
new_count += 1
|
|
|
|
# Find results in old_results missing in new_results
|
|
missing_count=0
|
|
for line in cdict:
|
|
if line not in ndict:
|
|
diff_file.writelines("- " + line)
|
|
missing_count += 1
|
|
|
|
logging.info(marker + " >>> " + str(new_count) + " new, " + str(missing_count) + " misses")
|
|
|
|
diff_file.writelines("\n\n")
|
|
|
|
old_file.close()
|
|
new_file.close()
|
|
diff_file.close()
|
|
return
|
|
|
|
def CompareResults(ref_dir, results_dir):
|
|
"""Compare results in two directories
|
|
|
|
Args:
|
|
ref_dir: the reference directory having layout results as references
|
|
results_dir: the results directory
|
|
"""
|
|
logging.info("Comparing results to " + ref_dir)
|
|
|
|
diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
|
|
if os.path.exists(diff_result):
|
|
os.remove(diff_result)
|
|
|
|
files=["passed", "failed", "nontext", "crashed"]
|
|
for f in files:
|
|
result_file_name = "layout_tests_" + f + ".txt"
|
|
DiffResults(f, os.path.join(results_dir, result_file_name),
|
|
os.path.join(ref_dir, result_file_name), diff_result,
|
|
f == "failed")
|
|
logging.info("Detailed diffs are in " + diff_result)
|
|
|
|
def main(options, args):
|
|
"""Run the tests. Will call sys.exit when complete.
|
|
|
|
Args:
|
|
options: a dictionary of command line options
|
|
args: a list of sub directories or files to test
|
|
"""
|
|
|
|
# Set up logging format.
|
|
log_level = logging.INFO
|
|
if options.verbose:
|
|
log_level = logging.DEBUG
|
|
logging.basicConfig(level=log_level,
|
|
format='%(message)s')
|
|
|
|
# Include all tests if none are specified.
|
|
if not args:
|
|
path = '/';
|
|
else:
|
|
path = ' '.join(args);
|
|
|
|
adb_cmd = "adb ";
|
|
if options.adb_options:
|
|
adb_cmd += options.adb_options
|
|
|
|
# Re-generate the test list if --refresh-test-list is on
|
|
if options.refresh_test_list:
|
|
logging.info("Generating test list.");
|
|
generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
|
|
adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
|
|
|
|
if adb_output.find('Process crashed') != -1:
|
|
logging.info("Aborting because cannot generate test list.\n" + adb_output)
|
|
sys.exit(1)
|
|
|
|
|
|
logging.info("Running tests")
|
|
|
|
# Count crashed tests.
|
|
crashed_tests = []
|
|
|
|
timeout_ms = '5000'
|
|
if options.time_out_ms:
|
|
timeout_ms = options.time_out_ms
|
|
|
|
# Run test until it's done
|
|
|
|
run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"
|
|
|
|
run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
|
|
if options.rebaseline:
|
|
run_layout_test_cmd_postfix += " -e rebaseline true"
|
|
run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
|
|
|
|
# Call LayoutTestsAutoTest::startLayoutTests.
|
|
run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix
|
|
|
|
adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
|
|
while not DumpRenderTreeFinished(adb_cmd):
|
|
# Get the running_test.txt
|
|
logging.error("DumpRenderTree crashed, output:\n" + adb_output)
|
|
|
|
shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
|
|
crashed_test = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE).communicate()[0]
|
|
|
|
logging.info(crashed_test + " CRASHED");
|
|
crashed_tests.append(crashed_test);
|
|
|
|
logging.info("Resuming layout test runner...");
|
|
# Call LayoutTestsAutoTest::resumeLayoutTests
|
|
run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix
|
|
|
|
adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
|
|
|
|
if adb_output.find('INSTRUMENTATION_FAILED') != -1:
|
|
logging.error("Error happened : " + adb_output)
|
|
sys.exit(1)
|
|
|
|
logging.debug(adb_output);
|
|
logging.info("Done\n");
|
|
|
|
# Pull results from /sdcard
|
|
results_dir = options.results_directory
|
|
if not os.path.exists(results_dir):
|
|
os.makedirs(results_dir)
|
|
if not os.path.isdir(results_dir):
|
|
logging.error("Cannot create results dir: " + results_dir);
|
|
sys.exit(1);
|
|
|
|
result_files = ["/sdcard/layout_tests_passed.txt",
|
|
"/sdcard/layout_tests_failed.txt",
|
|
"/sdcard/layout_tests_nontext.txt"]
|
|
for file in result_files:
|
|
shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
|
|
adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
|
|
logging.debug(adb_output)
|
|
|
|
# Create the crash list.
|
|
fp = open(results_dir + "/layout_tests_crashed.txt", "w");
|
|
fp.writelines('\n'.join(crashed_tests))
|
|
fp.close()
|
|
|
|
# Count the number of tests in each category.
|
|
passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
|
|
logging.info(str(passed_tests) + " passed")
|
|
failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
|
|
logging.info(str(failed_tests) + " failed")
|
|
crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
|
|
logging.info(str(crashed_tests) + " crashed")
|
|
nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
|
|
logging.info(str(nontext_tests) + " no dumpAsText")
|
|
|
|
logging.info("Results are stored under: " + results_dir + "\n")
|
|
|
|
# Comparing results to references to find new fixes and regressions.
|
|
results_dir = os.path.abspath(options.results_directory)
|
|
ref_dir = options.ref_directory
|
|
|
|
# if ref_dir is null, cannonify ref_dir to the script dir.
|
|
if not ref_dir:
|
|
script_self = sys.argv[0]
|
|
script_dir = os.path.dirname(script_self)
|
|
ref_dir = os.path.join(script_dir, "results")
|
|
|
|
ref_dir = os.path.abspath(ref_dir)
|
|
|
|
CompareResults(ref_dir, results_dir)
|
|
|
|
if '__main__' == __name__:
|
|
option_parser = optparse.OptionParser()
|
|
option_parser.add_option("", "--rebaseline", action="store_true",
|
|
default=False,
|
|
help="generate expected results for those tests not having one")
|
|
option_parser.add_option("", "--time-out-ms",
|
|
default=None,
|
|
help="set the timeout for each test")
|
|
option_parser.add_option("", "--verbose", action="store_true",
|
|
default=False,
|
|
help="include debug-level logging")
|
|
option_parser.add_option("", "--refresh-test-list", action="store_true",
|
|
default=False,
|
|
help="re-generate test list, it may take some time.")
|
|
option_parser.add_option("", "--adb-options",
|
|
default=None,
|
|
help="pass options to adb, such as -d -e, etc");
|
|
option_parser.add_option("", "--results-directory",
|
|
default="layout-test-results",
|
|
help="directory which results are stored.")
|
|
option_parser.add_option("", "--ref-directory",
|
|
default=None,
|
|
dest="ref_directory",
|
|
help="directory where reference results are stored.")
|
|
|
|
options, args = option_parser.parse_args();
|
|
main(options, args)
|