aaron/ispc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Updates to run_tests.py script.

Browse Source 
Add support for the generic targets (using the headers in examples/intrinsics
if none is provided.)

Provide option to run valgrind on the compiled code.

Print a list of all failing tests at the end.
This commit is contained in:
Matt Pharr
2012-01-04 12:40:29 -08:00
parent e3341176c5
commit f23dc5366a
1 changed files with 140 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
run_tests.py
View File

@@ -17,20 +17,55 @@ import subprocess
import shlex
import platform
is_windows = (platform.system() == 'Windows' or
'CYGWIN_NT' in platform.system())
parser = OptionParser()
parser.add_option("-r", "--random-shuffle", dest="random", help="Randomly order tests",
default=False, action="store_true")
parser.add_option("-g", "--generics-include", dest="include_file", help="Filename for header implementing functions for generics",
default=None)
parser.add_option('-t', '--target', dest='target',
help='Set compilation target (sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2)',
help='Set compilation target (sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2, generic-4, generic-8, generic-16)',
default="sse4")
parser.add_option('-a', '--arch', dest='arch',
help='Set architecture (x86, x86-64)',
default="x86-64")
parser.add_option("-c", "--compiler", dest="compiler_exe", help="Compiler binary to use to run tests",
default=None)
parser.add_option('-o', '--no-opt', dest='no_opt', help='Disable optimization',
default=False, action="store_true")
parser.add_option('-v', '--verbose', dest='verbose', help='Enable verbose output',
default=False, action="store_true")
if not is_windows:
parser.add_option('--valgrind', dest='valgrind', help='Run tests with valgrind',
default=False, action="store_true")
(options, args) = parser.parse_args()
if not is_windows and options.valgrind:
valgrind_cmd = "valgrind "
else:
valgrind_cmd = ""
is_generic_target = options.target.find("generic-") != -1
if is_generic_target and options.include_file == None:
if options.target == "generic-4":
print "No generics #include specified; using examples/intrinsics/sse4.h"
options.include_file = "examples/intrinsics/sse4.h"
elif options.target == "generic-8":
print "No generics #include specified and no default available for \"generic-8\" target.";
sys.exit(1)
elif options.target == "generic-16":
print "No generics #include specified; using examples/intrinsics/generic-16.h"
options.include_file = "examples/intrinsics/generic-16.h"
if options.compiler_exe == None:
if is_windows:
options.compiler_exe = "cl"
else:
options.compiler_exe = "g++"
# if no specific test files are specified, run all of the tests in tests/
# and failing_tests/
if len(args) == 0:
@@ -53,8 +88,7 @@ total_tests = 0
# http://bugs.python.org/issue5261. Therefore, we use the (deprecated but
# still available) mutex class.
#finished_tests_counter_lock = multiprocessing.Lock()
if not (platform.system() == 'Windows' or
'CYGWIN_NT' in platform.system()):
if not is_windows:
finished_tests_mutex = mutex.mutex()
finished_tests_counter = multiprocessing.Value(c_int)
@@ -71,48 +105,54 @@ def update_progress(fn):
sys.stdout.flush()
finished_tests_mutex.unlock()
fnull = open(os.devnull, 'w')
def run_command(cmd):
if options.verbose:
print "Running: %s" % cmd
sp = subprocess.Popen(shlex.split(cmd), stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out = sp.communicate()
output = ""
output += out[0]
output += out[1]
return (sp.returncode, output)
# run the commands in cmd_list
def run_cmds(cmd_list, filename, expect_failure):
output = ""
for cmd in cmd_list:
sp = subprocess.Popen(shlex.split(cmd), stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out = sp.communicate()
output += out[0]
output += out[1]
failed = (sp.returncode != 0)
if failed:
break
def run_cmds(compile_cmds, run_cmd, filename, expect_failure):
for cmd in compile_cmds:
(return_code, output) = run_command(cmd)
compile_failed = (return_code != 0)
if compile_failed:
print "Compilation of test %s failed " % filename
if output != "":
print "%s" % output
return (1, 0)
surprise = ((expect_failure and not failed) or
(not expect_failure and failed))
(return_code, output) = run_command(run_cmd)
run_failed = (return_code != 0)
surprise = ((expect_failure and not run_failed) or
(not expect_failure and run_failed))
if surprise == True:
print "Test %s %s (return code %d) " % \
(filename, "unexpectedly passed" if expect_failure else "failed",
sp.returncode)
return_code)
if output != "":
print "%s" % output
return surprise
if surprise == True:
return (0, 1)
else:
return (0, 0)
def run_test(filename):
# is this a test to make sure an error is issued?
error_count = 0
want_error = (filename.find("tests_errors") != -1)
if want_error == True:
ispc_cmd = "ispc --werror --nowrap %s --arch=%s --target=%s" % \
(filename, options.arch, options.target)
sp = subprocess.Popen(shlex.split(ispc_cmd), stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out = sp.communicate()
output = ""
output += out[0]
output += out[1]
got_error = (sp.returncode != 0)
(return_code, output) = run_command(ispc_cmd)
got_error = (return_code != 0)
# figure out the error message we're expecting
file = open(filename, 'r')
@@ -126,10 +166,12 @@ def run_test(filename):
print "OUT %s" % filename
print "Didnt see expected error message %s from test %s.\nActual output:\n%s" % \
(firstline, filename, output)
error_count += 1
return (1, 0)
elif got_error == False:
print "Unexpectedly no errors issued from test %s" % filename
error_count += 1
return (1, 0)
else:
return (0, 0)
else:
# do we expect this test to fail?
should_fail = (filename.find("failing_") != -1)
@@ -154,59 +196,80 @@ def run_test(filename):
if match == -1:
print "Fatal error: unable to find function signature " + \
"in test %s" % filename
error_count += 1
return (1, 0)
else:
if (platform.system() == 'Windows' or
'CYGWIN_NT' in platform.system()):
obj_name = "%s.obj" % filename
is_generic_target = options.target.find("generic-") != -1
if is_generic_target:
obj_name = "%s.cpp" % filename
global is_windows
if is_windows:
if not is_generic_target:
obj_name = "%s.obj" % filename
exe_name = "%s.exe" % filename
cc_cmd = "cl /nologo test_static.cpp /DTEST_SIG=%d %s.obj /Fe%s" % \
(match, filename, exe_name)
cc_cmd = "%s /I. /Zi /nologo /DTEST_SIG=%d test_static.cpp %s /Fe%s" % \
(options.compiler_exe, match, obj_name, exe_name)
if should_fail:
cc_cmd += " /DEXPECT_FAILURE"
else:
obj_name = "%s.o" % filename
if not is_generic_target:
obj_name = "%s.o" % filename
exe_name = "%s.run" % filename
if options.arch == 'x86':
gcc_arch = '-m32'
else:
gcc_arch = '-m64'
cc_cmd = "g++ %s test_static.cpp -DTEST_SIG=%d %s.o -o %s" % \
(gcc_arch, match, filename, exe_name)
cc_cmd = "%s -msse4.2 -I. %s test_static.cpp -DTEST_SIG=%d %s -o %s" % \
(options.compiler_exe, gcc_arch, match, obj_name, exe_name)
if platform.system() == 'Darwin':
cc_cmd += ' -Wl,-no_pie'
if should_fail:
cc_cmd += " -DEXPECT_FAILURE"
ispc_cmd = "ispc --woff %s -o %s --arch=%s --target=%s" % \
(filename, obj_name, options.arch, options.target)
if options.no_opt:
ispc_cmd += " -O0"
if is_generic_target:
ispc_cmd += " --emit-c++ --c++-include-file=%s" % options.include_file
# compile the ispc code, make the executable, and run it...
error_count += run_cmds([ispc_cmd, cc_cmd, exe_name], \
filename, should_fail)
global valgrind_cmd
(compile_error, run_error) = run_cmds([ispc_cmd, cc_cmd],
valgrind_cmd + " " + exe_name, \
filename, should_fail)
# clean up after running the test
try:
os.unlink(exe_name)
if not run_error:
os.unlink(exe_name)
if is_windows:
os.unlink(filename + ".pdb")
os.unlink(filename + ".ilk")
os.unlink(obj_name)
except:
None
return error_count
return (compile_error, run_error)
# pull tests to run from the given queue and run them. Multiple copies of
# this function will be running in parallel across all of the CPU cores of
# the system.
def run_tasks_from_queue(queue):
error_count = 0
def run_tasks_from_queue(queue, queue_ret):
compile_error_files = [ ]
run_error_files = [ ]
while True:
filename = queue.get()
if (filename == 'STOP'):
sys.exit(error_count)
queue_ret.put((compile_error_files, run_error_files))
sys.exit(0)
error_count += run_test(filename)
(compile_error, run_error) = run_test(filename)
if compile_error != 0:
compile_error_files += [ filename ]
if run_error != 0:
run_error_files += [ filename ]
# If not for http://bugs.python.org/issue5261 on OSX, we'd like to do this:
#with finished_tests_counter_lock:
@@ -214,7 +277,6 @@ def run_tasks_from_queue(queue):
# but instead we do this...
finished_tests_mutex.lock(update_progress, filename)
task_threads = []
def sigint(signum, frame):
@@ -224,10 +286,10 @@ def sigint(signum, frame):
if __name__ == '__main__':
total_tests = len(files)
error_count = 0
if (platform.system() == 'Windows' or
'CYGWIN_NT' in platform.system()):
compile_error_files = [ ]
run_error_files = [ ]
if is_windows:
# cl.exe gets itself all confused if we have multiple instances of
# it running concurrently and operating on the same .cpp file
# (test_static.cpp), even if we are generating a differently-named
@@ -236,7 +298,11 @@ if __name__ == '__main__':
num_done = 0
print "Running %d tests." % (total_tests)
for fn in files:
error_count += run_test(fn)
(compile_error, run_error) = run_test(fn)
if compile_error != 0:
compile_error_files += fn
if run_error != 0:
run_error_files += fn
num_done += 1
progress_str = " Done %d / %d [%s]" % (num_done, total_tests, fn)
@@ -256,6 +322,7 @@ if __name__ == '__main__':
q.put(fn)
for x in range(nthreads):
q.put('STOP')
qret = multiprocessing.Queue()
# need to catch sigint so that we can terminate all of the tasks if
# we're interrupted
@@ -263,18 +330,30 @@ if __name__ == '__main__':
# launch jobs to run tests
for x in range(nthreads):
t = multiprocessing.Process(target=run_tasks_from_queue, args=(q,))
t = multiprocessing.Process(target=run_tasks_from_queue, args=(q,qret))
task_threads.append(t)
t.start()
# wait for them to all finish and then return the number that failed
# (i.e. return 0 if all is ok)
error_count = 0
for t in task_threads:
t.join()
error_count += t.exitcode
print
if error_count > 0:
print "%d / %d tests FAILED!" % (error_count, total_tests)
sys.exit(error_count)
while not qret.empty():
(c, r) = qret.get()
compile_error_files += c
run_error_files += r
if len(compile_error_files) > 0:
compile_error_files.sort()
print "%d / %d tests FAILED compilation:" % (len(compile_error_files), total_tests)
for f in compile_error_files:
print "\t%s" % f
if len(run_error_files) > 0:
run_error_files.sort()
print "%d / %d tests FAILED execution:" % (len(run_error_files), total_tests)
for f in run_error_files:
print "\t%s" % f
sys.exit(len(compile_error_files) + len(run_error_files))