From f23dc5366a7c40a824f44325acfd55eef93332c3 Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Wed, 4 Jan 2012 12:40:29 -0800 Subject: [PATCH] Updates to run_tests.py script. Add support for the generic targets (using the headers in examples/intrinsics if none is provided.) Provide option to run valgrind on the compiled code. Print a list of all failing tests at the end. --- run_tests.py | 201 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 140 insertions(+), 61 deletions(-) diff --git a/run_tests.py b/run_tests.py index 7756524b..a7da640f 100755 --- a/run_tests.py +++ b/run_tests.py @@ -17,20 +17,55 @@ import subprocess import shlex import platform +is_windows = (platform.system() == 'Windows' or + 'CYGWIN_NT' in platform.system()) + parser = OptionParser() parser.add_option("-r", "--random-shuffle", dest="random", help="Randomly order tests", default=False, action="store_true") +parser.add_option("-g", "--generics-include", dest="include_file", help="Filename for header implementing functions for generics", + default=None) parser.add_option('-t', '--target', dest='target', - help='Set compilation target (sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2)', + help='Set compilation target (sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2, generic-4, generic-8, generic-16)', default="sse4") parser.add_option('-a', '--arch', dest='arch', help='Set architecture (x86, x86-64)', default="x86-64") +parser.add_option("-c", "--compiler", dest="compiler_exe", help="Compiler binary to use to run tests", + default=None) parser.add_option('-o', '--no-opt', dest='no_opt', help='Disable optimization', default=False, action="store_true") +parser.add_option('-v', '--verbose', dest='verbose', help='Enable verbose output', + default=False, action="store_true") +if not is_windows: + parser.add_option('--valgrind', dest='valgrind', help='Run tests with valgrind', + default=False, action="store_true") (options, args) = parser.parse_args() +if not is_windows and options.valgrind: + valgrind_cmd = "valgrind " +else: + valgrind_cmd = "" + +is_generic_target = options.target.find("generic-") != -1 +if is_generic_target and options.include_file == None: + if options.target == "generic-4": + print "No generics #include specified; using examples/intrinsics/sse4.h" + options.include_file = "examples/intrinsics/sse4.h" + elif options.target == "generic-8": + print "No generics #include specified and no default available for \"generic-8\" target."; + sys.exit(1) + elif options.target == "generic-16": + print "No generics #include specified; using examples/intrinsics/generic-16.h" + options.include_file = "examples/intrinsics/generic-16.h" + +if options.compiler_exe == None: + if is_windows: + options.compiler_exe = "cl" + else: + options.compiler_exe = "g++" + # if no specific test files are specified, run all of the tests in tests/ # and failing_tests/ if len(args) == 0: @@ -53,8 +88,7 @@ total_tests = 0 # http://bugs.python.org/issue5261. Therefore, we use the (deprecated but # still available) mutex class. #finished_tests_counter_lock = multiprocessing.Lock() -if not (platform.system() == 'Windows' or - 'CYGWIN_NT' in platform.system()): +if not is_windows: finished_tests_mutex = mutex.mutex() finished_tests_counter = multiprocessing.Value(c_int) @@ -71,48 +105,54 @@ def update_progress(fn): sys.stdout.flush() finished_tests_mutex.unlock() -fnull = open(os.devnull, 'w') +def run_command(cmd): + if options.verbose: + print "Running: %s" % cmd + sp = subprocess.Popen(shlex.split(cmd), stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out = sp.communicate() + output = "" + output += out[0] + output += out[1] + return (sp.returncode, output) # run the commands in cmd_list -def run_cmds(cmd_list, filename, expect_failure): - output = "" - for cmd in cmd_list: - sp = subprocess.Popen(shlex.split(cmd), stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out = sp.communicate() - output += out[0] - output += out[1] - failed = (sp.returncode != 0) - if failed: - break +def run_cmds(compile_cmds, run_cmd, filename, expect_failure): + for cmd in compile_cmds: + (return_code, output) = run_command(cmd) + compile_failed = (return_code != 0) + if compile_failed: + print "Compilation of test %s failed " % filename + if output != "": + print "%s" % output + return (1, 0) - surprise = ((expect_failure and not failed) or - (not expect_failure and failed)) + (return_code, output) = run_command(run_cmd) + run_failed = (return_code != 0) + + surprise = ((expect_failure and not run_failed) or + (not expect_failure and run_failed)) if surprise == True: print "Test %s %s (return code %d) " % \ (filename, "unexpectedly passed" if expect_failure else "failed", - sp.returncode) + return_code) if output != "": print "%s" % output - return surprise + if surprise == True: + return (0, 1) + else: + return (0, 0) def run_test(filename): # is this a test to make sure an error is issued? - error_count = 0 want_error = (filename.find("tests_errors") != -1) if want_error == True: ispc_cmd = "ispc --werror --nowrap %s --arch=%s --target=%s" % \ (filename, options.arch, options.target) - sp = subprocess.Popen(shlex.split(ispc_cmd), stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out = sp.communicate() - output = "" - output += out[0] - output += out[1] - got_error = (sp.returncode != 0) + (return_code, output) = run_command(ispc_cmd) + got_error = (return_code != 0) # figure out the error message we're expecting file = open(filename, 'r') @@ -126,10 +166,12 @@ def run_test(filename): print "OUT %s" % filename print "Didnt see expected error message %s from test %s.\nActual output:\n%s" % \ (firstline, filename, output) - error_count += 1 + return (1, 0) elif got_error == False: print "Unexpectedly no errors issued from test %s" % filename - error_count += 1 + return (1, 0) + else: + return (0, 0) else: # do we expect this test to fail? should_fail = (filename.find("failing_") != -1) @@ -154,59 +196,80 @@ def run_test(filename): if match == -1: print "Fatal error: unable to find function signature " + \ "in test %s" % filename - error_count += 1 + return (1, 0) else: - if (platform.system() == 'Windows' or - 'CYGWIN_NT' in platform.system()): - obj_name = "%s.obj" % filename + is_generic_target = options.target.find("generic-") != -1 + if is_generic_target: + obj_name = "%s.cpp" % filename + + global is_windows + if is_windows: + if not is_generic_target: + obj_name = "%s.obj" % filename exe_name = "%s.exe" % filename - cc_cmd = "cl /nologo test_static.cpp /DTEST_SIG=%d %s.obj /Fe%s" % \ - (match, filename, exe_name) + + cc_cmd = "%s /I. /Zi /nologo /DTEST_SIG=%d test_static.cpp %s /Fe%s" % \ + (options.compiler_exe, match, obj_name, exe_name) if should_fail: cc_cmd += " /DEXPECT_FAILURE" else: - obj_name = "%s.o" % filename + if not is_generic_target: + obj_name = "%s.o" % filename exe_name = "%s.run" % filename + if options.arch == 'x86': gcc_arch = '-m32' else: gcc_arch = '-m64' - cc_cmd = "g++ %s test_static.cpp -DTEST_SIG=%d %s.o -o %s" % \ - (gcc_arch, match, filename, exe_name) + cc_cmd = "%s -msse4.2 -I. %s test_static.cpp -DTEST_SIG=%d %s -o %s" % \ + (options.compiler_exe, gcc_arch, match, obj_name, exe_name) if platform.system() == 'Darwin': cc_cmd += ' -Wl,-no_pie' if should_fail: cc_cmd += " -DEXPECT_FAILURE" - + ispc_cmd = "ispc --woff %s -o %s --arch=%s --target=%s" % \ (filename, obj_name, options.arch, options.target) if options.no_opt: ispc_cmd += " -O0" + if is_generic_target: + ispc_cmd += " --emit-c++ --c++-include-file=%s" % options.include_file # compile the ispc code, make the executable, and run it... - error_count += run_cmds([ispc_cmd, cc_cmd, exe_name], \ - filename, should_fail) - + global valgrind_cmd + (compile_error, run_error) = run_cmds([ispc_cmd, cc_cmd], + valgrind_cmd + " " + exe_name, \ + filename, should_fail) # clean up after running the test try: - os.unlink(exe_name) + if not run_error: + os.unlink(exe_name) + if is_windows: + os.unlink(filename + ".pdb") + os.unlink(filename + ".ilk") os.unlink(obj_name) except: None - return error_count + return (compile_error, run_error) # pull tests to run from the given queue and run them. Multiple copies of # this function will be running in parallel across all of the CPU cores of # the system. -def run_tasks_from_queue(queue): - error_count = 0 +def run_tasks_from_queue(queue, queue_ret): + compile_error_files = [ ] + run_error_files = [ ] while True: filename = queue.get() if (filename == 'STOP'): - sys.exit(error_count) + queue_ret.put((compile_error_files, run_error_files)) + sys.exit(0) - error_count += run_test(filename) + (compile_error, run_error) = run_test(filename) + if compile_error != 0: + compile_error_files += [ filename ] + if run_error != 0: + run_error_files += [ filename ] # If not for http://bugs.python.org/issue5261 on OSX, we'd like to do this: #with finished_tests_counter_lock: @@ -214,7 +277,6 @@ def run_tasks_from_queue(queue): # but instead we do this... finished_tests_mutex.lock(update_progress, filename) - task_threads = [] def sigint(signum, frame): @@ -224,10 +286,10 @@ def sigint(signum, frame): if __name__ == '__main__': total_tests = len(files) - error_count = 0 - if (platform.system() == 'Windows' or - 'CYGWIN_NT' in platform.system()): + compile_error_files = [ ] + run_error_files = [ ] + if is_windows: # cl.exe gets itself all confused if we have multiple instances of # it running concurrently and operating on the same .cpp file # (test_static.cpp), even if we are generating a differently-named @@ -236,7 +298,11 @@ if __name__ == '__main__': num_done = 0 print "Running %d tests." % (total_tests) for fn in files: - error_count += run_test(fn) + (compile_error, run_error) = run_test(fn) + if compile_error != 0: + compile_error_files += fn + if run_error != 0: + run_error_files += fn num_done += 1 progress_str = " Done %d / %d [%s]" % (num_done, total_tests, fn) @@ -256,6 +322,7 @@ if __name__ == '__main__': q.put(fn) for x in range(nthreads): q.put('STOP') + qret = multiprocessing.Queue() # need to catch sigint so that we can terminate all of the tasks if # we're interrupted @@ -263,18 +330,30 @@ if __name__ == '__main__': # launch jobs to run tests for x in range(nthreads): - t = multiprocessing.Process(target=run_tasks_from_queue, args=(q,)) + t = multiprocessing.Process(target=run_tasks_from_queue, args=(q,qret)) task_threads.append(t) t.start() # wait for them to all finish and then return the number that failed # (i.e. return 0 if all is ok) - error_count = 0 for t in task_threads: t.join() - error_count += t.exitcode print - if error_count > 0: - print "%d / %d tests FAILED!" % (error_count, total_tests) - sys.exit(error_count) + while not qret.empty(): + (c, r) = qret.get() + compile_error_files += c + run_error_files += r + + if len(compile_error_files) > 0: + compile_error_files.sort() + print "%d / %d tests FAILED compilation:" % (len(compile_error_files), total_tests) + for f in compile_error_files: + print "\t%s" % f + if len(run_error_files) > 0: + run_error_files.sort() + print "%d / %d tests FAILED execution:" % (len(run_error_files), total_tests) + for f in run_error_files: + print "\t%s" % f + + sys.exit(len(compile_error_files) + len(run_error_files))