Updates to run_tests.py script.

Add support for the generic targets (using the headers in examples/intrinsics
if none is provided.)

Provide option to run valgrind on the compiled code.

Print a list of all failing tests at the end.

run_tests.py

@@ -17,20 +17,55 @@ import subprocess
 import shlex
 import platform
 
+is_windows = (platform.system() == 'Windows' or
+              'CYGWIN_NT' in platform.system())
+
 parser = OptionParser()
 parser.add_option("-r", "--random-shuffle", dest="random", help="Randomly order tests",
                   default=False, action="store_true")
+parser.add_option("-g", "--generics-include", dest="include_file", help="Filename for header implementing functions for generics",
+                  default=None)
 parser.add_option('-t', '--target', dest='target',
-                  help='Set compilation target (sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2)',
+                  help='Set compilation target (sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2, generic-4, generic-8, generic-16)',
                   default="sse4")
 parser.add_option('-a', '--arch', dest='arch',
                   help='Set architecture (x86, x86-64)',
                   default="x86-64")
+parser.add_option("-c", "--compiler", dest="compiler_exe", help="Compiler binary to use to run tests",
+                  default=None)
 parser.add_option('-o', '--no-opt', dest='no_opt', help='Disable optimization',
                   default=False, action="store_true")
+parser.add_option('-v', '--verbose', dest='verbose', help='Enable verbose output',
+                  default=False, action="store_true")
+if not is_windows:
+    parser.add_option('--valgrind', dest='valgrind', help='Run tests with valgrind',
+                      default=False, action="store_true")
 
 (options, args) = parser.parse_args()
 
+if not is_windows and options.valgrind:
+    valgrind_cmd = "valgrind "
+else:
+    valgrind_cmd = ""
+
+is_generic_target = options.target.find("generic-") != -1
+if is_generic_target and options.include_file == None:
+    if options.target == "generic-4":
+        print "No generics #include specified; using examples/intrinsics/sse4.h"
+        options.include_file = "examples/intrinsics/sse4.h"
+    elif options.target == "generic-8":
+        print "No generics #include specified and no default available for \"generic-8\" target.";
+        sys.exit(1)
+    elif options.target == "generic-16":
+        print "No generics #include specified; using examples/intrinsics/generic-16.h"
+        options.include_file = "examples/intrinsics/generic-16.h"
+
+if options.compiler_exe == None:
+    if is_windows:
+        options.compiler_exe = "cl"
+    else:
+        options.compiler_exe = "g++"
+
 # if no specific test files are specified, run all of the tests in tests/
 # and failing_tests/
 if len(args) == 0:
@@ -53,8 +88,7 @@ total_tests = 0
 # http://bugs.python.org/issue5261.  Therefore, we use the (deprecated but
 # still available) mutex class.
 #finished_tests_counter_lock = multiprocessing.Lock()
-if not (platform.system() == 'Windows' or
+if not is_windows:
-        'CYGWIN_NT' in platform.system()):
     finished_tests_mutex = mutex.mutex()
     finished_tests_counter = multiprocessing.Value(c_int)
 
@@ -71,48 +105,54 @@ def update_progress(fn):
     sys.stdout.flush()
     finished_tests_mutex.unlock()
 
-fnull = open(os.devnull, 'w')
+def run_command(cmd):
+    if options.verbose:
+        print "Running: %s" % cmd
+    sp = subprocess.Popen(shlex.split(cmd), stdin=None,
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+    out = sp.communicate()
+    output = ""
+    output += out[0]
+    output += out[1]
+    return (sp.returncode, output)
 
 # run the commands in cmd_list
-def run_cmds(cmd_list, filename, expect_failure):
+def run_cmds(compile_cmds, run_cmd, filename, expect_failure):
-    output = ""
+    for cmd in compile_cmds:
-    for cmd in cmd_list:
+        (return_code, output) = run_command(cmd)
-        sp = subprocess.Popen(shlex.split(cmd), stdin=None,
+        compile_failed = (return_code != 0)
-                              stdout=subprocess.PIPE,
+        if compile_failed:
-                              stderr=subprocess.PIPE)
+            print "Compilation of test %s failed            " % filename
-        out = sp.communicate()
+            if output != "":
-        output += out[0]
+                print "%s" % output
-        output += out[1]
+            return (1, 0)
-        failed = (sp.returncode != 0)
-        if failed:
-            break
 
-    surprise = ((expect_failure and not failed) or
+    (return_code, output) = run_command(run_cmd)
-                (not expect_failure and failed))
+    run_failed = (return_code != 0)
+
+    surprise = ((expect_failure and not run_failed) or
+                (not expect_failure and run_failed))
     if surprise == True:
         print "Test %s %s (return code %d)            " % \
             (filename, "unexpectedly passed" if expect_failure else "failed",
-             sp.returncode)
+             return_code)
     if output != "":
         print "%s" % output
-    return surprise
+    if surprise == True:
+        return (0, 1)
+    else:
+        return (0, 0)
 
 
 def run_test(filename):
     # is this a test to make sure an error is issued?
-    error_count = 0
     want_error = (filename.find("tests_errors") != -1)
     if want_error == True:
         ispc_cmd = "ispc --werror --nowrap %s --arch=%s --target=%s" % \
             (filename, options.arch, options.target)
-        sp = subprocess.Popen(shlex.split(ispc_cmd), stdin=None,
+        (return_code, output) = run_command(ispc_cmd)
-                              stdout=subprocess.PIPE,
+        got_error = (return_code != 0)
-                              stderr=subprocess.PIPE)
-        out = sp.communicate()
-        output = ""
-        output += out[0]
-        output += out[1]
-        got_error = (sp.returncode != 0)
 
         # figure out the error message we're expecting
         file = open(filename, 'r')
@@ -126,10 +166,12 @@ def run_test(filename):
             print "OUT %s" % filename
             print "Didnt see expected error message %s from test %s.\nActual output:\n%s" % \
                 (firstline, filename, output)
-            error_count += 1
+            return (1, 0)
         elif got_error == False:
             print "Unexpectedly no errors issued from test %s" % filename
-            error_count += 1
+            return (1, 0)
+        else:
+            return (0, 0)
     else:
         # do we expect this test to fail?
         should_fail = (filename.find("failing_") != -1)
@@ -154,59 +196,80 @@ def run_test(filename):
         if match == -1:
             print "Fatal error: unable to find function signature " + \
                   "in test %s" % filename
-            error_count += 1
+            return (1, 0)
         else:
-            if (platform.system() == 'Windows' or
+            is_generic_target = options.target.find("generic-") != -1
-                'CYGWIN_NT' in platform.system()):
+            if is_generic_target:
-                obj_name = "%s.obj" % filename
+                obj_name = "%s.cpp" % filename
+
+            global is_windows
+            if is_windows:
+                if not is_generic_target:
+                    obj_name = "%s.obj" % filename
                 exe_name = "%s.exe" % filename
-                cc_cmd = "cl /nologo test_static.cpp /DTEST_SIG=%d %s.obj /Fe%s" % \
+
-                         (match, filename, exe_name)
+                cc_cmd = "%s /I. /Zi /nologo /DTEST_SIG=%d test_static.cpp %s /Fe%s" % \
+                         (options.compiler_exe, match, obj_name, exe_name)
                 if should_fail:
                     cc_cmd += " /DEXPECT_FAILURE"
             else:
-                obj_name = "%s.o" % filename
+                if not is_generic_target:
+                    obj_name = "%s.o" % filename
                 exe_name = "%s.run" % filename
+
                 if options.arch == 'x86':
                     gcc_arch = '-m32'
                 else:
                     gcc_arch = '-m64'
-                cc_cmd = "g++ %s test_static.cpp -DTEST_SIG=%d %s.o -o %s" % \
+                cc_cmd = "%s -msse4.2 -I. %s test_static.cpp -DTEST_SIG=%d %s -o %s" % \
-                         (gcc_arch, match, filename, exe_name)
+                         (options.compiler_exe, gcc_arch, match, obj_name, exe_name)
                 if platform.system() == 'Darwin':
                     cc_cmd += ' -Wl,-no_pie'
                 if should_fail:
                     cc_cmd += " -DEXPECT_FAILURE"
-    
+
             ispc_cmd = "ispc --woff %s -o %s --arch=%s --target=%s" % \
                        (filename, obj_name, options.arch, options.target)
             if options.no_opt:
                 ispc_cmd += " -O0" 
+            if is_generic_target:
+                ispc_cmd += " --emit-c++ --c++-include-file=%s" % options.include_file
     
         # compile the ispc code, make the executable, and run it...
-        error_count += run_cmds([ispc_cmd, cc_cmd, exe_name], \
+        global valgrind_cmd
-                                filename, should_fail)
+        (compile_error, run_error) = run_cmds([ispc_cmd, cc_cmd], 
-    
+                                              valgrind_cmd + " " + exe_name, \
+                                              filename, should_fail)
         # clean up after running the test
         try:
-            os.unlink(exe_name)
+            if not run_error:
+                os.unlink(exe_name)
+                if is_windows:
+                    os.unlink(filename + ".pdb")
+                    os.unlink(filename + ".ilk")
             os.unlink(obj_name)
         except:
             None
 
-    return error_count
+        return (compile_error, run_error)
 
 # pull tests to run from the given queue and run them.  Multiple copies of
 # this function will be running in parallel across all of the CPU cores of
 # the system.
-def run_tasks_from_queue(queue):
+def run_tasks_from_queue(queue, queue_ret):
-    error_count = 0
+    compile_error_files = [ ]
+    run_error_files = [ ]
     while True:
         filename = queue.get()
         if (filename == 'STOP'):
-            sys.exit(error_count)
+            queue_ret.put((compile_error_files, run_error_files))
+            sys.exit(0)
 
-        error_count += run_test(filename)
+        (compile_error, run_error) = run_test(filename)
+        if compile_error != 0:
+            compile_error_files += [ filename ]
+        if run_error != 0:
+            run_error_files += [ filename ]
 
         # If not for http://bugs.python.org/issue5261 on OSX, we'd like to do this:
         #with finished_tests_counter_lock:
@@ -214,7 +277,6 @@ def run_tasks_from_queue(queue):
         # but instead we do this...
         finished_tests_mutex.lock(update_progress, filename)
 
-
 task_threads = []
 
 def sigint(signum, frame):
@@ -224,10 +286,10 @@ def sigint(signum, frame):
 
 if __name__ == '__main__':
     total_tests = len(files)
-    error_count = 0
 
-    if (platform.system() == 'Windows' or
+    compile_error_files = [ ]
-        'CYGWIN_NT' in platform.system()):
+    run_error_files = [ ]
+    if is_windows:
         # cl.exe gets itself all confused if we have multiple instances of
         # it running concurrently and operating on the same .cpp file
         # (test_static.cpp), even if we are generating a differently-named
@@ -236,7 +298,11 @@ if __name__ == '__main__':
         num_done = 0
         print "Running %d tests." % (total_tests)
         for fn in files:
-            error_count += run_test(fn)
+            (compile_error, run_error) = run_test(fn)
+            if compile_error != 0:
+                compile_error_files += fn
+            if run_error != 0:
+                run_error_files += fn
 
             num_done += 1
             progress_str = " Done %d / %d [%s]" % (num_done, total_tests, fn)
@@ -256,6 +322,7 @@ if __name__ == '__main__':
             q.put(fn)
         for x in range(nthreads):
             q.put('STOP')
+        qret = multiprocessing.Queue()
 
         # need to catch sigint so that we can terminate all of the tasks if
         # we're interrupted
@@ -263,18 +330,30 @@ if __name__ == '__main__':
 
         # launch jobs to run tests
         for x in range(nthreads):
-            t = multiprocessing.Process(target=run_tasks_from_queue, args=(q,))
+            t = multiprocessing.Process(target=run_tasks_from_queue, args=(q,qret))
             task_threads.append(t)
             t.start()
 
         # wait for them to all finish and then return the number that failed
         # (i.e. return 0 if all is ok)
-        error_count = 0
         for t in task_threads:
             t.join()
-            error_count += t.exitcode
         print
 
-    if error_count > 0:
+        while not qret.empty():
-        print "%d / %d tests FAILED!" % (error_count, total_tests)
+            (c, r) = qret.get()
-    sys.exit(error_count)
+            compile_error_files += c
+            run_error_files += r
+
+    if len(compile_error_files) > 0:
+        compile_error_files.sort()
+        print "%d / %d tests FAILED compilation:" % (len(compile_error_files), total_tests)
+        for f in compile_error_files:
+            print "\t%s" % f
+    if len(run_error_files) > 0:
+        run_error_files.sort()
+        print "%d / %d tests FAILED execution:" % (len(run_error_files), total_tests)
+        for f in run_error_files:
+            print "\t%s" % f
+
+    sys.exit(len(compile_error_files) + len(run_error_files))