aaron/ispc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add new test-driver script, run_tests.py.

Browse Source 
Old run_tests.sh still lives (for now).
Changes include:
- Tests are run in parallel across all of the available CPU cores
- Option to create a statically-linked executable for each test
  (rather than using the LLVM JIT).  This is in particular useful
  for AVX, which doesn't have good JIT support yet.
- Static executables also makes it possible to test x86, not
  just x86-64, codegen.
- Fixed a number of tests in failing_tests, which were actually
  failing due to the fact that the expected function signature of
  tests had changed.
This commit is contained in:
Matt Pharr
2011-08-29 14:15:09 -07:00
parent 33feeffe5d
commit 58e34ba4ae
9 changed files with 383 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
failing_tests/masked-scatter-vector.ispc
View File

@@ -14,7 +14,7 @@ export void f_fu(uniform float ret[], uniform float aa[], uniform float b) {
varying int3 vv = array[a];
++vv.y;
array[a] = vv;
print("fin %\n", array[programIndex].y);
//CO print("fin %\n", array[programIndex].y);
ret[programIndex] = array[programIndex].y;
}

23
failing_tests/max-uint-1.ispc
View File

@@ -1,19 +1,14 @@
static float float4(uniform float a, uniform float b, uniform float c,
uniform float d) {
float ret = 0;
for (uniform int i = 0; i < programCount; i += 4) {
ret = insert(ret, i + 0, a);
ret = insert(ret, i + 1, b);
ret = insert(ret, i + 2, c);
ret = insert(ret, i + 3, d);
}
return ret;
export uniform int width() { return programCount; }
export void f_f(uniform float r[], uniform float a[]) {
unsigned int i = (unsigned int)a[programIndex];
r[programIndex] = max((unsigned int)2, i);
}
export float f_f(float a) {
unsigned int i = (unsigned int)a;
return max((unsigned int)2, i);
export void result(uniform float r[]) {
r[programIndex] = 1+programIndex;
r[0] = 2;
}
export float result() { return float4(2,2,3,4); }

10
failing_tests/max-uint.ispc
View File

@@ -1,8 +1,10 @@
export float f_f(float a) {
unsigned int i = (unsigned int)a;
return max((unsigned int)10, i);
export uniform int width() { return programCount; }
export void f_f(uniform float result[], uniform float aa[]) {
unsigned int i = (unsigned int)aa[programIndex];
result[programIndex] = max((unsigned int)100, i);
}
export float result() { return 10; }
export void result(uniform float r[]) { r[programIndex] = 100; }

23
failing_tests/min-uint-1.ispc
View File

@@ -1,19 +1,14 @@
static float float4(uniform float a, uniform float b, uniform float c,
uniform float d) {
float ret = 0;
for (uniform int i = 0; i < programCount; i += 4) {
ret = insert(ret, i + 0, a);
ret = insert(ret, i + 1, b);
ret = insert(ret, i + 2, c);
ret = insert(ret, i + 3, d);
}
return ret;
export uniform int width() { return programCount; }
export void f_f(uniform float result[], uniform float aa[]) {
unsigned int i = (unsigned int)aa[programIndex];
result[programIndex] = min((unsigned int)2, i);
}
export float f_f(float a) {
unsigned int i = (unsigned int)a;
return min((unsigned int)2, i);
export void result(uniform float r[]) {
r[programIndex] = 2;
r[0] = 1;
}
export float result() { return float4(1,2,2,2); }

22
failing_tests/min-uint-2.ispc
View File

@@ -1,19 +1,13 @@
static float float4(uniform float a, uniform float b, uniform float c,
uniform float d) {
float ret = 0;
for (uniform int i = 0; i < programCount; i += 4) {
ret = insert(ret, i + 0, a);
ret = insert(ret, i + 1, b);
ret = insert(ret, i + 2, c);
ret = insert(ret, i + 3, d);
}
return ret;
export uniform int width() { return programCount; }
export void f_f(uniform float r[], uniform float a[]) {
unsigned int i = (unsigned int)a[programIndex];
r[programIndex] = min((unsigned int)20, i);
}
export float f_f(float a) {
unsigned int i = (unsigned int)a;
return min((unsigned int)20, i);
export void result(uniform float r[]) {
r[programIndex] = 1+programIndex;
}
export float result() { return float4(1,2,3,4); }

11
failing_tests/struct-array-assign.ispc
View File

@@ -1,11 +0,0 @@
struct Foo {
float f;
};
export float foo(Foo f[], int i, uniform int j) {
Foo x = f[i];
return x.f;
}

54
ispc_test.cpp
View File

@@ -85,6 +85,8 @@ extern "C" {
#include <llvm/Support/system_error.h>
#endif
bool shouldFail = false;
extern "C" {
void ISPCLaunch(void *, void *);
void ISPCSync();
@@ -117,6 +119,7 @@ void ISPCFree(void *ptr) {
static void usage(int ret) {
fprintf(stderr, "usage: ispc_test\n");
fprintf(stderr, "\t[-h/--help]\tprint help\n");
fprintf(stderr, "\t[-f]\t\tindicates that test is expected to fail\n");
fprintf(stderr, "\t<files>\n");
exit(ret);
}
@@ -267,7 +270,6 @@ static bool lRunTest(const char *fn) {
float result[16];
for (int i = 0; i < 16; ++i)
result[i] = 0;
bool ok = true;
if (foundResult) {
typedef void (*PFN)(float *);
PFN pfn = reinterpret_cast<PFN>(ee->getPointerToFunction(func));
@@ -324,32 +326,34 @@ static bool lRunTest(const char *fn) {
}
else {
fprintf(stderr, "Unable to find runnable function in file \"%s\"\n", fn);
ok = false;
return false;
}
// see if we got the right result
if (ok) {
if (foundResult) {
for (int i = 0; i < width; ++i)
if (returned[i] != result[i]) {
ok = false;
fprintf(stderr, "Test \"%s\" RETURNED %d: %g / %a EXPECTED %g / %a\n",
fn, i, returned[i], returned[i], result[i], result[i]);
}
}
else {
for (int i = 0; i < width; ++i)
fprintf(stderr, "Test \"%s\" returned %d: %g / %a\n",
fn, i, returned[i], returned[i]);
}
bool resultsMatch = true;
if (foundResult) {
for (int i = 0; i < width; ++i)
if (returned[i] != result[i]) {
resultsMatch = false;
fprintf(stderr, "Test \"%s\" RETURNED %d: %g / %a EXPECTED %g / %a\n",
fn, i, returned[i], returned[i], result[i], result[i]);
}
}
else {
for (int i = 0; i < width; ++i)
fprintf(stderr, "Test \"%s\" returned %d: %g / %a\n",
fn, i, returned[i], returned[i]);
}
if (foundResult && shouldFail && resultsMatch)
fprintf(stderr, "Test %s unexpectedly passed\n", fn);
delete ee;
delete ctx;
return ok && foundResult;
return foundResult && resultsMatch;
}
int main(int argc, char *argv[]) {
llvm::InitializeNativeTarget();
#if defined(LLVM_3_0) || defined(LLVM_3_0svn)
@@ -358,21 +362,15 @@ int main(int argc, char *argv[]) {
LLVMLinkInJIT();
#endif
std::vector<const char *> files;
const char *filename = NULL;
for (int i = 1; i < argc; ++i) {
if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-h"))
usage(0);
if (!strcmp(argv[i], "-f"))
shouldFail = true;
else
files.push_back(argv[i]);
filename = argv[i];
}
int passes = 0, fails = 0;
for (unsigned int i = 0; i < files.size(); ++i) {
if (lRunTest(files[i])) ++passes;
else ++fails;
}
if (fails > 0)
fprintf(stderr, "%d/%d tests passed\n", passes, passes+fails);
return fails > 0;
return (lRunTest(filename) == true) ? 0 : 1;
}

207
run_tests.py Executable file
View File

@@ -0,0 +1,207 @@
#!/usr/bin/python
# test-running driver for ispc
# TODO: windows support (mostly should be calling CL.exe rather than gcc
# for static linking?)
from optparse import OptionParser
import multiprocessing
from ctypes import c_int
import os
import sys
import glob
import re
import signal
import random
import string
import mutex
import subprocess
parser = OptionParser()
parser.add_option("-r", "--random-shuffle", dest="random", help="Randomly order tests",
default=False, action="store_true")
parser.add_option("-s", "--static-exe", dest="static_exe",
help="Create and run a regular executable for each test (rather than using the LLVM JIT).",
default=False, action="store_true")
parser.add_option('-t', '--target', dest='target',
help='Set compilation target (sse2, sse4, sse4x2, avx, avx-x2)',
default="sse4")
parser.add_option('-a', '--arch', dest='arch',
help='Set architecture (x86, x86-64)',
default="x86-64")
(options, args) = parser.parse_args()
# if no specific test files are specified, run all of the tests in tests/
# and failing_tests/
if len(args) == 0:
files = glob.glob("tests/*ispc") + glob.glob("failing_tests/*ispc")
else:
files = args
# randomly shuffle the tests if asked to do so
if (options.random):
random.seed()
random.shuffle(files)
# counter
total_tests = 0
finished_tests_counter = multiprocessing.Value(c_int)
# We'd like to use the Lock class from the multiprocessing package to
# serialize accesses to finished_tests_counter. Unfortunately, the version of
# python that ships with OSX 10.5 has this bug:
# http://bugs.python.org/issue5261. Therefore, we use the (deprecated but
# still available) mutex class.
#finished_tests_counter_lock = multiprocessing.Lock()
finished_tests_mutex = mutex.mutex()
# utility routine to print an update on the number of tests that have been
# finished. Should be called with the mutex (or lock) held..
def update_progress(fn):
finished_tests_counter.value = finished_tests_counter.value + 1
progress_str = " Done %d / %d [%s]" % (finished_tests_counter.value, total_tests, fn)
# spaces to clear out detrius from previous printing...
for x in range(30):
progress_str += ' '
progress_str += '\r'
sys.stdout.write(progress_str)
sys.stdout.flush()
finished_tests_mutex.unlock()
fnull = open(os.devnull, 'w')
# run the commands in cmd_list
def run_cmds(cmd_list, filename, expect_failure):
for cmd in cmd_list:
if expect_failure:
failed = (subprocess.call(cmd, shell = True, stdout = fnull, stderr = fnull) != 0)
else:
failed = (os.system(cmd) != 0)
if failed:
break
surprise = ((expect_failure and not failed) or (not expect_failure and failed))
if surprise == True:
print "Test %s %s " % \
(filename, "unexpectedly passed" if expect_failure else "failed")
return surprise
# pull tests to run from the given queue and run them. Multiple copies of
# this function will be running in parallel across all of the CPU cores of
# the system.
def run_tasks_from_queue(queue):
error_count = 0
while True:
filename = queue.get()
if (filename == 'STOP'):
sys.exit(error_count)
# do we expect this test to fail?
should_fail = (filename.find("failing_") != -1)
if options.static_exe == True:
# if the user wants us to build a static executable to run for
# this test, we need to figure out the signature of the test
# function that this test has.
sig2def = { "f_v(" : 0, "f_f(" : 1, "f_fu(" : 2, "f_fi(" : 3,
"f_du(" : 4, "f_duf(" : 5, "f_di(" : 6 }
file = open(filename, 'r')
match = -1
for line in file:
# look for lines with 'export'...
if line.find("export") == -1:
continue
# one of them should have a function with one of the
# declarations in sig2def
for pattern, ident in sig2def.items():
if line.find(pattern) != -1:
match = ident
break
file.close()
if match == -1:
print "Fatal error: unable to find function signature in test %s" % filename
error_count += 1
else:
obj_name = "%s.o" % filename
exe_name = "%s.run" % filename
ispc_cmd = "ispc --woff %s -o %s --arch=%s --target=%s" % \
(filename, obj_name, options.arch, options.target)
if options.arch == 'x86':
gcc_arch = '-m32'
else:
gcc_arch = '-m64'
gcc_cmd = "g++ -Wl,-no_pie %s test_static.cpp -DTEST_SIG=%d %s.o -o %s" % \
(gcc_arch, match, filename, exe_name)
if should_fail:
gcc_cmd += " -DEXPECT_FAILURE"
# compile the ispc code, make the executable, and run it...
error_count += run_cmds([ispc_cmd, gcc_cmd, exe_name], filename, should_fail)
# clean up after running the test
try:
os.unlink(exe_name)
os.unlink(obj_name)
except:
None
else:
# otherwise we'll use ispc_test + the LLVM JIT to run the test
bitcode_file = "%s.bc" % filename
compile_cmd = "ispc --woff --emit-llvm %s --target=%s -o %s" % \
(filename, options.target, bitcode_file)
test_cmd = "ispc_test %s" % bitcode_file
error_count += run_cmds([compile_cmd, test_cmd], filename, should_fail)
try:
os.unlink(bitcode_file)
except:
None
# If not for http://bugs.python.org/issue5261 on OSX, we'd like to do this:
#with finished_tests_counter_lock:
#update_progress(filename)
# but instead we do this...
finished_tests_mutex.lock(update_progress, filename)
task_threads = []
def sigint(signum, frame):
for t in task_threads:
t.terminate()
sys.exit(1)
if __name__ == '__main__':
nthreads = multiprocessing.cpu_count()
total_tests = len(files)
print "Found %d CPUs. Running %d tests." % (nthreads, total_tests)
# put each of the test filenames into a queue
q = multiprocessing.Queue()
for fn in files:
q.put(fn)
for x in range(nthreads):
q.put('STOP')
# need to catch sigint so that we can terminate all of the tasks if
# we're interrupted
signal.signal(signal.SIGINT, sigint)
# launch jobs to run tests
for x in range(nthreads):
t = multiprocessing.Process(target=run_tasks_from_queue, args=(q,))
task_threads.append(t)
t.start()
# wait for them to all finish and then return the number that failed
# (i.e. return 0 if all is ok)
error_count = 0
for t in task_threads:
t.join()
error_count += t.exitcode
print
sys.exit(error_count)

117
test_static.cpp Normal file
View File

@@ -0,0 +1,117 @@
/*
Copyright (c) 2010-2011, Intel Corporation
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of Intel Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <string.h>
#include <stdio.h>
#include <assert.h>
extern "C" {
extern int width();
extern void f_v(float *result);
extern void f_f(float *result, float *a);
extern void f_fu(float *result, float *a, float b);
extern void f_fi(float *result, float *a, int *b);
extern void f_du(float *result, double *a, double b);
extern void f_duf(float *result, double *a, float b);
extern void f_di(float *result, double *a, int *b);
extern void result(float *val);
void ISPCLaunch(void *f, void *d);
void ISPCSync();
}
void ISPCLaunch(void *f, void *d) {
typedef void (*TaskFuncType)(void *, int, int);
TaskFuncType func = (TaskFuncType)f;
func(d, 0, 1);
}
void ISPCSync() {
}
int main(int argc, char *argv[]) {
int w = width();
assert(w <= 16);
float r1[16];
memset(r1, 0, 16*sizeof(float));
float vfloat[16] = { 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
double vdouble[16] = { 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
int vint[16] = { 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32 };
int vint2[16] = { 5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20};
float b = 5.;
#if (TEST_SIG == 0)
f_v(r1);
#elif (TEST_SIG == 1)
f_f(r1, vfloat);
#elif (TEST_SIG == 2)
f_fu(r1, vfloat, b);
#elif (TEST_SIG == 3)
f_fi(r1, vfloat, vint);
#elif (TEST_SIG == 4)
f_du(r1, vdouble, 5.);
#elif (TEST_SIG == 5)
f_duf(r1, vdouble, 5.f);
#elif (TEST_SIG == 6)
f_di(r1, vdouble, vint2);
#else
#error "Unknown or unset TEST_SIG value"
#endif
float r2[16];
memset(r2, 0, 16*sizeof(float));
result(r2);
int errors = 0;
for (int i = 0; i < w; ++i) {
if (r1[i] != r2[i]) {
#ifdef EXPECT_FAILURE
// bingo, failed
return 1;
#else
printf("%s: value %d disagrees: should be %f [%a], returned %f [%a]\n",
argv[0], i, r1[i], r1[i], r2[i], r2[i]);
++errors;
#endif // EXPECT_FAILURE
}
}
#ifdef EXPECT_FAILURE
// Don't expect to get here
return 0;
#else
return errors > 0;
#endif
}