From ea18427d2939db2955efa621fcfaf5b0092ff263 Mon Sep 17 00:00:00 2001 From: Alex Reece Date: Tue, 7 Feb 2012 17:17:25 -0500 Subject: [PATCH 01/55] Remove UnwindInst Code no longer builds against head of LLVM branch after revision 149906 removed the unwind instruction. --- cbackend.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cbackend.cpp b/cbackend.cpp index 314b53d6..7f27f0cb 100644 --- a/cbackend.cpp +++ b/cbackend.cpp @@ -437,9 +437,11 @@ namespace { void visitInvokeInst(InvokeInst &I) { llvm_unreachable("Lowerinvoke pass didn't work!"); } +#if !defined(LLVM_3_1) && !defined(LLVM_3_1svn) void visitUnwindInst(UnwindInst &I) { llvm_unreachable("Lowerinvoke pass didn't work!"); } +#endif // !LLVM_3_1svn void visitResumeInst(ResumeInst &I) { llvm_unreachable("DwarfEHPrepare pass didn't work!"); } From ffba8580c1deb7d8fa1b1c2bb8484b9a84eab0b0 Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Wed, 8 Feb 2012 19:52:49 -0800 Subject: [PATCH 02/55] Make sure that non-zero exit code is returned when input file not found. Fixes issue #174. --- module.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/module.cpp b/module.cpp index acf57a90..e192c39d 100644 --- a/module.cpp +++ b/module.cpp @@ -1625,6 +1625,9 @@ Module::CompileAndOutput(const char *srcFile, const char *arch, const char *cpu, if (!m->writeOutput(Module::Header, headerFileName)) return 1; } + else + ++m->errorCount; + int errorCount = m->errorCount; delete m; m = NULL; From adb1e47a595520dad8f7b09efe14d5d5ee89a354 Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 10 Feb 2012 12:26:19 -0800 Subject: [PATCH 03/55] Add FAQ about how to cross-inline ispc and C/C++ code. --- docs/faq.rst | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 2ce7268b..2cdca136 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -19,6 +19,7 @@ distribution. + `How can I supply an initial execution mask in the call from the application?`_ + `How can I generate a single binary executable with support for multiple instruction sets?`_ + `How can I determine at run-time which vector instruction set's instructions were selected to execute?`_ + + `Is it possible to inline ispc functions in C/C++ code?`_ * Programming Techniques @@ -346,6 +347,50 @@ In a similar fashion, it's possible to find out at run-time the value of export uniform int width() { return programCount; } +Is it possible to inline ispc functions in C/C++ code? +------------------------------------------------------ + +If you're willing to use the ``clang`` C/C++ compiler that's part of the +LLVM tool suite, then it is possible to inline ``ispc`` code with C/C++ +(and conversely, to inline C/C++ calls in ``ispc``). Doing so can provide +performance advantages when calling out to short functions written in the +"other" language. Note that you don't need to use ``clang`` to compile all +of your C/C++ code, but only for the files where you want to be able to +inline. In order to do this, you must have a full installation of LLVM +version 3.0 or later, including the ``clang`` compiler. + +The basic approach is to have the various compilers emit LLVM intermediate +representation (IR) code and to then use tools from LLVM to link together +the IR from the compilers and then re-optimize it, which gives the LLVM +optimizer the opportunity to do additional inlining and cross-function +optimizations. If you have source files ``foo.ispc`` and ``foo.cpp``, +first emit LLVM IR: + +:: + + ispc --emit-llvm -o foo_ispc.bc foo.ispc + clang -O2 -c -emit-llvm -o foo_cpp.bc foo.cpp + +Next, link the two IR files into a single file and run the LLVM optimizer +on the result: + +:: + + llvm-link foo_ispc.bc foo_cpp.bc -o - | opt -O3 -o foo_opt.bc + +And finally, generate a native object file: + +:: + + llc -filetype=obj foo_opt.bc -o foo.o + +This file can in turn be linked in with the rest of your object files when +linking your applicaiton. + +(Note that if you're using the AVX instruction set, you must provide the +``-mattr=+avx`` flag to ``llc``.) + + Programming Techniques ====================== From 1dead425e410bc46434f56c0de3bcb621a19fe1f Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 10 Feb 2012 12:26:35 -0800 Subject: [PATCH 04/55] Don't indent *too* much on continued lines with warnings/errors. --- util.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util.cpp b/util.cpp index b7976e64..7057755b 100644 --- a/util.cpp +++ b/util.cpp @@ -171,8 +171,8 @@ lPrintWithWordBreaks(const char *buf, int columnWidth, FILE *out) { column = indent; outStr.push_back('\n'); // Indent to the same column as the ":" at the start of the - // message - for (int i = 0; i < indent; ++i) + // message, unless doing so would be too far in. + for (int i = 0; i < std::min(16, indent); ++i) outStr.push_back(' '); } From fe2d9aa6004a65080fc3309330401fcb5dbcd7e1 Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 10 Feb 2012 12:27:13 -0800 Subject: [PATCH 05/55] Add perfbench to examples: a few small microbenchmarks. --- examples/README.txt | 10 +- examples/perfbench/Makefile | 7 + examples/perfbench/perfbench.cpp | 108 +++++++++++++++ examples/perfbench/perfbench.ispc | 170 +++++++++++++++++++++++ examples/perfbench/perfbench.vcxproj | 175 ++++++++++++++++++++++++ examples/perfbench/perfbench_serial.cpp | 61 +++++++++ 6 files changed, 528 insertions(+), 3 deletions(-) create mode 100644 examples/perfbench/Makefile create mode 100644 examples/perfbench/perfbench.cpp create mode 100644 examples/perfbench/perfbench.ispc create mode 100644 examples/perfbench/perfbench.vcxproj create mode 100644 examples/perfbench/perfbench_serial.cpp diff --git a/examples/README.txt b/examples/README.txt index 7dcae8bf..25aa3a21 100644 --- a/examples/README.txt +++ b/examples/README.txt @@ -39,9 +39,6 @@ example implementation of this function that counts the number of times the callback is made and records some statistics about control flow coherence is provided in the instrument.cpp file. -*** Note: on Linux, this example currently hits an assertion in LLVM during -*** compilation - Deferred ======== @@ -110,6 +107,13 @@ This program implements both the Black-Scholes and Binomial options pricing models in both ispc and regular serial C++ code. +Perfbench +========= + +This runs a number of microbenchmarks to measure system performance and +code generation quality. + + RT == diff --git a/examples/perfbench/Makefile b/examples/perfbench/Makefile new file mode 100644 index 00000000..43684c71 --- /dev/null +++ b/examples/perfbench/Makefile @@ -0,0 +1,7 @@ + +EXAMPLE=perbench +CPP_SRC=perfbench.cpp perfbench_serial.cpp +ISPC_SRC=perfbench.ispc +ISPC_TARGETS=sse2,sse4,avx + +include ../common.mk diff --git a/examples/perfbench/perfbench.cpp b/examples/perfbench/perfbench.cpp new file mode 100644 index 00000000..04e72bd9 --- /dev/null +++ b/examples/perfbench/perfbench.cpp @@ -0,0 +1,108 @@ +/* + Copyright (c) 2012, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef _MSC_VER +#define _CRT_SECURE_NO_WARNINGS +#define NOMINMAX +#pragma warning (disable: 4244) +#pragma warning (disable: 4305) +#endif + +#include +#include +#include "../timing.h" + +#include "perfbench_ispc.h" + +typedef void (FuncType)(float *, int, float *, float *); + +struct PerfTest { + FuncType *aFunc; + const char *aName; + FuncType *bFunc; + const char *bName; + const char *testName; +}; + +extern void xyzSumAOS(float *a, int count, float *zeros, float *result); +extern void xyzSumSOA(float *a, int count, float *zeros, float *result); + + +static void +lInitData(float *ptr, int count) { + for (int i = 0; i < count; ++i) + ptr[i] = float(i) / (1024.f * 1024.f); +} + +static PerfTest tests[] = { + { xyzSumAOS, "serial", ispc::xyzSumAOS, "ispc", "AOS vector element sum (with coalescing)" }, + { xyzSumAOS, "serial", ispc::xyzSumAOSStdlib, "ispc", "AOS vector element sum (stdlib swizzle)" }, + { xyzSumAOS, "serial", ispc::xyzSumAOSNoCoalesce, "ispc", "AOS vector element sum (no coalescing)" }, + { xyzSumSOA, "serial", ispc::xyzSumSOA, "ispc", "SOA vector element sum" }, + { ispc::gathers, "gather", ispc::loads, "vector load", "Memory reads" }, + { ispc::scatters, "scatter", ispc::stores, "vector store", "Memory writes" }, +}; + +int main() { + int count = 3*64*1024; + float *a = new float[count]; + float zeros[32] = { 0 }; + + int nTests = sizeof(tests) / sizeof(tests[0]); + for (int i = 0; i < nTests; ++i) { + lInitData(a, count); + reset_and_start_timer(); + float resultA[3] = { 0, 0, 0 }; + for (int j = 0; j < 100; ++j) + tests[i].aFunc(a, count, zeros, resultA); + double aTime = get_elapsed_mcycles(); + + lInitData(a, count); + reset_and_start_timer(); + float resultB[3] = { 0, 0, 0 }; + for (int j = 0; j < 100; ++j) + tests[i].bFunc(a, count, zeros, resultB); + double bTime = get_elapsed_mcycles(); + + printf("%-40s: [%.2f] M cycles %s, [%.2f] M cycles %s (%.2fx speedup).\n", + tests[i].testName, aTime, tests[i].aName, bTime, tests[i].bName, + aTime/bTime); +#if 0 + printf("\t(%f %f %f) - (%f %f %f)\n", resultSerial[0], resultSerial[1], + resultSerial[2], resultISPC[0], resultISPC[1], resultISPC[2]); +#endif + } + + return 0; +} + diff --git a/examples/perfbench/perfbench.ispc b/examples/perfbench/perfbench.ispc new file mode 100644 index 00000000..38fe6cee --- /dev/null +++ b/examples/perfbench/perfbench.ispc @@ -0,0 +1,170 @@ +/* + Copyright (c) 2012, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +export void xyzSumAOS(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float xsum = 0, ysum = 0, zsum = 0; + foreach (i = 0 ... count/3) { + float x = array[3*i]; + float y = array[3*i+1]; + float z = array[3*i+2]; + + xsum += x; + ysum += y; + zsum += z; + } + result[0] = reduce_add(xsum); + result[1] = reduce_add(ysum); + result[2] = reduce_add(zsum); +} + +export void xyzSumAOSStdlib(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float xsum = 0, ysum = 0, zsum = 0; + for (uniform int i = 0; i < 64*1024 /*count/3*/; i += programCount) { + float x, y, z; + aos_to_soa3(&array[3*i], &x, &y, &z); + + xsum += x; + ysum += y; + zsum += z; + } + result[0] = reduce_add(xsum); + result[1] = reduce_add(ysum); + result[2] = reduce_add(zsum); +} + +export void xyzSumAOSNoCoalesce(uniform float array[], uniform int count, + uniform float zerosArray[], uniform float result[]) { + int zeros = zerosArray[programIndex]; + float xsum = 0, ysum = 0, zsum = 0; + foreach (i = 0 ... count/3) { + float x = array[3*i+zeros]; + float y = array[3*i+1+zeros]; + float z = array[3*i+2+zeros]; + + xsum += x; + ysum += y; + zsum += z; + } + result[0] = reduce_add(xsum); + result[1] = reduce_add(ysum); + result[2] = reduce_add(zsum); +} + +export void xyzSumSOA(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float xsum = 0, ysum = 0, zsum = 0; + uniform float * uniform ap = array; + assert(programCount <= 8); + + for (uniform int i = 0; i < count/3; i += 8, ap += 24) { + for (uniform int j = 0; j < 8; j += programCount) { + float x = ap[j + programIndex]; + float y = ap[8 + j + programIndex]; + float z = ap[16 + j + programIndex]; + + xsum += x; + ysum += y; + zsum += z; + } + } + result[0] = reduce_add(xsum); + result[1] = reduce_add(ysum); + result[2] = reduce_add(zsum); +} + +export void gathers(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float sum = 0; + int zero = zeros[programIndex]; + foreach (i = 0 ... count) + sum += array[i + zero]; + result[0] = reduce_add(sum); +} + + +export void loads(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float sum = 0; + foreach (i = 0 ... count) + sum += array[i]; + result[0] = reduce_add(sum); +} + + +export void scatters(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + int zero = zeros[programIndex]; + foreach (i = 0 ... count) + array[i + zero] = zero; +} + + +export void stores(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + int zero = zeros[programIndex]; + foreach (i = 0 ... count) + array[i] = zero; +} + +export void normalizeAOSNoCoalesce(uniform float array[], uniform int count, + uniform float zeroArray[]) { + float zeros = zeroArray[programIndex]; + foreach (i = 0 ... count/3) { + float x = array[3*i+zeros]; + float y = array[3*i+1+zeros]; + float z = array[3*i+2+zeros]; + + float l2 = x*x + y*y + z*z; + + array[3*i] /= l2; + array[3*i+1] /= l2; + array[3*i+2] /= l2; + } +} + +export void normalizeSOA(uniform float array[], uniform int count, + uniform float zeros[]) { + foreach (i = 0 ... count/3) { + float x = array[3*i]; + float y = array[3*i+1]; + float z = array[3*i+2]; + + float l2 = x*x + y*y + z*z; + + array[3*i] /= l2; + array[3*i+1] /= l2; + array[3*i+2] /= l2; + } +} diff --git a/examples/perfbench/perfbench.vcxproj b/examples/perfbench/perfbench.vcxproj new file mode 100644 index 00000000..31974ac7 --- /dev/null +++ b/examples/perfbench/perfbench.vcxproj @@ -0,0 +1,175 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {d923bb7e-a7c8-4850-8fcf-0eb9ce35b4e8} + Win32Proj + perfbench + + + + Application + true + Unicode + + + Application + true + Unicode + + + Application + false + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(ProjectDir)..\..;$(ExecutablePath) + + + true + $(ProjectDir)..\..;$(ExecutablePath) + + + false + $(ProjectDir)..\..;$(ExecutablePath) + + + false + $(ProjectDir)..\..;$(ExecutablePath) + + + + + + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(TargetDir) + true + Fast + + + Console + true + + + + + + + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(TargetDir) + true + Fast + + + Console + true + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(TargetDir) + Fast + + + Console + true + true + true + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(TargetDir) + Fast + + + Console + true + true + true + + + + + + + + + Document + ispc -O2 %(Filename).ispc -o $(TargetDir)%(Filename).obj -h $(TargetDir)%(Filename)_ispc.h --arch=x86 --target=sse2,sse4,avx + + ispc -O2 %(Filename).ispc -o $(TargetDir)%(Filename).obj -h $(TargetDir)%(Filename)_ispc.h --target=sse2,sse4,avx + + $(TargetDir)%(Filename).obj;$(TargetDir)%(Filename)_sse2.obj;$(TargetDir)%(Filename)_sse4.obj;$(TargetDir)%(Filename)_avx.obj;$(TargetDir)%(Filename)_ispc.h + $(TargetDir)%(Filename).obj;$(TargetDir)%(Filename)_sse2.obj;$(TargetDir)%(Filename)_sse4.obj;$(TargetDir)%(Filename)_avx.obj;$(TargetDir)%(Filename)_ispc.h + ispc -O2 %(Filename).ispc -o $(TargetDir)%(Filename).obj -h $(TargetDir)%(Filename)_ispc.h --arch=x86 --target=sse2,sse4,avx + + ispc -O2 %(Filename).ispc -o $(TargetDir)%(Filename).obj -h $(TargetDir)%(Filename)_ispc.h --target=sse2,sse4,avx + + $(TargetDir)%(Filename).obj;$(TargetDir)%(Filename)_sse2.obj;$(TargetDir)%(Filename)_sse4.obj;$(TargetDir)%(Filename)_avx.obj;$(TargetDir)%(Filename)_ispc.h + $(TargetDir)%(Filename).obj;$(TargetDir)%(Filename)_sse2.obj;$(TargetDir)%(Filename)_sse4.obj;$(TargetDir)%(Filename)_avx.obj;$(TargetDir)%(Filename)_ispc.h + + + + + + diff --git a/examples/perfbench/perfbench_serial.cpp b/examples/perfbench/perfbench_serial.cpp new file mode 100644 index 00000000..dfd8e370 --- /dev/null +++ b/examples/perfbench/perfbench_serial.cpp @@ -0,0 +1,61 @@ +/* + Copyright (c) 2012, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include + +void +xyzSumAOS(float *a, int count, float *zeros, float *result) { + float xsum = 0, ysum = 0, zsum = 0; + for (int i = 0; i < count; i += 3) { + xsum += a[i]; + ysum += a[i+1]; + zsum += a[i+2]; + } + result[0] = xsum; + result[1] = ysum; + result[2] = zsum; +} + +void +xyzSumSOA(float *a, int count, float *zeros, float *result) { + float xsum = 0, ysum = 0, zsum = 0; + for (int i = 0; i < count/3; ++i) { + float *p = a + (i >> 3) * 24 + (i & 7); + xsum += p[0]; + ysum += p[8]; + zsum += p[16]; + } + result[0] = xsum; + result[1] = ysum; + result[2] = zsum; +} From 49880ab761fbc37f8a3893b27ca522d3e7a10394 Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 10 Feb 2012 12:28:54 -0800 Subject: [PATCH 06/55] Constant fold more cases in SelectExpr::Optimize() Specifically, if both of the expressions are compile-time constants and the condition is a varying compile-time constant (even if not all true or all false), then we can assemble a compile-time constant result. --- expr.cpp | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/expr.cpp b/expr.cpp index ec242fea..8925de2b 100644 --- a/expr.cpp +++ b/expr.cpp @@ -2924,10 +2924,77 @@ SelectExpr::Optimize() { // Varying test: see if all of the values are the same; if so, then // return the corresponding expression bool first = bv[0]; + bool mismatch = false; for (int i = 0; i < count; ++i) - if (bv[i] != first) - return this; - return (bv[0] == true) ? expr1 : expr2; + if (bv[i] != first) { + mismatch = true; + break; + } + if (mismatch == false) + return (bv[0] == true) ? expr1 : expr2; + + // Last chance: see if the two expressions are constants; if so, + // then we can do an element-wise selection based on the constant + // condition.. + ConstExpr *constExpr1 = dynamic_cast(expr1); + ConstExpr *constExpr2 = dynamic_cast(expr2); + if (constExpr1 == NULL || constExpr2 == NULL) + return this; + + Assert(constExpr1->GetType() == constExpr2->GetType()); + const Type *exprType = constExpr1->GetType()->GetAsNonConstType(); + Assert(exprType->IsVaryingType()); + + // FIXME: it's annoying to have to have all of this replicated code. + if (exprType == AtomicType::VaryingInt32 || + exprType == AtomicType::VaryingUInt32) { + int32_t v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + int32_t result[ISPC_MAX_NVEC]; + constExpr1->AsInt32(v1); + constExpr2->AsInt32(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + else if (exprType == AtomicType::VaryingInt64 || + exprType == AtomicType::VaryingUInt64) { + int64_t v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + int64_t result[ISPC_MAX_NVEC]; + constExpr1->AsInt64(v1); + constExpr2->AsInt64(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + else if (exprType == AtomicType::VaryingFloat) { + float v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + float result[ISPC_MAX_NVEC]; + constExpr1->AsFloat(v1); + constExpr2->AsFloat(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + else if (exprType == AtomicType::VaryingDouble) { + double v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + double result[ISPC_MAX_NVEC]; + constExpr1->AsDouble(v1); + constExpr2->AsDouble(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + else if (exprType == AtomicType::VaryingBool) { + bool v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + bool result[ISPC_MAX_NVEC]; + constExpr1->AsBool(v1); + constExpr2->AsBool(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + + return this; } } From 0c8ad09040739794a7c4ac08a14614a9a1f09104 Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 10 Feb 2012 12:29:57 -0800 Subject: [PATCH 07/55] Fix placement of ParserInit() call This makes it possible to use fuzz testing even without --nostdlib! --- module.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module.cpp b/module.cpp index e192c39d..102445cc 100644 --- a/module.cpp +++ b/module.cpp @@ -153,6 +153,9 @@ Module::CompileFile() { llvm::UnsafeFPMath = true; #endif // !LLVM_3_1svn + extern void ParserInit(); + ParserInit(); + // FIXME: it'd be nice to do this in the Module constructor, but this // function ends up calling into routines that expect the global // variable 'm' to be initialized and available (which it isn't until @@ -161,9 +164,6 @@ Module::CompileFile() { bool runPreprocessor = g->runCPP; - extern void ParserInit(); - ParserInit(); - if (runPreprocessor) { if (filename != NULL) { // Try to open the file first, since otherwise we crash in the From db72781d2a9f921506deb5d0fd8ad32c6f7e941d Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 10 Feb 2012 12:30:31 -0800 Subject: [PATCH 08/55] Fix C++ backend to not assert with LLVM 3.1 svn builds. --- cbackend.cpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cbackend.cpp b/cbackend.cpp index 7f27f0cb..4550cd87 100644 --- a/cbackend.cpp +++ b/cbackend.cpp @@ -363,6 +363,9 @@ namespace { bool printConstExprCast(const ConstantExpr *CE, bool Static); void printConstantArray(ConstantArray *CPA, bool Static); void printConstantVector(ConstantVector *CV, bool Static); +#ifdef LLVM_3_1svn + void printConstantDataVector(ConstantDataVector *CV, bool Static); +#endif /// isAddressExposed - Return true if the specified value's name needs to /// have its address taken in order to get a C value of the correct type. @@ -885,6 +888,19 @@ void CWriter::printConstantVector(ConstantVector *CP, bool Static) { } } +#ifdef LLVM_3_1svn +void CWriter::printConstantDataVector(ConstantDataVector *CP, bool Static) { + if (CP->getNumElements()) { + Out << ' '; + printConstant(cast(CP->getElementAsConstant(0)), Static); + for (unsigned i = 1, e = CP->getNumElements(); i != e; ++i) { + Out << ", "; + printConstant(cast(CP->getElementAsConstant(i)), Static); + } + } +} +#endif // LLVM_3_1svn + // isFPCSafeToPrint - Returns true if we may assume that CFP may be written out // textually as a double (rather than as a reference to a stack-allocated // variable). We decide this by converting CFP to a string and back into a @@ -1376,7 +1392,13 @@ void CWriter::printConstant(Constant *CPV, bool Static) { if (ConstantVector *CV = dyn_cast(CPV)) { printConstantVector(CV, Static); - } else { + } +#ifdef LLVM_3_1svn + else if (ConstantDataVector *CDV = dyn_cast(CPV)) { + printConstantDataVector(CDV, Static); + } +#endif + else { assert(isa(CPV) || isa(CPV)); VectorType *VT = cast(CPV->getType()); Constant *CZ = Constant::getNullValue(VT->getElementType()); From 0c25bc063c93ad7b494efaffd10a37ffec5c8fde Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 10 Feb 2012 12:32:15 -0800 Subject: [PATCH 09/55] Add lGEPInst() utility routine to opt.cpp. Deal with the messiness of LLVM API changes when creating these in a single place. --- opt.cpp | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/opt.cpp b/opt.cpp index 0cfb360c..74733804 100644 --- a/opt.cpp +++ b/opt.cpp @@ -250,6 +250,22 @@ lCallInst(llvm::Function *func, llvm::Value *arg0, llvm::Value *arg1, #endif } + +static llvm::Instruction * +lGEPInst(llvm::Value *ptr, llvm::Value *offset, const char *name, + llvm::Instruction *insertBefore) { + llvm::Value *index[1] = { offset }; +#if defined(LLVM_3_0) || defined(LLVM_3_0svn) || defined(LLVM_3_1svn) + llvm::ArrayRef arrayRef(&index[0], &index[1]); + return llvm::GetElementPtrInst::Create(ptr, arrayRef, name, + insertBefore); +#else + return llvm::GetElementPtrInst::Create(ptr, &index[0], &index[1], + name, insertBefore); +#endif +} + + /////////////////////////////////////////////////////////////////////////// void @@ -1585,16 +1601,7 @@ lExtractUniformsFromOffset(llvm::Value **basePtr, llvm::Value **offsetVector, if (uniformDelta == NULL) return; - llvm::Value *index[1] = { uniformDelta }; -#if defined(LLVM_3_0) || defined(LLVM_3_0svn) || defined(LLVM_3_1svn) - llvm::ArrayRef arrayRef(&index[0], &index[1]); - *basePtr = llvm::GetElementPtrInst::Create(*basePtr, arrayRef, "new_base", - insertBefore); -#else - *basePtr = llvm::GetElementPtrInst::Create(*basePtr, &index[0], - &index[1], "new_base", - insertBefore); -#endif + *basePtr = lGEPInst(*basePtr, arrayRef, "new_base", insertBefore); // this should only happen if we have only uniforms, but that in turn // shouldn't be a gather/scatter! @@ -2392,17 +2399,7 @@ lComputeCommonPointer(llvm::Value *base, llvm::Value *offsets, llvm::Value *firstOffset = llvm::ExtractElementInst::Create(offsets, LLVMInt32(0), "first_offset", insertBefore); - - llvm::Value *offsetIndex[1] = { firstOffset }; -#if defined(LLVM_3_0) || defined(LLVM_3_0svn) || defined(LLVM_3_1svn) - llvm::ArrayRef arrayRef(&offsetIndex[0], &offsetIndex[1]); - return - llvm::GetElementPtrInst::Create(base, arrayRef, "ptr", insertBefore); -#else - return - llvm::GetElementPtrInst::Create(base, &offsetIndex[0], &offsetIndex[1], - "ptr", insertBefore); -#endif + return lGEPInst(base, firstOffset, "ptr", insertBefore); } From f20a2d2ee970c21dc508ba20a2f65230b410c81b Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 10 Feb 2012 12:35:44 -0800 Subject: [PATCH 10/55] Generalize code to extract scales by 2/4/8 from addressing calculations. Now, if we have a scale by 16, say, we extract out the scalar scale of 8 and leave an explicit scale by 2. --- opt.cpp | 56 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/opt.cpp b/opt.cpp index 74733804..d64cdcbd 100644 --- a/opt.cpp +++ b/opt.cpp @@ -1389,10 +1389,10 @@ lExtractConstantOffset(llvm::Value *vec, llvm::Value **constOffset, /* Returns true if the given value is a constant vector of integers with - the value 2, 4, 8 in all of the elements. (Returns the splatted value - in *splat, if so). */ + the same value in all of the elements. (Returns the splatted value in + *splat, if so). */ static bool -lIs248Splat(llvm::Value *v, int *splat) { +lIsIntegerSplat(llvm::Value *v, int *splat) { #ifdef LLVM_3_1svn llvm::ConstantDataVector *cvec = llvm::dyn_cast(v); @@ -1412,14 +1412,46 @@ lIs248Splat(llvm::Value *v, int *splat) { return false; int64_t splatVal = ci->getSExtValue(); - if (splatVal != 2 && splatVal != 4 && splatVal != 8) - return false; - *splat = (int)splatVal; return true; } +static llvm::Value * +lExtract248Scale(llvm::Value *splatOperand, int splatValue, + llvm::Value *otherOperand, llvm::Value **result) { + if (splatValue == 2 || splatValue == 4 || splatValue == 8) { + *result = otherOperand; + return LLVMInt32(splatValue); + } + // Even if we don't have a common scale by exactly 2, 4, or 8, we'll + // see if we can pull out that much of the scale anyway; this may in + // turn allow other optimizations later. + for (int scale = 8; scale >= 2; scale /= 2) { + llvm::Instruction *insertBefore = + llvm::dyn_cast(*result); + Assert(insertBefore != NULL); + + if ((splatValue % scale) == 0) { + // *result = otherOperand + splatOperand / splatValue; + llvm::Value *splatScaleVec = + (splatOperand->getType() == LLVMTypes::Int32VectorType) ? + LLVMInt32Vector(scale) : LLVMInt64Vector(scale); + llvm::Value *splatDiv = + llvm::BinaryOperator::Create(llvm::Instruction::SDiv, + splatOperand, splatScaleVec, + "div", insertBefore); + *result = + llvm::BinaryOperator::Create(llvm::Instruction::Add, + splatDiv, otherOperand, + "add", insertBefore); + return LLVMInt32(scale); + } + } + return LLVMInt32(1); +} + + /** Given a vector of integer offsets to a base pointer being used for a gather or a scatter, see if its root operation is a multiply by a vector of some value by all 2s/4s/8s. If not, return NULL. @@ -1482,14 +1514,10 @@ lExtractOffsetVector248Scale(llvm::Value **vec) { else if (bop->getOpcode() == llvm::Instruction::Mul) { // Check each operand for being one of the scale factors we care about. int splat; - if (lIs248Splat(op0, &splat)) { - *vec = op1; - return LLVMInt32(splat); - } - else if (lIs248Splat(op1, &splat)) { - *vec = op0; - return LLVMInt32(splat); - } + if (lIsIntegerSplat(op0, &splat)) + return lExtract248Scale(op0, splat, op1, vec); + else if (lIsIntegerSplat(op1, &splat)) + return lExtract248Scale(op1, splat, op0, vec); else return LLVMInt32(1); } From 73bf552cd6e517bd96589dd83086b80be83172f8 Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 10 Feb 2012 12:46:59 -0800 Subject: [PATCH 11/55] Add support for coalescing memory accesses from gathers. There are two related optimizations that happen now. (These currently only apply for gathers where the mask is known to be all on, and to gathers that are accessing 32-bit sized elements, but both of these may be generalized in the future.) First, for any single gather, we are now more flexible in mapping it to individual memory operations. Previously, we would only either map it to a general gather (one scalar load per SIMD lane), or an unaligned vector load (if the program instances could be determined to be accessing a sequential set of locations in memory.) Now, we are able to break gathers into scalar, 2-wide (i.e. 64-bit), 4-wide, or 8-wide loads. Further, we now generate code that shuffles these loads around. Doing fewer, larger loads in this manner, when possible, can be more efficient. Second, we can coalesce memory accesses across multiple gathers. If we have a series of gathers without any memory writes in the middle, then we try to analyze their reads collectively and choose an efficient set of loads for them. Not only does this help if different gathers reuse values from the same location in memory, but it's specifically helpful when data with AOS layout is being accessed; in this case, we're often able to generate wide vector loads and appropriate shuffles automatically. --- ispc.cpp | 1 + ispc.h | 4 + main.cpp | 11 +- opt.cpp | 1156 ++++++++++++++++++++++++++++++++++++++++- tests/coalesce-1.ispc | 14 + tests/coalesce-2.ispc | 14 + tests/coalesce-3.ispc | 14 + tests/coalesce-4.ispc | 17 + tests/coalesce-5.ispc | 20 + tests/coalesce-6.ispc | 21 + tests/coalesce-7.ispc | 21 + tests/coalesce-8.ispc | 19 + 12 files changed, 1307 insertions(+), 5 deletions(-) create mode 100644 tests/coalesce-1.ispc create mode 100644 tests/coalesce-2.ispc create mode 100644 tests/coalesce-3.ispc create mode 100644 tests/coalesce-4.ispc create mode 100644 tests/coalesce-5.ispc create mode 100644 tests/coalesce-6.ispc create mode 100644 tests/coalesce-7.ispc create mode 100644 tests/coalesce-8.ispc diff --git a/ispc.cpp b/ispc.cpp index 6729da92..49623be4 100644 --- a/ispc.cpp +++ b/ispc.cpp @@ -497,6 +497,7 @@ Opt::Opt() { disableMaskedStoreToStore = false; disableGatherScatterFlattening = false; disableUniformMemoryOptimizations = false; + disableCoalescing = false; } /////////////////////////////////////////////////////////////////////////// diff --git a/ispc.h b/ispc.h index 59c9140f..c28a9f67 100644 --- a/ispc.h +++ b/ispc.h @@ -339,6 +339,10 @@ struct Opt { than gathers/scatters. This is likely only useful for measuring the impact of this optimization. */ bool disableUniformMemoryOptimizations; + + /** Disables optimizations that coalesce incoherent scalar memory + access from gathers into wider vector operations, when possible. */ + bool disableCoalescing; }; /** @brief This structure collects together a number of global variables. diff --git a/main.cpp b/main.cpp index 7b8c66d5..faee437f 100644 --- a/main.cpp +++ b/main.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2010-2011, Intel Corporation + Copyright (c) 2010-2012, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -141,16 +141,17 @@ devUsage(int ret) { printf(" [--fuzz-test]\t\t\tRandomly perturb program input to test error conditions\n"); printf(" [--fuzz-seed=]\t\tSeed value for RNG for fuzz testing\n"); printf(" [--opt=