From d7b0c5794ef819a4c039a725681bdabbac946223 Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Fri, 19 Jul 2013 11:06:11 -0700 Subject: [PATCH] Add support for ARM NEON targets. Initial support for ARM NEON on Cortex-A9 and A15 CPUs. All but ~10 tests pass, and all examples compile and run correctly. Most of the examples show a ~2x speedup on a single A15 core versus scalar code. Current open issues/TODOs - Code quality looks decent, but hasn't been carefully examined. Known issues/opportunities for improvement include: - fp32 vector divide is done as a series of scalar divides rather than a vector divide (which I believe exists, but I may be mistaken.) This is particularly harmful to examples/rt, which only runs ~1.5x faster with ispc, likely due to long chains of scalar divides. - The compiler isn't generating a vmin.f32 for e.g. the final scalar min in reduce_min(); instead it's generating a compare and then a select instruction (and similarly elsewhere). - There are some additional FIXMEs in builtins/target-neon.ll that include both a few pieces of missing functionality (e.g. rounding doubles) as well as places that deserve attention for possible code quality improvements. - Currently only the "cortex-a9" and "cortex-15" CPU targets are supported; LLVM supports many other ARM CPUs and ispc should provide access to all of the ones that have NEON support (and aren't too obscure.) - ~5 of the reduce-* tests hit an assertion inside LLVM (unfortunately only when the compiler runs on an ARM host, though). - The Windows build hasn't been tested (though I've tried to update ispc.vcxproj appropriately). It may just work, but will more likely have various small issues.) - Anything related to 64-bit ARM has seen no attention. --- Makefile | 4 +- builtins.cpp | 67 ++- builtins/target-neon.ll | 706 +++++++++++++++++++++++++++++ examples/aobench/Makefile | 3 +- examples/common.mk | 24 +- examples/deferred/Makefile | 3 +- examples/gmres/Makefile | 3 +- examples/mandelbrot/Makefile | 3 +- examples/mandelbrot_tasks/Makefile | 3 +- examples/noise/Makefile | 3 +- examples/options/Makefile | 3 +- examples/perfbench/Makefile | 3 +- examples/rt/Makefile | 3 +- examples/sort/Makefile | 3 +- examples/stencil/Makefile | 3 +- examples/timing.h | 52 ++- examples/volume_rendering/Makefile | 3 +- ispc.cpp | 47 +- ispc.h | 2 +- ispc.vcxproj | 15 + main.cpp | 12 + run_tests.py | 16 +- 22 files changed, 914 insertions(+), 67 deletions(-) create mode 100644 builtins/target-neon.ll diff --git a/Makefile b/Makefile index d0466fdf..835f8e15 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ LLVM_CXXFLAGS=$(shell $(LLVM_CONFIG) --cppflags) LLVM_VERSION=LLVM_$(shell $(LLVM_CONFIG) --version | sed -e s/\\./_/ -e s/svn//) LLVM_VERSION_DEF=-D$(LLVM_VERSION) -LLVM_COMPONENTS = engine ipo bitreader bitwriter instrumentation linker +LLVM_COMPONENTS = engine ipo bitreader bitwriter instrumentation linker arm # Component "option" was introduced in 3.3 and starting with 3.4 it is required for the link step. # We check if it's available before adding it (to not break 3.2 and earlier). ifeq ($(shell $(LLVM_CONFIG) --components |grep -c option), 1) @@ -122,7 +122,7 @@ CXX_SRC=ast.cpp builtins.cpp cbackend.cpp ctx.cpp decl.cpp expr.cpp func.cpp \ type.cpp util.cpp HEADERS=ast.h builtins.h ctx.h decl.h expr.h func.h ispc.h llvmutil.h module.h \ opt.h stmt.h sym.h type.h util.h -TARGETS=avx1 avx1-x2 avx11 avx11-x2 avx2 avx2-x2 sse2 sse2-x2 sse4 sse4-x2 \ +TARGETS=neon avx1 avx1-x2 avx11 avx11-x2 avx2 avx2-x2 sse2 sse2-x2 sse4 sse4-x2 \ generic-4 generic-8 generic-16 generic-32 generic-64 generic-1 # These files need to be compiled in two versions - 32 and 64 bits. BUILTINS_SRC_TARGET=$(addprefix builtins/target-, $(addsuffix .ll, $(TARGETS))) diff --git a/builtins.cpp b/builtins.cpp index ab0e0cd8..3e03de10 100644 --- a/builtins.cpp +++ b/builtins.cpp @@ -638,26 +638,50 @@ AddBitcodeToModule(const unsigned char *bitcode, int length, // linking together modules with incompatible target triples.. llvm::Triple mTriple(m->module->getTargetTriple()); llvm::Triple bcTriple(bcModule->getTargetTriple()); - Assert(bcTriple.getArch() == llvm::Triple::UnknownArch || - mTriple.getArch() == bcTriple.getArch()); - Assert(bcTriple.getVendor() == llvm::Triple::UnknownVendor || - mTriple.getVendor() == bcTriple.getVendor()); - bcModule->setTargetTriple(mTriple.str()); + Debug(SourcePos(), "module triple: %s\nbitcode triple: %s\n", + mTriple.str().c_str(), bcTriple.str().c_str()); +#ifndef __arm__ + // FIXME: More ugly and dangerous stuff. We really haven't set up + // proper build and runtime infrastructure for ispc to do + // cross-compilation, yet it's at minimum useful to be able to emit + // ARM code from x86 for ispc development. One side-effect is that + // when the build process turns builtins/builtins.c to LLVM bitcode + // for us to link in at runtime, that bitcode has been compiled for + // an IA target, which in turn causes the checks in the following + // code to (appropraitely) fail. + // + // In order to be able to have some ability to generate ARM code on + // IA, we'll just skip those tests in that case and allow the + // setTargetTriple() and setDataLayout() calls below to shove in + // the values for an ARM target. This maybe won't cause problems + // in the generated code, since bulitins.c doesn't do anything too + // complex w.r.t. struct layouts, etc. + if (g->target->getISA() != Target::NEON) +#endif // !__arm__ + { + Assert(bcTriple.getArch() == llvm::Triple::UnknownArch || + mTriple.getArch() == bcTriple.getArch()); + Assert(bcTriple.getVendor() == llvm::Triple::UnknownVendor || + mTriple.getVendor() == bcTriple.getVendor()); - // We unconditionally set module DataLayout to library, but we must - // ensure that library and module DataLayouts are compatible. - // If they are not, we should recompile the library for problematic - // architecture and investigate what happened. - // Generally we allow library DataLayout to be subset of module - // DataLayout or library DataLayout to be empty. - if (!VerifyDataLayoutCompatibility(module->getDataLayout(), - bcModule->getDataLayout())) { - Error(SourcePos(), "Module DataLayout is incompatible with library DataLayout:\n" - "Module DL: %s\n" - "Library DL: %s\n", - module->getDataLayout().c_str(), bcModule->getDataLayout().c_str()); + // We unconditionally set module DataLayout to library, but we must + // ensure that library and module DataLayouts are compatible. + // If they are not, we should recompile the library for problematic + // architecture and investigate what happened. + // Generally we allow library DataLayout to be subset of module + // DataLayout or library DataLayout to be empty. + if (!VerifyDataLayoutCompatibility(module->getDataLayout(), + bcModule->getDataLayout())) { + Warning(SourcePos(), "Module DataLayout is incompatible with " + "library DataLayout:\n" + "Module DL: %s\n" + "Library DL: %s\n", + module->getDataLayout().c_str(), + bcModule->getDataLayout().c_str()); + } } + bcModule->setTargetTriple(mTriple.str()); bcModule->setDataLayout(module->getDataLayout()); std::string(linkError); @@ -795,6 +819,15 @@ DefineStdlib(SymbolTable *symbolTable, llvm::LLVMContext *ctx, llvm::Module *mod // Next, add the target's custom implementations of the various needed // builtin functions (e.g. __masked_store_32(), etc). switch (g->target->getISA()) { + case Target::NEON: { + if (runtime32) { + EXPORT_MODULE(builtins_bitcode_neon_32bit); + } + else { + EXPORT_MODULE(builtins_bitcode_neon_64bit); + } + break; + } case Target::SSE2: { switch (g->target->getVectorWidth()) { case 4: diff --git a/builtins/target-neon.ll b/builtins/target-neon.ll new file mode 100644 index 00000000..e70b774b --- /dev/null +++ b/builtins/target-neon.ll @@ -0,0 +1,706 @@ +;; +;; target-neon.ll +;; +;; Copyright(c) 2012-2013 Matt Pharr +;; Copyright(c) 2013 Google, Inc. +;; +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; * Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; +;; * Neither the name of Matt Pharr nor the names of its +;; contributors may be used to endorse or promote products derived from +;; this software without specific prior written permission. +;; +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +;; IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +;; TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +;; PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +;; OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +;; EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +;; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +target datalayout = "e-p:32:32:32-S32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f16:16:16-f32:32:32-f64:32:64-f128:128:128-v64:32:64-v128:32:128-a0:0:64-n32" + +define(`WIDTH',`4') + +define(`MASK',`i32') + +include(`util.m4') + +stdlib_core() +scans() +reduce_equal(WIDTH) +rdrand_decls() +define_shuffles() +aossoa() +ctlztz() + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; half conversion routines + +declare <4 x i16> @llvm.arm.neon.vcvtfp2hf(<4 x float>) nounwind readnone +declare <4 x float> @llvm.arm.neon.vcvthf2fp(<4 x i16>) nounwind readnone + +define float @__half_to_float_uniform(i16 %v) nounwind readnone { + %v1 = bitcast i16 %v to <1 x i16> + %vec = shufflevector <1 x i16> %v1, <1 x i16> undef, + <4 x i32> + %h = call <4 x float> @llvm.arm.neon.vcvthf2fp(<4 x i16> %vec) + %r = extractelement <4 x float> %h, i32 0 + ret float %r +} + +define <4 x float> @__half_to_float_varying(<4 x i16> %v) nounwind readnone { + %r = call <4 x float> @llvm.arm.neon.vcvthf2fp(<4 x i16> %v) + ret <4 x float> %r +} + +define i16 @__float_to_half_uniform(float %v) nounwind readnone { + %v1 = bitcast float %v to <1 x float> + %vec = shufflevector <1 x float> %v1, <1 x float> undef, + <4 x i32> + %h = call <4 x i16> @llvm.arm.neon.vcvtfp2hf(<4 x float> %vec) + %r = extractelement <4 x i16> %h, i32 0 + ret i16 %r +} + + +define <4 x i16> @__float_to_half_varying(<4 x float> %v) nounwind readnone { + %r = call <4 x i16> @llvm.arm.neon.vcvtfp2hf(<4 x float> %v) + ret <4 x i16> %r +} + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; math + +define void @__fastmath() nounwind { + ret void +} + +;; round/floor/ceil + +;; FIXME: grabbed these from the sse2 target, which does not have native +;; instructions for these. Is there a better approach for NEON? + +define float @__round_uniform_float(float) nounwind readonly alwaysinline { + %float_to_int_bitcast.i.i.i.i = bitcast float %0 to i32 + %bitop.i.i = and i32 %float_to_int_bitcast.i.i.i.i, -2147483648 + %bitop.i = xor i32 %bitop.i.i, %float_to_int_bitcast.i.i.i.i + %int_to_float_bitcast.i.i40.i = bitcast i32 %bitop.i to float + %binop.i = fadd float %int_to_float_bitcast.i.i40.i, 8.388608e+06 + %binop21.i = fadd float %binop.i, -8.388608e+06 + %float_to_int_bitcast.i.i.i = bitcast float %binop21.i to i32 + %bitop31.i = xor i32 %float_to_int_bitcast.i.i.i, %bitop.i.i + %int_to_float_bitcast.i.i.i = bitcast i32 %bitop31.i to float + ret float %int_to_float_bitcast.i.i.i +} + +define float @__floor_uniform_float(float) nounwind readonly alwaysinline { + %calltmp.i = tail call float @__round_uniform_float(float %0) nounwind + %bincmp.i = fcmp ogt float %calltmp.i, %0 + %selectexpr.i = sext i1 %bincmp.i to i32 + %bitop.i = and i32 %selectexpr.i, -1082130432 + %int_to_float_bitcast.i.i.i = bitcast i32 %bitop.i to float + %binop.i = fadd float %calltmp.i, %int_to_float_bitcast.i.i.i + ret float %binop.i +} + +define float @__ceil_uniform_float(float) nounwind readonly alwaysinline { + %calltmp.i = tail call float @__round_uniform_float(float %0) nounwind + %bincmp.i = fcmp olt float %calltmp.i, %0 + %selectexpr.i = sext i1 %bincmp.i to i32 + %bitop.i = and i32 %selectexpr.i, 1065353216 + %int_to_float_bitcast.i.i.i = bitcast i32 %bitop.i to float + %binop.i = fadd float %calltmp.i, %int_to_float_bitcast.i.i.i + ret float %binop.i +} + +define <4 x float> @__round_varying_float(<4 x float>) nounwind readonly alwaysinline { + %float_to_int_bitcast.i.i.i.i = bitcast <4 x float> %0 to <4 x i32> + %bitop.i.i = and <4 x i32> %float_to_int_bitcast.i.i.i.i, + %bitop.i = xor <4 x i32> %float_to_int_bitcast.i.i.i.i, %bitop.i.i + %int_to_float_bitcast.i.i40.i = bitcast <4 x i32> %bitop.i to <4 x float> + %binop.i = fadd <4 x float> %int_to_float_bitcast.i.i40.i, + %binop21.i = fadd <4 x float> %binop.i, + %float_to_int_bitcast.i.i.i = bitcast <4 x float> %binop21.i to <4 x i32> + %bitop31.i = xor <4 x i32> %float_to_int_bitcast.i.i.i, %bitop.i.i + %int_to_float_bitcast.i.i.i = bitcast <4 x i32> %bitop31.i to <4 x float> + ret <4 x float> %int_to_float_bitcast.i.i.i +} + +define <4 x float> @__floor_varying_float(<4 x float>) nounwind readonly alwaysinline { + %calltmp.i = tail call <4 x float> @__round_varying_float(<4 x float> %0) nounwind + %bincmp.i = fcmp ogt <4 x float> %calltmp.i, %0 + %val_to_boolvec32.i = sext <4 x i1> %bincmp.i to <4 x i32> + %bitop.i = and <4 x i32> %val_to_boolvec32.i, + %int_to_float_bitcast.i.i.i = bitcast <4 x i32> %bitop.i to <4 x float> + %binop.i = fadd <4 x float> %calltmp.i, %int_to_float_bitcast.i.i.i + ret <4 x float> %binop.i +} + +define <4 x float> @__ceil_varying_float(<4 x float>) nounwind readonly alwaysinline { + %calltmp.i = tail call <4 x float> @__round_varying_float(<4 x float> %0) nounwind + %bincmp.i = fcmp olt <4 x float> %calltmp.i, %0 + %val_to_boolvec32.i = sext <4 x i1> %bincmp.i to <4 x i32> + %bitop.i = and <4 x i32> %val_to_boolvec32.i, + %int_to_float_bitcast.i.i.i = bitcast <4 x i32> %bitop.i to <4 x float> + %binop.i = fadd <4 x float> %calltmp.i, %int_to_float_bitcast.i.i.i + ret <4 x float> %binop.i +} + +;; FIXME: rounding doubles and double vectors needs to be implemented +declare double @__round_uniform_double(double) nounwind readnone +declare double @__floor_uniform_double(double) nounwind readnone +declare double @__ceil_uniform_double(double) nounwind readnone + +declare @__round_varying_double() nounwind readnone +declare @__floor_varying_double() nounwind readnone +declare @__ceil_varying_double() nounwind readnone + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; min/max + +define float @__max_uniform_float(float, float) nounwind readnone { + %cmp = fcmp ugt float %0, %1 + %r = select i1 %cmp, float %0, float %1 + ret float %r +} + +define float @__min_uniform_float(float, float) nounwind readnone { + %cmp = fcmp ult float %0, %1 + %r = select i1 %cmp, float %0, float %1 + ret float %r +} + +define i32 @__min_uniform_int32(i32, i32) nounwind readnone { + %cmp = icmp slt i32 %0, %1 + %r = select i1 %cmp, i32 %0, i32 %1 + ret i32 %r +} + +define i32 @__max_uniform_int32(i32, i32) nounwind readnone { + %cmp = icmp sgt i32 %0, %1 + %r = select i1 %cmp, i32 %0, i32 %1 + ret i32 %r +} + +define i32 @__min_uniform_uint32(i32, i32) nounwind readnone { + %cmp = icmp ult i32 %0, %1 + %r = select i1 %cmp, i32 %0, i32 %1 + ret i32 %r +} + +define i32 @__max_uniform_uint32(i32, i32) nounwind readnone { + %cmp = icmp ugt i32 %0, %1 + %r = select i1 %cmp, i32 %0, i32 %1 + ret i32 %r +} + +define i64 @__min_uniform_int64(i64, i64) nounwind readnone { + %cmp = icmp slt i64 %0, %1 + %r = select i1 %cmp, i64 %0, i64 %1 + ret i64 %r +} + +define i64 @__max_uniform_int64(i64, i64) nounwind readnone { + %cmp = icmp sgt i64 %0, %1 + %r = select i1 %cmp, i64 %0, i64 %1 + ret i64 %r +} + +define i64 @__min_uniform_uint64(i64, i64) nounwind readnone { + %cmp = icmp ult i64 %0, %1 + %r = select i1 %cmp, i64 %0, i64 %1 + ret i64 %r +} + +define i64 @__max_uniform_uint64(i64, i64) nounwind readnone { + %cmp = icmp ugt i64 %0, %1 + %r = select i1 %cmp, i64 %0, i64 %1 + ret i64 %r +} + +define double @__min_uniform_double(double, double) nounwind readnone { + %cmp = fcmp olt double %0, %1 + %r = select i1 %cmp, double %0, double %1 + ret double %r +} + +define double @__max_uniform_double(double, double) nounwind readnone { + %cmp = fcmp ogt double %0, %1 + %r = select i1 %cmp, double %0, double %1 + ret double %r +} + +declare <4 x float> @llvm.arm.neon.vmins.v4f32(<4 x float>, <4 x float>) nounwind readnone +declare <4 x float> @llvm.arm.neon.vmaxs.v4f32(<4 x float>, <4 x float>) nounwind readnone + +define @__max_varying_float(, + ) nounwind readnone { + %r = call <4 x float> @llvm.arm.neon.vmaxs.v4f32(<4 x float> %0, <4 x float> %1) + ret %r +} + +define @__min_varying_float(, + ) nounwind readnone { + %r = call <4 x float> @llvm.arm.neon.vmins.v4f32(<4 x float> %0, <4 x float> %1) + ret %r +} + +declare <4 x i32> @llvm.arm.neon.vmins.v4i32(<4 x i32>, <4 x i32>) nounwind readnone +declare <4 x i32> @llvm.arm.neon.vminu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone +declare <4 x i32> @llvm.arm.neon.vmaxs.v4i32(<4 x i32>, <4 x i32>) nounwind readnone +declare <4 x i32> @llvm.arm.neon.vmaxu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone + +define @__min_varying_int32(, ) nounwind readnone { + %r = call <4 x i32> @llvm.arm.neon.vmins.v4i32(<4 x i32> %0, <4 x i32> %1) + ret <4 x i32> %r +} + +define @__max_varying_int32(, ) nounwind readnone { + %r = call <4 x i32> @llvm.arm.neon.vmaxs.v4i32(<4 x i32> %0, <4 x i32> %1) + ret <4 x i32> %r +} + +define @__min_varying_uint32(, ) nounwind readnone { + %r = call <4 x i32> @llvm.arm.neon.vminu.v4i32(<4 x i32> %0, <4 x i32> %1) + ret <4 x i32> %r +} + +define @__max_varying_uint32(, ) nounwind readnone { + %r = call <4 x i32> @llvm.arm.neon.vmaxu.v4i32(<4 x i32> %0, <4 x i32> %1) + ret <4 x i32> %r +} + +define @__min_varying_int64(, ) nounwind readnone { + %m = icmp slt %0, %1 + %r = select %m, %0, %1 + ret %r +} + +define @__max_varying_int64(, ) nounwind readnone { + %m = icmp sgt %0, %1 + %r = select %m, %0, %1 + ret %r +} + +define @__min_varying_uint64(, ) nounwind readnone { + %m = icmp ult %0, %1 + %r = select %m, %0, %1 + ret %r +} + +define @__max_varying_uint64(, ) nounwind readnone { + %m = icmp ugt %0, %1 + %r = select %m, %0, %1 + ret %r +} + +define @__min_varying_double(, + ) nounwind readnone { + %m = fcmp olt %0, %1 + %r = select %m, %0, %1 + ret %r +} + +define @__max_varying_double(, + ) nounwind readnone { + %m = fcmp ogt %0, %1 + %r = select %m, %0, %1 + ret %r +} + +;; sqrt/rsqrt/rcp + +declare <4 x float> @llvm.arm.neon.vrecpe.v4f32(<4 x float>) nounwind readnone +declare <4 x float> @llvm.arm.neon.vrecps.v4f32(<4 x float>, <4 x float>) nounwind readnone + +define @__rcp_varying_float( %d) nounwind readnone { + %x0 = call <4 x float> @llvm.arm.neon.vrecpe.v4f32(<4 x float> %d) + %x0_nr = call <4 x float> @llvm.arm.neon.vrecps.v4f32(<4 x float> %d, <4 x float> %x0) + %x1 = fmul <4 x float> %x0, %x0_nr + %x1_nr = call <4 x float> @llvm.arm.neon.vrecps.v4f32(<4 x float> %d, <4 x float> %x1) + %x2 = fmul <4 x float> %x1, %x1_nr + ret <4 x float> %x2 +} + +declare <4 x float> @llvm.arm.neon.vrsqrte.v4f32(<4 x float>) nounwind readnone +declare <4 x float> @llvm.arm.neon.vrsqrts.v4f32(<4 x float>, <4 x float>) nounwind readnone + +define @__rsqrt_varying_float( %d) nounwind readnone { + %x0 = call <4 x float> @llvm.arm.neon.vrsqrte.v4f32(<4 x float> %d) + %x0_2 = fmul <4 x float> %x0, %x0 + %x0_nr = call <4 x float> @llvm.arm.neon.vrsqrts.v4f32(<4 x float> %d, <4 x float> %x0_2) + %x1 = fmul <4 x float> %x0, %x0_nr + %x1_2 = fmul <4 x float> %x1, %x1 + %x1_nr = call <4 x float> @llvm.arm.neon.vrsqrts.v4f32(<4 x float> %d, <4 x float> %x1_2) + %x2 = fmul <4 x float> %x1, %x1_nr + ret <4 x float> %x2 +} + +define float @__rsqrt_uniform_float(float) nounwind readnone { + %v1 = bitcast float %0 to <1 x float> + %vs = shufflevector <1 x float> %v1, <1 x float> undef, + <4 x i32> + %vr = call <4 x float> @__rsqrt_varying_float(<4 x float> %vs) + %r = extractelement <4 x float> %vr, i32 0 + ret float %r +} + +define float @__rcp_uniform_float(float) nounwind readnone { + %v1 = bitcast float %0 to <1 x float> + %vs = shufflevector <1 x float> %v1, <1 x float> undef, + <4 x i32> + %vr = call <4 x float> @__rcp_varying_float(<4 x float> %vs) + %r = extractelement <4 x float> %vr, i32 0 + ret float %r +} + +declare float @llvm.sqrt.f32(float) + +define float @__sqrt_uniform_float(float) nounwind readnone { + %r = call float @llvm.sqrt.f32(float %0) + ret float %r +} + +declare <4 x float> @llvm.sqrt.v4f32(<4 x float>) + +define @__sqrt_varying_float() nounwind readnone { + %result = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %0) +;; this returns nan for v=0, which is undesirable.. +;; %rsqrt = call @__rsqrt_varying_float( %0) +;; %result = fmul <4 x float> %rsqrt, %0 + ret <4 x float> %result +} + +declare double @llvm.sqrt.f64(double) + +define double @__sqrt_uniform_double(double) nounwind readnone { + %r = call double @llvm.sqrt.f64(double %0) + ret double %r +} + +declare <4 x double> @llvm.sqrt.v4f64(<4 x double>) + +define @__sqrt_varying_double() nounwind readnone { + %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %0) + ret <4 x double> %r +} + +;; bit ops + +declare i32 @llvm.ctpop.i32(i32) nounwind readnone +declare i64 @llvm.ctpop.i64(i64) nounwind readnone + +define i32 @__popcnt_int32(i32) nounwind readnone { + %v = call i32 @llvm.ctpop.i32(i32 %0) + ret i32 %v +} + +define i64 @__popcnt_int64(i64) nounwind readnone { + %v = call i64 @llvm.ctpop.i64(i64 %0) + ret i64 %v +} + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; reductions + +define i64 @__movmsk(<4 x MASK>) nounwind readnone { + %and_mask = and <4 x MASK> %0, + %v01 = shufflevector <4 x i32> %and_mask, <4 x i32> undef, <2 x i32> + %v23 = shufflevector <4 x i32> %and_mask, <4 x i32> undef, <2 x i32> + %vor = or <2 x i32> %v01, %v23 + %v0 = extractelement <2 x i32> %vor, i32 0 + %v1 = extractelement <2 x i32> %vor, i32 1 + %v = or i32 %v0, %v1 + %mask64 = zext i32 %v to i64 + ret i64 %mask64 +} + +define i1 @__any(<4 x i32>) nounwind readnone alwaysinline { + %v01 = shufflevector <4 x i32> %0, <4 x i32> undef, <2 x i32> + %v23 = shufflevector <4 x i32> %0, <4 x i32> undef, <2 x i32> + %vor = or <2 x i32> %v01, %v23 + %v0 = extractelement <2 x i32> %vor, i32 0 + %v1 = extractelement <2 x i32> %vor, i32 1 + %v = or i32 %v0, %v1 + %cmp = icmp ne i32 %v, 0 + ret i1 %cmp +} + +define i1 @__all(<4 x i32>) nounwind readnone alwaysinline { + %v01 = shufflevector <4 x i32> %0, <4 x i32> undef, <2 x i32> + %v23 = shufflevector <4 x i32> %0, <4 x i32> undef, <2 x i32> + %vor = and <2 x i32> %v01, %v23 + %v0 = extractelement <2 x i32> %vor, i32 0 + %v1 = extractelement <2 x i32> %vor, i32 1 + %v = and i32 %v0, %v1 + %cmp = icmp ne i32 %v, 0 + ret i1 %cmp +} + +define i1 @__none(<4 x i32>) nounwind readnone alwaysinline { + %any = call i1 @__any(<4 x i32> %0) + %none = icmp eq i1 %any, 0 + ret i1 %none +} + +;; $1: scalar type +;; $2: vector reduce function (2 x <2 x vec> -> <2 x vec>) +;; $3 scalar reduce function + +define(`neon_reduce', ` + %v0 = shufflevector <4 x $1> %0, <4 x $1> undef, <2 x i32> + %v1 = shufflevector <4 x $1> %0, <4 x $1> undef, <2 x i32> + %vh = call <2 x $1> $2(<2 x $1> %v0, <2 x $1> %v1) + %vh0 = extractelement <2 x $1> %vh, i32 0 + %vh1 = extractelement <2 x $1> %vh, i32 1 + %r = call $1$3 ($1 %vh0, $1 %vh1) + ret $1 %r +') + +declare <2 x float> @llvm.arm.neon.vpadd.v2f32(<2 x float>, <2 x float>) nounwind readnone + +define internal float @add_f32(float, float) { + %r = fadd float %0, %1 + ret float %r +} + +define float @__reduce_add_float(<4 x float>) nounwind readnone { + neon_reduce(float, @llvm.arm.neon.vpadd.v2f32, @add_f32) +} + +declare <2 x float> @llvm.arm.neon.vpmins.v2f32(<2 x float>, <2 x float>) nounwind readnone + +define internal float @min_f32(float, float) { + %cmp = fcmp olt float %0, %1 + %r = select i1 %cmp, float %0, float %1 + ret float %r +} + +define float @__reduce_min_float(<4 x float>) nounwind readnone { + neon_reduce(float, @llvm.arm.neon.vpmins.v2f32, @min_f32) +} + +declare <2 x float> @llvm.arm.neon.vpmaxs.v2f32(<2 x float>, <2 x float>) nounwind readnone + +define internal float @max_f32(float, float) { + %cmp = fcmp ugt float %0, %1 + %r = select i1 %cmp, float %0, float %1 + ret float %r +} + +define float @__reduce_max_float(<4 x float>) nounwind readnone { + neon_reduce(float, @llvm.arm.neon.vpmaxs.v2f32, @max_f32) +} + +define internal i32 @add_i32(i32, i32) { + %r = add i32 %0, %1 + ret i32 %r +} + +declare <2 x i32> @llvm.arm.neon.vpadd.v2i32(<2 x i32>, <2 x i32>) nounwind readnone + +define i32 @__reduce_add_int32() nounwind readnone { + neon_reduce(i32, @llvm.arm.neon.vpadd.v2i32, @add_i32) +} + +declare <2 x i32> @llvm.arm.neon.vpmins.v2i32(<2 x i32>, <2 x i32>) nounwind readnone + +define internal i32 @min_si32(i32, i32) { + %cmp = icmp slt i32 %0, %1 + %r = select i1 %cmp, i32 %0, i32 %1 + ret i32 %r +} + +define i32 @__reduce_min_int32(<4 x i32>) nounwind readnone { + neon_reduce(i32, @llvm.arm.neon.vpmins.v2i32, @min_si32) +} + +declare <2 x i32> @llvm.arm.neon.vpmaxs.v2i32(<2 x i32>, <2 x i32>) nounwind readnone + +define internal i32 @max_si32(i32, i32) { + %cmp = icmp sgt i32 %0, %1 + %r = select i1 %cmp, i32 %0, i32 %1 + ret i32 %r +} + +define i32 @__reduce_max_int32(<4 x i32>) nounwind readnone { + neon_reduce(i32, @llvm.arm.neon.vpmaxs.v2i32, @max_si32) +} + +declare <2 x i32> @llvm.arm.neon.vpminu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone + +define internal i32 @min_ui32(i32, i32) { + %cmp = icmp ult i32 %0, %1 + %r = select i1 %cmp, i32 %0, i32 %1 + ret i32 %r +} + +define i32 @__reduce_min_uint32(<4 x i32>) nounwind readnone { + neon_reduce(i32, @llvm.arm.neon.vpmins.v2i32, @min_ui32) +} + +declare <2 x i32> @llvm.arm.neon.vpmaxu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone + +define internal i32 @max_ui32(i32, i32) { + %cmp = icmp ugt i32 %0, %1 + %r = select i1 %cmp, i32 %0, i32 %1 + ret i32 %r +} + +define i32 @__reduce_max_uint32(<4 x i32>) nounwind readnone { + neon_reduce(i32, @llvm.arm.neon.vpmaxs.v2i32, @max_ui32) +} + +define double @__reduce_add_double(<4 x double>) nounwind readnone { + %v0 = shufflevector <4 x double> %0, <4 x double> undef, + <2 x i32> + %v1 = shufflevector <4 x double> %0, <4 x double> undef, + <2 x i32> + %sum = fadd <2 x double> %v0, %v1 + %e0 = extractelement <2 x double> %sum, i32 0 + %e1 = extractelement <2 x double> %sum, i32 1 + %m = fadd double %e0, %e1 + ret double %m +} + +define double @__reduce_min_double(<4 x double>) nounwind readnone { + reduce4(double, @__min_varying_double, @__min_uniform_double) +} + +define double @__reduce_max_double(<4 x double>) nounwind readnone { + reduce4(double, @__max_varying_double, @__max_uniform_double) +} + +define i64 @__reduce_add_int64(<4 x i64>) nounwind readnone { + %v0 = shufflevector <4 x i64> %0, <4 x i64> undef, + <2 x i32> + %v1 = shufflevector <4 x i64> %0, <4 x i64> undef, + <2 x i32> + %sum = add <2 x i64> %v0, %v1 + %e0 = extractelement <2 x i64> %sum, i32 0 + %e1 = extractelement <2 x i64> %sum, i32 1 + %m = add i64 %e0, %e1 + ret i64 %m +} + +define i64 @__reduce_min_int64(<4 x i64>) nounwind readnone { + reduce4(i64, @__min_varying_int64, @__min_uniform_int64) +} + +define i64 @__reduce_max_int64(<4 x i64>) nounwind readnone { + reduce4(i64, @__max_varying_int64, @__max_uniform_int64) +} + +define i64 @__reduce_min_uint64(<4 x i64>) nounwind readnone { + reduce4(i64, @__min_varying_uint64, @__min_uniform_uint64) +} + +define i64 @__reduce_max_uint64(<4 x i64>) nounwind readnone { + reduce4(i64, @__max_varying_uint64, @__max_uniform_uint64) +} + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; unaligned loads/loads+broadcasts + +masked_load(i8, 1) +masked_load(i16, 2) +masked_load(i32, 4) +masked_load(float, 4) +masked_load(i64, 8) +masked_load(double, 8) + +gen_masked_store(i8) +gen_masked_store(i16) +gen_masked_store(i32) +gen_masked_store(i64) +masked_store_float_double() + +define void @__masked_store_blend_i8(* nocapture %ptr, %new, + %mask) nounwind alwaysinline { + %old = load * %ptr + %mask1 = trunc <4 x MASK> %mask to <4 x i1> + %result = select <4 x i1> %mask1, <4 x i8> %new, <4 x i8> %old + store %result, * %ptr + ret void +} + +define void @__masked_store_blend_i16(* nocapture %ptr, %new, + %mask) nounwind alwaysinline { + %old = load * %ptr + %mask1 = trunc <4 x MASK> %mask to <4 x i1> + %result = select <4 x i1> %mask1, <4 x i16> %new, <4 x i16> %old + store %result, * %ptr + ret void +} + +define void @__masked_store_blend_i32(* nocapture %ptr, %new, + %mask) nounwind alwaysinline { + %old = load * %ptr + %mask1 = trunc <4 x MASK> %mask to <4 x i1> + %result = select <4 x i1> %mask1, <4 x i32> %new, <4 x i32> %old + store %result, * %ptr + ret void +} + +define void @__masked_store_blend_i64(* nocapture %ptr, + %new, %mask) nounwind alwaysinline { + %old = load * %ptr + %mask1 = trunc <4 x MASK> %mask to <4 x i1> + %result = select <4 x i1> %mask1, <4 x i64> %new, <4 x i64> %old + store %result, * %ptr + ret void +} + +;; yuck. We need declarations of these, even though we shouldnt ever +;; actually generate calls to them for the NEON target... + +declare @__svml_sin() +declare @__svml_cos() +declare void @__svml_sincos(, *, *) +declare @__svml_tan() +declare @__svml_atan() +declare @__svml_atan2(, ) +declare @__svml_exp() +declare @__svml_log() +declare @__svml_pow(, ) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; gather + +gen_gather_factored(i8) +gen_gather_factored(i16) +gen_gather_factored(i32) +gen_gather_factored(float) +gen_gather_factored(i64) +gen_gather_factored(double) + +gen_scatter(i8) +gen_scatter(i16) +gen_scatter(i32) +gen_scatter(float) +gen_scatter(i64) +gen_scatter(double) + +packed_load_and_store(4) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; prefetch + +define_prefetches() diff --git a/examples/aobench/Makefile b/examples/aobench/Makefile index 126e8ed3..7aba4f01 100644 --- a/examples/aobench/Makefile +++ b/examples/aobench/Makefile @@ -2,6 +2,7 @@ EXAMPLE=ao CPP_SRC=ao.cpp ao_serial.cpp ISPC_SRC=ao.ispc -ISPC_TARGETS=sse2,sse4,avx +ISPC_IA_TARGETS=sse2,sse4,avx +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/examples/common.mk b/examples/common.mk index 48adb943..cdfc4c6a 100644 --- a/examples/common.mk +++ b/examples/common.mk @@ -4,16 +4,30 @@ TASK_LIB=-lpthread TASK_OBJ=objs/tasksys.o CXX=g++ -CXXFLAGS=-Iobjs/ -O2 -m64 +CXXFLAGS=-Iobjs/ -O2 CC=gcc -CCFLAGS=-Iobjs/ -O2 -m64 +CCFLAGS=-Iobjs/ -O2 LIBS=-lm $(TASK_LIB) -lstdc++ -ISPC=ispc -O2 --arch=x86-64 $(ISPC_FLAGS) -ISPC_OBJS=$(addprefix objs/, $(ISPC_SRC:.ispc=)_ispc.o $(ISPC_SRC:.ispc=)_ispc_sse2.o \ - $(ISPC_SRC:.ispc=)_ispc_sse4.o $(ISPC_SRC:.ispc=)_ispc_avx.o) +ISPC=ispc -O2 $(ISPC_FLAGS) ISPC_HEADER=objs/$(ISPC_SRC:.ispc=_ispc.h) +ARCH:=$(shell uname -m | sed -e s/x86_64/x86/ -e s/arm.*/arm/ -e s/sa110/arm/) + +ifeq ($(ARCH),x86) + ISPC_OBJS=$(addprefix objs/, $(ISPC_SRC:.ispc=)_ispc.o $(ISPC_SRC:.ispc=)_ispc_sse2.o \ + $(ISPC_SRC:.ispc=)_ispc_sse4.o $(ISPC_SRC:.ispc=)_ispc_avx.o) + ISPC_TARGETS=$(ISPC_IA_TARGETS) + ISPC_FLAGS += --arch=x86-64 + CXXFLAGS += -m64 + CCFLAGS += -m64 +else ifeq ($(ARCH),arm) + ISPC_OBJS=$(addprefix objs/, $(ISPC_SRC:.ispc=_ispc.o)) + ISPC_TARGETS=$(ISPC_ARM_TARGETS) +else + $(error Unknown architecture $(ARCH) from uname -m) +endif + CPP_OBJS=$(addprefix objs/, $(CPP_SRC:.cpp=.o)) CC_OBJS=$(addprefix objs/, $(CC_SRC:.c=.o)) OBJS=$(CPP_OBJS) $(CC_OBJS) $(TASK_OBJ) $(ISPC_OBJS) diff --git a/examples/deferred/Makefile b/examples/deferred/Makefile index d98b1c4e..09fa56f0 100644 --- a/examples/deferred/Makefile +++ b/examples/deferred/Makefile @@ -2,7 +2,8 @@ EXAMPLE=deferred_shading CPP_SRC=common.cpp main.cpp dynamic_c.cpp dynamic_cilk.cpp ISPC_SRC=kernels.ispc -ISPC_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_IA_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_ARM_TARGETS=neon ISPC_FLAGS=--opt=fast-math include ../common.mk diff --git a/examples/gmres/Makefile b/examples/gmres/Makefile index 3e1031c4..5b57cbf8 100644 --- a/examples/gmres/Makefile +++ b/examples/gmres/Makefile @@ -3,6 +3,7 @@ EXAMPLE=gmres CPP_SRC=algorithm.cpp main.cpp matrix.cpp CC_SRC=mmio.c ISPC_SRC=matrix.ispc -ISPC_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_IA_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/examples/mandelbrot/Makefile b/examples/mandelbrot/Makefile index 8cff944e..7e83e618 100644 --- a/examples/mandelbrot/Makefile +++ b/examples/mandelbrot/Makefile @@ -2,6 +2,7 @@ EXAMPLE=mandelbrot CPP_SRC=mandelbrot.cpp mandelbrot_serial.cpp ISPC_SRC=mandelbrot.ispc -ISPC_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_IA_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/examples/mandelbrot_tasks/Makefile b/examples/mandelbrot_tasks/Makefile index 8cff944e..7e83e618 100644 --- a/examples/mandelbrot_tasks/Makefile +++ b/examples/mandelbrot_tasks/Makefile @@ -2,6 +2,7 @@ EXAMPLE=mandelbrot CPP_SRC=mandelbrot.cpp mandelbrot_serial.cpp ISPC_SRC=mandelbrot.ispc -ISPC_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_IA_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/examples/noise/Makefile b/examples/noise/Makefile index 4bda4527..8cc72689 100644 --- a/examples/noise/Makefile +++ b/examples/noise/Makefile @@ -2,6 +2,7 @@ EXAMPLE=noise CPP_SRC=$(EXAMPLE).cpp $(EXAMPLE)_serial.cpp ISPC_SRC=noise.ispc -ISPC_TARGETS=sse2,sse4,avx-x2 +ISPC_IA_TARGETS=sse2,sse4,avx-x2 +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/examples/options/Makefile b/examples/options/Makefile index 1a4f0a59..11d3d790 100644 --- a/examples/options/Makefile +++ b/examples/options/Makefile @@ -2,6 +2,7 @@ EXAMPLE=options CPP_SRC=options.cpp options_serial.cpp ISPC_SRC=options.ispc -ISPC_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_IA_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/examples/perfbench/Makefile b/examples/perfbench/Makefile index 43684c71..02507c84 100644 --- a/examples/perfbench/Makefile +++ b/examples/perfbench/Makefile @@ -2,6 +2,7 @@ EXAMPLE=perbench CPP_SRC=perfbench.cpp perfbench_serial.cpp ISPC_SRC=perfbench.ispc -ISPC_TARGETS=sse2,sse4,avx +ISPC_IA_TARGETS=sse2,sse4,avx +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/examples/rt/Makefile b/examples/rt/Makefile index 1e1219e4..647086cb 100644 --- a/examples/rt/Makefile +++ b/examples/rt/Makefile @@ -2,6 +2,7 @@ EXAMPLE=rt CPP_SRC=rt.cpp rt_serial.cpp ISPC_SRC=rt.ispc -ISPC_TARGETS=sse2,sse4-x2,avx +ISPC_IA_TARGETS=sse2,sse4-x2,avx +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/examples/sort/Makefile b/examples/sort/Makefile index 2ead0854..cf6bffa4 100644 --- a/examples/sort/Makefile +++ b/examples/sort/Makefile @@ -2,7 +2,8 @@ EXAMPLE=sort CPP_SRC=sort.cpp sort_serial.cpp ISPC_SRC=sort.ispc -ISPC_TARGETS=sse2,sse4-x2,avx +ISPC_IA_TARGETS=sse2,sse4-x2,avx +ISPC_ARM_TARGETS=neon #ISPC_FLAGS=-DDEBUG include ../common.mk diff --git a/examples/stencil/Makefile b/examples/stencil/Makefile index 7b58fe0b..097cd597 100644 --- a/examples/stencil/Makefile +++ b/examples/stencil/Makefile @@ -2,6 +2,7 @@ EXAMPLE=stencil CPP_SRC=stencil.cpp stencil_serial.cpp ISPC_SRC=stencil.ispc -ISPC_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_IA_TARGETS=sse2,sse4-x2,avx-x2 +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/examples/timing.h b/examples/timing.h index 7d746d45..8569d439 100644 --- a/examples/timing.h +++ b/examples/timing.h @@ -33,33 +33,47 @@ #include +#ifdef __arm__ +#include +// There's no easy way to get a hardware clock counter on ARM, so instead +// we'll pretend it's a 1GHz processor and then compute pretend cycles +// based on elapsed time from gettimeofday(). +__inline__ uint64_t rdtsc() { + static bool first = true; + static struct timeval tv_start; + if (first) { + gettimeofday(&tv_start, NULL); + first = false; + return 0; + } + + struct timeval tv; + gettimeofday(&tv, NULL); + tv.tv_sec -= tv_start.tv_sec; + tv.tv_usec -= tv_start.tv_usec; + return (1000000ull * tv.tv_sec + tv.tv_usec) * 1000ull; +} + +#else // __arm__ #ifdef WIN32 #include #define rdtsc __rdtsc -#else -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - __inline__ uint64_t rdtsc() { - uint32_t low, high; +#else // WIN32 +__inline__ uint64_t rdtsc() { + uint32_t low, high; #ifdef __x86_64 - __asm__ __volatile__ ( - "xorl %%eax,%%eax \n cpuid" - ::: "%rax", "%rbx", "%rcx", "%rdx" ); + __asm__ __volatile__ ("xorl %%eax,%%eax \n cpuid" + ::: "%rax", "%rbx", "%rcx", "%rdx" ); #else - __asm__ __volatile__ ( - "xorl %%eax,%%eax \n cpuid" - ::: "%eax", "%ebx", "%ecx", "%edx" ); + __asm__ __volatile__ ("xorl %%eax,%%eax \n cpuid" + ::: "%eax", "%ebx", "%ecx", "%edx" ); #endif - __asm__ __volatile__ ( - "rdtsc" : "=a" (low), "=d" (high)); - return (uint64_t)high << 32 | low; - } -#ifdef __cplusplus + __asm__ __volatile__ ("rdtsc" : "=a" (low), "=d" (high)); + return (uint64_t)high << 32 | low; } -#endif /* __cplusplus */ -#endif +#endif // !WIN32 +#endif // !__arm__ static uint64_t start, end; diff --git a/examples/volume_rendering/Makefile b/examples/volume_rendering/Makefile index 0ae93b58..7bb86e10 100644 --- a/examples/volume_rendering/Makefile +++ b/examples/volume_rendering/Makefile @@ -2,6 +2,7 @@ EXAMPLE=volume CPP_SRC=volume.cpp volume_serial.cpp ISPC_SRC=volume.ispc -ISPC_TARGETS=sse2,sse4-x2,avx +ISPC_IA_TARGETS=sse2,sse4-x2,avx +ISPC_ARM_TARGETS=neon include ../common.mk diff --git a/ispc.cpp b/ispc.cpp index 89c6898e..e4590b7e 100644 --- a/ispc.cpp +++ b/ispc.cpp @@ -85,7 +85,7 @@ Module *m; /////////////////////////////////////////////////////////////////////////// // Target -#ifndef ISPC_IS_WINDOWS +#if !defined(ISPC_IS_WINDOWS) && !defined(__arm__) static void __cpuid(int info[4], int infoType) { __asm__ __volatile__ ("cpuid" : "=a" (info[0]), "=b" (info[1]), "=c" (info[2]), "=d" (info[3]) @@ -100,11 +100,14 @@ static void __cpuidex(int info[4], int level, int count) { : "=a" (info[0]), "=r" (info[1]), "=c" (info[2]), "=d" (info[3]) : "0" (level), "2" (count)); } -#endif // ISPC_IS_WINDOWS +#endif // !ISPC_IS_WINDOWS && !__ARM__ static const char * lGetSystemISA() { +#ifdef __arm__ + return "neon"; +#else int info[4]; __cpuid(info, 1); @@ -133,10 +136,15 @@ lGetSystemISA() { fprintf(stderr, "Unable to detect supported SSE/AVX ISA. Exiting.\n"); exit(1); } +#endif } static const char *supportedCPUs[] = { + // FIXME: LLVM supports a ton of different ARM CPU variants--not just + // cortex-a9 and a15. We should be able to handle any of them that also + // have NEON support. + "cortex-a9", "cortex-a15", "atom", "penryn", "core2", "corei7", "corei7-avx" #if !defined(LLVM_3_1) , "core-avx-i", "core-avx2" @@ -177,6 +185,9 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic) : // possible ISA based on that. if (!strcmp(cpu, "core-avx2")) isa = "avx2"; + else if (!strcmp(cpu, "cortex-a9") || + !strcmp(cpu, "cortex-a15")) + isa = "neon"; else if (!strcmp(cpu, "core-avx-i")) isa = "avx1.1"; else if (!strcmp(cpu, "sandybridge") || @@ -200,6 +211,13 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic) : } } +#if !defined(__arm__) + if (cpu == NULL && !strcmp(isa, "neon")) + // If we're compiling NEON on an x86 host and the CPU wasn't + // supplied, don't go and set the CPU based on the host... + cpu = "cortex-a9"; +#endif + if (cpu == NULL) { std::string hostCPU = llvm::sys::getHostCPUName(); if (hostCPU.size() > 0) @@ -227,8 +245,12 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic) : this->m_cpu = cpu; - if (arch == NULL) - arch = "x86-64"; + if (arch == NULL) { + if (!strcmp(isa, "neon")) + arch = "arm"; + else + arch = "x86-64"; + } bool error = false; @@ -423,6 +445,15 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic) : this->m_hasGather = true; #endif } + else if (!strcasecmp(isa, "neon")) { + this->m_isa = Target::NEON; + this->m_nativeVectorWidth = 4; + this->m_vectorWidth = 4; + this->m_attributes = "+neon,+fp16"; + this->m_hasHalf = true; // ?? + this->m_maskingIsFree = false; + this->m_maskBitCount = 32; + } else { fprintf(stderr, "Target ISA \"%s\" is unknown. Choices are: %s\n", isa, SupportedTargetISAs()); @@ -437,6 +468,8 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic) : llvm::Reloc::Default; std::string featuresString = m_attributes; llvm::TargetOptions options; + if (m_isa == Target::NEON) + options.FloatABIType = llvm::FloatABI::Hard; #if !defined(LLVM_3_1) if (g->opt.disableFMA == false) options.AllowFPOpFusion = llvm::FPOpFusion::Fast; @@ -528,13 +561,13 @@ Target::SupportedTargetCPUs() { const char * Target::SupportedTargetArchs() { - return "x86, x86-64"; + return "arm, x86, x86-64"; } const char * Target::SupportedTargetISAs() { - return "sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2" + return "neon, sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2" ", avx1.1, avx1.1-x2, avx2, avx2-x2" ", generic-1, generic-4, generic-8, generic-16, generic-32"; } @@ -565,6 +598,8 @@ Target::GetTripleString() const { const char * Target::ISAToString(ISA isa) { switch (isa) { + case Target::NEON: + return "neon"; case Target::SSE2: return "sse2"; case Target::SSE4: diff --git a/ispc.h b/ispc.h index 043e9bab..7d10b908 100644 --- a/ispc.h +++ b/ispc.h @@ -175,7 +175,7 @@ public: flexible/performant of them will apear last in the enumerant. Note also that __best_available_isa() needs to be updated if ISAs are added or the enumerant values are reordered. */ - enum ISA { SSE2, SSE4, AVX, AVX11, AVX2, GENERIC, NUM_ISAS }; + enum ISA { NEON, SSE2, SSE4, AVX, AVX11, AVX2, GENERIC, NUM_ISAS }; /** Initializes the given Target pointer for a target of the given name, if the name is a known target. Returns true if the diff --git a/ispc.vcxproj b/ispc.vcxproj index 36fbad5d..96682fe3 100755 --- a/ispc.vcxproj +++ b/ispc.vcxproj @@ -45,6 +45,8 @@ + + @@ -185,6 +187,19 @@ Building gen-bitcode-sse2-x2-64bit.cpp + + + Document + m4 -Ibuiltins/ -DLLVM_VERSION=%LLVM_VERSION% builtins\target-neon.ll | python bitcode2cpp.py builtins\target-neon.ll > gen-bitcode-neon.cpp + gen-bitcode-neon.cpp + builtins\util.m4 + m4 -Ibuiltins/ -DLLVM_VERSION=%LLVM_VERSION% builtins\target-neon.ll | python bitcode2cpp.py builtins\target-neon.ll > gen-bitcode-neon.cpp + gen-bitcode-neon.cpp + builtins\util.m4 + Building gen-bitcode-neon.cpp + Building gen-bitcode-neon.cpp + + Document diff --git a/main.cpp b/main.cpp index 80f77683..de2bb620 100644 --- a/main.cpp +++ b/main.cpp @@ -243,12 +243,24 @@ int main(int Argc, char *Argv[]) { llvm::sys::AddSignalHandler(lSignal, NULL); // initialize available LLVM targets +#ifndef __arm__ + // FIXME: LLVM build on ARM doesn't build the x86 targets by default. + // It's not clear that anyone's going to want to generate x86 from an + // ARM host, though... LLVMInitializeX86TargetInfo(); LLVMInitializeX86Target(); LLVMInitializeX86AsmPrinter(); LLVMInitializeX86AsmParser(); LLVMInitializeX86Disassembler(); LLVMInitializeX86TargetMC(); +#endif // !__ARM__ + // Generating ARM from x86 is more likely to be useful, though. + LLVMInitializeARMTargetInfo(); + LLVMInitializeARMTarget(); + LLVMInitializeARMAsmPrinter(); + LLVMInitializeARMAsmParser(); + LLVMInitializeARMDisassembler(); + LLVMInitializeARMTargetMC(); char *file = NULL; const char *headerFileName = NULL; diff --git a/run_tests.py b/run_tests.py index 2bdf04de..7c6b1eb8 100755 --- a/run_tests.py +++ b/run_tests.py @@ -37,10 +37,10 @@ parser.add_option("-g", "--generics-include", dest="include_file", help="Filenam parser.add_option("-f", "--ispc-flags", dest="ispc_flags", help="Additional flags for ispc (-g, -O1, ...)", default="") parser.add_option('-t', '--target', dest='target', - help='Set compilation target (sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2, generic-4, generic-8, generic-16, generic-32)', + help='Set compilation target (neon, sse2, sse2-x2, sse4, sse4-x2, avx, avx-x2, generic-4, generic-8, generic-16, generic-32)', default="sse4") parser.add_option('-a', '--arch', dest='arch', - help='Set architecture (x86, x86-64)', + help='Set architecture (arm, x86, x86-64)', default="x86-64") parser.add_option("-c", "--compiler", dest="compiler_exe", help="Compiler binary to use to run tests", default=None) @@ -58,6 +58,9 @@ parser.add_option('--time', dest='time', help='Enable time output', (options, args) = parser.parse_args() +if options.target == 'neon': + options.arch = 'arm' + # use relative path to not depend on host directory, which may possibly # have white spaces and unicode characters. if not is_windows: @@ -345,10 +348,13 @@ def run_test(testname): obj_name = "%s.o" % testname exe_name = "%s.run" % testname - if options.arch == 'x86': - gcc_arch = '-m32' + if options.arch == 'arm': + gcc_arch = '--with-fpu=hardfp -marm -mfpu=neon -mfloat-abi=hard' else: - gcc_arch = '-m64' + if options.arch == 'x86': + gcc_arch = '-m32' + else: + gcc_arch = '-m64' gcc_isa="" if options.target == 'generic-4':