diff --git a/.gitignore b/.gitignore index 70d6fcd9..0469cf7d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,11 @@ ispc ispc_test objs docs/doxygen -docs/ispc.html +docs/*.html +tests*/*cpp +tests*/*run +examples/*/*.png +examples/*/*.ppm +examples/*/objs/* + + diff --git a/Makefile b/Makefile index 08e487f9..e39eb831 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,9 @@ CLANG=clang CLANG_LIBS = -lclangFrontend -lclangDriver \ -lclangSerialization -lclangParse -lclangSema \ -lclangAnalysis -lclangAST -lclangLex -lclangBasic +ifeq ($(shell llvm-config --version), 3.1svn) + CLANG_LIBS += -lclangEdit +endif ISPC_LIBS=$(shell llvm-config --ldflags) $(CLANG_LIBS) $(LLVM_LIBS) \ -lpthread diff --git a/builtins.cpp b/builtins.cpp index dd910c9a..0e34596d 100644 --- a/builtins.cpp +++ b/builtins.cpp @@ -438,6 +438,12 @@ lSetInternalFunctions(llvm::Module *module) { "__max_varying_uint32", "__max_varying_uint64", "__memory_barrier", + "__memcpy32", + "__memcpy64", + "__memmove32", + "__memmove64", + "__memset32", + "__memset64", "__min_uniform_double", "__min_uniform_float", "__min_uniform_int32", @@ -527,6 +533,8 @@ lSetInternalFunctions(llvm::Module *module) { "__sqrt_uniform_float", "__sqrt_varying_double", "__sqrt_varying_float", + "__stdlib_acosf", + "__stdlib_asinf", "__stdlib_atan", "__stdlib_atan2", "__stdlib_atan2f", @@ -627,8 +635,9 @@ AddBitcodeToModule(const unsigned char *bitcode, int length, static void lDefineConstantInt(const char *name, int val, llvm::Module *module, SymbolTable *symbolTable) { - Symbol *pw = new Symbol(name, SourcePos(), AtomicType::UniformConstInt32, - SC_STATIC); + Symbol *pw = + new Symbol(name, SourcePos(), AtomicType::UniformInt32->GetAsConstType(), + SC_STATIC); pw->constValue = new ConstExpr(pw->type, val, SourcePos()); LLVM_TYPE_CONST llvm::Type *ltype = LLVMTypes::Int32Type; llvm::Constant *linit = LLVMInt32(val); @@ -661,8 +670,9 @@ lDefineConstantIntFunc(const char *name, int val, llvm::Module *module, static void lDefineProgramIndex(llvm::Module *module, SymbolTable *symbolTable) { - Symbol *pidx = new Symbol("programIndex", SourcePos(), - AtomicType::VaryingConstInt32, SC_STATIC); + Symbol *pidx = + new Symbol("programIndex", SourcePos(), + AtomicType::VaryingInt32->GetAsConstType(), SC_STATIC); int pi[ISPC_MAX_NVEC]; for (int i = 0; i < g->target.vectorWidth; ++i) diff --git a/builtins/util.m4 b/builtins/util.m4 index 7c022e94..26cbfafb 100644 --- a/builtins/util.m4 +++ b/builtins/util.m4 @@ -1768,6 +1768,55 @@ define @__sext_varying_bool() nounwind readnone alwa ret %0') } +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; memcpy/memmove/memset + +declare void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, + i32 %len, i32 %align, i1 %isvolatile) +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* %dest, i8* %src, + i64 %len, i32 %align, i1 %isvolatile) + +define void @__memcpy32(i8 * %dst, i8 * %src, i32 %len) alwaysinline { + call void @llvm.memcpy.p0i8.p0i8.i32(i8 * %dst, i8 * %src, i32 %len, i32 0, i1 0) + ret void +} + +define void @__memcpy64(i8 * %dst, i8 * %src, i64 %len) alwaysinline { + call void @llvm.memcpy.p0i8.p0i8.i64(i8 * %dst, i8 * %src, i64 %len, i32 0, i1 0) + ret void +} + +declare void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, + i32 %len, i32 %align, i1 %isvolatile) +declare void @llvm.memmove.p0i8.p0i8.i64(i8* %dest, i8* %src, + i64 %len, i32 %align, i1 %isvolatile) + +define void @__memmove32(i8 * %dst, i8 * %src, i32 %len) alwaysinline { + call void @llvm.memmove.p0i8.p0i8.i32(i8 * %dst, i8 * %src, i32 %len, i32 0, i1 0) + ret void +} + +define void @__memmove64(i8 * %dst, i8 * %src, i64 %len) alwaysinline { + call void @llvm.memmove.p0i8.p0i8.i64(i8 * %dst, i8 * %src, i64 %len, i32 0, i1 0) + ret void +} + + +declare void @llvm.memset.p0i8.i32(i8* %dest, i8 %val, i32 %len, i32 %align, + i1 %isvolatile) +declare void @llvm.memset.p0i8.i64(i8* %dest, i8 %val, i64 %len, i32 %align, + i1 %isvolatile) + +define void @__memset32(i8 * %dst, i8 %val, i32 %len) alwaysinline { + call void @llvm.memset.p0i8.i32(i8 * %dst, i8 %val, i32 %len, i32 0, i1 0) + ret void +} + +define void @__memset64(i8 * %dst, i8 %val, i64 %len) alwaysinline { + call void @llvm.memset.p0i8.i64(i8 * %dst, i8 %val, i64 %len, i32 0, i1 0) + ret void +} + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; assert @@ -1890,6 +1939,8 @@ entry: declare float @sinf(float) nounwind readnone declare float @cosf(float) nounwind readnone declare void @sincosf(float, float *, float *) nounwind readnone +declare float @asinf(float) nounwind readnone +declare float @acosf(float) nounwind readnone declare float @tanf(float) nounwind readnone declare float @atanf(float) nounwind readnone declare float @atan2f(float, float) nounwind readnone @@ -1912,6 +1963,16 @@ define void @__stdlib_sincosf(float, float *, float *) nounwind readnone alwaysi ret void } +define float @__stdlib_asinf(float) nounwind readnone alwaysinline { + %r = call float @asinf(float %0) + ret float %r +} + +define float @__stdlib_acosf(float) nounwind readnone alwaysinline { + %r = call float @acosf(float %0) + ret float %r +} + define float @__stdlib_tanf(float) nounwind readnone alwaysinline { %r = call float @tanf(float %0) ret float %r diff --git a/cbackend.cpp b/cbackend.cpp index 314b53d6..6aade4ed 100644 --- a/cbackend.cpp +++ b/cbackend.cpp @@ -363,6 +363,9 @@ namespace { bool printConstExprCast(const ConstantExpr *CE, bool Static); void printConstantArray(ConstantArray *CPA, bool Static); void printConstantVector(ConstantVector *CV, bool Static); +#ifdef LLVM_3_1svn + void printConstantDataSequential(ConstantDataSequential *CDS, bool Static); +#endif /// isAddressExposed - Return true if the specified value's name needs to /// have its address taken in order to get a C value of the correct type. @@ -437,9 +440,11 @@ namespace { void visitInvokeInst(InvokeInst &I) { llvm_unreachable("Lowerinvoke pass didn't work!"); } +#if !defined(LLVM_3_1) && !defined(LLVM_3_1svn) void visitUnwindInst(UnwindInst &I) { llvm_unreachable("Lowerinvoke pass didn't work!"); } +#endif // !LLVM_3_1svn void visitResumeInst(ResumeInst &I) { llvm_unreachable("DwarfEHPrepare pass didn't work!"); } @@ -783,7 +788,7 @@ raw_ostream &CWriter::printType(raw_ostream &Out, Type *Ty, if (ATy->getElementType() == LLVMTypes::Int8Type) { Out << " static " << NameSoFar << " init(const char *p) {\n"; Out << " " << NameSoFar << " ret;\n"; - Out << " strncpy((char *)ret.array, p, " << NumElements << ");\n"; + Out << " memcpy((uint8_t *)ret.array, (uint8_t *)p, " << NumElements << ");\n"; Out << " return ret;\n"; Out << " }\n"; } @@ -796,19 +801,13 @@ raw_ostream &CWriter::printType(raw_ostream &Out, Type *Ty, default: llvm_unreachable("Unhandled case in getTypeProps!"); } - - return Out; } void CWriter::printConstantArray(ConstantArray *CPA, bool Static) { - - // As a special case, print the array as a string if it is an array of - // ubytes or an array of sbytes with positive values. - // +#ifndef LLVM_3_1svn Type *ETy = CPA->getType()->getElementType(); // MMP: this looks like a bug: both sides of the || are the same - bool isString = (ETy == Type::getInt8Ty(CPA->getContext()) || - ETy == Type::getInt8Ty(CPA->getContext())); + bool isString = ETy == Type::getInt8Ty(CPA->getContext()); // Make sure the last character is a null char, as automatically added by C if (isString && (CPA->getNumOperands() == 0 || @@ -816,7 +815,7 @@ void CWriter::printConstantArray(ConstantArray *CPA, bool Static) { isString = false; if (isString) { - Out << '\"'; + Out << "\""; // Keep track of whether the last number was a hexadecimal escape. bool LastWasHex = false; @@ -839,49 +838,100 @@ void CWriter::printConstantArray(ConstantArray *CPA, bool Static) { } else { LastWasHex = false; switch (C) { - case '\n': Out << "\\n"; break; - case '\t': Out << "\\t"; break; - case '\r': Out << "\\r"; break; - case '\v': Out << "\\v"; break; - case '\a': Out << "\\a"; break; - case '\"': Out << "\\\""; break; - case '\'': Out << "\\\'"; break; - default: - Out << "\\x"; - Out << (char)(( C/16 < 10) ? ( C/16 +'0') : ( C/16 -10+'A')); - Out << (char)(((C&15) < 10) ? ((C&15)+'0') : ((C&15)-10+'A')); - LastWasHex = true; - break; + case '\n': Out << "\\n"; break; + case '\t': Out << "\\t"; break; + case '\r': Out << "\\r"; break; + case '\v': Out << "\\v"; break; + case '\a': Out << "\\a"; break; + case '\"': Out << "\\\""; break; + case '\'': Out << "\\\'"; break; + default: + Out << "\\x"; + Out << (char)(( C/16 < 10) ? ( C/16 +'0') : ( C/16 -10+'A')); + Out << (char)(((C&15) < 10) ? ((C&15)+'0') : ((C&15)-10+'A')); + LastWasHex = true; + break; + } + } + } + Out << "\""; + return; + } +#endif // !LLVM_3_1 + + printConstant(cast(CPA->getOperand(0)), Static); + for (unsigned i = 1, e = CPA->getNumOperands(); i != e; ++i) { + Out << ", "; + printConstant(cast(CPA->getOperand(i)), Static); + } +} + +void CWriter::printConstantVector(ConstantVector *CP, bool Static) { + printConstant(cast(CP->getOperand(0)), Static); + for (unsigned i = 1, e = CP->getNumOperands(); i != e; ++i) { + Out << ", "; + printConstant(cast(CP->getOperand(i)), Static); + } +} + +#ifdef LLVM_3_1svn +void CWriter::printConstantDataSequential(ConstantDataSequential *CDS, + bool Static) { + // As a special case, print the array as a string if it is an array of + // ubytes or an array of sbytes with positive values. + // + if (CDS->isCString()) { + Out << '\"'; + // Keep track of whether the last number was a hexadecimal escape. + bool LastWasHex = false; + + StringRef Bytes = CDS->getAsCString(); + + // Do not include the last character, which we know is null + for (unsigned i = 0, e = Bytes.size(); i != e; ++i) { + unsigned char C = Bytes[i]; + + // Print it out literally if it is a printable character. The only thing + // to be careful about is when the last letter output was a hex escape + // code, in which case we have to be careful not to print out hex digits + // explicitly (the C compiler thinks it is a continuation of the previous + // character, sheesh...) + // + if (isprint(C) && (!LastWasHex || !isxdigit(C))) { + LastWasHex = false; + if (C == '"' || C == '\\') + Out << "\\" << (char)C; + else + Out << (char)C; + } else { + LastWasHex = false; + switch (C) { + case '\n': Out << "\\n"; break; + case '\t': Out << "\\t"; break; + case '\r': Out << "\\r"; break; + case '\v': Out << "\\v"; break; + case '\a': Out << "\\a"; break; + case '\"': Out << "\\\""; break; + case '\'': Out << "\\\'"; break; + default: + Out << "\\x"; + Out << (char)(( C/16 < 10) ? ( C/16 +'0') : ( C/16 -10+'A')); + Out << (char)(((C&15) < 10) ? ((C&15)+'0') : ((C&15)-10+'A')); + LastWasHex = true; + break; } } } Out << '\"'; } else { - if (Static) - Out << '{'; - if (CPA->getNumOperands()) { - Out << ' '; - printConstant(cast(CPA->getOperand(0)), Static); - for (unsigned i = 1, e = CPA->getNumOperands(); i != e; ++i) { - Out << ", "; - printConstant(cast(CPA->getOperand(i)), Static); - } - } - if (Static) - Out << " }"; - } -} - -void CWriter::printConstantVector(ConstantVector *CP, bool Static) { - if (CP->getNumOperands()) { - Out << ' '; - printConstant(cast(CP->getOperand(0)), Static); - for (unsigned i = 1, e = CP->getNumOperands(); i != e; ++i) { + printConstant(CDS->getElementAsConstant(0), Static); + for (unsigned i = 1, e = CDS->getNumElements(); i != e; ++i) { Out << ", "; - printConstant(cast(CP->getOperand(i)), Static); + printConstant(CDS->getElementAsConstant(i), Static); } } } +#endif // LLVM_3_1svn // isFPCSafeToPrint - Returns true if we may assume that CFP may be written out // textually as a double (rather than as a reference to a stack-allocated @@ -1305,7 +1355,7 @@ void CWriter::printConstant(Constant *CPV, bool Static) { char Buffer[100]; uint64_t ll = DoubleToBits(V); - sprintf(Buffer, "0x%"PRIx64, static_cast(ll)); + sprintf(Buffer, "0x%"PRIx64, ll); std::string Num(&Buffer[0], &Buffer[6]); unsigned long Val = strtoul(Num.c_str(), 0, 16); @@ -1350,6 +1400,11 @@ void CWriter::printConstant(Constant *CPV, bool Static) { } if (ConstantArray *CA = dyn_cast(CPV)) { printConstantArray(CA, Static); +#ifdef LLVM_3_1svn + } else if (ConstantDataSequential *CDS = + dyn_cast(CPV)) { + printConstantDataSequential(CDS, Static); +#endif // LLVM_3_1svn } else { assert(isa(CPV) || isa(CPV)); if (AT->getNumElements()) { @@ -1374,6 +1429,11 @@ void CWriter::printConstant(Constant *CPV, bool Static) { if (ConstantVector *CV = dyn_cast(CPV)) { printConstantVector(CV, Static); +#ifdef LLVM_3_1svn + } else if (ConstantDataSequential *CDS = + dyn_cast(CPV)) { + printConstantDataSequential(CDS, Static); +#endif } else { assert(isa(CPV) || isa(CPV)); VectorType *VT = cast(CPV->getType()); @@ -1995,7 +2055,6 @@ bool CWriter::doInitialization(Module &M) { Out << "#include \n"; // Unwind support Out << "#include \n"; // With overflow intrinsics support. Out << "#include \n"; - Out << "#include \n"; Out << "#ifdef _MSC_VER\n"; Out << " #define NOMINMAX\n"; Out << " #include \n"; @@ -2014,6 +2073,68 @@ bool CWriter::doInitialization(Module &M) { generateCompilerSpecificCode(Out, TD); + // Function declarations + Out << "\n/* Function Declarations */\n"; + Out << "extern \"C\" {\n"; + Out << "int puts(unsigned char *);\n"; + Out << "unsigned int putchar(unsigned int);\n"; + Out << "int fflush(void *);\n"; + Out << "int printf(const unsigned char *, ...);\n"; + Out << "uint8_t *memcpy(uint8_t *, uint8_t *, uint64_t );\n"; + + // Store the intrinsics which will be declared/defined below. + SmallVector intrinsicsToDefine; + + for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) { + // Don't print declarations for intrinsic functions. + // Store the used intrinsics, which need to be explicitly defined. + if (I->isIntrinsic()) { + switch (I->getIntrinsicID()) { + default: + break; + case Intrinsic::uadd_with_overflow: + case Intrinsic::sadd_with_overflow: + intrinsicsToDefine.push_back(I); + break; + } + continue; + } + + if (I->getName() == "setjmp" || I->getName() == "abort" || + I->getName() == "longjmp" || I->getName() == "_setjmp" || + I->getName() == "memset" || I->getName() == "memset_pattern16" || + I->getName() == "puts" || + I->getName() == "printf" || I->getName() == "putchar" || + I->getName() == "fflush" || I->getName() == "malloc" || + I->getName() == "free") + continue; + + // Don't redeclare ispc's own intrinsics + std::string name = I->getName(); + if (name.size() > 2 && name[0] == '_' && name[1] == '_') + continue; + + if (I->hasExternalWeakLinkage()) + Out << "extern "; + printFunctionSignature(I, true); + if (I->hasWeakLinkage() || I->hasLinkOnceLinkage()) + Out << " __ATTRIBUTE_WEAK__"; + if (I->hasExternalWeakLinkage()) + Out << " __EXTERNAL_WEAK__"; + if (StaticCtors.count(I)) + Out << " __ATTRIBUTE_CTOR__"; + if (StaticDtors.count(I)) + Out << " __ATTRIBUTE_DTOR__"; + if (I->hasHiddenVisibility()) + Out << " __HIDDEN__"; + + if (I->hasName() && I->getName()[0] == 1) + Out << " LLVM_ASM(\"" << I->getName().substr(1) << "\")"; + + Out << ";\n"; + } + Out << "}\n"; + // Provide a definition for `bool' if not compiling with a C++ compiler. Out << "\n" << "#ifndef __cplusplus\ntypedef unsigned char bool;\n#endif\n" @@ -2083,67 +2204,6 @@ bool CWriter::doInitialization(Module &M) { } } - // Function declarations - Out << "\n/* Function Declarations */\n"; - Out << "extern \"C\" {\n"; - Out << "int puts(unsigned char *);\n"; - Out << "unsigned int putchar(unsigned int);\n"; - Out << "int fflush(void *);\n"; - Out << "int printf(const unsigned char *, ...);\n"; - - // Store the intrinsics which will be declared/defined below. - SmallVector intrinsicsToDefine; - - for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) { - // Don't print declarations for intrinsic functions. - // Store the used intrinsics, which need to be explicitly defined. - if (I->isIntrinsic()) { - switch (I->getIntrinsicID()) { - default: - break; - case Intrinsic::uadd_with_overflow: - case Intrinsic::sadd_with_overflow: - intrinsicsToDefine.push_back(I); - break; - } - continue; - } - - if (I->getName() == "setjmp" || I->getName() == "abort" || - I->getName() == "longjmp" || I->getName() == "_setjmp" || - I->getName() == "memset" || I->getName() == "memset_pattern16" || - I->getName() == "puts" || - I->getName() == "printf" || I->getName() == "putchar" || - I->getName() == "fflush" || I->getName() == "malloc" || - I->getName() == "free") - continue; - - // Don't redeclare ispc's own intrinsics - std::string name = I->getName(); - if (name.size() > 2 && name[0] == '_' && name[1] == '_') - continue; - - if (I->hasExternalWeakLinkage()) - Out << "extern "; - printFunctionSignature(I, true); - if (I->hasWeakLinkage() || I->hasLinkOnceLinkage()) - Out << " __ATTRIBUTE_WEAK__"; - if (I->hasExternalWeakLinkage()) - Out << " __EXTERNAL_WEAK__"; - if (StaticCtors.count(I)) - Out << " __ATTRIBUTE_CTOR__"; - if (StaticDtors.count(I)) - Out << " __ATTRIBUTE_DTOR__"; - if (I->hasHiddenVisibility()) - Out << " __HIDDEN__"; - - if (I->hasName() && I->getName()[0] == 1) - Out << " LLVM_ASM(\"" << I->getName().substr(1) << "\")"; - - Out << ";\n"; - } - Out << "}\n"; - // Output the global variable declarations if (!M.global_empty()) { Out << "\n\n/* Global Variable Declarations */\n"; @@ -2763,11 +2823,17 @@ void CWriter::visitSwitchInst(SwitchInst &SI) { printBranchToBlock(SI.getParent(), SI.getDefaultDest(), 2); Out << ";\n"; - unsigned NumCases = SI.getNumCases(); +#ifdef LLVM_3_1svn + for (SwitchInst::CaseIt i = SI.case_begin(), e = SI.case_end(); i != e; ++i) { + ConstantInt* CaseVal = i.getCaseValue(); + BasicBlock* Succ = i.getCaseSuccessor(); +#else // Skip the first item since that's the default case. + unsigned NumCases = SI.getNumCases(); for (unsigned i = 1; i < NumCases; ++i) { ConstantInt* CaseVal = SI.getCaseValue(i); BasicBlock* Succ = SI.getSuccessor(i); +#endif // LLVM_3_1svn Out << " case "; writeOperand(CaseVal); Out << ":\n"; diff --git a/ctx.cpp b/ctx.cpp index 41178a5b..34de1a17 100644 --- a/ctx.cpp +++ b/ctx.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2010-2011, Intel Corporation + Copyright (c) 2010-2012, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -246,7 +246,7 @@ FunctionEmitContext::FunctionEmitContext(Function *func, Symbol *funSym, launchGroupHandlePtr); const Type *returnType = function->GetReturnType(); - if (!returnType || returnType == AtomicType::Void) + if (!returnType || Type::Equal(returnType, AtomicType::Void)) returnValuePtr = NULL; else { LLVM_TYPE_CONST llvm::Type *ftype = returnType->LLVMType(g->ctx); @@ -1144,7 +1144,7 @@ FunctionEmitContext::GetLabeledBasicBlock(const std::string &label) { void FunctionEmitContext::CurrentLanesReturned(Expr *expr, bool doCoherenceCheck) { const Type *returnType = function->GetReturnType(); - if (returnType == AtomicType::Void) { + if (Type::Equal(returnType, AtomicType::Void)) { if (expr != NULL) Error(expr->pos, "Can't return non-void type \"%s\" from void function.", expr->GetType()->GetString().c_str()); @@ -1169,7 +1169,7 @@ FunctionEmitContext::CurrentLanesReturned(Expr *expr, bool doCoherenceCheck) { // values from other lanes that may have executed return // statements previously. StoreInst(retVal, returnValuePtr, GetInternalMask(), - PointerType::GetUniform(returnType)); + returnType, PointerType::GetUniform(returnType)); } } } @@ -1904,17 +1904,146 @@ FunctionEmitContext::applyVaryingGEP(llvm::Value *basePtr, llvm::Value *index, } +void +FunctionEmitContext::MatchIntegerTypes(llvm::Value **v0, llvm::Value **v1) { + LLVM_TYPE_CONST llvm::Type *type0 = (*v0)->getType(); + LLVM_TYPE_CONST llvm::Type *type1 = (*v1)->getType(); + + // First, promote to a vector type if one of the two values is a vector + // type + if (llvm::isa(type0) && + !llvm::isa(type1)) { + *v1 = SmearUniform(*v1, "smear_v1"); + type1 = (*v1)->getType(); + } + if (!llvm::isa(type0) && + llvm::isa(type1)) { + *v0 = SmearUniform(*v0, "smear_v0"); + type0 = (*v0)->getType(); + } + + // And then update to match bit widths + if (type0 == LLVMTypes::Int32VectorType && + type1 == LLVMTypes::Int64VectorType) + *v0 = SExtInst(*v0, LLVMTypes::Int64VectorType); + else if (type1 == LLVMTypes::Int32VectorType && + type0 == LLVMTypes::Int64VectorType) + *v1 = SExtInst(*v1, LLVMTypes::Int64VectorType); +} + + +/** Given an integer index in indexValue that's indexing into an array of + soa<> structures with given soaWidth, compute the two sub-indices we + need to do the actual indexing calculation: + + subIndices[0] = (indexValue >> log(soaWidth)) + subIndices[1] = (indexValue & (soaWidth-1)) + */ +static llvm::Value * +lComputeSliceIndex(FunctionEmitContext *ctx, int soaWidth, + llvm::Value *indexValue, llvm::Value *ptrSliceOffset, + llvm::Value **newSliceOffset) { + // Compute the log2 of the soaWidth. + Assert(soaWidth > 0); + int logWidth = 0, sw = soaWidth; + while (sw > 1) { + ++logWidth; + sw >>= 1; + } + Assert((1 << logWidth) == soaWidth); + + ctx->MatchIntegerTypes(&indexValue, &ptrSliceOffset); + + LLVM_TYPE_CONST llvm::Type *indexType = indexValue->getType(); + llvm::Value *shift = LLVMIntAsType(logWidth, indexType); + llvm::Value *mask = LLVMIntAsType(soaWidth-1, indexType); + + llvm::Value *indexSum = + ctx->BinaryOperator(llvm::Instruction::Add, indexValue, ptrSliceOffset, + "index_sum"); + + // minor index = (index & (soaWidth - 1)) + *newSliceOffset = ctx->BinaryOperator(llvm::Instruction::And, indexSum, + mask, "slice_index_minor"); + // slice offsets are always 32 bits... + if ((*newSliceOffset)->getType() == LLVMTypes::Int64Type) + *newSliceOffset = ctx->TruncInst(*newSliceOffset, LLVMTypes::Int32Type); + else if ((*newSliceOffset)->getType() == LLVMTypes::Int64VectorType) + *newSliceOffset = ctx->TruncInst(*newSliceOffset, LLVMTypes::Int32VectorType); + + // major index = (index >> logWidth) + return ctx->BinaryOperator(llvm::Instruction::AShr, indexSum, + shift, "slice_index_major"); +} + + +llvm::Value * +FunctionEmitContext::MakeSlicePointer(llvm::Value *ptr, llvm::Value *offset) { + // Create a small struct where the first element is the type of the + // given pointer and the second element is the type of the offset + // value. + std::vector eltTypes; + eltTypes.push_back(ptr->getType()); + eltTypes.push_back(offset->getType()); + LLVM_TYPE_CONST llvm::StructType *st = + llvm::StructType::get(*g->ctx, eltTypes); + + llvm::Value *ret = llvm::UndefValue::get(st); + ret = InsertInst(ret, ptr, 0); + ret = InsertInst(ret, offset, 1); + return ret; +} + + llvm::Value * FunctionEmitContext::GetElementPtrInst(llvm::Value *basePtr, llvm::Value *index, - const Type *ptrType, const char *name) { + const Type *ptrRefType, const char *name) { if (basePtr == NULL || index == NULL) { Assert(m->errorCount > 0); return NULL; } - if (dynamic_cast(ptrType) != NULL) - ptrType = PointerType::GetUniform(ptrType->GetReferenceTarget()); - Assert(dynamic_cast(ptrType) != NULL); + // Regularize to a standard pointer type for basePtr's type + const PointerType *ptrType; + if (dynamic_cast(ptrRefType) != NULL) + ptrType = PointerType::GetUniform(ptrRefType->GetReferenceTarget()); + else { + ptrType = dynamic_cast(ptrRefType); + Assert(ptrType != NULL); + } + + if (ptrType->IsSlice()) { + Assert(llvm::isa(basePtr->getType())); + + llvm::Value *ptrSliceOffset = ExtractInst(basePtr, 1); + if (ptrType->IsFrozenSlice() == false) { + // For slice pointers that aren't frozen, we compute a new + // index based on the given index plus the offset in the slice + // pointer. This gives us an updated integer slice index for + // the resulting slice pointer and then an index to index into + // the soa<> structs with. + llvm::Value *newSliceOffset; + int soaWidth = ptrType->GetBaseType()->GetSOAWidth(); + index = lComputeSliceIndex(this, soaWidth, index, + ptrSliceOffset, &newSliceOffset); + ptrSliceOffset = newSliceOffset; + } + + // Handle the indexing into the soa<> structs with the major + // component of the index through a recursive call + llvm::Value *p = GetElementPtrInst(ExtractInst(basePtr, 0), index, + ptrType->GetAsNonSlice(), name); + + // And mash the results together for the return value + return MakeSlicePointer(p, ptrSliceOffset); + } + + // Double-check consistency between the given pointer type and its LLVM + // type. + if (ptrType->IsUniformType()) + Assert(llvm::isa(basePtr->getType())); + else if (ptrType->IsVaryingType()) + Assert(llvm::isa(basePtr->getType())); bool indexIsVaryingType = llvm::isa(index->getType()); @@ -1943,16 +2072,41 @@ FunctionEmitContext::GetElementPtrInst(llvm::Value *basePtr, llvm::Value *index, llvm::Value * FunctionEmitContext::GetElementPtrInst(llvm::Value *basePtr, llvm::Value *index0, - llvm::Value *index1, const Type *ptrType, + llvm::Value *index1, const Type *ptrRefType, const char *name) { if (basePtr == NULL || index0 == NULL || index1 == NULL) { Assert(m->errorCount > 0); return NULL; } - if (dynamic_cast(ptrType) != NULL) - ptrType = PointerType::GetUniform(ptrType->GetReferenceTarget()); - Assert(dynamic_cast(ptrType) != NULL); + // Regaularize the pointer type for basePtr + const PointerType *ptrType = NULL; + if (dynamic_cast(ptrRefType) != NULL) + ptrType = PointerType::GetUniform(ptrRefType->GetReferenceTarget()); + else { + ptrType = dynamic_cast(ptrRefType); + Assert(ptrType != NULL); + } + + if (ptrType->IsSlice()) { + // Similar to the 1D GEP implementation above, for non-frozen slice + // pointers we do the two-step indexing calculation and then pass + // the new major index on to a recursive GEP call. + Assert(llvm::isa(basePtr->getType())); + llvm::Value *ptrSliceOffset = ExtractInst(basePtr, 1); + if (ptrType->IsFrozenSlice() == false) { + llvm::Value *newSliceOffset; + int soaWidth = ptrType->GetBaseType()->GetSOAWidth(); + index1 = lComputeSliceIndex(this, soaWidth, index1, + ptrSliceOffset, &newSliceOffset); + ptrSliceOffset = newSliceOffset; + } + + llvm::Value *p = GetElementPtrInst(ExtractInst(basePtr, 0), index0, + index1, ptrType->GetAsNonSlice(), + name); + return MakeSlicePointer(p, ptrSliceOffset); + } bool index0IsVaryingType = llvm::isa(index0->getType()); @@ -2000,62 +2154,113 @@ FunctionEmitContext::GetElementPtrInst(llvm::Value *basePtr, llvm::Value *index0 llvm::Value * -FunctionEmitContext::AddElementOffset(llvm::Value *basePtr, int elementNum, - const Type *ptrType, const char *name) { - if (ptrType == NULL || ptrType->IsUniformType() || - dynamic_cast(ptrType) != NULL) { - // If the pointer is uniform or we have a reference (which is a - // uniform pointer in the end), we can use the regular LLVM GEP. +FunctionEmitContext::AddElementOffset(llvm::Value *fullBasePtr, int elementNum, + const Type *ptrRefType, const char *name, + const PointerType **resultPtrType) { + if (resultPtrType != NULL) + Assert(ptrRefType != NULL); + + // (Unfortunately) it's not required to pass a non-NULL ptrRefType, but + // if we have one, regularize into a pointer type. + const PointerType *ptrType = NULL; + if (ptrRefType != NULL) { + // Normalize references to uniform pointers + if (dynamic_cast(ptrRefType) != NULL) + ptrType = PointerType::GetUniform(ptrRefType->GetReferenceTarget()); + else + ptrType = dynamic_cast(ptrRefType); + Assert(ptrType != NULL); + } + + // Similarly, we have to see if the pointer type is a struct to see if + // we have a slice pointer instead of looking at ptrType; this is also + // unfortunate... + llvm::Value *basePtr = fullBasePtr; + bool baseIsSlicePtr = + llvm::isa(fullBasePtr->getType()); + const PointerType *rpt; + if (baseIsSlicePtr) { + Assert(ptrType != NULL); + // Update basePtr to just be the part that actually points to the + // start of an soa<> struct for now; the element offset computation + // doesn't change the slice offset, so we'll incorporate that into + // the final value right before this method returns. + basePtr = ExtractInst(fullBasePtr, 0); + if (resultPtrType == NULL) + resultPtrType = &rpt; + } + + // Return the pointer type of the result of this call, for callers that + // want it. + if (resultPtrType != NULL) { + Assert(ptrType != NULL); + const CollectionType *ct = + dynamic_cast(ptrType->GetBaseType()); + Assert(ct != NULL); + *resultPtrType = new PointerType(ct->GetElementType(elementNum), + ptrType->GetVariability(), + ptrType->IsConstType(), + ptrType->IsSlice()); + } + + llvm::Value *resultPtr = NULL; + if (ptrType == NULL || ptrType->IsUniformType()) { + // If the pointer is uniform, we can use the regular LLVM GEP. llvm::Value *offsets[2] = { LLVMInt32(0), LLVMInt32(elementNum) }; #if defined(LLVM_3_0) || defined(LLVM_3_0svn) || defined(LLVM_3_1svn) llvm::ArrayRef arrayRef(&offsets[0], &offsets[2]); - return llvm::GetElementPtrInst::Create(basePtr, arrayRef, - name ? name : "struct_offset", bblock); + resultPtr = + llvm::GetElementPtrInst::Create(basePtr, arrayRef, + name ? name : "struct_offset", bblock); #else - return llvm::GetElementPtrInst::Create(basePtr, &offsets[0], &offsets[2], - name ? name : "struct_offset", bblock); + resultPtr = + llvm::GetElementPtrInst::Create(basePtr, &offsets[0], &offsets[2], + name ? name : "struct_offset", bblock); #endif - } - - if (dynamic_cast(ptrType) != NULL) - ptrType = PointerType::GetUniform(ptrType->GetReferenceTarget()); - Assert(dynamic_cast(ptrType) != NULL); - - // Otherwise do the math to find the offset and add it to the given - // varying pointers - const StructType *st = - dynamic_cast(ptrType->GetBaseType()); - llvm::Value *offset = NULL; - if (st != NULL) - // If the pointer is to a structure, Target::StructOffset() gives - // us the offset in bytes to the given element of the structure - offset = g->target.StructOffset(st->LLVMType(g->ctx), elementNum, - bblock); else { - // Otherwise we should have a vector or array here and the offset - // is given by the element number times the size of the element - // type of the vector. - const SequentialType *st = - dynamic_cast(ptrType->GetBaseType()); - Assert(st != NULL); - llvm::Value *size = - g->target.SizeOf(st->GetElementType()->LLVMType(g->ctx), bblock); - llvm::Value *scale = (g->target.is32Bit || g->opt.force32BitAddressing) ? - LLVMInt32(elementNum) : LLVMInt64(elementNum); - offset = BinaryOperator(llvm::Instruction::Mul, size, scale); + // Otherwise do the math to find the offset and add it to the given + // varying pointers + const StructType *st = + dynamic_cast(ptrType->GetBaseType()); + llvm::Value *offset = NULL; + if (st != NULL) + // If the pointer is to a structure, Target::StructOffset() gives + // us the offset in bytes to the given element of the structure + offset = g->target.StructOffset(st->LLVMType(g->ctx), elementNum, + bblock); + else { + // Otherwise we should have a vector or array here and the offset + // is given by the element number times the size of the element + // type of the vector. + const SequentialType *st = + dynamic_cast(ptrType->GetBaseType()); + Assert(st != NULL); + llvm::Value *size = + g->target.SizeOf(st->GetElementType()->LLVMType(g->ctx), bblock); + llvm::Value *scale = (g->target.is32Bit || g->opt.force32BitAddressing) ? + LLVMInt32(elementNum) : LLVMInt64(elementNum); + offset = BinaryOperator(llvm::Instruction::Mul, size, scale); + } + + offset = SmearUniform(offset, "offset_smear"); + + if (g->target.is32Bit == false && g->opt.force32BitAddressing == true) + // If we're doing 32 bit addressing with a 64 bit target, although + // we did the math above in 32 bit, we need to go to 64 bit before + // we add the offset to the varying pointers. + offset = SExtInst(offset, LLVMTypes::Int64VectorType, "offset_to_64"); + + resultPtr = BinaryOperator(llvm::Instruction::Add, basePtr, offset, + "struct_ptr_offset"); } - offset = SmearUniform(offset, "offset_smear"); - - if (g->target.is32Bit == false && g->opt.force32BitAddressing == true) - // If we're doing 32 bit addressing with a 64 bit target, although - // we did the math above in 32 bit, we need to go to 64 bit before - // we add the offset to the varying pointers. - offset = SExtInst(offset, LLVMTypes::Int64VectorType, "offset_to_64"); - - return BinaryOperator(llvm::Instruction::Add, basePtr, offset, - "struct_ptr_offset"); + // Finally, if had a slice pointer going in, mash back together with + // the original (unchanged) slice offset. + if (baseIsSlicePtr) + return MakeSlicePointer(resultPtr, ExtractInst(fullBasePtr, 1)); + else + return resultPtr; } @@ -2085,43 +2290,124 @@ FunctionEmitContext::LoadInst(llvm::Value *ptr, const char *name) { } +/** Given a slice pointer to soa'd data that is a basic type (atomic, + pointer, or enum type), use the slice offset to compute pointer(s) to + the appropriate individual data element(s). + */ +static llvm::Value * +lFinalSliceOffset(FunctionEmitContext *ctx, llvm::Value *ptr, + const PointerType **ptrType) { + Assert(dynamic_cast(*ptrType) != NULL); + + llvm::Value *slicePtr = ctx->ExtractInst(ptr, 0, "slice_ptr"); + llvm::Value *sliceOffset = ctx->ExtractInst(ptr, 1, "slice_offset"); + + // slicePtr should be a pointer to an soa-width wide array of the + // final atomic/enum/pointer type + const Type *unifBaseType = (*ptrType)->GetBaseType()->GetAsUniformType(); + Assert(Type::IsBasicType(unifBaseType)); + + // The final pointer type is a uniform or varying pointer to the + // underlying uniform type, depending on whether the given pointer is + // uniform or varying. + *ptrType = (*ptrType)->IsUniformType() ? + PointerType::GetUniform(unifBaseType) : + PointerType::GetVarying(unifBaseType); + + // For uniform pointers, bitcast to a pointer to the uniform element + // type, so that the GEP below does the desired indexing + if ((*ptrType)->IsUniformType()) + slicePtr = ctx->BitCastInst(slicePtr, (*ptrType)->LLVMType(g->ctx)); + + // And finally index based on the slice offset + return ctx->GetElementPtrInst(slicePtr, sliceOffset, *ptrType, + "final_slice_gep"); +} + + +/** Utility routine that loads from a uniform pointer to soa<> data, + returning a regular uniform (non-SOA result). + */ +llvm::Value * +FunctionEmitContext::loadUniformFromSOA(llvm::Value *ptr, llvm::Value *mask, + const PointerType *ptrType, + const char *name) { + const Type *unifType = ptrType->GetBaseType()->GetAsUniformType(); + + const CollectionType *ct = + dynamic_cast(ptrType->GetBaseType()); + if (ct != NULL) { + // If we have a struct/array, we need to decompose it into + // individual element loads to fill in the result structure since + // the SOA slice of values we need isn't contiguous in memory... + LLVM_TYPE_CONST llvm::Type *llvmReturnType = unifType->LLVMType(g->ctx); + llvm::Value *retValue = llvm::UndefValue::get(llvmReturnType); + + for (int i = 0; i < ct->GetElementCount(); ++i) { + const PointerType *eltPtrType; + llvm::Value *eltPtr = AddElementOffset(ptr, i, ptrType, + "elt_offset", &eltPtrType); + llvm::Value *eltValue = LoadInst(eltPtr, mask, eltPtrType, name); + retValue = InsertInst(retValue, eltValue, i, "set_value"); + } + + return retValue; + } + else { + // Otherwise we've made our way to a slice pointer to a basic type; + // we need to apply the slice offset into this terminal SOA array + // and then perform the final load + ptr = lFinalSliceOffset(this, ptr, &ptrType); + return LoadInst(ptr, mask, ptrType, name); + } +} + + llvm::Value * FunctionEmitContext::LoadInst(llvm::Value *ptr, llvm::Value *mask, - const Type *ptrType, const char *name) { + const Type *ptrRefType, const char *name) { if (ptr == NULL) { Assert(m->errorCount > 0); return NULL; } - Assert(ptrType != NULL && mask != NULL); + Assert(ptrRefType != NULL && mask != NULL); - if (dynamic_cast(ptrType) != NULL) - ptrType = PointerType::GetUniform(ptrType->GetReferenceTarget()); - - Assert(dynamic_cast(ptrType) != NULL); + const PointerType *ptrType; + if (dynamic_cast(ptrRefType) != NULL) + ptrType = PointerType::GetUniform(ptrRefType->GetReferenceTarget()); + else { + ptrType = dynamic_cast(ptrRefType); + Assert(ptrType != NULL); + } if (ptrType->IsUniformType()) { - // FIXME: same issue as above load inst regarding alignment... - // - // If the ptr is a straight up regular pointer, then just issue - // a regular load. First figure out the alignment; in general we - // can just assume the natural alignment (0 here), but for varying - // atomic types, we need to make sure that the compiler emits - // unaligned vector loads, so we specify a reduced alignment here. - int align = 0; - const AtomicType *atomicType = - dynamic_cast(ptrType->GetBaseType()); - if (atomicType != NULL && atomicType->IsVaryingType()) - // We actually just want to align to the vector element - // alignment, but can't easily get that here, so just tell LLVM - // it's totally unaligned. (This shouldn't make any difference - // vs the proper alignment in practice.) - align = 1; - llvm::Instruction *inst = new llvm::LoadInst(ptr, name ? name : "load", - false /* not volatile */, - align, bblock); - AddDebugPos(inst); - return inst; + if (ptrType->IsSlice()) { + return loadUniformFromSOA(ptr, mask, ptrType, name); + } + else { + // FIXME: same issue as above load inst regarding alignment... + // + // If the ptr is a straight up regular pointer, then just issue + // a regular load. First figure out the alignment; in general we + // can just assume the natural alignment (0 here), but for varying + // atomic types, we need to make sure that the compiler emits + // unaligned vector loads, so we specify a reduced alignment here. + int align = 0; + const AtomicType *atomicType = + dynamic_cast(ptrType->GetBaseType()); + if (atomicType != NULL && atomicType->IsVaryingType()) + // We actually just want to align to the vector element + // alignment, but can't easily get that here, so just tell LLVM + // it's totally unaligned. (This shouldn't make any difference + // vs the proper alignment in practice.) + align = 1; + llvm::Instruction *inst = new llvm::LoadInst(ptr, name ? name : "load", + false /* not volatile */, + align, bblock); + AddDebugPos(inst); + return inst; + } } else { // Otherwise we should have a varying ptr and it's time for a @@ -2132,11 +2418,10 @@ FunctionEmitContext::LoadInst(llvm::Value *ptr, llvm::Value *mask, llvm::Value * -FunctionEmitContext::gather(llvm::Value *ptr, const Type *ptrType, +FunctionEmitContext::gather(llvm::Value *ptr, const PointerType *ptrType, llvm::Value *mask, const char *name) { - // We should have a varying lvalue if we get here... - Assert(ptrType->IsVaryingType() && - ptr->getType() == LLVMTypes::VoidPointerVectorType); + // We should have a varying pointer if we get here... + Assert(ptrType->IsVaryingType()); const Type *returnType = ptrType->GetBaseType()->GetAsVaryingType(); LLVM_TYPE_CONST llvm::Type *llvmReturnType = returnType->LLVMType(g->ctx); @@ -2147,10 +2432,12 @@ FunctionEmitContext::gather(llvm::Value *ptr, const Type *ptrType, // For collections, recursively gather element wise to find the // result. llvm::Value *retValue = llvm::UndefValue::get(llvmReturnType); + for (int i = 0; i < collectionType->GetElementCount(); ++i) { - llvm::Value *eltPtr = AddElementOffset(ptr, i, ptrType); - const Type *eltPtrType = - PointerType::GetVarying(collectionType->GetElementType(i)); + const PointerType *eltPtrType; + llvm::Value *eltPtr = + AddElementOffset(ptr, i, ptrType, "gather_elt_ptr", &eltPtrType); + eltPtr = addVaryingOffsetsIfNeeded(eltPtr, eltPtrType); // This in turn will be another gather @@ -2160,7 +2447,15 @@ FunctionEmitContext::gather(llvm::Value *ptr, const Type *ptrType, } return retValue; } - + else if (ptrType->IsSlice()) { + // If we have a slice pointer, we need to add the final slice + // offset here right before issuing the actual gather + // + // FIXME: would it be better to do the corresponding same thing for + // all of the varying offsets stuff here (and in scatter)? + ptr = lFinalSliceOffset(this, ptr, &ptrType); + } + // Otherwise we should just have a basic scalar or pointer type and we // can go and do the actual gather AddInstrumentationPoint("gather"); @@ -2303,29 +2598,41 @@ FunctionEmitContext::maskedStore(llvm::Value *value, llvm::Value *ptr, llvm::Value *eltValue = ExtractInst(value, i, "value_member"); llvm::Value *eltPtr = AddElementOffset(ptr, i, ptrType, "struct_ptr_ptr"); - const Type *eltPtrType = - PointerType::GetUniform(collectionType->GetElementType(i)); - StoreInst(eltValue, eltPtr, mask, eltPtrType); + const Type *eltType = collectionType->GetElementType(i); + const Type *eltPtrType = PointerType::GetUniform(eltType); + StoreInst(eltValue, eltPtr, mask, eltType, eltPtrType); } return; } // We must have a regular atomic, enumerator, or pointer type at this // point. - Assert(dynamic_cast(valueType) != NULL || - dynamic_cast(valueType) != NULL || - dynamic_cast(valueType) != NULL); + Assert(Type::IsBasicType(valueType)); valueType = valueType->GetAsNonConstType(); - llvm::Function *maskedStoreFunc = NULL; // Figure out if we need a 8, 16, 32 or 64-bit masked store. - if (dynamic_cast(valueType) != NULL) { + llvm::Function *maskedStoreFunc = NULL; + + const PointerType *pt = dynamic_cast(valueType); + if (pt != NULL) { + if (pt->IsSlice()) { + // For masked stores of (varying) slice pointers to memory, we + // grab the equivalent StructType and make a recursive call to + // maskedStore, giving it that type for the pointer type; that + // in turn will lead to the base pointer and offset index being + // mask stored to memory.. + const StructType *sliceStructType = pt->GetSliceStructType(); + ptrType = PointerType::GetUniform(sliceStructType); + maskedStore(value, ptr, ptrType, mask); + return; + } + if (g->target.is32Bit) maskedStoreFunc = m->module->getFunction("__pseudo_masked_store_32"); else maskedStoreFunc = m->module->getFunction("__pseudo_masked_store_64"); } - else if (valueType == AtomicType::VaryingBool && + else if (Type::Equal(valueType, AtomicType::VaryingBool) && g->target.maskBitCount == 1) { llvm::Value *notMask = BinaryOperator(llvm::Instruction::Xor, mask, LLVMMaskAllOn, "~mask"); @@ -2339,35 +2646,35 @@ FunctionEmitContext::maskedStore(llvm::Value *value, llvm::Value *ptr, StoreInst(final, ptr); return; } - else if (valueType == AtomicType::VaryingDouble || - valueType == AtomicType::VaryingInt64 || - valueType == AtomicType::VaryingUInt64) { + else if (Type::Equal(valueType, AtomicType::VaryingDouble) || + Type::Equal(valueType, AtomicType::VaryingInt64) || + Type::Equal(valueType, AtomicType::VaryingUInt64)) { maskedStoreFunc = m->module->getFunction("__pseudo_masked_store_64"); ptr = BitCastInst(ptr, LLVMTypes::Int64VectorPointerType, "ptr_to_int64vecptr"); value = BitCastInst(value, LLVMTypes::Int64VectorType, "value_to_int64"); } - else if (valueType == AtomicType::VaryingFloat || - valueType == AtomicType::VaryingBool || - valueType == AtomicType::VaryingInt32 || - valueType == AtomicType::VaryingUInt32 || + else if (Type::Equal(valueType, AtomicType::VaryingFloat) || + Type::Equal(valueType, AtomicType::VaryingBool) || + Type::Equal(valueType, AtomicType::VaryingInt32) || + Type::Equal(valueType, AtomicType::VaryingUInt32) || dynamic_cast(valueType) != NULL) { maskedStoreFunc = m->module->getFunction("__pseudo_masked_store_32"); ptr = BitCastInst(ptr, LLVMTypes::Int32VectorPointerType, "ptr_to_int32vecptr"); - if (valueType == AtomicType::VaryingFloat) + if (Type::Equal(valueType, AtomicType::VaryingFloat)) value = BitCastInst(value, LLVMTypes::Int32VectorType, "value_to_int32"); } - else if (valueType == AtomicType::VaryingInt16 || - valueType == AtomicType::VaryingUInt16) { + else if (Type::Equal(valueType, AtomicType::VaryingInt16) || + Type::Equal(valueType, AtomicType::VaryingUInt16)) { maskedStoreFunc = m->module->getFunction("__pseudo_masked_store_16"); ptr = BitCastInst(ptr, LLVMTypes::Int16VectorPointerType, "ptr_to_int16vecptr"); } - else if (valueType == AtomicType::VaryingInt8 || - valueType == AtomicType::VaryingUInt8) { + else if (Type::Equal(valueType, AtomicType::VaryingInt8) || + Type::Equal(valueType, AtomicType::VaryingUInt8)) { maskedStoreFunc = m->module->getFunction("__pseudo_masked_store_8"); ptr = BitCastInst(ptr, LLVMTypes::Int8VectorPointerType, "ptr_to_int8vecptr"); @@ -2391,28 +2698,68 @@ FunctionEmitContext::maskedStore(llvm::Value *value, llvm::Value *ptr, */ void FunctionEmitContext::scatter(llvm::Value *value, llvm::Value *ptr, - const Type *ptrType, llvm::Value *mask) { - Assert(dynamic_cast(ptrType) != NULL); + const Type *valueType, const Type *origPt, + llvm::Value *mask) { + const PointerType *ptrType = dynamic_cast(origPt); + Assert(ptrType != NULL); Assert(ptrType->IsVaryingType()); - const Type *valueType = ptrType->GetBaseType(); - - // I think this should be impossible - Assert(dynamic_cast(valueType) == NULL); - - const CollectionType *collectionType = dynamic_cast(valueType); - if (collectionType != NULL) { + const CollectionType *srcCollectionType = + dynamic_cast(valueType); + if (srcCollectionType != NULL) { + // We're scattering a collection type--we need to keep track of the + // source type (the type of the data values to be stored) and the + // destination type (the type of objects in memory that will be + // stored into) separately. This is necessary so that we can get + // all of the addressing calculations right if we're scattering + // from a varying struct to an array of uniform instances of the + // same struct type, versus scattering into an array of varying + // instances of the struct type, etc. + const CollectionType *dstCollectionType = + dynamic_cast(ptrType->GetBaseType()); + Assert(dstCollectionType != NULL); + // Scatter the collection elements individually - for (int i = 0; i < collectionType->GetElementCount(); ++i) { - llvm::Value *eltPtr = AddElementOffset(ptr, i, ptrType); + for (int i = 0; i < srcCollectionType->GetElementCount(); ++i) { + // First, get the values for the current element out of the + // source. llvm::Value *eltValue = ExtractInst(value, i); - const Type *eltPtrType = - PointerType::GetVarying(collectionType->GetElementType(i)); - eltPtr = addVaryingOffsetsIfNeeded(eltPtr, eltPtrType); - scatter(eltValue, eltPtr, eltPtrType, mask); + const Type *srcEltType = srcCollectionType->GetElementType(i); + + // We may be scattering a uniform atomic element; in this case + // we'll smear it out to be varying before making the recursive + // scatter() call below. + if (srcEltType->IsUniformType() && Type::IsBasicType(srcEltType)) { + eltValue = SmearUniform(eltValue, "to_varying"); + srcEltType = srcEltType->GetAsVaryingType(); + } + + // Get the (varying) pointer to the i'th element of the target + // collection + llvm::Value *eltPtr = AddElementOffset(ptr, i, ptrType); + + // The destination element type may be uniform (e.g. if we're + // scattering to an array of uniform structs). Thus, we need + // to be careful about passing the correct type to + // addVaryingOffsetsIfNeeded() here. + const Type *dstEltType = dstCollectionType->GetElementType(i); + const PointerType *dstEltPtrType = PointerType::GetVarying(dstEltType); + if (ptrType->IsSlice()) + dstEltPtrType = dstEltPtrType->GetAsSlice(); + + eltPtr = addVaryingOffsetsIfNeeded(eltPtr, dstEltPtrType); + + // And recursively scatter() until we hit a basic type, at + // which point the actual memory operations can be performed... + scatter(eltValue, eltPtr, srcEltType, dstEltPtrType, mask); } return; } + else if (ptrType->IsSlice()) { + // As with gather, we need to add the final slice offset finally + // once we get to a terminal SOA array of basic types.. + ptr = lFinalSliceOffset(this, ptr, &ptrType); + } const PointerType *pt = dynamic_cast(valueType); @@ -2483,19 +2830,28 @@ FunctionEmitContext::StoreInst(llvm::Value *value, llvm::Value *ptr) { void FunctionEmitContext::StoreInst(llvm::Value *value, llvm::Value *ptr, - llvm::Value *mask, const Type *ptrType) { + llvm::Value *mask, const Type *valueType, + const Type *ptrRefType) { if (value == NULL || ptr == NULL) { // may happen due to error elsewhere Assert(m->errorCount > 0); return; } - if (dynamic_cast(ptrType) != NULL) - ptrType = PointerType::GetUniform(ptrType->GetReferenceTarget()); + const PointerType *ptrType; + if (dynamic_cast(ptrRefType) != NULL) + ptrType = PointerType::GetUniform(ptrRefType->GetReferenceTarget()); + else { + ptrType = dynamic_cast(ptrRefType); + Assert(ptrType != NULL); + } // Figure out what kind of store we're doing here if (ptrType->IsUniformType()) { - if (ptrType->GetBaseType()->IsUniformType()) + if (ptrType->IsSlice()) + // storing a uniform value to a single slice of a SOA type + storeUniformToSOA(value, ptr, mask, valueType, ptrType); + else if (ptrType->GetBaseType()->IsUniformType()) // the easy case StoreInst(value, ptr); else if (mask == LLVMMaskAllOn && !g->opt.disableMaskAllOnOptimizations) @@ -2509,11 +2865,72 @@ FunctionEmitContext::StoreInst(llvm::Value *value, llvm::Value *ptr, Assert(ptrType->IsVaryingType()); // We have a varying ptr (an array of pointers), so it's time to // scatter - scatter(value, ptr, ptrType, GetFullMask()); + scatter(value, ptr, valueType, ptrType, GetFullMask()); } } +/** Store a uniform type to SOA-laid-out memory. + */ +void +FunctionEmitContext::storeUniformToSOA(llvm::Value *value, llvm::Value *ptr, + llvm::Value *mask, const Type *valueType, + const PointerType *ptrType) { + Assert(Type::Equal(ptrType->GetBaseType()->GetAsUniformType(), valueType)); + + const CollectionType *ct = dynamic_cast(valueType); + if (ct != NULL) { + // Handle collections element wise... + for (int i = 0; i < ct->GetElementCount(); ++i) { + llvm::Value *eltValue = ExtractInst(value, i); + const Type *eltType = ct->GetElementType(i); + const PointerType *dstEltPtrType; + llvm::Value *dstEltPtr = + AddElementOffset(ptr, i, ptrType, "slice_offset", + &dstEltPtrType); + StoreInst(eltValue, dstEltPtr, mask, eltType, dstEltPtrType); + } + } + else { + // We're finally at a leaf SOA array; apply the slice offset and + // then we can do a final regular store + Assert(Type::IsBasicType(valueType)); + ptr = lFinalSliceOffset(this, ptr, &ptrType); + StoreInst(value, ptr); + } +} + + +void +FunctionEmitContext::MemcpyInst(llvm::Value *dest, llvm::Value *src, + llvm::Value *count, llvm::Value *align) { + dest = BitCastInst(dest, LLVMTypes::VoidPointerType); + src = BitCastInst(src, LLVMTypes::VoidPointerType); + if (count->getType() != LLVMTypes::Int64Type) { + Assert(count->getType() == LLVMTypes::Int32Type); + count = ZExtInst(count, LLVMTypes::Int64Type, "count_to_64"); + } + if (align == NULL) + align = LLVMInt32(1); + + llvm::Constant *mcFunc = + m->module->getOrInsertFunction("llvm.memcpy.p0i8.p0i8.i64", + LLVMTypes::VoidType, LLVMTypes::VoidPointerType, + LLVMTypes::VoidPointerType, LLVMTypes::Int64Type, + LLVMTypes::Int32Type, LLVMTypes::BoolType, NULL); + Assert(mcFunc != NULL); + Assert(llvm::isa(mcFunc)); + + std::vector args; + args.push_back(dest); + args.push_back(src); + args.push_back(count); + args.push_back(align); + args.push_back(LLVMFalse); /* not volatile */ + CallInst(mcFunc, NULL, args, ""); +} + + void FunctionEmitContext::BranchInst(llvm::BasicBlock *dest) { llvm::Instruction *b = llvm::BranchInst::Create(dest, bblock); @@ -2761,7 +3178,7 @@ FunctionEmitContext::CallInst(llvm::Value *func, const FunctionType *funcType, // accumulate the result using the call mask. if (callResult != NULL) { Assert(resultPtr != NULL); - StoreInst(callResult, resultPtr, callMask, + StoreInst(callResult, resultPtr, callMask, returnType, PointerType::GetUniform(returnType)); } else @@ -2825,7 +3242,7 @@ FunctionEmitContext::ReturnInst() { rinst = llvm::ReturnInst::Create(*g->ctx, retVal, bblock); } else { - Assert(function->GetReturnType() == AtomicType::Void); + Assert(Type::Equal(function->GetReturnType(), AtomicType::Void)); rinst = llvm::ReturnInst::Create(*g->ctx, bblock); } @@ -2945,11 +3362,10 @@ FunctionEmitContext::addVaryingOffsetsIfNeeded(llvm::Value *ptr, Assert(pt && pt->IsVaryingType()); const Type *baseType = ptrType->GetBaseType(); - if (dynamic_cast(baseType) == NULL && - dynamic_cast(baseType) == NULL && - dynamic_cast(baseType) == NULL) + if (Type::IsBasicType(baseType) == false) return ptr; - if (baseType->IsUniformType()) + + if (baseType->IsVaryingType() == false) return ptr; // Find the size of a uniform element of the varying type diff --git a/ctx.h b/ctx.h index adf26560..74b19596 100644 --- a/ctx.h +++ b/ctx.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2010-2011, Intel Corporation + Copyright (c) 2010-2012, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -392,6 +392,16 @@ public: llvm::Instruction *ZExtInst(llvm::Value *value, LLVM_TYPE_CONST llvm::Type *type, const char *name = NULL); + /** Given two integer-typed values (but possibly one vector and the + other not, and or of possibly-different bit-widths), update their + values as needed so that the two have the same (more general) + type. */ + void MatchIntegerTypes(llvm::Value **v0, llvm::Value **v1); + + /** Create a new slice pointer out of the given pointer to an soa type + and an integer offset to a slice within that type. */ + llvm::Value *MakeSlicePointer(llvm::Value *ptr, llvm::Value *offset); + /** These GEP methods are generalizations of the standard ones in LLVM; they support both uniform and varying basePtr values as well as uniform and varying index values (arrays of indices). Varying base @@ -412,7 +422,8 @@ public: the type of the pointer, though it may be NULL if the base pointer is uniform. */ llvm::Value *AddElementOffset(llvm::Value *basePtr, int elementNum, - const Type *ptrType, const char *name = NULL); + const Type *ptrType, const char *name = NULL, + const PointerType **resultPtrType = NULL); /** Load from the memory location(s) given by lvalue, using the given mask. The lvalue may be varying, in which case this corresponds to @@ -443,7 +454,14 @@ public: varying, the given storeMask is used to mask the stores so that they only execute for the active program instances. */ void StoreInst(llvm::Value *value, llvm::Value *ptr, - llvm::Value *storeMask, const Type *ptrType); + llvm::Value *storeMask, const Type *valueType, + const Type *ptrType); + + /** Copy count bytes of memory from the location pointed to by src to + the location pointed to by dest. (src and dest must not be + overlapping.) */ + void MemcpyInst(llvm::Value *dest, llvm::Value *src, llvm::Value *count, + llvm::Value *align = NULL); void BranchInst(llvm::BasicBlock *block); void BranchInst(llvm::BasicBlock *trueBlock, llvm::BasicBlock *falseBlock, @@ -646,12 +664,19 @@ private: CFInfo *popCFState(); - void scatter(llvm::Value *value, llvm::Value *ptr, const Type *ptrType, - llvm::Value *mask); + void scatter(llvm::Value *value, llvm::Value *ptr, const Type *valueType, + const Type *ptrType, llvm::Value *mask); void maskedStore(llvm::Value *value, llvm::Value *ptr, const Type *ptrType, llvm::Value *mask); - llvm::Value *gather(llvm::Value *ptr, const Type *ptrType, llvm::Value *mask, - const char *name); + void storeUniformToSOA(llvm::Value *value, llvm::Value *ptr, + llvm::Value *mask, const Type *valueType, + const PointerType *ptrType); + llvm::Value *loadUniformFromSOA(llvm::Value *ptr, llvm::Value *mask, + const PointerType *ptrType, const char *name); + + llvm::Value *gather(llvm::Value *ptr, const PointerType *ptrType, + llvm::Value *mask, const char *name); + llvm::Value *addVaryingOffsetsIfNeeded(llvm::Value *ptr, const Type *ptrType); }; diff --git a/decl.cpp b/decl.cpp index f5c0eb88..44867b4e 100644 --- a/decl.cpp +++ b/decl.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2010-2011, Intel Corporation + Copyright (c) 2010-2012, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -69,12 +69,21 @@ lApplyTypeQualifiers(int typeQualifiers, const Type *type, SourcePos pos) { if ((typeQualifiers & TYPEQUAL_CONST) != 0) type = type->GetAsConstType(); - if ((typeQualifiers & TYPEQUAL_UNIFORM) != 0) - type = type->GetAsUniformType(); - else if ((typeQualifiers & TYPEQUAL_VARYING) != 0) - type = type->GetAsVaryingType(); + if ((typeQualifiers & TYPEQUAL_UNIFORM) != 0) { + if (Type::Equal(type, AtomicType::Void)) + Error(pos, "\"uniform\" qualifier is illegal with \"void\" type."); + else + type = type->GetAsUniformType(); + } + else if ((typeQualifiers & TYPEQUAL_VARYING) != 0) { + if (Type::Equal(type, AtomicType::Void)) + Error(pos, "\"varying\" qualifier is illegal with \"void\" type."); + else + type = type->GetAsVaryingType(); + } else - type = type->GetAsUnboundVariabilityType(); + if (Type::Equal(type, AtomicType::Void) == false) + type = type->GetAsUnboundVariabilityType(); if ((typeQualifiers & TYPEQUAL_UNSIGNED) != 0) { if ((typeQualifiers & TYPEQUAL_SIGNED) != 0) @@ -84,15 +93,20 @@ lApplyTypeQualifiers(int typeQualifiers, const Type *type, SourcePos pos) { const Type *unsignedType = type->GetAsUnsignedType(); if (unsignedType != NULL) type = unsignedType; - else + else { + const Type *resolvedType = + type->ResolveUnboundVariability(Variability::Varying); Error(pos, "\"unsigned\" qualifier is illegal with \"%s\" type.", - type->ResolveUnboundVariability(Type::Varying)->GetString().c_str()); + resolvedType->GetString().c_str()); + } } - if ((typeQualifiers & TYPEQUAL_SIGNED) != 0 && type->IsIntType() == false) + if ((typeQualifiers & TYPEQUAL_SIGNED) != 0 && type->IsIntType() == false) { + const Type *resolvedType = + type->ResolveUnboundVariability(Variability::Varying); Error(pos, "\"signed\" qualifier is illegal with non-integer type " - "\"%s\".", - type->ResolveUnboundVariability(Type::Varying)->GetString().c_str()); + "\"%s\".", resolvedType->GetString().c_str()); + } return type; } @@ -112,24 +126,54 @@ DeclSpecs::DeclSpecs(const Type *t, StorageClass sc, int tq) { const Type * DeclSpecs::GetBaseType(SourcePos pos) const { - const Type *bt = baseType; + const Type *retType = baseType; - if (bt == NULL) { + if (retType == NULL) { Warning(pos, "No type specified in declaration. Assuming int32."); - bt = AtomicType::UnboundInt32; + retType = AtomicType::UniformInt32->GetAsUnboundVariabilityType(); } if (vectorSize > 0) { - const AtomicType *atomicType = dynamic_cast(bt); + const AtomicType *atomicType = dynamic_cast(retType); if (atomicType == NULL) { Error(pos, "Only atomic types (int, float, ...) are legal for vector " "types."); return NULL; } - bt = new VectorType(atomicType, vectorSize); + retType = new VectorType(atomicType, vectorSize); } - return lApplyTypeQualifiers(typeQualifiers, bt, pos); + retType = lApplyTypeQualifiers(typeQualifiers, retType, pos); + + if (soaWidth > 0) { + const StructType *st = dynamic_cast(retType); + + if (st == NULL) { + Error(pos, "Illegal to provide soa<%d> qualifier with non-struct " + "type \"%s\".", soaWidth, retType->GetString().c_str()); + return NULL; + } + else if (soaWidth <= 0 || (soaWidth & (soaWidth - 1)) != 0) { + Error(pos, "soa<%d> width illegal. Value must be positive power " + "of two.", soaWidth); + return NULL; + } + + if (st->IsUniformType()) { + Error(pos, "\"uniform\" qualifier and \"soa<%d>\" qualifier can't " + "both be used in a type declaration.", soaWidth); + return NULL; + } + else if (st->IsVaryingType()) { + Error(pos, "\"varying\" qualifier and \"soa<%d>\" qualifier can't " + "both be used in a type declaration.", soaWidth); + return NULL; + } + else + retType = st->GetAsSOAType(soaWidth); + } + + return retType; } @@ -280,13 +324,13 @@ Declarator::GetFunctionInfo(DeclSpecs *ds, std::vector *funArgs) { continue; } else - sym->type = sym->type->ResolveUnboundVariability(Type::Varying); + sym->type = sym->type->ResolveUnboundVariability(Variability::Varying); funArgs->push_back(sym); } if (funSym != NULL) - funSym->type = funSym->type->ResolveUnboundVariability(Type::Varying); + funSym->type = funSym->type->ResolveUnboundVariability(Variability::Varying); return funSym; } @@ -306,11 +350,11 @@ Declarator::GetType(const Type *base, DeclSpecs *ds) const { if (kind != DK_FUNCTION && isTask) Error(pos, "\"task\" qualifier illegal in variable declaration."); - Type::Variability variability = Type::Unbound; + Variability variability(Variability::Unbound); if (hasUniformQual) - variability = Type::Uniform; + variability = Variability::Uniform; else if (hasVaryingQual) - variability = Type::Varying; + variability = Variability::Varying; const Type *type = base; switch (kind) { @@ -322,7 +366,10 @@ Declarator::GetType(const Type *base, DeclSpecs *ds) const { return type; case DK_POINTER: - type = new PointerType(type, variability, isConst); + /* For now, any pointer to an SOA type gets the slice property; if + we add the capability to declare pointers as slices or not, + we'll want to set this based on a type qualifier here. */ + type = new PointerType(type, variability, isConst, type->IsSOAType()); if (child != NULL) return child->GetType(type, ds); else @@ -351,7 +398,7 @@ Declarator::GetType(const Type *base, DeclSpecs *ds) const { break; case DK_ARRAY: - if (type == AtomicType::Void) { + if (Type::Equal(type, AtomicType::Void)) { Error(pos, "Arrays of \"void\" type are illegal."); return NULL; } @@ -387,7 +434,7 @@ Declarator::GetType(const Type *base, DeclSpecs *ds) const { "function parameter declaration for parameter \"%s\".", lGetStorageClassName(d->declSpecs->storageClass), sym->name.c_str()); - if (sym->type == AtomicType::Void) { + if (Type::Equal(sym->type, AtomicType::Void)) { Error(sym->pos, "Parameter with type \"void\" illegal in function " "parameter list."); sym->type = NULL; @@ -408,7 +455,11 @@ Declarator::GetType(const Type *base, DeclSpecs *ds) const { return NULL; } - sym->type = PointerType::GetUniform(at->GetElementType()); + const Type *targetType = at->GetElementType(); + targetType = + targetType->ResolveUnboundVariability(Variability::Varying); + sym->type = PointerType::GetUniform(targetType); + // Make sure there are no unsized arrays (other than the // first dimension) in function parameter lists. at = dynamic_cast(at->GetElementType()); @@ -485,36 +536,13 @@ Declarator::GetType(const Type *base, DeclSpecs *ds) const { const Type *functionType = new FunctionType(returnType, args, argNames, argDefaults, argPos, isTask, isExported, isExternC); - functionType = functionType->ResolveUnboundVariability(Type::Varying); + functionType = functionType->ResolveUnboundVariability(Variability::Varying); return child->GetType(functionType, ds); } default: FATAL("Unexpected decl kind"); return NULL; } - -#if 0 - // Make sure we actually have an array of structs .. - const StructType *childStructType = - dynamic_cast(childType); - if (childStructType == NULL) { - Error(pos, "Illegal to provide soa<%d> qualifier with non-struct " - "type \"%s\".", soaWidth, childType->GetString().c_str()); - return new ArrayType(childType, arraySize == -1 ? 0 : arraySize); - } - else if ((soaWidth & (soaWidth - 1)) != 0) { - Error(pos, "soa<%d> width illegal. Value must be power of two.", - soaWidth); - return NULL; - } - else if (arraySize != -1 && (arraySize % soaWidth) != 0) { - Error(pos, "soa<%d> width must evenly divide array size %d.", - soaWidth, arraySize); - return NULL; - } - return new SOAArrayType(childStructType, arraySize == -1 ? 0 : arraySize, - soaWidth); -#endif } @@ -596,9 +624,9 @@ Declaration::GetVariableDeclarations() const { Assert(m->errorCount > 0); continue; } - sym->type = sym->type->ResolveUnboundVariability(Type::Varying); + sym->type = sym->type->ResolveUnboundVariability(Variability::Varying); - if (sym->type == AtomicType::Void) + if (Type::Equal(sym->type, AtomicType::Void)) Error(sym->pos, "\"void\" type variable illegal in declaration."); else if (dynamic_cast(sym->type) == NULL) { m->symbolTable->AddVariable(sym); @@ -627,7 +655,7 @@ Declaration::DeclareFunctions() { Assert(m->errorCount > 0); continue; } - sym->type = sym->type->ResolveUnboundVariability(Type::Varying); + sym->type = sym->type->ResolveUnboundVariability(Variability::Varying); if (dynamic_cast(sym->type) == NULL) continue; @@ -674,7 +702,7 @@ GetStructTypesNamesPositions(const std::vector &sd, Symbol *sym = d->GetSymbol(); - if (sym->type == AtomicType::Void) + if (Type::Equal(sym->type, AtomicType::Void)) Error(d->pos, "\"void\" type illegal for struct member."); const ArrayType *arrayType = diff --git a/docs/build.sh b/docs/build.sh index f5700c38..dd20be5e 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -5,6 +5,9 @@ for i in ispc perfguide faq; do --stylesheet-path=css/style.css $i.rst > $i.html done +rst2html.py --template=template-news.txt --link-stylesheet \ + --stylesheet-path=css/style.css news.rst > news.html + rst2html.py --template=template-perf.txt --link-stylesheet \ --stylesheet-path=css/style.css perf.rst > perf.html diff --git a/docs/faq.rst b/docs/faq.rst index 2ce7268b..2cdca136 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -19,6 +19,7 @@ distribution. + `How can I supply an initial execution mask in the call from the application?`_ + `How can I generate a single binary executable with support for multiple instruction sets?`_ + `How can I determine at run-time which vector instruction set's instructions were selected to execute?`_ + + `Is it possible to inline ispc functions in C/C++ code?`_ * Programming Techniques @@ -346,6 +347,50 @@ In a similar fashion, it's possible to find out at run-time the value of export uniform int width() { return programCount; } +Is it possible to inline ispc functions in C/C++ code? +------------------------------------------------------ + +If you're willing to use the ``clang`` C/C++ compiler that's part of the +LLVM tool suite, then it is possible to inline ``ispc`` code with C/C++ +(and conversely, to inline C/C++ calls in ``ispc``). Doing so can provide +performance advantages when calling out to short functions written in the +"other" language. Note that you don't need to use ``clang`` to compile all +of your C/C++ code, but only for the files where you want to be able to +inline. In order to do this, you must have a full installation of LLVM +version 3.0 or later, including the ``clang`` compiler. + +The basic approach is to have the various compilers emit LLVM intermediate +representation (IR) code and to then use tools from LLVM to link together +the IR from the compilers and then re-optimize it, which gives the LLVM +optimizer the opportunity to do additional inlining and cross-function +optimizations. If you have source files ``foo.ispc`` and ``foo.cpp``, +first emit LLVM IR: + +:: + + ispc --emit-llvm -o foo_ispc.bc foo.ispc + clang -O2 -c -emit-llvm -o foo_cpp.bc foo.cpp + +Next, link the two IR files into a single file and run the LLVM optimizer +on the result: + +:: + + llvm-link foo_ispc.bc foo_cpp.bc -o - | opt -O3 -o foo_opt.bc + +And finally, generate a native object file: + +:: + + llc -filetype=obj foo_opt.bc -o foo.o + +This file can in turn be linked in with the rest of your object files when +linking your applicaiton. + +(Note that if you're using the AVX instruction set, you must provide the +``-mattr=+avx`` flag to ``llc``.) + + Programming Techniques ====================== diff --git a/docs/ispc.rst b/docs/ispc.rst index aa15158d..61f4a21a 100644 --- a/docs/ispc.rst +++ b/docs/ispc.rst @@ -92,7 +92,9 @@ Contents: * `Reference Types`_ * `Enumeration Types`_ * `Short Vector Types`_ - * `Struct and Array Types`_ + * `Array Types`_ + * `Struct Types`_ + * `Structure of Array Types`_ + `Declarations and Initializers`_ + `Expressions`_ @@ -132,9 +134,13 @@ Contents: * `Reductions`_ - + `Data Conversions And Storage`_ + + `Data Movement`_ + * `Setting and Copying Values In Memory`_ * `Packed Load and Store Operations`_ + + + `Data Conversions`_ + * `Converting Between Array-of-Structures and Structure-of-Arrays Layout`_ * `Conversions To and From Half-Precision Floats`_ @@ -150,7 +156,6 @@ Contents: + `Data Layout`_ + `Data Alignment and Aliasing`_ + `Restructuring Existing Programs to Use ISPC`_ - + `Understanding How to Interoperate With the Application's Data`_ * `Disclaimer and Legal Information`_ @@ -859,7 +864,9 @@ A variable that is declared with the ``uniform`` qualifier represents a single value that is shared across the entire gang. (In contrast, the default variability qualifier for variables in ``ispc``, ``varying``, represents a variable that has a distinct storage location for each program -instance in the gang.) +instance in the gang.) (Though see the discussion in `Struct Types`_ for +some subtleties related to ``uniform`` and ``varying`` when used with +structures.) It is an error to try to assign a ``varying`` value to a ``uniform`` variable, though ``uniform`` values can be assigned to ``uniform`` @@ -1499,56 +1506,72 @@ Pointer Types It is possible to have pointers to data in memory; pointer arithmetic, changing values in memory with pointers, and so forth is supported as in C. +As with other basic types, pointers can be both ``uniform`` and +``varying``. + +** Like other types in ``ispc``, pointers are ``varying`` by default, if an +explicit ``uniform`` qualifier isn't provided. However, the default +variability of the pointed-to type is uniform. ** This rule will be +illustrated and explained in examples below. + +For example, the ``ptr`` variable in the code below is a varying pointer to +``uniform float`` values. Each program instance has a separate pointer +value and the assignment to ``*ptr`` generally represents a scatter to +memory. :: - float a = 0; - float *pa = &a; - *pa = 1; // now a == 1 + uniform float a[] = ...; + int index = ...; + float * ptr = &a[index]; + *ptr = 1; +A ``uniform`` pointer can be declared with an appropriately-placed +qualifier: + +:: + + float f = 0; + varying float * uniform pf = &f; // uniform pointer to a varying float + *pf = 1; + +The placement of the ``uniform`` qualifier to declare a ``uniform`` pointer +may be initially surprising, but it matches the form of how, for example, a +pointer that is itself ``const`` (as opposed to pointing to a ``const`` +type) is declared in C. (Reading the declaration from right to left gives +its meaning: a uniform pointer to a float that is varying.) + +A subtlety comes in in cases like the where a uniform pointer points to a +varying datatype. In this case, each program instance accesses a distinct +location in memory (because the underlying varying datatype is itself laid +out with a separate location in memory for each program instance.) + +:: + + float a; + varying float * uniform pa = &a; + *pa = programIndex; // same as (a = programIndex) + Also as in C, arrays are silently converted into pointers: :: float a[10] = { ... }; - float *pa = a; // pointer to first element of a - float *pb = a + 5; // pointer to 5th element of a - -As with other basic types, pointers can be both ``uniform`` and -``varying``. By default, they are varying. The placement of the -``uniform`` qualifier to declare a ``uniform`` pointer may be initially -surprising, but it matches the form of how for example a pointer that is -itself ``const`` (as opposed to pointing to a ``const`` type) is declared -in C. - -:: - - uniform float f = 0; - uniform float * uniform pf = &f; - *pf = 1; - -A subtlety comes in when a uniform pointer points to a varying datatype. -In this case, each program instance accesses a distinct location in memory -(because the underlying varying datatype is itself laid out with a separate -location in memory for each program instance.) - -:: - - float a; - float * uniform pa = &a; - *pa = programIndex; // same as (a = programIndex) - + varying float * uniform pa = a; // pointer to first element of a + varying float * uniform pb = a + 5; // pointer to 5th element of a Any pointer type can be explicitly typecast to another pointer type, as long as the source type isn't a ``varying`` pointer when the destination -type is a ``uniform`` pointer. Like other types, ``uniform`` pointers can -be typecast to be ``varying`` pointers, however. +type is a ``uniform`` pointer. :: float *pa = ...; int *pb = (int *)pa; // legal, but beware +Like other types, ``uniform`` pointers can be typecast to be ``varying`` +pointers, however. + Any pointer type can be assigned to a ``void`` pointer without a type cast: :: @@ -1815,20 +1838,19 @@ expressions: int8<2> bar = ...; foo.yz = bar; // Error: can't assign to left-hand side of expression -Struct and Array Types ----------------------- -More complex data structures can be built using ``struct`` and arrays. +Array Types +----------- + +Arrays of any type can be declared just as in C and C++: :: - struct Foo { - float time; - int flags[10]; - }; + float a[10]; + uniform int * varying b[20]; -Like in C, multidimensional arrays can be specified; the following declares -an array of 5 arrays of 15 floats. +Multidimensional arrays can be specified as arrays of arrays; the following +declares an array of 5 arrays of 15 floats. :: @@ -1843,6 +1865,27 @@ that functions can be declared to take "unsized arrays" as parameters: void foo(float array[], int length); +Finally, the name of an array will be automatically implicitly converted to +a uniform pointer to the array type if needed: + +:: + + int a[10]; + int * uniform ap = a; + + +Struct Types +------------ + +Aggregate data structures can be built using ``struct``. + +:: + + struct Foo { + float time; + int flags[10]; + }; + As in C++, after a ``struct`` is declared, an instance can be created using the ``struct``'s name: @@ -1856,6 +1899,131 @@ Alternatively, ``struct`` can be used before the structure name: struct Foo f; +Members in a structure declaration may each have ``uniform`` or ``varying`` +qualifiers, or may have no rate qualifier, in which case their variability +is initially "unbound". + +:: + + struct Bar { + uniform int a; + varying int b; + int c; + }; + + +In the declaration above, the variability of ``c`` is unbound. The +variability of struct members that are unbound is resolved when a struct is +defined; if the ``struct`` is ``uniform``, then unbound members are +``uniform``, and if the ``struct`` is ``varying``, then unbound members are +varying. + +:: + + Bar vb; + uniform Bar ub; + +Here, ``b`` is a ``varying Bar`` (since ``varying`` is the default +variability). If ``Bar`` is defined as above, then ``vb.a`` is still a +``uniform int``, since its varaibility was bound in the original +declaration of the ``Bar`` type. Similarly, ``vb.b`` is ``varying``. The +variability fo ``vb.c`` is ``varying``, since ``vb`` is ``varying``. + +(Similarly, ``ub.a`` is ``uniform``, ``ub.b`` is ``varying``, and ``ub.c`` +is ``uniform``.) + +In most cases, it's worthwhile to declare ``struct`` members with unbound +variability so that all have the same variability for both ``uniform`` and +``varying`` structs. In particular, if a ``struct`` has a member with +bound ``uniform`` type, it's not possible to index into an array of the +struct type with a ``varying`` index. Consider the following example: + +:: + + struct Foo { uniform int a; }; + uniform Foo f[...] = ...; + int index = ...; + Foo fv = f[index]; // ERROR + +Here, the ``Foo`` type has a member with bound ``uniform`` variability. +Because ``index`` has a different value for each program instance in the +above code, the value of ``f[index]`` needs to be able to store a different +value of ``Foo::a`` for each program instance. However, a ``varying Foo`` +still has only a single ``a`` member, since ``a`` was declared with +``uniform`` variability in the declaration of ``Foo``. Therefore, the +indexing operation in the last line results in an error. + + +Structure of Array Types +------------------------ + +If data can be laid out in memory so that the executing program instances +access it via loads and stores of contiguous sections of memory, overall +performance can be improved noticably. One way to improve this memory +access coherence is to lay out structures in "structure of arrays" (SOA) +format in memory; the benefits from SOA layout are discussed in more detail +in the `Use "Structure of Arrays" Layout When Possible`_ section in the +ispc Performance Guide. + +.. _Use "Structure of Arrays" Layout When Possible: perf.html#use-structure-of-arrays-layout-when-possible + +``ispc`` provides two key language-level capabilities for laying out and +accessing data in SOA format: + +* An ``soa`` keyword that transforms a regular ``struct`` into an SOA version + of the struct. +* Array indexing syntax for SOA arrays that transparently handles SOA + indexing. + +As an example, consider a simple struct declaration: + +:: + + struct Point { float x, y, z; }; + +With the ``soa`` rate qualifier, an array of SOA variants of this structure +can be declared: + +:: + + soa<8> Point pts[...]; + +The in-memory layout of the ``Point``s has had the SOA transformation +applied, such that there are 8 ``x`` values in memory followed by 8 ``y`` +values, and so forth. Here is the effective declaration of ``soa<8> +Point``: + +:: + + struct { uniform float x[8], y[8], z[8]; }; + +Given an array of SOA data, array indexing (and pointer arithmetic) is done +so that the appropriate values from the SOA array are accessed. For +example, given: + +:: + + soa<8> Point pts[...]; + uniform float x = pts[10].x; + +The generated code effectively accesses the second 8-wide SOA structure and +then loads the third ``x`` value from it. In general, one can write the +same code to access arrays of SOA elements as one would write to access +them in AOS layout. + +Note that it directly follows from SOA layout that the layout of a single +element of the array isn't contiguous in memory--``pts[1].x`` and +``pts[1].y`` are separated by 7 ``float`` values in the above example. + +There are a few limitations to the current implementation of SOA types in +``ispc``; these may be relaxed in future releases: + +* It's illegal to typecast to ``soa`` data to ``void`` pointers. +* Reference types are illegal in SOA structures +* All members of SOA structures must have no rate qualifiers--specifically, + it's illegal to have an explicitly-qualified ``uniform`` or ``varying`` + member of a structure that has ``soa`` applied to it. + Declarations and Initializers ----------------------------- @@ -1992,7 +2160,7 @@ based on C++'s ``new`` and ``delete`` operators: :: int count = ...; - int *ptr = new uniform int[count]; + int *ptr = new int[count]; // use ptr... delete[] ptr; @@ -2002,13 +2170,22 @@ that memory. Uses of ``new`` and ``delete`` in ``ispc`` programs are serviced by corresponding calls the system C library's ``malloc()`` and ``free()`` functions. +Note that the rules for ``uniform`` and ``varying`` for ``new`` are +analogous to the corresponding rules are for pointers (as described in +`Pointer Types`_.) Specifically, if a specific rate qualifier isn't +provided with the ``new`` expression, then the default is that a "varying" +``new`` is performed, where each program instance performs a unique +allocation. The allocated type, in turn, is by default ``uniform`` for +``varying`` ``new`` expressions, and ``varying`` for ``uniform`` new +expressions. + After a pointer has been deleted, it is illegal to access the memory it -points to. However, note that deletion happens on a per-program-instance -basis. In other words, consider the following code: +points to. However, that deletion happens on a per-program-instance basis. +In other words, consider the following code: :: - int *ptr = new uniform int[count]; + int *ptr = new int[count]; // use ptr if (count > 1000) delete[] ptr; @@ -2033,7 +2210,9 @@ gang of program instances. A ``new`` statement can be qualified with While a regular call to ``new`` returns a ``varying`` pointer (i.e. a distinct pointer to separately-allocated memory for each program instance), a ``uniform new`` performs a single allocation and returns a ``uniform`` -pointer. +pointer. Recall that with a ``uniform`` ``new``, the default variability +of the allocated type is ``varying``, so the above code is allocating an +array of ten ``varying float`` values. When using ``uniform new``, it's important to be aware of a subtlety; if the returned pointer is stored in a varying pointer variable (as may be @@ -2043,7 +2222,7 @@ statement, which is an error: effectively :: - float *ptr = uniform new float[10]; + varying float * ptr = uniform new float[10]; // use ptr... delete ptr; // ERROR: varying pointer is deleted @@ -2052,28 +2231,31 @@ executing program instance, which is an error (unless it happens that only a single program instance is active in the above code.) When using ``new`` statements, it's important to make an appropriate choice -of ``uniform`` or ``varying`` (as always, the default), for both the -``new`` operator itself as well as the type of data being allocated, based -on the program's needs. Consider the following four memory allocations: +of ``uniform`` or ``varying``, for both the ``new`` operator itself as well +as the type of data being allocated, based on the program's needs. +Consider the following four memory allocations: :: uniform float * uniform p1 = uniform new uniform float[10]; float * uniform p2 = uniform new float[10]; - uniform float * p3 = new uniform float[10]; - float * p4 = new float[10]; + float * p3 = new float[10]; + varying float * p4 = new varying float[10]; Assuming that a ``float`` is 4 bytes in memory and if the gang size is 8 program instances, then the first allocation represents a single allocation -of 40 bytes, the second is a single allocation of 8*4*10 = 320 bytes, the -third is 8 allocations of 40 bytes, and the last performs 8 allocations of -80 bytes each. +of 10 ``uniform float`` values (40 bytes), the second is a single +allocation of 10 ``varying float`` values (8*4*10 = 320 bytes), the third +is 8 allocations of 10 ``uniform float`` values (8 allocations of 40 bytes +each), and the last performs 8 allocations of 320 bytes each. Note in particular that varying allocations of varying data types are rarely desirable in practice. In that case, each program instance is performing a separate allocation of ``varying float`` memory. In this case, it's likely that the program instances will only access a single element of each -``varying float``, which is wasteful. +``varying float``, which is wasteful. (This in turn is partially why the +allocated type is uniform by default with both pointers and ``new`` +statements.) Although ``ispc`` doesn't support constructors or destructors like C++, it is possible to provide initializer values with ``new`` statements: @@ -2835,13 +3017,17 @@ architectures. float tan(float x) uniform float tan(uniform float x) -Arctangent functions are also available: +The corresponding inverse functions are also available: :: + float asin(float x) + uniform float asin(uniformfloat x) + float acos(float x) + uniform float acos(uniform float x) float atan(float x) - float atan2(float x, float y) uniform float atan(uniform float x) + float atan2(float x, float y) uniform float atan2(uniform float x, uniform float y) If both sine and cosine are needed, then the ``sincos()`` call computes @@ -2850,7 +3036,7 @@ functions: :: - void sincos(float x, float * uniform s, float * uniform c) + void sincos(float x, varying float * uniform s, varying float * uniform c) void sincos(uniform float x, uniform float * uniform s, uniform float * uniform c) @@ -2876,7 +3062,7 @@ normalized exponent as a power of two in the ``pw2`` parameter. float ldexp(float x, int n) uniform float ldexp(uniform float x, uniform int n) - float frexp(float x, int * uniform pw2) + float frexp(float x, varying int * uniform pw2) uniform float frexp(uniform float x, uniform int * uniform pw2) @@ -2891,7 +3077,8 @@ library. State for the RNG is maintained in an instance of the :: struct RNGState; - void seed_rng(RNGState * uniform state, uniform int seed) + void seed_rng(varying RNGState * uniform state, uniform int seed) + void seed_rng(uniform RNGState * uniform state, uniform int seed) After the RNG is seeded, the ``random()`` function can be used to get a pseudo-random ``unsigned int32`` value and the ``frandom()`` function can @@ -2899,8 +3086,10 @@ be used to get a pseudo-random ``float`` value. :: - unsigned int32 random(RNGState * uniform state) - float frandom(RNGState * uniform state) + unsigned int32 random(varying RNGState * uniform state) + float frandom(varying RNGState * uniform state) + uniform unsigned int32 random(RNGState * uniform state) + uniform float frandom(uniform RNGState * uniform state) Output Functions ---------------- @@ -3202,8 +3391,52 @@ program instances into a compact output buffer is `discussed in the FAQ`_. .. _discussed in the FAQ: faq.html#how-can-a-gang-of-program-instances-generate-variable-amounts-of-output-efficiently -Data Conversions And Storage ----------------------------- +Data Movement +------------- + +Setting and Copying Values In Memory +------------------------------------ + +There are a few functions for copying blocks of memory and initializing +values in memory. Along the lines of the equivalently-named routines in +the C Standard libary, ``memcpy`` copies a given number of bytes starting +from a source location in memory to a destination locaiton, where the two +regions of memory are guaranteed by the caller to be non-overlapping. +Alternatively, ``memmove`` can be used to copy data if the buffers may +overlap. + +:: + + void memcpy(void * uniform dst, void * uniform src, uniform int32 count) + void memmove(void * uniform dst, void * uniform src, uniform int32 count) + void memcpy(void * varying dst, void * varying src, int32 count) + void memmove(void * varying dst, void * varying src, int32 count) + +Note that there are variants of these functions that take both ``uniform`` +and ``varying`` pointers. + +To initialize values in memory, the ``memset`` routine can be used. (It +also behaves like the function of the same name in the C Standard Library.) +It sets the given number of bytes of memory starting at the given location +to the value provided. + +:: + + void memset(void * uniform ptr, uniform int8 val, uniform int32 count) + void memset(void * varying ptr, int8 val, int32 count) + +There are also variants of all of these functions that take 64-bit values +for the number of bytes of memory to operate on: + +:: + + void memcpy64(void * uniform dst, void * uniform src, uniform int64 count) + void memcpy64(void * varying dst, void * varying src, int64 count) + void memmove64(void * uniform dst, void * uniform src, uniform int64 count) + void memmove64(void * varying dst, void * varying src, int64 count) + void memset64(void * uniform ptr, uniform int8 val, uniform int64 count) + void memset64(void * varying ptr, int8 val, int64 count) + Packed Load and Store Operations -------------------------------- @@ -3218,9 +3451,9 @@ variable. They return the total number of values loaded. :: uniform int packed_load_active(uniform int * uniform base, - int * uniform val) + varying int * uniform val) uniform int packed_load_active(uniform unsigned int * uniform base, - unsigned int * uniform val) + varying unsigned int * uniform val) Similarly, the ``packed_store_active()`` functions store the ``val`` values for each program instances that executed the ``packed_store_active()`` @@ -3262,14 +3495,18 @@ of four negative values, and initializes the first four elements of indices where ``a[i]`` was less than zero. +Data Conversions +---------------- + Converting Between Array-of-Structures and Structure-of-Arrays Layout --------------------------------------------------------------------- Applications often lay data out in memory in "array of structures" form. Though convenient in C/C++ code, this layout can make ``ispc`` programs less efficient than they would be if the data was laid out in "structure of -arrays" form. (See the section `Understanding How to Interoperate With the -Application's Data`_ for extended discussion of this topic.) +arrays" form. (See the section `Use "Structure of Arrays" Layout When +Possible`_ in the performance guide for extended discussion of this topic.) + The standard library does provide a few functions that efficiently convert between these two formats, for cases where it's not possible to change the @@ -3309,10 +3546,10 @@ are both ``int32`` and ``float`` variants of this function: :: - void aos_to_soa3(uniform float a[], float * uniform v0, - float * uniform v1, float * uniform v2) - void aos_to_soa3(uniform int32 a[], int32 * uniform v0, - int32 * uniform v1, int32 * uniform v2) + void aos_to_soa3(uniform float a[], varying float * uniform v0, + varying float * uniform v1, varying float * uniform v2) + void aos_to_soa3(uniform int32 a[], varying int32 * uniform v0, + varying int32 * uniform v1, varying int32 * uniform v2) After computation is done, corresponding functions convert back from the SoA values in ``ispc`` ``varying`` variables and write the values back to @@ -3341,10 +3578,12 @@ the given array, starting at the given offset. :: - void aos_to_soa4(uniform float a[], float * uniform v0, float * uniform v1, - float * uniform v2, float * uniform v3) - void aos_to_soa4(uniform int32 a[], int32 * uniform v0, int32 * uniform v1, - int32 * uniform v2, int32 * uniform v3) + void aos_to_soa4(uniform float a[], varying float * uniform v0, + varying float * uniform v1, varying float * uniform v2, + varying float * uniform v3) + void aos_to_soa4(uniform int32 a[], varying int32 * uniform v0, + varying int32 * uniform v1, varying int32 * uniform v2, + varying int32 * uniform v3) void soa_to_aos4(float v0, float v1, float v2, float v3, uniform float a[]) void soa_to_aos4(int32 v0, int32 v1, int32 v2, int32 v3, uniform int32 a[]) @@ -3793,12 +4032,12 @@ equivalents of them.) For example, given a structure in ``ispc``: // ispc code struct Node { - uniform int count; - uniform float pos[3]; + int count; + float pos[3]; }; -If the ``Node`` structure is used in the parameters to an ``export`` ed -function, then the header file generated by the ``ispc`` compiler will +If a ``uniform Node`` structure is used in the parameters to an ``export`` +ed function, then the header file generated by the ``ispc`` compiler will have a declaration like: :: @@ -3814,9 +4053,9 @@ program instances, ``ispc`` prohibits any varying types from being used in parameters to functions with the ``export`` qualifier. (``ispc`` also prohibits passing structures that themselves have varying types as members, etc.) Thus, all datatypes that is shared with the application must have -the ``uniform`` qualifier applied to them. (See `Understanding How to -Interoperate With the Application's Data`_ for more discussion of how to -load vectors of SoA or AoSoA data from the application.) +the ``uniform`` or ``soa`` rate qualifier applied to them. (See `Use +"Structure of Arrays" Layout When Possible`_ in the Performance Guide for +more discussion of how to load vectors of SOA data from the application.) Similarly, ``struct`` types shared with the application can also have embedded pointers. @@ -3834,7 +4073,7 @@ On the ``ispc`` side, the corresponding ``struct`` declaration is: // ispc struct Foo { - uniform float * uniform foo, * uniform bar; + float * uniform foo, * uniform bar; }; There is one subtlety related to data layout to be aware of: ``ispc`` @@ -3911,156 +4150,6 @@ program instances improves performance. .. _ispc Performance Tuning Guide: http://ispc.github.com/perf.html -Understanding How to Interoperate With the Application's Data -------------------------------------------------------------- - -One of ``ispc``'s key goals is to be able to interoperate with the -application's data, in whatever layout it is stored in. You don't need to -worry about reformatting of data or the overhead of a driver model that -abstracts the data layout. This section illustrates some of the -alternatives with a simple example of computing the length of a large -number of vectors. - -Consider for starters a ``Vector`` data-type, defined in C as: - -:: - - struct Vector { float x, y, z; }; - -We might have (still in C) an array of ``Vector`` s defined like this: - -:: - - Vector vectors[1024]; - -This is called an "array of structures" (AoS) layout. To compute the -lengths of these vectors in parallel, you can write ``ispc`` code like -this: - -:: - - export void length(Vector vectors[1024], uniform float len[]) { - foreach (index = 0 ... 1024) { - float x = vectors[index].x; - float y = vectors[index].y; - float z = vectors[index].z; - float l = sqrt(x*x + y*y + z*z); - len[index] = l; - } - } - -The problem with this implementation is that the indexing into the array of -structures, ``vectors[index].x`` is relatively expensive. On a target -machine that supports four-wide Intel® SSE, this turns into four loads of -single ``float`` values from non-contiguous memory locations, which are -then packed into a four-wide register corresponding to ``float x``. Once the -values are loaded into the local ``x``, ``y``, and ``z`` variables, -SIMD-efficient computation can proceed; getting to that point is -relatively inefficient. - -(As described previously in `Converting Between Array-of-Structures and -Structure-of-Arrays Layout`_, this computation could be written more -efficiently using standard library routines to convert from the AoS layout, -if we were given a flat array of ``float`` values.) - -An alternative data layout would be the "structure of arrays" (SoA). In C, -the data would be declared as: - -:: - - float x[1024], y[1024], z[1024]; - -The ``ispc`` code might be: - -:: - - export void length(uniform float x[1024], uniform float y[1024], - uniform float z[1024], uniform float len[]) { - foreach (index = 0 ... 1024) { - float xx = x[index]; - float yy = y[index]; - float zz = z[index]; - float l = sqrt(xx*xx + yy*yy + zz*zz); - len[index] = l; - } - } - -In this example, the loads into ``xx``, ``yy``, and ``zz`` are single -vector loads of an entire gang's worth of values into the corresponding -registers. This processing is more efficient than the multiple scalar -loads that are required with the AoS layout above. - -A final alternative is "array of structures of arrays" (AoSoA), a hybrid -between these two. A structure is declared that stores a small number of -``x``, ``y``, and ``z`` values in contiguous memory locations: - -:: - - struct Vector16 { - float x[16], y[16], z[16]; - }; - - -The ``ispc`` code has an outer loop over ``Vector16`` elements and -then an inner loop that peels off values from the element members: - -:: - - #define N_VEC (1024/16) - export void length(Vector16 v[N_VEC], uniform float len[]) { - foreach (i = 0 ... N_VEC, j = 0 ... 16) { - float x = v[i].x[j]; - float y = v[i].y[j]; - float z = v[i].z[j]; - float l = sqrt(x*x + y*y + z*z); - len[16*i+j] = l; - } - } - } - -One advantage of the AoSoA layout is that the memory accesses to load -values are to nearby memory locations, where as with SoA, each of the three -loads above is to locations separated by a few thousand bytes. Thus, AoSoA -can be more cache friendly. For structures with many members, this -difference can lead to a substantial improvement. - -With some additional complexity, ``ispc`` can also generate code that -efficiently processes data in AoSoA layout where the inner array length is -less than the machine vector width. For example, consider doing -computation with this AoSoA structure definition on a machine with an -8-wide vector unit (for example, an Intel® AVX target): - -:: - - struct Vector4 { - float x[4], y[4], z[4]; - }; - - -The ``ispc`` code to process this loads elements four at a time from -``Vector4`` instances until it has a full ``programCount`` number of -elements to work with and then proceeds with the computation. - -:: - - #define N_VEC (1024/4) - export void length(Vector4 v[N_VEC], uniform float len[]) { - for (uniform int i = 0; i < N_VEC; i += programCount / 4) { - float x, y, z; - for (uniform int j = 0; j < programCount / 4; ++j) { - if (programIndex >= 4 * j && - programIndex < 4 * (j+1)) { - int index = (programIndex & 0x3); - x = v[i+j].x[index]; - y = v[i+j].y[index]; - z = v[i+j].z[index]; - } - } - float l = sqrt(x*x + y*y + z*z); - len[4*i + programIndex] = l; - } - } - Disclaimer and Legal Information ================================ diff --git a/docs/news.rst b/docs/news.rst new file mode 100644 index 00000000..68800299 --- /dev/null +++ b/docs/news.rst @@ -0,0 +1,28 @@ +========= +ispc News +========= + +ispc 1.1.4 is Released +---------------------- + +On February 4, 2012, the 1.1.4 release of ``ispc`` was posted; new features +include ``new`` and ``delete`` for dynamic memory allocation in ``ispc`` +programs, "local" atomic operations in the standard library, and a new +scalar compilation target. See the `1.1.4 release notes`_ for details. + +.. _1.1.4 release notes: https://github.com/ispc/ispc/tree/master/docs/ReleaseNotes.txt + + +ispc 1.1.3 is Released +---------------------- + +With this release, the language now supports "switch" statements, with the same semantics and syntax as in C. + +This release includes fixes for two important performance related issues: +the quality of code generated for "foreach" statements has been +substantially improved, and performance regression with code for "gathers" +that was introduced in v1.1.2 has been fixed in this release. + +Thanks to Jean-Luc Duprat for a number of patches that improve support for +building on various platforms, and to Pierre-Antoine Lacaze for patches so +that ispc builds under MinGW. diff --git a/docs/perfguide.rst b/docs/perfguide.rst index 80c7d8f8..6e8555bf 100644 --- a/docs/perfguide.rst +++ b/docs/perfguide.rst @@ -13,6 +13,7 @@ the most out of ``ispc`` in practice. + `Improving Control Flow Coherence With "foreach_tiled"`_ + `Using Coherent Control Flow Constructs`_ + `Use "uniform" Whenever Appropriate`_ + + `Use "Structure of Arrays" Layout When Possible`_ * `Tips and Techniques`_ @@ -247,6 +248,76 @@ but it's always best to provide the compiler with as much help as possible to understand the actual form of your computation. +Use "Structure of Arrays" Layout When Possible +---------------------------------------------- + +In general, memory access performance (for both reads and writes) is best +when the running program instances access a contiguous region of memory; in +this case efficient vector load and store instructions can often be used +rather than gathers and scatters. As an example of this issue, consider an +array of a simple point datatype laid out and accessed in conventional +"array of structures" (AOS) layout: + +:: + + struct Point { float x, y, z; }; + uniform Point pts[...]; + float v = pts[programIndex].x; + +In the above code, the access to ``pts[programIndex].x`` accesses +non-sequential memory locations, due to the ``y`` and ``z`` values between +the desired ``x`` values in memory. A "gather" is required to get the +value of ``v``, with a corresponding decrease in performance. + +If ``Point`` was defined as a "structure of arrays" (SOA) type, the access +can be much more efficient: + +:: + + struct Point8 { float x[8], y[8], z[8]; }; + uniform Point8 pts8[...]; + int majorIndex = programIndex / 8; + int minorIndex = programIndex % 8; + float v = pts8[majorIndex].x[minorIndex]; + +In this case, each ``Point8`` has 8 ``x`` values contiguous in memory +before 8 ``y`` values and then 8 ``z`` values. If the gang size is 8 or +less, the access for ``v`` will have the same value of ``majorIndex`` for +all program instances and will access consecutive elements of the ``x[8]`` +array with a vector load. (For larger gang sizes, two 8-wide vector loads +would be issues, which is also quite efficient.) + +However, the syntax in the above code is messy; accessing SOA data in this +fashion is much less elegant than the corresponding code for accessing the +data with AOS layout. The ``soa`` qualifier in ``ispc`` can be used to +cause the corresponding transformation to be made to the ``Point`` type, +while preserving the clean syntax for data access that comes with AOS +layout: + +:: + + soa<8> Point pts[...]; + float v = pts[programIndex].x; + +Thanks to having SOA layout a first-class concept in the language's type +system, it's easy to write functions that convert data between the +layouts. For example, the ``aos_to_soa`` function below converts ``count`` +elements of the given ``Point`` type from AOS to 8-wide SOA layout. (It +assumes that the caller has pre-allocated sufficient space in the +``pts_soa`` output array. + +:: + + void aos_to_soa(uniform Point pts_aos[], uniform int count, + soa<8> pts_soa[]) { + foreach (i = 0 ... count) + pts_soa[i] = pts_aos[i]; + } + +Analogously, a function could be written to convert back from SOA to AOS if +needed. + + Tips and Techniques =================== @@ -339,6 +410,12 @@ based on the index, it can be worth doing. See the example ``examples/volume_rendering`` in the ``ispc`` distribution for the use of this technique in an instance where it is beneficial to performance. +Understanding Memory Read Coalescing +------------------------------------ + +XXXX todo + + Avoid 64-bit Addressing Calculations When Possible -------------------------------------------------- diff --git a/docs/template-news.txt b/docs/template-news.txt new file mode 100644 index 00000000..da249106 --- /dev/null +++ b/docs/template-news.txt @@ -0,0 +1,65 @@ +%(head_prefix)s +%(head)s + +%(stylesheet)s +%(body_prefix)s +
+
+ + +
+ +%(body_pre_docinfo)s +%(docinfo)s +
+%(body)s +
+
+ +
+
+
+%(body_suffix)s diff --git a/docs/template-perf.txt b/docs/template-perf.txt index 3dd8c3e0..d3b6d888 100644 --- a/docs/template-perf.txt +++ b/docs/template-perf.txt @@ -26,6 +26,7 @@
- diff --git a/docs/template.txt b/docs/template.txt index 56ebb7cc..a9618e5d 100644 --- a/docs/template.txt +++ b/docs/template.txt @@ -26,6 +26,7 @@
- diff --git a/examples/README.txt b/examples/README.txt index 7dcae8bf..25aa3a21 100644 --- a/examples/README.txt +++ b/examples/README.txt @@ -39,9 +39,6 @@ example implementation of this function that counts the number of times the callback is made and records some statistics about control flow coherence is provided in the instrument.cpp file. -*** Note: on Linux, this example currently hits an assertion in LLVM during -*** compilation - Deferred ======== @@ -110,6 +107,13 @@ This program implements both the Black-Scholes and Binomial options pricing models in both ispc and regular serial C++ code. +Perfbench +========= + +This runs a number of microbenchmarks to measure system performance and +code generation quality. + + RT == diff --git a/examples/aobench/ao.ispc b/examples/aobench/ao.ispc index 61c2dc7d..556e4cde 100644 --- a/examples/aobench/ao.ispc +++ b/examples/aobench/ao.ispc @@ -50,7 +50,6 @@ struct Isect { struct Sphere { vec center; float radius; - }; struct Plane { @@ -83,7 +82,7 @@ static inline void vnormalize(vec &v) { static void -ray_plane_intersect(Isect &isect, Ray &ray, Plane &plane) { +ray_plane_intersect(Isect &isect, Ray &ray, uniform Plane &plane) { float d = -dot(plane.p, plane.n); float v = dot(ray.dir, plane.n); @@ -103,7 +102,7 @@ ray_plane_intersect(Isect &isect, Ray &ray, Plane &plane) { static inline void -ray_sphere_intersect(Isect &isect, Ray &ray, Sphere &sphere) { +ray_sphere_intersect(Isect &isect, Ray &ray, uniform Sphere &sphere) { vec rs = ray.org - sphere.center; float B = dot(rs, ray.dir); @@ -148,7 +147,7 @@ orthoBasis(vec basis[3], vec n) { static float -ambient_occlusion(Isect &isect, Plane &plane, Sphere spheres[3], +ambient_occlusion(Isect &isect, uniform Plane &plane, uniform Sphere spheres[3], RNGState &rngstate) { float eps = 0.0001f; vec p, n; @@ -204,8 +203,8 @@ ambient_occlusion(Isect &isect, Plane &plane, Sphere spheres[3], static void ao_scanlines(uniform int y0, uniform int y1, uniform int w, uniform int h, uniform int nsubsamples, uniform float image[]) { - static Plane plane = { { 0.0f, -0.5f, 0.0f }, { 0.f, 1.f, 0.f } }; - static Sphere spheres[3] = { + static uniform Plane plane = { { 0.0f, -0.5f, 0.0f }, { 0.f, 1.f, 0.f } }; + static uniform Sphere spheres[3] = { { { -2.0f, 0.0f, -3.5f }, 0.5f }, { { -0.5f, 0.0f, -3.0f }, 0.5f }, { { 1.0f, 0.0f, -2.2f }, 0.5f } }; diff --git a/examples/deferred/common.cpp b/examples/deferred/common.cpp index dc2b928f..fa4ee57b 100644 --- a/examples/deferred/common.cpp +++ b/examples/deferred/common.cpp @@ -204,6 +204,7 @@ void WriteFrame(const char *filename, const InputData *input, fprintf(out, "P6 %d %d 255\n", input->header.framebufferWidth, input->header.framebufferHeight); fwrite(framebufferAOS, imageBytes, 1, out); + fclose(out); lAlignedFree(framebufferAOS); } diff --git a/examples/deferred/kernels.ispc b/examples/deferred/kernels.ispc index ae0542b2..6a20d45e 100644 --- a/examples/deferred/kernels.ispc +++ b/examples/deferred/kernels.ispc @@ -35,35 +35,35 @@ struct InputDataArrays { - uniform float * uniform zBuffer; - uniform unsigned int16 * uniform normalEncoded_x; // half float - uniform unsigned int16 * uniform normalEncoded_y; // half float - uniform unsigned int16 * uniform specularAmount; // half float - uniform unsigned int16 * uniform specularPower; // half float - uniform unsigned int8 * uniform albedo_x; // unorm8 - uniform unsigned int8 * uniform albedo_y; // unorm8 - uniform unsigned int8 * uniform albedo_z; // unorm8 - uniform float * uniform lightPositionView_x; - uniform float * uniform lightPositionView_y; - uniform float * uniform lightPositionView_z; - uniform float * uniform lightAttenuationBegin; - uniform float * uniform lightColor_x; - uniform float * uniform lightColor_y; - uniform float * uniform lightColor_z; - uniform float * uniform lightAttenuationEnd; + float *zBuffer; + unsigned int16 *normalEncoded_x; // half float + unsigned int16 *normalEncoded_y; // half float + unsigned int16 *specularAmount; // half float + unsigned int16 *specularPower; // half float + unsigned int8 *albedo_x; // unorm8 + unsigned int8 *albedo_y; // unorm8 + unsigned int8 *albedo_z; // unorm8 + float *lightPositionView_x; + float *lightPositionView_y; + float *lightPositionView_z; + float *lightAttenuationBegin; + float *lightColor_x; + float *lightColor_y; + float *lightColor_z; + float *lightAttenuationEnd; }; struct InputHeader { - uniform float cameraProj[4][4]; - uniform float cameraNear; - uniform float cameraFar; + float cameraProj[4][4]; + float cameraNear; + float cameraFar; - uniform int32 framebufferWidth; - uniform int32 framebufferHeight; - uniform int32 numLights; - uniform int32 inputDataChunkSize; - uniform int32 inputDataArrayOffsets[idaNum]; + int32 framebufferWidth; + int32 framebufferHeight; + int32 numLights; + int32 inputDataChunkSize; + int32 inputDataArrayOffsets[idaNum]; }; @@ -575,8 +575,6 @@ SplitTileMinMax( uniform float light_positionView_z_array[], uniform float light_attenuationEnd_array[], // Outputs - // TODO: ISPC doesn't currently like multidimensionsal arrays so we'll do the - // indexing math ourselves uniform int32 subtileIndices[], uniform int32 subtileIndicesPitch, uniform int32 subtileNumLights[] diff --git a/examples/examples.sln b/examples/examples.sln index 102dbade..e9992f76 100755 --- a/examples/examples.sln +++ b/examples/examples.sln @@ -23,6 +23,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "stencil", "stencil\stencil. EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "deferred_shading", "deferred\deferred_shading.vcxproj", "{87F53C53-957E-4E91-878A-BC27828FB9EB}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "perfbench", "perfbench\perfbench.vcxproj", "{D923BB7E-A7C8-4850-8FCF-0EB9CE35B4E8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -119,6 +121,14 @@ Global {87F53C53-957E-4E91-878A-BC27828FB9EB}.Release|Win32.Build.0 = Release|Win32 {87F53C53-957E-4E91-878A-BC27828FB9EB}.Release|x64.ActiveCfg = Release|x64 {87F53C53-957E-4E91-878A-BC27828FB9EB}.Release|x64.Build.0 = Release|x64 + {D923BB7E-A7C8-4850-8FCF-0EB9CE35B4E8}.Debug|Win32.ActiveCfg = Debug|Win32 + {D923BB7E-A7C8-4850-8FCF-0EB9CE35B4E8}.Debug|Win32.Build.0 = Debug|Win32 + {D923BB7E-A7C8-4850-8FCF-0EB9CE35B4E8}.Debug|x64.ActiveCfg = Debug|x64 + {D923BB7E-A7C8-4850-8FCF-0EB9CE35B4E8}.Debug|x64.Build.0 = Debug|x64 + {D923BB7E-A7C8-4850-8FCF-0EB9CE35B4E8}.Release|Win32.ActiveCfg = Release|Win32 + {D923BB7E-A7C8-4850-8FCF-0EB9CE35B4E8}.Release|Win32.Build.0 = Release|Win32 + {D923BB7E-A7C8-4850-8FCF-0EB9CE35B4E8}.Release|x64.ActiveCfg = Release|x64 + {D923BB7E-A7C8-4850-8FCF-0EB9CE35B4E8}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/examples/perfbench/Makefile b/examples/perfbench/Makefile new file mode 100644 index 00000000..43684c71 --- /dev/null +++ b/examples/perfbench/Makefile @@ -0,0 +1,7 @@ + +EXAMPLE=perbench +CPP_SRC=perfbench.cpp perfbench_serial.cpp +ISPC_SRC=perfbench.ispc +ISPC_TARGETS=sse2,sse4,avx + +include ../common.mk diff --git a/examples/perfbench/perfbench.cpp b/examples/perfbench/perfbench.cpp new file mode 100644 index 00000000..04e72bd9 --- /dev/null +++ b/examples/perfbench/perfbench.cpp @@ -0,0 +1,108 @@ +/* + Copyright (c) 2012, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef _MSC_VER +#define _CRT_SECURE_NO_WARNINGS +#define NOMINMAX +#pragma warning (disable: 4244) +#pragma warning (disable: 4305) +#endif + +#include +#include +#include "../timing.h" + +#include "perfbench_ispc.h" + +typedef void (FuncType)(float *, int, float *, float *); + +struct PerfTest { + FuncType *aFunc; + const char *aName; + FuncType *bFunc; + const char *bName; + const char *testName; +}; + +extern void xyzSumAOS(float *a, int count, float *zeros, float *result); +extern void xyzSumSOA(float *a, int count, float *zeros, float *result); + + +static void +lInitData(float *ptr, int count) { + for (int i = 0; i < count; ++i) + ptr[i] = float(i) / (1024.f * 1024.f); +} + +static PerfTest tests[] = { + { xyzSumAOS, "serial", ispc::xyzSumAOS, "ispc", "AOS vector element sum (with coalescing)" }, + { xyzSumAOS, "serial", ispc::xyzSumAOSStdlib, "ispc", "AOS vector element sum (stdlib swizzle)" }, + { xyzSumAOS, "serial", ispc::xyzSumAOSNoCoalesce, "ispc", "AOS vector element sum (no coalescing)" }, + { xyzSumSOA, "serial", ispc::xyzSumSOA, "ispc", "SOA vector element sum" }, + { ispc::gathers, "gather", ispc::loads, "vector load", "Memory reads" }, + { ispc::scatters, "scatter", ispc::stores, "vector store", "Memory writes" }, +}; + +int main() { + int count = 3*64*1024; + float *a = new float[count]; + float zeros[32] = { 0 }; + + int nTests = sizeof(tests) / sizeof(tests[0]); + for (int i = 0; i < nTests; ++i) { + lInitData(a, count); + reset_and_start_timer(); + float resultA[3] = { 0, 0, 0 }; + for (int j = 0; j < 100; ++j) + tests[i].aFunc(a, count, zeros, resultA); + double aTime = get_elapsed_mcycles(); + + lInitData(a, count); + reset_and_start_timer(); + float resultB[3] = { 0, 0, 0 }; + for (int j = 0; j < 100; ++j) + tests[i].bFunc(a, count, zeros, resultB); + double bTime = get_elapsed_mcycles(); + + printf("%-40s: [%.2f] M cycles %s, [%.2f] M cycles %s (%.2fx speedup).\n", + tests[i].testName, aTime, tests[i].aName, bTime, tests[i].bName, + aTime/bTime); +#if 0 + printf("\t(%f %f %f) - (%f %f %f)\n", resultSerial[0], resultSerial[1], + resultSerial[2], resultISPC[0], resultISPC[1], resultISPC[2]); +#endif + } + + return 0; +} + diff --git a/examples/perfbench/perfbench.ispc b/examples/perfbench/perfbench.ispc new file mode 100644 index 00000000..38fe6cee --- /dev/null +++ b/examples/perfbench/perfbench.ispc @@ -0,0 +1,170 @@ +/* + Copyright (c) 2012, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +export void xyzSumAOS(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float xsum = 0, ysum = 0, zsum = 0; + foreach (i = 0 ... count/3) { + float x = array[3*i]; + float y = array[3*i+1]; + float z = array[3*i+2]; + + xsum += x; + ysum += y; + zsum += z; + } + result[0] = reduce_add(xsum); + result[1] = reduce_add(ysum); + result[2] = reduce_add(zsum); +} + +export void xyzSumAOSStdlib(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float xsum = 0, ysum = 0, zsum = 0; + for (uniform int i = 0; i < 64*1024 /*count/3*/; i += programCount) { + float x, y, z; + aos_to_soa3(&array[3*i], &x, &y, &z); + + xsum += x; + ysum += y; + zsum += z; + } + result[0] = reduce_add(xsum); + result[1] = reduce_add(ysum); + result[2] = reduce_add(zsum); +} + +export void xyzSumAOSNoCoalesce(uniform float array[], uniform int count, + uniform float zerosArray[], uniform float result[]) { + int zeros = zerosArray[programIndex]; + float xsum = 0, ysum = 0, zsum = 0; + foreach (i = 0 ... count/3) { + float x = array[3*i+zeros]; + float y = array[3*i+1+zeros]; + float z = array[3*i+2+zeros]; + + xsum += x; + ysum += y; + zsum += z; + } + result[0] = reduce_add(xsum); + result[1] = reduce_add(ysum); + result[2] = reduce_add(zsum); +} + +export void xyzSumSOA(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float xsum = 0, ysum = 0, zsum = 0; + uniform float * uniform ap = array; + assert(programCount <= 8); + + for (uniform int i = 0; i < count/3; i += 8, ap += 24) { + for (uniform int j = 0; j < 8; j += programCount) { + float x = ap[j + programIndex]; + float y = ap[8 + j + programIndex]; + float z = ap[16 + j + programIndex]; + + xsum += x; + ysum += y; + zsum += z; + } + } + result[0] = reduce_add(xsum); + result[1] = reduce_add(ysum); + result[2] = reduce_add(zsum); +} + +export void gathers(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float sum = 0; + int zero = zeros[programIndex]; + foreach (i = 0 ... count) + sum += array[i + zero]; + result[0] = reduce_add(sum); +} + + +export void loads(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + float sum = 0; + foreach (i = 0 ... count) + sum += array[i]; + result[0] = reduce_add(sum); +} + + +export void scatters(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + int zero = zeros[programIndex]; + foreach (i = 0 ... count) + array[i + zero] = zero; +} + + +export void stores(uniform float array[], uniform int count, + uniform float zeros[], uniform float result[]) { + int zero = zeros[programIndex]; + foreach (i = 0 ... count) + array[i] = zero; +} + +export void normalizeAOSNoCoalesce(uniform float array[], uniform int count, + uniform float zeroArray[]) { + float zeros = zeroArray[programIndex]; + foreach (i = 0 ... count/3) { + float x = array[3*i+zeros]; + float y = array[3*i+1+zeros]; + float z = array[3*i+2+zeros]; + + float l2 = x*x + y*y + z*z; + + array[3*i] /= l2; + array[3*i+1] /= l2; + array[3*i+2] /= l2; + } +} + +export void normalizeSOA(uniform float array[], uniform int count, + uniform float zeros[]) { + foreach (i = 0 ... count/3) { + float x = array[3*i]; + float y = array[3*i+1]; + float z = array[3*i+2]; + + float l2 = x*x + y*y + z*z; + + array[3*i] /= l2; + array[3*i+1] /= l2; + array[3*i+2] /= l2; + } +} diff --git a/examples/perfbench/perfbench.vcxproj b/examples/perfbench/perfbench.vcxproj new file mode 100644 index 00000000..31974ac7 --- /dev/null +++ b/examples/perfbench/perfbench.vcxproj @@ -0,0 +1,175 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {d923bb7e-a7c8-4850-8fcf-0eb9ce35b4e8} + Win32Proj + perfbench + + + + Application + true + Unicode + + + Application + true + Unicode + + + Application + false + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(ProjectDir)..\..;$(ExecutablePath) + + + true + $(ProjectDir)..\..;$(ExecutablePath) + + + false + $(ProjectDir)..\..;$(ExecutablePath) + + + false + $(ProjectDir)..\..;$(ExecutablePath) + + + + + + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(TargetDir) + true + Fast + + + Console + true + + + + + + + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(TargetDir) + true + Fast + + + Console + true + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(TargetDir) + Fast + + + Console + true + true + true + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(TargetDir) + Fast + + + Console + true + true + true + + + + + + + + + Document + ispc -O2 %(Filename).ispc -o $(TargetDir)%(Filename).obj -h $(TargetDir)%(Filename)_ispc.h --arch=x86 --target=sse2,sse4,avx + + ispc -O2 %(Filename).ispc -o $(TargetDir)%(Filename).obj -h $(TargetDir)%(Filename)_ispc.h --target=sse2,sse4,avx + + $(TargetDir)%(Filename).obj;$(TargetDir)%(Filename)_sse2.obj;$(TargetDir)%(Filename)_sse4.obj;$(TargetDir)%(Filename)_avx.obj;$(TargetDir)%(Filename)_ispc.h + $(TargetDir)%(Filename).obj;$(TargetDir)%(Filename)_sse2.obj;$(TargetDir)%(Filename)_sse4.obj;$(TargetDir)%(Filename)_avx.obj;$(TargetDir)%(Filename)_ispc.h + ispc -O2 %(Filename).ispc -o $(TargetDir)%(Filename).obj -h $(TargetDir)%(Filename)_ispc.h --arch=x86 --target=sse2,sse4,avx + + ispc -O2 %(Filename).ispc -o $(TargetDir)%(Filename).obj -h $(TargetDir)%(Filename)_ispc.h --target=sse2,sse4,avx + + $(TargetDir)%(Filename).obj;$(TargetDir)%(Filename)_sse2.obj;$(TargetDir)%(Filename)_sse4.obj;$(TargetDir)%(Filename)_avx.obj;$(TargetDir)%(Filename)_ispc.h + $(TargetDir)%(Filename).obj;$(TargetDir)%(Filename)_sse2.obj;$(TargetDir)%(Filename)_sse4.obj;$(TargetDir)%(Filename)_avx.obj;$(TargetDir)%(Filename)_ispc.h + + + + + + diff --git a/examples/perfbench/perfbench_serial.cpp b/examples/perfbench/perfbench_serial.cpp new file mode 100644 index 00000000..dfd8e370 --- /dev/null +++ b/examples/perfbench/perfbench_serial.cpp @@ -0,0 +1,61 @@ +/* + Copyright (c) 2012, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include + +void +xyzSumAOS(float *a, int count, float *zeros, float *result) { + float xsum = 0, ysum = 0, zsum = 0; + for (int i = 0; i < count; i += 3) { + xsum += a[i]; + ysum += a[i+1]; + zsum += a[i+2]; + } + result[0] = xsum; + result[1] = ysum; + result[2] = zsum; +} + +void +xyzSumSOA(float *a, int count, float *zeros, float *result) { + float xsum = 0, ysum = 0, zsum = 0; + for (int i = 0; i < count/3; ++i) { + float *p = a + (i >> 3) * 24 + (i & 7); + xsum += p[0]; + ysum += p[8]; + zsum += p[16]; + } + result[0] = xsum; + result[1] = ysum; + result[2] = zsum; +} diff --git a/examples/rt/rt.ispc b/examples/rt/rt.ispc index 97d63d43..490dc5c1 100644 --- a/examples/rt/rt.ispc +++ b/examples/rt/rt.ispc @@ -43,17 +43,17 @@ struct Ray { }; struct Triangle { - uniform float p[3][4]; - uniform int id; - uniform int pad[3]; + float p[3][4]; + int id; + int pad[3]; }; struct LinearBVHNode { - uniform float bounds[2][3]; - uniform unsigned int offset; // num primitives for leaf, second child for interior - uniform unsigned int8 nPrimitives; - uniform unsigned int8 splitAxis; - uniform unsigned int16 pad; + float bounds[2][3]; + unsigned int offset; // num primitives for leaf, second child for interior + unsigned int8 nPrimitives; + unsigned int8 splitAxis; + unsigned int16 pad; }; static inline float3 Cross(const float3 v1, const float3 v2) { @@ -88,9 +88,12 @@ static void generateRay(uniform const float raster2camera[4][4], camy /= camw; camz /= camw; - ray.dir.x = camera2world[0][0] * camx + camera2world[0][1] * camy + camera2world[0][2] * camz; - ray.dir.y = camera2world[1][0] * camx + camera2world[1][1] * camy + camera2world[1][2] * camz; - ray.dir.z = camera2world[2][0] * camx + camera2world[2][1] * camy + camera2world[2][2] * camz; + ray.dir.x = camera2world[0][0] * camx + camera2world[0][1] * camy + + camera2world[0][2] * camz; + ray.dir.y = camera2world[1][0] * camx + camera2world[1][1] * camy + + camera2world[1][2] * camz; + ray.dir.z = camera2world[2][0] * camx + camera2world[2][1] * camy + + camera2world[2][2] * camz; ray.origin.x = camera2world[0][3] / camera2world[3][3]; ray.origin.y = camera2world[1][3] / camera2world[3][3]; @@ -143,7 +146,7 @@ static bool BBoxIntersect(const uniform float bounds[2][3], -static bool TriIntersect(const Triangle &tri, Ray &ray) { +static bool TriIntersect(const uniform Triangle &tri, Ray &ray) { uniform float3 p0 = { tri.p[0][0], tri.p[0][1], tri.p[0][2] }; uniform float3 p1 = { tri.p[1][0], tri.p[1][1], tri.p[1][2] }; uniform float3 p2 = { tri.p[2][0], tri.p[2][1], tri.p[2][2] }; @@ -183,8 +186,8 @@ static bool TriIntersect(const Triangle &tri, Ray &ray) { } -bool BVHIntersect(const LinearBVHNode nodes[], const Triangle tris[], - Ray &r) { +bool BVHIntersect(const uniform LinearBVHNode nodes[], + const uniform Triangle tris[], Ray &r) { Ray ray = r; bool hit = false; // Follow ray through BVH nodes to find primitive intersections @@ -193,7 +196,7 @@ bool BVHIntersect(const LinearBVHNode nodes[], const Triangle tris[], while (true) { // Check ray against BVH node - LinearBVHNode node = nodes[nodeNum]; + uniform LinearBVHNode node = nodes[nodeNum]; if (any(BBoxIntersect(node.bounds, ray))) { uniform unsigned int nPrimitives = node.nPrimitives; if (nPrimitives > 0) { @@ -239,8 +242,8 @@ static void raytrace_tile(uniform int x0, uniform int x1, const uniform float raster2camera[4][4], const uniform float camera2world[4][4], uniform float image[], uniform int id[], - const LinearBVHNode nodes[], - const Triangle triangles[]) { + const uniform LinearBVHNode nodes[], + const uniform Triangle triangles[]) { uniform float widthScale = (float)(baseWidth) / (float)(width); uniform float heightScale = (float)(baseHeight) / (float)(height); @@ -262,8 +265,8 @@ export void raytrace_ispc(uniform int width, uniform int height, const uniform float raster2camera[4][4], const uniform float camera2world[4][4], uniform float image[], uniform int id[], - const LinearBVHNode nodes[], - const Triangle triangles[]) { + const uniform LinearBVHNode nodes[], + const uniform Triangle triangles[]) { raytrace_tile(0, width, 0, height, width, height, baseWidth, baseHeight, raster2camera, camera2world, image, id, nodes, triangles); @@ -275,8 +278,8 @@ task void raytrace_tile_task(uniform int width, uniform int height, const uniform float raster2camera[4][4], const uniform float camera2world[4][4], uniform float image[], uniform int id[], - const LinearBVHNode nodes[], - const Triangle triangles[]) { + const uniform LinearBVHNode nodes[], + const uniform Triangle triangles[]) { uniform int dx = 16, dy = 16; // must match dx, dy below uniform int xBuckets = (width + (dx-1)) / dx; uniform int x0 = (taskIndex % xBuckets) * dx; @@ -295,8 +298,8 @@ export void raytrace_ispc_tasks(uniform int width, uniform int height, const uniform float raster2camera[4][4], const uniform float camera2world[4][4], uniform float image[], uniform int id[], - const LinearBVHNode nodes[], - const Triangle triangles[]) { + const uniform LinearBVHNode nodes[], + const uniform Triangle triangles[]) { uniform int dx = 16, dy = 16; uniform int xBuckets = (width + (dx-1)) / dx; uniform int yBuckets = (height + (dy-1)) / dy; diff --git a/examples/rt/rt_serial.cpp b/examples/rt/rt_serial.cpp index cc413dea..535f25e4 100644 --- a/examples/rt/rt_serial.cpp +++ b/examples/rt/rt_serial.cpp @@ -123,9 +123,12 @@ static void generateRay(const float raster2camera[4][4], camy /= camw; camz /= camw; - ray.dir.x = camera2world[0][0] * camx + camera2world[0][1] * camy + camera2world[0][2] * camz; - ray.dir.y = camera2world[1][0] * camx + camera2world[1][1] * camy + camera2world[1][2] * camz; - ray.dir.z = camera2world[2][0] * camx + camera2world[2][1] * camy + camera2world[2][2] * camz; + ray.dir.x = camera2world[0][0] * camx + camera2world[0][1] * camy + + camera2world[0][2] * camz; + ray.dir.y = camera2world[1][0] * camx + camera2world[1][1] * camy + + camera2world[1][2] * camz; + ray.dir.z = camera2world[2][0] * camx + camera2world[2][1] * camy + + camera2world[2][2] * camz; ray.origin.x = camera2world[0][3] / camera2world[3][3]; ray.origin.y = camera2world[1][3] / camera2world[3][3]; diff --git a/expr.cpp b/expr.cpp index ec242fea..4ed26877 100644 --- a/expr.cpp +++ b/expr.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2010-2011, Intel Corporation + Copyright (c) 2010-2012, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -197,14 +197,14 @@ lDoTypeConv(const Type *fromType, const Type *toType, Expr **expr, if (Type::Equal(toType, fromType)) return true; - if (fromType == AtomicType::Void) { + if (Type::Equal(fromType, AtomicType::Void)) { if (!failureOk) Error(pos, "Can't convert from \"void\" to \"%s\" for %s.", toType->GetString().c_str(), errorMsgBase); return false; } - if (toType == AtomicType::Void) { + if (Type::Equal(toType, AtomicType::Void)) { if (!failureOk) Error(pos, "Can't convert type \"%s\" to \"void\" for %s.", fromType->GetString().c_str(), errorMsgBase); @@ -226,6 +226,16 @@ lDoTypeConv(const Type *fromType, const Type *toType, Expr **expr, return false; } + if ((toType->GetSOAWidth() > 0 || fromType->GetSOAWidth() > 0) && + Type::Equal(toType->GetAsUniformType(), fromType->GetAsUniformType()) && + toType->GetSOAWidth() != fromType->GetSOAWidth()) { + if (!failureOk) + Error(pos, "Can't convert between types \"%s\" and \"%s\" with " + "different SOA widths for %s.", fromType->GetString().c_str(), + toType->GetString().c_str(), errorMsgBase); + return false; + } + const ArrayType *toArrayType = dynamic_cast(toType); const ArrayType *fromArrayType = dynamic_cast(fromType); const VectorType *toVectorType = dynamic_cast(toType); @@ -264,9 +274,9 @@ lDoTypeConv(const Type *fromType, const Type *toType, Expr **expr, if (toType->IsUniformType() && fromType->IsVaryingType()) { if (!failureOk) - Error(pos, "Can't convert from varying type \"%s\" to uniform " - "type \"%s\" for %s.", fromType->GetString().c_str(), - toType->GetString().c_str(), errorMsgBase); + Error(pos, "Can't convert from type \"%s\" to type \"%s\" for %s.", + fromType->GetString().c_str(), toType->GetString().c_str(), + errorMsgBase); return false; } @@ -289,6 +299,15 @@ lDoTypeConv(const Type *fromType, const Type *toType, Expr **expr, toType->GetString().c_str(), errorMsgBase); return false; } + else if (fromPointerType->IsSlice() == true && + toPointerType->IsSlice() == false) { + if (!failureOk) + Error(pos, "Can't convert from pointer to SOA type " + "\"%s\" to pointer to non-SOA type \"%s\" for %s.", + fromPointerType->GetAsNonSlice()->GetString().c_str(), + toType->GetString().c_str(), errorMsgBase); + return false; + } else if (PointerType::IsVoidPointer(toPointerType)) { // any pointer type can be converted to a void * goto typecast_ok; @@ -314,6 +333,10 @@ lDoTypeConv(const Type *fromType, const Type *toType, Expr **expr, if (toType->IsVaryingType() && fromType->IsUniformType()) goto typecast_ok; + if (toPointerType->IsSlice() == true && + fromPointerType->IsSlice() == false) + goto typecast_ok; + // Otherwise there's nothing to do return true; } @@ -332,7 +355,19 @@ lDoTypeConv(const Type *fromType, const Type *toType, Expr **expr, } return false; } - + + // Need to check this early, since otherwise the [sic] "unbound" + // variability of SOA struct types causes things to get messy if that + // hasn't been detected... + if (toStructType && fromStructType && + (toStructType->GetSOAWidth() != fromStructType->GetSOAWidth())) { + if (!failureOk) + Error(pos, "Can't convert between incompatible struct types \"%s\" " + "and \"%s\" for %s.", fromType->GetString().c_str(), + toType->GetString().c_str(), errorMsgBase); + return false; + } + // Convert from type T -> const T; just return a TypeCast expr, which // can handle this if (Type::Equal(toType, fromType->GetAsConstType())) @@ -469,23 +504,31 @@ lDoTypeConv(const Type *fromType, const Type *toType, Expr **expr, // other... if (fromAtomicType == NULL) { if (!failureOk) - Error(pos, "Type conversion only possible from atomic types, not " - "from \"%s\" to \"%s\", for %s.", fromType->GetString().c_str(), + Error(pos, "Type conversion from \"%s\" to \"%s\" for %s is not " + "possible.", fromType->GetString().c_str(), toType->GetString().c_str(), errorMsgBase); return false; } // scalar -> short-vector conversions - if (toVectorType != NULL) + if (toVectorType != NULL && + (fromType->GetSOAWidth() == toType->GetSOAWidth())) goto typecast_ok; // ok, it better be a scalar->scalar conversion of some sort by now if (toAtomicType == NULL) { if (!failureOk) - Error(pos, "Type conversion only possible to atomic types, not " - "from \"%s\" to \"%s\", for %s.", - fromType->GetString().c_str(), toType->GetString().c_str(), - errorMsgBase); + Error(pos, "Type conversion from \"%s\" to \"%s\" for %s is " + "not possible", fromType->GetString().c_str(), + toType->GetString().c_str(), errorMsgBase); + return false; + } + + if (fromType->GetSOAWidth() != toType->GetSOAWidth()) { + if (!failureOk) + Error(pos, "Can't convert between types \"%s\" and \"%s\" with " + "different SOA widths for %s.", fromType->GetString().c_str(), + toType->GetString().c_str(), errorMsgBase); return false; } @@ -509,6 +552,9 @@ TypeConvertExpr(Expr *expr, const Type *toType, const char *errorMsgBase) { if (expr == NULL) return NULL; + Debug(expr->pos, "type convert %s -> %s.", expr->GetType()->GetString().c_str(), + toType->GetString().c_str()); + const Type *fromType = expr->GetType(); Expr *e = expr; if (lDoTypeConv(fromType, toType, &e, false, errorMsgBase, @@ -545,7 +591,7 @@ PossiblyResolveFunctionOverloads(Expr *expr, const Type *type) { /** Utility routine that emits code to initialize a symbol given an initializer expression. - @param lvalue Memory location of storage for the symbol's data + @param ptr Memory location of storage for the symbol's data @param symName Name of symbol (used in error messages) @param symType Type of variable being initialized @param initExpr Expression for the initializer @@ -553,12 +599,38 @@ PossiblyResolveFunctionOverloads(Expr *expr, const Type *type) { @param pos Source file position of the variable being initialized */ void -InitSymbol(llvm::Value *lvalue, const Type *symType, Expr *initExpr, +InitSymbol(llvm::Value *ptr, const Type *symType, Expr *initExpr, FunctionEmitContext *ctx, SourcePos pos) { if (initExpr == NULL) // leave it uninitialized return; + // See if we have a constant initializer a this point + llvm::Constant *constValue = initExpr->GetConstant(symType); + if (constValue != NULL) { + // It'd be nice if we could just do a StoreInst(constValue, ptr) + // at this point, but unfortunately that doesn't generate great + // code (e.g. a bunch of scalar moves for a constant array.) So + // instead we'll make a constant static global that holds the + // constant value and emit a memcpy to put its value into the + // pointer we have. + LLVM_TYPE_CONST llvm::Type *llvmType = symType->LLVMType(g->ctx); + if (llvmType == NULL) { + Assert(m->errorCount > 0); + return; + } + + llvm::Value *constPtr = + new llvm::GlobalVariable(*m->module, llvmType, true /* const */, + llvm::GlobalValue::InternalLinkage, + constValue, "const_initializer"); + llvm::Value *size = g->target.SizeOf(llvmType, + ctx->GetCurrentBasicBlock()); + ctx->MemcpyInst(ptr, constPtr, size); + + return; + } + // If the initializer is a straight up expression that isn't an // ExprList, then we'll see if we can type convert it to the type of // the variable. @@ -567,28 +639,28 @@ InitSymbol(llvm::Value *lvalue, const Type *symType, Expr *initExpr, return; initExpr = TypeConvertExpr(initExpr, symType, "initializer"); - if (initExpr != NULL) { - llvm::Value *initializerValue = initExpr->GetValue(ctx); - if (initializerValue != NULL) - // Bingo; store the value in the variable's storage - ctx->StoreInst(initializerValue, lvalue); + if (initExpr == NULL) return; - } + + llvm::Value *initializerValue = initExpr->GetValue(ctx); + if (initializerValue != NULL) + // Bingo; store the value in the variable's storage + ctx->StoreInst(initializerValue, ptr); + return; } - // Atomic types and enums can't be initialized with { ... } initializer - // expressions, so print an error and return if that's what we've got - // here.. - if (dynamic_cast(symType) != NULL || - dynamic_cast(symType) != NULL || - dynamic_cast(symType) != NULL) { + // Atomic types and enums can be initialized with { ... } initializer + // expressions if they have a single element (except for SOA types, + // which are handled below). + if (symType->IsSOAType() == false && Type::IsBasicType(symType)) { ExprList *elist = dynamic_cast(initExpr); if (elist != NULL) { if (elist->exprs.size() == 1) - InitSymbol(lvalue, symType, elist->exprs[0], ctx, pos); + InitSymbol(ptr, symType, elist->exprs[0], ctx, pos); else - Error(initExpr->pos, "Expression list initializers can't be used " - "with type \"%s\".", symType->GetString().c_str()); + Error(initExpr->pos, "Expression list initializers with " + "multiple values can't be used with type \"%s\".", + symType->GetString().c_str()); } return; } @@ -604,19 +676,18 @@ InitSymbol(llvm::Value *lvalue, const Type *symType, Expr *initExpr, llvm::Value *initializerValue = initExpr->GetValue(ctx); if (initializerValue) - ctx->StoreInst(initializerValue, lvalue); + ctx->StoreInst(initializerValue, ptr); return; } - // There are two cases for initializing structs, arrays and vectors; - // either a single initializer may be provided (float foo[3] = 0;), in - // which case all of the elements are initialized to the given value, - // or an initializer list may be provided (float foo[3] = { 1,2,3 }), - // in which case the elements are initialized with the corresponding - // values. + // Handle initiailizers for SOA types as well as for structs, arrays, + // and vectors. const CollectionType *collectionType = dynamic_cast(symType); - if (collectionType != NULL) { + if (collectionType != NULL || symType->IsSOAType()) { + int nElements = collectionType ? collectionType->GetElementCount() : + symType->GetSOAWidth(); + std::string name; if (dynamic_cast(symType) != NULL) name = "struct"; @@ -624,35 +695,64 @@ InitSymbol(llvm::Value *lvalue, const Type *symType, Expr *initExpr, name = "array"; else if (dynamic_cast(symType) != NULL) name = "vector"; - else + else if (symType->IsSOAType()) + name = symType->GetVariability().GetString(); + else FATAL("Unexpected CollectionType in InitSymbol()"); + // There are two cases for initializing these types; either a + // single initializer may be provided (float foo[3] = 0;), in which + // case all of the elements are initialized to the given value, or + // an initializer list may be provided (float foo[3] = { 1,2,3 }), + // in which case the elements are initialized with the + // corresponding values. ExprList *exprList = dynamic_cast(initExpr); if (exprList != NULL) { - // The { ... } case; make sure we have the same number of - // expressions in the ExprList as we have struct members + // The { ... } case; make sure we have the no more expressions + // in the ExprList as we have struct members int nInits = exprList->exprs.size(); - if (nInits != collectionType->GetElementCount()) { + if (nInits > nElements) { Error(initExpr->pos, "Initializer for %s type \"%s\" requires " - "%d values; %d provided.", name.c_str(), - symType->GetString().c_str(), - collectionType->GetElementCount(), nInits); + "no more than %d values; %d provided.", name.c_str(), + symType->GetString().c_str(), nElements, nInits); return; } // Initialize each element with the corresponding value from // the ExprList - for (int i = 0; i < nInits; ++i) { + for (int i = 0; i < nElements; ++i) { + // For SOA types, the element type is the uniform variant + // of the underlying type + const Type *elementType = + collectionType ? collectionType->GetElementType(i) : + symType->GetAsUniformType(); + if (elementType == NULL) { + Assert(m->errorCount > 0); + return; + } + llvm::Value *ep; if (dynamic_cast(symType) != NULL) - ep = ctx->AddElementOffset(lvalue, i, NULL, "element"); + ep = ctx->AddElementOffset(ptr, i, NULL, "element"); else - ep = ctx->GetElementPtrInst(lvalue, LLVMInt32(0), LLVMInt32(i), - PointerType::GetUniform(collectionType->GetElementType(i)), + ep = ctx->GetElementPtrInst(ptr, LLVMInt32(0), LLVMInt32(i), + PointerType::GetUniform(elementType), "gep"); - InitSymbol(ep, collectionType->GetElementType(i), - exprList->exprs[i], ctx, pos); + if (i < nInits) + InitSymbol(ep, elementType, exprList->exprs[i], ctx, pos); + else { + // If we don't have enough initializer values, initialize the + // rest as zero. + LLVM_TYPE_CONST llvm::Type *llvmType = elementType->LLVMType(g->ctx); + if (llvmType == NULL) { + Assert(m->errorCount > 0); + return; + } + + llvm::Constant *zeroInit = llvm::ConstantAggregateZero::get(llvmType); + ctx->StoreInst(zeroInit, ep); + } } } else @@ -831,8 +931,9 @@ lMaskForSymbol(Symbol *baseSym, FunctionEmitContext *ctx) { /** Store the result of an assignment to the given location. */ static void -lStoreAssignResult(llvm::Value *value, llvm::Value *ptr, const Type *ptrType, - FunctionEmitContext *ctx, Symbol *baseSym) { +lStoreAssignResult(llvm::Value *value, llvm::Value *ptr, const Type *valueType, + const Type *ptrType, FunctionEmitContext *ctx, + Symbol *baseSym) { Assert(baseSym == NULL || baseSym->varyingCFDepth <= ctx->VaryingCFDepth()); if (!g->opt.disableMaskedStoreToStore && @@ -850,10 +951,10 @@ lStoreAssignResult(llvm::Value *value, llvm::Value *ptr, const Type *ptrType, // never be accessed, since those lanes aren't executing, and won't // be executing at this scope or any other one before the variable // goes out of scope. - ctx->StoreInst(value, ptr, LLVMMaskAllOn, ptrType); + ctx->StoreInst(value, ptr, LLVMMaskAllOn, valueType, ptrType); } else { - ctx->StoreInst(value, ptr, lMaskForSymbol(baseSym, ctx), ptrType); + ctx->StoreInst(value, ptr, lMaskForSymbol(baseSym, ctx), valueType, ptrType); } } @@ -919,7 +1020,7 @@ lEmitPrePostIncDec(UnaryExpr::Op op, Expr *expr, SourcePos pos, // And store the result out to the lvalue Symbol *baseSym = expr->GetBaseSymbol(); - lStoreAssignResult(binop, lvalue, lvalueType, ctx, baseSym); + lStoreAssignResult(binop, lvalue, type, lvalueType, ctx, baseSym); // And then if it's a pre increment/decrement, return the final // computed result; otherwise return the previously-grabbed expression @@ -1029,12 +1130,12 @@ UnaryExpr::Optimize() { bool isEnumType = dynamic_cast(type) != NULL; const Type *baseType = type->GetAsNonConstType()->GetAsUniformType(); - if (baseType == AtomicType::UniformInt8 || - baseType == AtomicType::UniformUInt8 || - baseType == AtomicType::UniformInt16 || - baseType == AtomicType::UniformUInt16 || - baseType == AtomicType::UniformInt64 || - baseType == AtomicType::UniformUInt64) + if (Type::Equal(baseType, AtomicType::UniformInt8) || + Type::Equal(baseType, AtomicType::UniformUInt8) || + Type::Equal(baseType, AtomicType::UniformInt16) || + Type::Equal(baseType, AtomicType::UniformUInt16) || + Type::Equal(baseType, AtomicType::UniformInt64) || + Type::Equal(baseType, AtomicType::UniformUInt64)) // FIXME: should handle these at some point; for now we only do // constant folding for bool, int32 and float types... return this; @@ -1059,20 +1160,16 @@ UnaryExpr::Optimize() { return new ConstExpr(constExpr, v); } case BitNot: { - if (type == AtomicType::UniformInt32 || - type == AtomicType::VaryingInt32 || - type == AtomicType::UniformConstInt32 || - type == AtomicType::VaryingConstInt32) { + if (Type::EqualIgnoringConst(type, AtomicType::UniformInt32) || + Type::EqualIgnoringConst(type, AtomicType::VaryingInt32)) { int32_t v[ISPC_MAX_NVEC]; int count = constExpr->AsInt32(v); for (int i = 0; i < count; ++i) v[i] = ~v[i]; return new ConstExpr(type, v, pos); } - else if (type == AtomicType::UniformUInt32 || - type == AtomicType::VaryingUInt32 || - type == AtomicType::UniformConstUInt32 || - type == AtomicType::VaryingConstUInt32 || + else if (Type::EqualIgnoringConst(type, AtomicType::UniformUInt32) || + Type::EqualIgnoringConst(type, AtomicType::VaryingUInt32) || isEnumType == true) { uint32_t v[ISPC_MAX_NVEC]; int count = constExpr->AsUInt32(v); @@ -1084,10 +1181,8 @@ UnaryExpr::Optimize() { FATAL("unexpected type in UnaryExpr::Optimize() / BitNot case"); } case LogicalNot: { - Assert(type == AtomicType::UniformBool || - type == AtomicType::VaryingBool || - type == AtomicType::UniformConstBool || - type == AtomicType::VaryingConstBool); + Assert(Type::EqualIgnoringConst(type, AtomicType::UniformBool) || + Type::EqualIgnoringConst(type, AtomicType::VaryingBool)); bool v[ISPC_MAX_NVEC]; int count = constExpr->AsBool(v); for (int i = 0; i < count; ++i) @@ -1108,6 +1203,12 @@ UnaryExpr::TypeCheck() { // something went wrong in type checking... return NULL; + if (type->IsSOAType()) { + Error(pos, "Can't apply unary operator to SOA type \"%s\".", + type->GetString().c_str()); + return NULL; + } + if (op == PreInc || op == PreDec || op == PostInc || op == PostDec) { if (type->IsConstType()) { Error(pos, "Can't assign to type \"%s\" on left-hand side of " @@ -1118,7 +1219,8 @@ UnaryExpr::TypeCheck() { if (type->IsNumericType()) return this; - if (dynamic_cast(type) == NULL) { + const PointerType *pt = dynamic_cast(type); + if (pt == NULL) { Error(expr->pos, "Can only pre/post increment numeric and " "pointer types, not \"%s\".", type->GetString().c_str()); return NULL; @@ -1250,6 +1352,106 @@ lEmitBinaryBitOp(BinaryExpr::Op op, llvm::Value *arg0Val, } +static llvm::Value * +lEmitBinaryPointerArith(BinaryExpr::Op op, llvm::Value *value0, + llvm::Value *value1, const Type *type0, + const Type *type1, FunctionEmitContext *ctx, + SourcePos pos) { + const PointerType *ptrType = dynamic_cast(type0); + + switch (op) { + case BinaryExpr::Add: + // ptr + integer + return ctx->GetElementPtrInst(value0, value1, ptrType, "ptrmath"); + break; + case BinaryExpr::Sub: { + if (dynamic_cast(type1) != NULL) { + Assert(Type::Equal(type0, type1)); + + if (ptrType->IsSlice()) { + llvm::Value *p0 = ctx->ExtractInst(value0, 0); + llvm::Value *p1 = ctx->ExtractInst(value1, 0); + const Type *majorType = ptrType->GetAsNonSlice(); + llvm::Value *majorDelta = + lEmitBinaryPointerArith(op, p0, p1, majorType, majorType, + ctx, pos); + + int soaWidth = ptrType->GetBaseType()->GetSOAWidth(); + Assert(soaWidth > 0); + llvm::Value *soaScale = LLVMIntAsType(soaWidth, + majorDelta->getType()); + + llvm::Value *majorScale = + ctx->BinaryOperator(llvm::Instruction::Mul, majorDelta, + soaScale, "major_soa_scaled"); + + llvm::Value *m0 = ctx->ExtractInst(value0, 1); + llvm::Value *m1 = ctx->ExtractInst(value1, 1); + llvm::Value *minorDelta = + ctx->BinaryOperator(llvm::Instruction::Sub, m0, m1, + "minor_soa_delta"); + + ctx->MatchIntegerTypes(&majorScale, &minorDelta); + return ctx->BinaryOperator(llvm::Instruction::Add, majorScale, + minorDelta, "soa_ptrdiff"); + } + + // ptr - ptr + if (ptrType->IsUniformType()) { + value0 = ctx->PtrToIntInst(value0); + value1 = ctx->PtrToIntInst(value1); + } + + // Compute the difference in bytes + llvm::Value *delta = + ctx->BinaryOperator(llvm::Instruction::Sub, value0, value1, + "ptr_diff"); + + // Now divide by the size of the type that the pointer + // points to in order to return the difference in elements. + LLVM_TYPE_CONST llvm::Type *llvmElementType = + ptrType->GetBaseType()->LLVMType(g->ctx); + llvm::Value *size = g->target.SizeOf(llvmElementType, + ctx->GetCurrentBasicBlock()); + if (ptrType->IsVaryingType()) + size = ctx->SmearUniform(size); + + if (g->target.is32Bit == false && + g->opt.force32BitAddressing == true) { + // If we're doing 32-bit addressing math on a 64-bit + // target, then trunc the delta down to a 32-bit value. + // (Thus also matching what will be a 32-bit value + // returned from SizeOf above.) + if (ptrType->IsUniformType()) + delta = ctx->TruncInst(delta, LLVMTypes::Int32Type, + "trunc_ptr_delta"); + else + delta = ctx->TruncInst(delta, LLVMTypes::Int32VectorType, + "trunc_ptr_delta"); + } + + // And now do the actual division + return ctx->BinaryOperator(llvm::Instruction::SDiv, delta, size, + "element_diff"); + } + else { + // ptr - integer + llvm::Value *zero = lLLVMConstantValue(type1, g->ctx, 0.); + llvm::Value *negOffset = + ctx->BinaryOperator(llvm::Instruction::Sub, zero, value1, + "negate"); + // Do a GEP as ptr + -integer + return ctx->GetElementPtrInst(value0, negOffset, ptrType, + "ptrmath"); + } + } + default: + FATAL("Logic error in lEmitBinaryArith() for pointer type case"); + return NULL; + } + +} + /** Utility routine to emit binary arithmetic operator based on the given BinaryExpr::Op. */ @@ -1259,68 +1461,9 @@ lEmitBinaryArith(BinaryExpr::Op op, llvm::Value *value0, llvm::Value *value1, FunctionEmitContext *ctx, SourcePos pos) { const PointerType *ptrType = dynamic_cast(type0); - if (ptrType != NULL) { - switch (op) { - case BinaryExpr::Add: - // ptr + integer - return ctx->GetElementPtrInst(value0, value1, ptrType, "ptrmath"); - break; - case BinaryExpr::Sub: { - if (dynamic_cast(type1) != NULL) { - // ptr - ptr - if (ptrType->IsUniformType()) { - value0 = ctx->PtrToIntInst(value0); - value1 = ctx->PtrToIntInst(value1); - } - - // Compute the difference in bytes - llvm::Value *delta = - ctx->BinaryOperator(llvm::Instruction::Sub, value0, value1, - "ptr_diff"); - - // Now divide by the size of the type that the pointer - // points to in order to return the difference in elements. - LLVM_TYPE_CONST llvm::Type *llvmElementType = - ptrType->GetBaseType()->LLVMType(g->ctx); - llvm::Value *size = g->target.SizeOf(llvmElementType, - ctx->GetCurrentBasicBlock()); - if (ptrType->IsVaryingType()) - size = ctx->SmearUniform(size); - - if (g->target.is32Bit == false && - g->opt.force32BitAddressing == true) { - // If we're doing 32-bit addressing math on a 64-bit - // target, then trunc the delta down to a 32-bit value. - // (Thus also matching what will be a 32-bit value - // returned from SizeOf above.) - if (ptrType->IsUniformType()) - delta = ctx->TruncInst(delta, LLVMTypes::Int32Type, - "trunc_ptr_delta"); - else - delta = ctx->TruncInst(delta, LLVMTypes::Int32VectorType, - "trunc_ptr_delta"); - } - - // And now do the actual division - return ctx->BinaryOperator(llvm::Instruction::SDiv, delta, size, - "element_diff"); - } - else { - // ptr - integer - llvm::Value *zero = lLLVMConstantValue(type1, g->ctx, 0.); - llvm::Value *negOffset = - ctx->BinaryOperator(llvm::Instruction::Sub, zero, value1, - "negate"); - // Do a GEP as ptr + -integer - return ctx->GetElementPtrInst(value0, negOffset, ptrType, - "ptrmath"); - } - } - default: - FATAL("Logic error in lEmitBinaryArith() for pointer type case"); - return NULL; - } - } + if (ptrType != NULL) + return lEmitBinaryPointerArith(op, value0, value1, type0, type1, + ctx, pos); else { Assert(Type::EqualIgnoringConst(type0, type1)); @@ -1934,10 +2077,8 @@ BinaryExpr::Optimize() { // transform x / const -> x * (1/const) if (op == Div && constArg1 != NULL) { const Type *type1 = constArg1->GetType(); - if (Type::Equal(type1, AtomicType::UniformFloat) || - Type::Equal(type1, AtomicType::VaryingFloat) || - Type::Equal(type1, AtomicType::UniformConstFloat) || - Type::Equal(type1, AtomicType::VaryingConstFloat)) { + if (Type::EqualIgnoringConst(type1, AtomicType::UniformFloat) || + Type::EqualIgnoringConst(type1, AtomicType::VaryingFloat)) { float inv[ISPC_MAX_NVEC]; int count = constArg1->AsFloat(inv); for (int i = 0; i < count; ++i) @@ -1954,10 +2095,8 @@ BinaryExpr::Optimize() { // transform x / y -> x * rcp(y) if (op == Div) { const Type *type1 = arg1->GetType(); - if (Type::Equal(type1, AtomicType::UniformFloat) || - Type::Equal(type1, AtomicType::VaryingFloat) || - Type::Equal(type1, AtomicType::UniformConstFloat) || - Type::Equal(type1, AtomicType::VaryingConstFloat)) { + if (Type::EqualIgnoringConst(type1, AtomicType::UniformFloat) || + Type::EqualIgnoringConst(type1, AtomicType::VaryingFloat)) { // Get the symbol for the appropriate builtin std::vector rcpFuns; m->symbolTable->LookupFunction("rcp", &rcpFuns); @@ -1994,7 +2133,8 @@ BinaryExpr::Optimize() { Assert(Type::EqualIgnoringConst(arg0->GetType(), arg1->GetType())); const Type *type = arg0->GetType()->GetAsNonConstType(); - if (type == AtomicType::UniformFloat || type == AtomicType::VaryingFloat) { + if (Type::Equal(type, AtomicType::UniformFloat) || + Type::Equal(type, AtomicType::VaryingFloat)) { float v0[ISPC_MAX_NVEC], v1[ISPC_MAX_NVEC]; constArg0->AsFloat(v0); constArg1->AsFloat(v1); @@ -2006,7 +2146,8 @@ BinaryExpr::Optimize() { else return this; } - if (type == AtomicType::UniformDouble || type == AtomicType::VaryingDouble) { + if (Type::Equal(type, AtomicType::UniformDouble) || + Type::Equal(type, AtomicType::VaryingDouble)) { double v0[ISPC_MAX_NVEC], v1[ISPC_MAX_NVEC]; constArg0->AsDouble(v0); constArg1->AsDouble(v1); @@ -2018,7 +2159,8 @@ BinaryExpr::Optimize() { else return this; } - if (type == AtomicType::UniformInt32 || type == AtomicType::VaryingInt32) { + if (Type::Equal(type, AtomicType::UniformInt32) || + Type::Equal(type, AtomicType::VaryingInt32)) { int32_t v0[ISPC_MAX_NVEC], v1[ISPC_MAX_NVEC]; constArg0->AsInt32(v0); constArg1->AsInt32(v1); @@ -2032,7 +2174,8 @@ BinaryExpr::Optimize() { else return this; } - else if (type == AtomicType::UniformUInt32 || type == AtomicType::VaryingUInt32 || + else if (Type::Equal(type, AtomicType::UniformUInt32) || + Type::Equal(type, AtomicType::VaryingUInt32) || dynamic_cast(type) != NULL) { uint32_t v0[ISPC_MAX_NVEC], v1[ISPC_MAX_NVEC]; constArg0->AsUInt32(v0); @@ -2047,7 +2190,8 @@ BinaryExpr::Optimize() { else return this; } - else if (type == AtomicType::UniformBool || type == AtomicType::VaryingBool) { + else if (Type::Equal(type, AtomicType::UniformBool) || + Type::Equal(type, AtomicType::VaryingBool)) { bool v0[ISPC_MAX_NVEC], v1[ISPC_MAX_NVEC]; constArg0->AsBool(v0); constArg1->AsBool(v1); @@ -2073,6 +2217,8 @@ BinaryExpr::TypeCheck() { if (type0 == NULL || type1 == NULL) return NULL; + // If either operand is a reference, dereference it before we move + // forward if (dynamic_cast(type0) != NULL) { arg0 = new DereferenceExpr(arg0, arg0->pos); type0 = arg0->GetType(); @@ -2094,9 +2240,22 @@ BinaryExpr::TypeCheck() { type1 = arg1->GetType(); } + // Prohibit binary operators with SOA types + if (type0->GetSOAWidth() > 0) { + Error(arg0->pos, "Illegal to use binary operator %s with SOA type " + "\"%s\".", lOpString(op), type0->GetString().c_str()); + return NULL; + } + if (type1->GetSOAWidth() > 0) { + Error(arg1->pos, "Illegal to use binary operator %s with SOA type " + "\"%s\".", lOpString(op), type1->GetString().c_str()); + return NULL; + } + const PointerType *pt0 = dynamic_cast(type0); const PointerType *pt1 = dynamic_cast(type1); if (pt0 != NULL && pt1 != NULL && op == Sub) { + // Pointer subtraction if (PointerType::IsVoidPointer(type0)) { Error(pos, "Illegal to perform pointer arithmetic " "on \"%s\" type.", type0->GetString().c_str()); @@ -2111,6 +2270,7 @@ BinaryExpr::TypeCheck() { const Type *t = Type::MoreGeneralType(type0, type1, pos, "-"); if (t == NULL) return NULL; + arg0 = TypeConvertExpr(arg0, t, "pointer subtraction"); arg1 = TypeConvertExpr(arg1, t, "pointer subtraction"); if (arg0 == NULL || arg1 == NULL) @@ -2364,7 +2524,7 @@ BinaryExpr::Print() const { static const char * lOpString(AssignExpr::Op op) { switch (op) { - case AssignExpr::Assign: return "="; + case AssignExpr::Assign: return "assignment operator"; case AssignExpr::MulAssign: return "*="; case AssignExpr::DivAssign: return "/="; case AssignExpr::ModAssign: return "%%="; @@ -2394,7 +2554,8 @@ lEmitOpAssign(AssignExpr::Op op, Expr *arg0, Expr *arg1, const Type *type, return NULL; } const Type *lvalueType = arg0->GetLValueType(); - if (lvalueType == NULL) + const Type *resultType = arg0->GetType(); + if (lvalueType == NULL || resultType == NULL) return NULL; // Get the value on the right-hand side of the assignment+operation @@ -2447,7 +2608,7 @@ lEmitOpAssign(AssignExpr::Op op, Expr *arg0, Expr *arg1, const Type *type, } // And store the result back to the lvalue. - lStoreAssignResult(newValue, lv, lvalueType, ctx, baseSym); + lStoreAssignResult(newValue, lv, resultType, lvalueType, ctx, baseSym); return newValue; } @@ -2472,29 +2633,30 @@ AssignExpr::GetValue(FunctionEmitContext *ctx) const { switch (op) { case Assign: { - llvm::Value *lv = lvalue->GetLValue(ctx); - if (lv == NULL) { + llvm::Value *ptr = lvalue->GetLValue(ctx); + if (ptr == NULL) { Error(lvalue->pos, "Left hand side of assignment expression can't " "be assigned to."); return NULL; } - const Type *lvalueType = lvalue->GetLValueType(); - if (lvalueType == NULL) { + const Type *ptrType = lvalue->GetLValueType(); + const Type *valueType = rvalue->GetType(); + if (ptrType == NULL || valueType == NULL) { Assert(m->errorCount > 0); return NULL; } - llvm::Value *rv = rvalue->GetValue(ctx); - if (rv == NULL) { + llvm::Value *value = rvalue->GetValue(ctx); + if (value == NULL) { Assert(m->errorCount > 0); return NULL; } ctx->SetDebugPos(pos); - lStoreAssignResult(rv, lv, lvalueType, ctx, baseSym); + lStoreAssignResult(value, ptr, valueType, ptrType, ctx, baseSym); - return rv; + return value; } case MulAssign: case DivAssign: @@ -2582,7 +2744,7 @@ AssignExpr::TypeCheck() { const FunctionType *ftype; if (dynamic_cast(lvalueType) == NULL || (ftype = dynamic_cast(lvalueType->GetBaseType())) == NULL) { - Error(pos, "Can't assign function pointer to type \"%s\".", + Error(lvalue->pos, "Can't assign function pointer to type \"%s\".", lvalue->GetType()->GetString().c_str()); return NULL; } @@ -2627,13 +2789,13 @@ AssignExpr::TypeCheck() { else if (op == Assign) rvalue = TypeConvertExpr(rvalue, lhsType, "assignment"); else { - Error(pos, "Assignment operator \"%s\" is illegal with pointer types.", + Error(lvalue->pos, "Assignment operator \"%s\" is illegal with pointer types.", lOpString(op)); return NULL; } } else if (dynamic_cast(lhsType) != NULL) { - Error(pos, "Illegal to assign to array type \"%s\".", + Error(lvalue->pos, "Illegal to assign to array type \"%s\".", lhsType->GetString().c_str()); return NULL; } @@ -2709,7 +2871,7 @@ lEmitVaryingSelect(FunctionEmitContext *ctx, llvm::Value *test, // Use masking to conditionally store the expr1 values Assert(resultPtr->getType() == PointerType::GetUniform(type)->LLVMType(g->ctx)); - ctx->StoreInst(expr1, resultPtr, test, PointerType::GetUniform(type)); + ctx->StoreInst(expr1, resultPtr, test, type, PointerType::GetUniform(type)); return ctx->LoadInst(resultPtr, "selectexpr_final"); } @@ -2751,12 +2913,12 @@ SelectExpr::GetValue(FunctionEmitContext *ctx) const { const Type *testType = test->GetType()->GetAsNonConstType(); // This should be taken care of during typechecking - Assert(testType->GetBaseType() == AtomicType::UniformBool || - testType->GetBaseType() == AtomicType::VaryingBool); + Assert(Type::Equal(testType->GetBaseType(), AtomicType::UniformBool) || + Type::Equal(testType->GetBaseType(), AtomicType::VaryingBool)); const Type *type = expr1->GetType(); - if (testType == AtomicType::UniformBool) { + if (Type::Equal(testType, AtomicType::UniformBool)) { // Simple case of a single uniform bool test expression; we just // want one of the two expressions. In this case, we can be // careful to evaluate just the one of the expressions that we need @@ -2924,10 +3086,77 @@ SelectExpr::Optimize() { // Varying test: see if all of the values are the same; if so, then // return the corresponding expression bool first = bv[0]; + bool mismatch = false; for (int i = 0; i < count; ++i) - if (bv[i] != first) - return this; - return (bv[0] == true) ? expr1 : expr2; + if (bv[i] != first) { + mismatch = true; + break; + } + if (mismatch == false) + return (bv[0] == true) ? expr1 : expr2; + + // Last chance: see if the two expressions are constants; if so, + // then we can do an element-wise selection based on the constant + // condition.. + ConstExpr *constExpr1 = dynamic_cast(expr1); + ConstExpr *constExpr2 = dynamic_cast(expr2); + if (constExpr1 == NULL || constExpr2 == NULL) + return this; + + Assert(Type::Equal(constExpr1->GetType(), constExpr2->GetType())); + const Type *exprType = constExpr1->GetType()->GetAsNonConstType(); + Assert(exprType->IsVaryingType()); + + // FIXME: it's annoying to have to have all of this replicated code. + if (Type::Equal(exprType, AtomicType::VaryingInt32) || + Type::Equal(exprType, AtomicType::VaryingUInt32)) { + int32_t v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + int32_t result[ISPC_MAX_NVEC]; + constExpr1->AsInt32(v1); + constExpr2->AsInt32(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + else if (Type::Equal(exprType, AtomicType::VaryingInt64) || + Type::Equal(exprType, AtomicType::VaryingUInt64)) { + int64_t v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + int64_t result[ISPC_MAX_NVEC]; + constExpr1->AsInt64(v1); + constExpr2->AsInt64(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + else if (Type::Equal(exprType, AtomicType::VaryingFloat)) { + float v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + float result[ISPC_MAX_NVEC]; + constExpr1->AsFloat(v1); + constExpr2->AsFloat(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + else if (Type::Equal(exprType, AtomicType::VaryingDouble)) { + double v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + double result[ISPC_MAX_NVEC]; + constExpr1->AsDouble(v1); + constExpr2->AsDouble(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + else if (Type::Equal(exprType, AtomicType::VaryingBool)) { + bool v1[ISPC_MAX_NVEC], v2[ISPC_MAX_NVEC]; + bool result[ISPC_MAX_NVEC]; + constExpr1->AsBool(v1); + constExpr2->AsBool(v2); + for (int i = 0; i < count; ++i) + result[i] = bv[i] ? v1[i] : v2[i]; + return new ConstExpr(exprType, result, pos); + } + + return this; } } @@ -3046,7 +3275,7 @@ FunctionCallExpr::GetValue(FunctionEmitContext *ctx) const { const FunctionType *ft = lGetFunctionType(func); Assert(ft != NULL); - bool isVoidFunc = (ft->GetReturnType() == AtomicType::Void); + bool isVoidFunc = Type::Equal(ft->GetReturnType(), AtomicType::Void); // Automatically convert function call args to references if needed. // FIXME: this should move to the TypeCheck() method... (but the @@ -3369,9 +3598,10 @@ ExprList::GetConstant(const Type *type) const { else FATAL("Unexpected CollectionType in ExprList::GetConstant()"); - if ((int)exprs.size() != collectionType->GetElementCount()) { - Error(pos, "Initializer list for %s \"%s\" must have %d elements " - "(has %d).", name.c_str(), collectionType->GetString().c_str(), + if ((int)exprs.size() > collectionType->GetElementCount()) { + Error(pos, "Initializer list for %s \"%s\" must have no more than %d " + "elements (has %d).", name.c_str(), + collectionType->GetString().c_str(), collectionType->GetElementCount(), (int)exprs.size()); return NULL; } @@ -3389,6 +3619,24 @@ ExprList::GetConstant(const Type *type) const { cv.push_back(c); } + // If there are too few, then treat missing ones as if they were zero + for (int i = (int)exprs.size(); i < collectionType->GetElementCount(); ++i) { + const Type *elementType = collectionType->GetElementType(i); + if (elementType == NULL) { + Assert(m->errorCount > 0); + return NULL; + } + + LLVM_TYPE_CONST llvm::Type *llvmType = elementType->LLVMType(g->ctx); + if (llvmType == NULL) { + Assert(m->errorCount > 0); + return NULL; + } + + llvm::Constant *c = llvm::Constant::getNullValue(llvmType); + cv.push_back(c); + } + if (dynamic_cast(type) != NULL) { #if defined(LLVM_2_9) return llvm::ConstantStruct::get(*g->ctx, cv, false); @@ -3403,11 +3651,24 @@ ExprList::GetConstant(const Type *type) const { LLVM_TYPE_CONST llvm::Type *lt = type->LLVMType(g->ctx); LLVM_TYPE_CONST llvm::ArrayType *lat = llvm::dyn_cast(lt); - // FIXME: should the assert below validly fail for uniform vectors - // now? Need a test case to reproduce it and then to be sure we - // have the right fix; leave the assert until we can hit it... - Assert(lat != NULL); - return llvm::ConstantArray::get(lat, cv); + if (lat != NULL) + return llvm::ConstantArray::get(lat, cv); + else { + // uniform short vector type + Assert(type->IsUniformType() && + dynamic_cast(type) != NULL); + LLVM_TYPE_CONST llvm::VectorType *lvt = + llvm::dyn_cast(lt); + Assert(lvt != NULL); + + // Uniform short vectors are stored as vectors of length + // rounded up to the native vector width. So we add additional + // undef values here until we get the right size. + while ((cv.size() % g->target.nativeVectorWidth) != 0) + cv.push_back(llvm::UndefValue::get(lvt->getElementType())); + + return llvm::ConstantVector::get(cv); + } } return NULL; } @@ -3463,23 +3724,22 @@ IndexExpr::IndexExpr(Expr *a, Expr *i, SourcePos p) */ static llvm::Value * lAddVaryingOffsetsIfNeeded(FunctionEmitContext *ctx, llvm::Value *ptr, - const Type *ptrType) { - if (dynamic_cast(ptrType) != NULL) + const Type *ptrRefType) { + if (dynamic_cast(ptrRefType) != NULL) // References are uniform pointers, so no offsetting is needed return ptr; - Assert(dynamic_cast(ptrType) != NULL); - if (ptrType->IsUniformType()) + const PointerType *ptrType = dynamic_cast(ptrRefType); + Assert(ptrType != NULL); + if (ptrType->IsUniformType() || ptrType->IsSlice()) return ptr; const Type *baseType = ptrType->GetBaseType(); - if (baseType->IsUniformType()) + if (baseType->IsVaryingType() == false) return ptr; // must be indexing into varying atomic, enum, or pointer types - if (dynamic_cast(baseType) == NULL && - dynamic_cast(baseType) == NULL && - dynamic_cast(baseType) == NULL) + if (Type::IsBasicType(baseType) == false) return ptr; // Onward: compute the per lane offsets. @@ -3499,45 +3759,113 @@ lAddVaryingOffsetsIfNeeded(FunctionEmitContext *ctx, llvm::Value *ptr, } +/** Check to see if the given type is an array of or pointer to a varying + struct type that in turn has a member with bound 'uniform' variability. + Issue an error and return true if such a member is found. + */ +static bool +lVaryingStructHasUniformMember(const Type *type, SourcePos pos) { + if (dynamic_cast(type) != NULL || + dynamic_cast(type) != NULL) + return false; + + const StructType *st = dynamic_cast(type); + if (st == NULL) { + const ArrayType *at = dynamic_cast(type); + if (at != NULL) + st = dynamic_cast(at->GetElementType()); + else { + const PointerType *pt = dynamic_cast(type); + if (pt == NULL) + return false; + + st = dynamic_cast(pt->GetBaseType()); + } + + if (st == NULL) + return false; + } + + if (st->IsVaryingType() == false) + return false; + + for (int i = 0; i < st->GetElementCount(); ++i) { + const Type *eltType = st->GetElementType(i); + if (eltType == NULL) { + Assert(m->errorCount > 0); + continue; + } + + if (dynamic_cast(eltType) != NULL) { + // We know that the enclosing struct is varying at this point, + // so push that down to the enclosed struct before makign the + // recursive call. + eltType = eltType->GetAsVaryingType(); + if (lVaryingStructHasUniformMember(eltType, pos)) + return true; + } + else if (eltType->IsUniformType()) { + Error(pos, "Gather operation is impossible due to the presence of " + "struct member \"%s\" with uniform type \"%s\" in the " + "varying struct type \"%s\".", + st->GetElementName(i).c_str(), eltType->GetString().c_str(), + st->GetString().c_str()); + return true; + } + } + + return false; +} + + llvm::Value * IndexExpr::GetValue(FunctionEmitContext *ctx) const { - const Type *baseExprType; + const Type *indexType, *returnType; if (baseExpr == NULL || index == NULL || - ((baseExprType = baseExpr->GetType()) == NULL)) + ((indexType = index->GetType()) == NULL) || + ((returnType = GetType()) == NULL)) { + Assert(m->errorCount > 0); + return NULL; + } + + // If this is going to be a gather, make sure that the varying return + // type can represent the result (i.e. that we don't have a bound + // 'uniform' member in a varying struct...) + if (indexType->IsVaryingType() && + lVaryingStructHasUniformMember(returnType, pos)) return NULL; ctx->SetDebugPos(pos); - llvm::Value *lvalue = GetLValue(ctx); + llvm::Value *ptr = GetLValue(ctx); llvm::Value *mask = NULL; const Type *lvalueType = GetLValueType(); - if (lvalue == NULL) { + if (ptr == NULL) { // We may be indexing into a temporary that hasn't hit memory, so // get the full value and stuff it into temporary alloca'd space so // that we can index from there... + const Type *baseExprType = baseExpr->GetType(); llvm::Value *val = baseExpr->GetValue(ctx); - if (val == NULL) { + if (baseExprType == NULL || val == NULL) { Assert(m->errorCount > 0); return NULL; } ctx->SetDebugPos(pos); - llvm::Value *ptr = ctx->AllocaInst(baseExprType->LLVMType(g->ctx), - "array_tmp"); - ctx->StoreInst(val, ptr); - - lvalue = ctx->GetElementPtrInst(ptr, LLVMInt32(0), index->GetValue(ctx), - PointerType::GetUniform(baseExprType)); + llvm::Value *tmpPtr = ctx->AllocaInst(baseExprType->LLVMType(g->ctx), + "array_tmp"); + ctx->StoreInst(val, tmpPtr); + // Get a pointer type to the underlying elements const SequentialType *st = dynamic_cast(baseExprType); - if (st == NULL) { - Assert(m->errorCount > 0); - return NULL; - } + Assert(st != NULL); lvalueType = PointerType::GetUniform(st->GetElementType()); - lvalue = lAddVaryingOffsetsIfNeeded(ctx, lvalue, lvalueType); - + // And do the indexing calculation into the temporary array in memory + ptr = ctx->GetElementPtrInst(tmpPtr, LLVMInt32(0), index->GetValue(ctx), + PointerType::GetUniform(baseExprType)); + ptr = lAddVaryingOffsetsIfNeeded(ctx, ptr, lvalueType); + mask = LLVMMaskAllOn; } else { @@ -3547,7 +3875,7 @@ IndexExpr::GetValue(FunctionEmitContext *ctx) const { } ctx->SetDebugPos(pos); - return ctx->LoadInst(lvalue, mask, lvalueType, "index"); + return ctx->LoadInst(ptr, mask, lvalueType, "index"); } @@ -3574,12 +3902,23 @@ IndexExpr::GetType() const { elementType = sequentialType->GetElementType(); } - if (indexType->IsUniformType()) - // If the index is uniform, the resulting type is just whatever the - // element type is + // If we're indexing into a sequence of SOA types, the result type is + // actually the underlying type, as a uniform or varying. Get the + // uniform variant of it for starters, then below we'll make it varying + // if the index is varying. + // (If we ever provide a way to index into SOA types and get an entire + // SOA'd struct out of the array, then we won't want to do this in that + // case..) + if (elementType->IsSOAType()) + elementType = elementType->GetAsUniformType(); + + // If either the index is varying or we're indexing into a varying + // pointer, then the result type is the varying variant of the indexed + // type. + if (indexType->IsUniformType() && + (pointerType == NULL || pointerType->IsUniformType())) return elementType; else - // A varying index into even a uniform base type -> varying type return elementType->GetAsVaryingType(); } @@ -3590,104 +3929,207 @@ IndexExpr::GetBaseSymbol() const { } +/** Utility routine that takes a regualr pointer (either uniform or + varying) and returns a slice pointer with zero offsets. + */ +static llvm::Value * +lConvertToSlicePointer(FunctionEmitContext *ctx, llvm::Value *ptr, + const PointerType *slicePtrType) { + LLVM_TYPE_CONST llvm::Type *llvmSlicePtrType = + slicePtrType->LLVMType(g->ctx); + LLVM_TYPE_CONST llvm::StructType *sliceStructType = + llvm::dyn_cast(llvmSlicePtrType); + Assert(sliceStructType != NULL && + sliceStructType->getElementType(0) == ptr->getType()); + + // Get a null-initialized struct to take care of having zeros for the + // offsets + llvm::Value *result = llvm::Constant::getNullValue(sliceStructType); + // And replace the pointer in the struct with the given pointer + return ctx->InsertInst(result, ptr, 0); +} + + +/** If the given array index is a compile time constant, check to see if it + value/values don't go past the end of the array; issue a warning if + so. +*/ +static void +lCheckIndicesVersusBounds(const Type *baseExprType, Expr *index) { + const SequentialType *seqType = + dynamic_cast(baseExprType); + if (seqType == NULL) + return; + + int nElements = seqType->GetElementCount(); + if (nElements == 0) + // Unsized array... + return; + + // If it's an array of soa<> items, then the number of elements to + // worry about w.r.t. index values is the product of the array size and + // the soa width. + int soaWidth = seqType->GetElementType()->GetSOAWidth(); + if (soaWidth > 0) + nElements *= soaWidth; + + ConstExpr *ce = dynamic_cast(index); + if (ce == NULL) + return; + + int32_t indices[ISPC_MAX_NVEC]; + int count = ce->AsInt32(indices); + for (int i = 0; i < count; ++i) { + if (indices[i] < 0 || indices[i] >= nElements) + Warning(index->pos, "Array index \"%d\" may be out of bounds for %d " + "element array.", indices[i], nElements); + } +} + + +/** Converts the given pointer value to a slice pointer if the pointer + points to SOA'ed data. +*/ +static llvm::Value * +lConvertPtrToSliceIfNeeded(FunctionEmitContext *ctx, + llvm::Value *ptr, + const Type **type) { + Assert(*type != NULL); + const PointerType *ptrType = dynamic_cast(*type); + bool convertToSlice = (ptrType->GetBaseType()->IsSOAType() && + ptrType->IsSlice() == false); + if (convertToSlice == false) + return ptr; + + *type = ptrType->GetAsSlice(); + return lConvertToSlicePointer(ctx, ptr, ptrType->GetAsSlice()); +} + + llvm::Value * IndexExpr::GetLValue(FunctionEmitContext *ctx) const { const Type *baseExprType; if (baseExpr == NULL || index == NULL || - ((baseExprType = baseExpr->GetType()) == NULL)) + ((baseExprType = baseExpr->GetType()) == NULL)) { + Assert(m->errorCount > 0); return NULL; + } + + ctx->SetDebugPos(pos); + llvm::Value *indexValue = index->GetValue(ctx); + if (indexValue == NULL) { + Assert(m->errorCount > 0); + return NULL; + } ctx->SetDebugPos(pos); if (dynamic_cast(baseExprType) != NULL) { - // We're indexing off of a base pointer - llvm::Value *baseValue = baseExpr->GetValue(ctx); - llvm::Value *indexValue = index->GetValue(ctx); - if (baseValue == NULL || indexValue == NULL) + // We're indexing off of a pointer + llvm::Value *basePtrValue = baseExpr->GetValue(ctx); + if (basePtrValue == NULL) { + Assert(m->errorCount > 0); return NULL; + } ctx->SetDebugPos(pos); - llvm::Value *ptr = ctx->GetElementPtrInst(baseValue, indexValue, + + // Convert to a slice pointer if we're indexing into SOA data + basePtrValue = lConvertPtrToSliceIfNeeded(ctx, basePtrValue, + &baseExprType); + + llvm::Value *ptr = ctx->GetElementPtrInst(basePtrValue, indexValue, baseExprType, "ptr_offset"); - ptr = lAddVaryingOffsetsIfNeeded(ctx, ptr, GetLValueType()); - return ptr; + return lAddVaryingOffsetsIfNeeded(ctx, ptr, GetLValueType()); } - // Otherwise it's an array or vector + // Not a pointer: we must be indexing an array or vector (and possibly + // a reference thereuponfore.) llvm::Value *basePtr = NULL; - const Type *basePtrType = NULL; + const PointerType *basePtrType = NULL; if (dynamic_cast(baseExprType) || dynamic_cast(baseExprType)) { basePtr = baseExpr->GetLValue(ctx); - basePtrType = baseExpr->GetLValueType(); + basePtrType = dynamic_cast(baseExpr->GetLValueType()); + if (baseExpr->GetLValueType()) Assert(basePtrType != NULL); } else { baseExprType = baseExprType->GetReferenceTarget(); Assert(dynamic_cast(baseExprType) || dynamic_cast(baseExprType)); basePtr = baseExpr->GetValue(ctx); - basePtrType = baseExpr->GetType(); + basePtrType = PointerType::GetUniform(baseExprType); } if (!basePtr) return NULL; - // If the array index is a compile time constant, check to see if it - // may lead to an out-of-bounds access. - ConstExpr *ce = dynamic_cast(index); - const SequentialType *seqType = - dynamic_cast(baseExprType); - if (seqType != NULL) { - int nElements = seqType->GetElementCount(); - if (ce != NULL && nElements > 0) { - int32_t indices[ISPC_MAX_NVEC]; - int count = ce->AsInt32(indices); - for (int i = 0; i < count; ++i) { - if (indices[i] < 0 || indices[i] >= nElements) - Warning(index->pos, "Array index \"%d\" may be out of bounds for " - "%d element array.", indices[i], nElements); - } - } - } + // If possible, check the index value(s) against the size of the array + lCheckIndicesVersusBounds(baseExprType, index); + + // Convert to a slice pointer if indexing into SOA data + basePtr = lConvertPtrToSliceIfNeeded(ctx, basePtr, + (const Type **)&basePtrType); ctx->SetDebugPos(pos); + + // And do the actual indexing calculation.. llvm::Value *ptr = - ctx->GetElementPtrInst(basePtr, LLVMInt32(0), index->GetValue(ctx), + ctx->GetElementPtrInst(basePtr, LLVMInt32(0), indexValue, basePtrType); - ptr = lAddVaryingOffsetsIfNeeded(ctx, ptr, GetLValueType()); - return ptr; + return lAddVaryingOffsetsIfNeeded(ctx, ptr, GetLValueType()); } const Type * IndexExpr::GetLValueType() const { - const Type *baseExprLValueType, *indexType; - if (baseExpr == NULL || index == NULL || + const Type *baseExprType, *baseExprLValueType, *indexType; + if (baseExpr == NULL || index == NULL || + ((baseExprType = baseExpr->GetType()) == NULL) || ((baseExprLValueType = baseExpr->GetLValueType()) == NULL) || ((indexType = index->GetType()) == NULL)) return NULL; - if (dynamic_cast(baseExprLValueType) != NULL) - baseExprLValueType = PointerType::GetUniform(baseExprLValueType->GetReferenceTarget()); + // regularize to a PointerType + if (dynamic_cast(baseExprLValueType) != NULL) { + const Type *refTarget = baseExprLValueType->GetReferenceTarget(); + baseExprLValueType = PointerType::GetUniform(refTarget); + } Assert(dynamic_cast(baseExprLValueType) != NULL); - // FIXME: can we do something in the type system that unifies the - // concept of a sequential type's element type and a pointer type's - // base type? The code below is identical but for handling that - // difference. IndexableType? + // Find the type of thing that we're indexing into + const Type *elementType; const SequentialType *st = dynamic_cast(baseExprLValueType->GetBaseType()); - if (st != NULL) { - if (baseExprLValueType->IsUniformType() && indexType->IsUniformType()) - return PointerType::GetUniform(st->GetElementType()); - else - return PointerType::GetVarying(st->GetElementType()); + if (st != NULL) + elementType = st->GetElementType(); + else { + const PointerType *pt = + dynamic_cast(baseExprLValueType->GetBaseType()); + Assert(pt != NULL); + elementType = pt->GetBaseType(); } - const PointerType *pt = - dynamic_cast(baseExprLValueType->GetBaseType()); - Assert(pt != NULL); - if (baseExprLValueType->IsUniformType() && indexType->IsUniformType()) - return PointerType::GetUniform(pt->GetBaseType()); + // Are we indexing into a varying type, or are we indexing with a + // varying pointer? + bool baseVarying; + if (dynamic_cast(baseExprType) != NULL) + baseVarying = baseExprType->IsVaryingType(); else - return PointerType::GetVarying(pt->GetBaseType()); + baseVarying = baseExprLValueType->IsVaryingType(); + + // The return type is uniform iff. the base is a uniform pointer / a + // collection of uniform typed elements and the index is uniform. + const PointerType *retType; + if (baseVarying == false && indexType->IsUniformType()) + retType = PointerType::GetUniform(elementType); + else + retType = PointerType::GetVarying(elementType); + + // Finally, if we're indexing into an SOA type, then the resulting + // pointer must (currently) be a slice pointer; we don't allow indexing + // the soa-width-wide structs directly. + if (elementType->IsSOAType()) + retType = retType->GetAsSlice(); + + return retType; } @@ -3701,12 +4143,18 @@ IndexExpr::Optimize() { Expr * IndexExpr::TypeCheck() { - if (baseExpr == NULL || index == NULL || index->GetType() == NULL) + const Type *indexType; + if (baseExpr == NULL || index == NULL || + ((indexType = index->GetType()) == NULL)) { + Assert(m->errorCount > 0); return NULL; + } const Type *baseExprType = baseExpr->GetType(); - if (baseExprType == NULL) + if (baseExprType == NULL) { + Assert(m->errorCount > 0); return NULL; + } if (!dynamic_cast(baseExprType->GetReferenceTarget()) && !dynamic_cast(baseExprType)) { @@ -3717,11 +4165,20 @@ IndexExpr::TypeCheck() { bool isUniform = (index->GetType()->IsUniformType() && !g->opt.disableUniformMemoryOptimizations); - const Type *indexType = isUniform ? AtomicType::UniformInt32 : - AtomicType::VaryingInt32; - index = TypeConvertExpr(index, indexType, "array index"); - if (index == NULL) - return NULL; + + // Unless we have an explicit 64-bit index and are compiling to a + // 64-bit target with 64-bit addressing, convert the index to an int32 + // type. + if (Type::EqualIgnoringConst(indexType->GetAsUniformType(), + AtomicType::UniformInt64) == false || + g->target.is32Bit || + g->opt.force32BitAddressing) { + const Type *indexType = isUniform ? AtomicType::UniformInt32 : + AtomicType::VaryingInt32; + index = TypeConvertExpr(index, indexType, "array index"); + if (index == NULL) + return NULL; + } return this; } @@ -3798,6 +4255,7 @@ public: SourcePos idpos, bool derefLValue); const Type *GetType() const; + const Type *GetLValueType() const; int getElementNumber() const; const Type *getElementType() const; @@ -3816,9 +4274,15 @@ const Type * StructMemberExpr::GetType() const { // It's a struct, and the result type is the element type, possibly // promoted to varying if the struct type / lvalue is varying. - const StructType *structType = getStructType(); - if (structType == NULL) + const Type *exprType, *lvalueType; + const StructType *structType; + if (expr == NULL || + ((exprType = expr->GetType()) == NULL) || + ((structType = getStructType()) == NULL) || + ((lvalueType = GetLValueType()) == NULL)) { + Assert(m->errorCount > 0); return NULL; + } const Type *elementType = structType->GetElementType(identifier); if (elementType == NULL) { @@ -3828,13 +4292,63 @@ StructMemberExpr::GetType() const { getCandidateNearMatches().c_str()); return NULL; } + Assert(Type::Equal(lvalueType->GetBaseType(), elementType)); - const PointerType *pt = dynamic_cast(expr->GetType()); - if (structType->IsVaryingType() || - (pt != NULL && pt->IsVaryingType())) - return elementType->GetAsVaryingType(); - else - return elementType; + bool isSlice = (dynamic_cast(lvalueType) && + dynamic_cast(lvalueType)->IsSlice()); + if (isSlice) { + // FIXME: not true if we allow bound unif/varying for soa<> + // structs?... + Assert(elementType->IsSOAType()); + + // If we're accessing a member of an soa structure via a uniform + // slice pointer, then the result type is the uniform variant of + // the element type. + if (lvalueType->IsUniformType()) + elementType = elementType->GetAsUniformType(); + } + + if (lvalueType->IsVaryingType()) + // If the expression we're getting the member of has an lvalue that + // is a varying pointer type (be it slice or non-slice), then the + // result type must be the varying version of the element type. + elementType = elementType->GetAsVaryingType(); + + return elementType; +} + + +const Type * +StructMemberExpr::GetLValueType() const { + if (expr == NULL) { + Assert(m->errorCount > 0); + return NULL; + } + + const Type *exprLValueType = dereferenceExpr ? expr->GetType() : + expr->GetLValueType(); + if (exprLValueType == NULL) { + Assert(m->errorCount > 0); + return NULL; + } + + // The pointer type is varying if the lvalue type of the expression is + // varying (and otherwise uniform) + const PointerType *ptrType = + (exprLValueType->IsUniformType() || + dynamic_cast(exprLValueType) != NULL) ? + PointerType::GetUniform(getElementType()) : + PointerType::GetVarying(getElementType()); + + // If struct pointer is a slice pointer, the resulting member pointer + // needs to be a frozen slice pointer--i.e. any further indexing with + // the result shouldn't modify the minor slice offset, but it should be + // left unchanged until we get to a leaf SOA value. + if (dynamic_cast(exprLValueType) && + dynamic_cast(exprLValueType)->IsSlice()) + ptrType = ptrType->GetAsFrozenSlice(); + + return ptrType; } @@ -3850,6 +4364,7 @@ StructMemberExpr::getElementNumber() const { "Element name \"%s\" not present in struct type \"%s\".%s", identifier.c_str(), structType->GetString().c_str(), getCandidateNearMatches().c_str()); + return elementNumber; } @@ -3860,30 +4375,32 @@ StructMemberExpr::getElementType() const { if (structType == NULL) return NULL; - return structType->GetAsUniformType()->GetElementType(identifier); + return structType->GetElementType(identifier); } +/** Returns the type of the underlying struct that we're returning a member + of. */ const StructType * StructMemberExpr::getStructType() const { - const Type *exprType = expr->GetType(); - if (exprType == NULL) + const Type *type = dereferenceExpr ? expr->GetType() : + expr->GetLValueType(); + if (type == NULL) return NULL; - - const StructType *structType = dynamic_cast(exprType); - if (structType == NULL) { - const PointerType *pt = dynamic_cast(exprType); - if (pt != NULL) - structType = dynamic_cast(pt->GetBaseType()); - else { - const ReferenceType *rt = - dynamic_cast(exprType); - Assert(rt != NULL); - structType = dynamic_cast(rt->GetReferenceTarget()); - } - Assert(structType != NULL); + + const Type *structType; + const ReferenceType *rt = dynamic_cast(type); + if (rt != NULL) + structType = rt->GetReferenceTarget(); + else { + const PointerType *pt = dynamic_cast(type); + Assert(pt != NULL); + structType = pt->GetBaseType(); } - return structType; + + const StructType *ret = dynamic_cast(structType); + Assert(ret != NULL); + return ret; } @@ -3942,8 +4459,19 @@ VectorMemberExpr::GetType() const { (const Type *)memberType; const Type *lvalueType = GetLValueType(); - if (lvalueType != NULL && lvalueType->IsVaryingType()) - type = type->GetAsVaryingType(); + if (lvalueType != NULL) { + bool isSlice = (dynamic_cast(lvalueType) && + dynamic_cast(lvalueType)->IsSlice()); + if (isSlice) { +//CO Assert(type->IsSOAType()); + if (lvalueType->IsUniformType()) + type = type->GetAsUniformType(); + } + + if (lvalueType->IsVaryingType()) + type = type->GetAsVaryingType(); + } + return type; } @@ -3961,8 +4489,10 @@ VectorMemberExpr::GetLValue(FunctionEmitContext* ctx) const { const Type * VectorMemberExpr::GetLValueType() const { if (identifier.length() == 1) { - if (expr == NULL) + if (expr == NULL) { + Assert(m->errorCount > 0); return NULL; + } const Type *exprLValueType = dereferenceExpr ? expr->GetType() : expr->GetLValueType(); @@ -3977,14 +4507,20 @@ VectorMemberExpr::GetLValueType() const { Assert(vt != NULL); // we don't want to report that it's e.g. a pointer to a float<1>, - // but ta pointer to a float, etc. + // but a pointer to a float, etc. const Type *elementType = vt->GetElementType(); if (dynamic_cast(exprLValueType) != NULL) return new ReferenceType(elementType); - else - return exprLValueType->IsUniformType() ? + else { + const PointerType *ptrType = exprLValueType->IsUniformType() ? PointerType::GetUniform(elementType) : PointerType::GetVarying(elementType); + // FIXME: replicated logic with structmemberexpr.... + if (dynamic_cast(exprLValueType) && + dynamic_cast(exprLValueType)->IsSlice()) + ptrType = ptrType->GetAsFrozenSlice(); + return ptrType; + } } else return NULL; @@ -4226,22 +4762,6 @@ MemberExpr::GetLValue(FunctionEmitContext *ctx) const { } -const Type * -MemberExpr::GetLValueType() const { - if (expr == NULL) - return NULL; - - const Type *exprLValueType = dereferenceExpr ? expr->GetType() : - expr->GetLValueType(); - if (exprLValueType == NULL) - return NULL; - - return exprLValueType->IsUniformType() ? - PointerType::GetUniform(getElementType()) : - PointerType::GetVarying(getElementType()); -} - - Expr * MemberExpr::TypeCheck() { return expr ? this : NULL; @@ -4311,7 +4831,7 @@ ConstExpr::ConstExpr(const Type *t, int8_t i, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstInt8); + Assert(Type::Equal(type, AtomicType::UniformInt8->GetAsConstType())); int8Val[0] = i; } @@ -4320,8 +4840,8 @@ ConstExpr::ConstExpr(const Type *t, int8_t *i, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstInt8 || - type == AtomicType::VaryingConstInt8); + Assert(Type::Equal(type, AtomicType::UniformInt8->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingInt8->GetAsConstType())); for (int j = 0; j < Count(); ++j) int8Val[j] = i[j]; } @@ -4331,7 +4851,7 @@ ConstExpr::ConstExpr(const Type *t, uint8_t u, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstUInt8); + Assert(Type::Equal(type, AtomicType::UniformUInt8->GetAsConstType())); uint8Val[0] = u; } @@ -4340,8 +4860,8 @@ ConstExpr::ConstExpr(const Type *t, uint8_t *u, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstUInt8 || - type == AtomicType::VaryingConstUInt8); + Assert(Type::Equal(type, AtomicType::UniformUInt8->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingUInt8->GetAsConstType())); for (int j = 0; j < Count(); ++j) uint8Val[j] = u[j]; } @@ -4351,7 +4871,7 @@ ConstExpr::ConstExpr(const Type *t, int16_t i, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstInt16); + Assert(Type::Equal(type, AtomicType::UniformInt16->GetAsConstType())); int16Val[0] = i; } @@ -4360,8 +4880,8 @@ ConstExpr::ConstExpr(const Type *t, int16_t *i, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstInt16 || - type == AtomicType::VaryingConstInt16); + Assert(Type::Equal(type, AtomicType::UniformInt16->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingInt16->GetAsConstType())); for (int j = 0; j < Count(); ++j) int16Val[j] = i[j]; } @@ -4371,7 +4891,7 @@ ConstExpr::ConstExpr(const Type *t, uint16_t u, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstUInt16); + Assert(Type::Equal(type, AtomicType::UniformUInt16->GetAsConstType())); uint16Val[0] = u; } @@ -4380,8 +4900,8 @@ ConstExpr::ConstExpr(const Type *t, uint16_t *u, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstUInt16 || - type == AtomicType::VaryingConstUInt16); + Assert(Type::Equal(type, AtomicType::UniformUInt16->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingUInt16->GetAsConstType())); for (int j = 0; j < Count(); ++j) uint16Val[j] = u[j]; } @@ -4391,7 +4911,7 @@ ConstExpr::ConstExpr(const Type *t, int32_t i, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstInt32); + Assert(Type::Equal(type, AtomicType::UniformInt32->GetAsConstType())); int32Val[0] = i; } @@ -4400,8 +4920,8 @@ ConstExpr::ConstExpr(const Type *t, int32_t *i, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstInt32 || - type == AtomicType::VaryingConstInt32); + Assert(Type::Equal(type, AtomicType::UniformInt32->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingInt32->GetAsConstType())); for (int j = 0; j < Count(); ++j) int32Val[j] = i[j]; } @@ -4411,7 +4931,7 @@ ConstExpr::ConstExpr(const Type *t, uint32_t u, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstUInt32 || + Assert(Type::Equal(type, AtomicType::UniformUInt32->GetAsConstType()) || (dynamic_cast(type) != NULL && type->IsUniformType())); uint32Val[0] = u; @@ -4422,8 +4942,8 @@ ConstExpr::ConstExpr(const Type *t, uint32_t *u, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstUInt32 || - type == AtomicType::VaryingConstUInt32 || + Assert(Type::Equal(type, AtomicType::UniformUInt32->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingUInt32->GetAsConstType()) || (dynamic_cast(type) != NULL)); for (int j = 0; j < Count(); ++j) uint32Val[j] = u[j]; @@ -4434,7 +4954,7 @@ ConstExpr::ConstExpr(const Type *t, float f, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstFloat); + Assert(Type::Equal(type, AtomicType::UniformFloat->GetAsConstType())); floatVal[0] = f; } @@ -4443,8 +4963,8 @@ ConstExpr::ConstExpr(const Type *t, float *f, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstFloat || - type == AtomicType::VaryingConstFloat); + Assert(Type::Equal(type, AtomicType::UniformFloat->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingFloat->GetAsConstType())); for (int j = 0; j < Count(); ++j) floatVal[j] = f[j]; } @@ -4454,7 +4974,7 @@ ConstExpr::ConstExpr(const Type *t, int64_t i, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstInt64); + Assert(Type::Equal(type, AtomicType::UniformInt64->GetAsConstType())); int64Val[0] = i; } @@ -4463,8 +4983,8 @@ ConstExpr::ConstExpr(const Type *t, int64_t *i, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstInt64 || - type == AtomicType::VaryingConstInt64); + Assert(Type::Equal(type, AtomicType::UniformInt64->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingInt64->GetAsConstType())); for (int j = 0; j < Count(); ++j) int64Val[j] = i[j]; } @@ -4474,7 +4994,7 @@ ConstExpr::ConstExpr(const Type *t, uint64_t u, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstUInt64); + Assert(Type::Equal(type, AtomicType::UniformUInt64->GetAsConstType())); uint64Val[0] = u; } @@ -4483,8 +5003,8 @@ ConstExpr::ConstExpr(const Type *t, uint64_t *u, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstUInt64 || - type == AtomicType::VaryingConstUInt64); + Assert(Type::Equal(type, AtomicType::UniformUInt64->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingUInt64->GetAsConstType())); for (int j = 0; j < Count(); ++j) uint64Val[j] = u[j]; } @@ -4494,7 +5014,7 @@ ConstExpr::ConstExpr(const Type *t, double f, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstDouble); + Assert(Type::Equal(type, AtomicType::UniformDouble->GetAsConstType())); doubleVal[0] = f; } @@ -4503,8 +5023,8 @@ ConstExpr::ConstExpr(const Type *t, double *f, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstDouble || - type == AtomicType::VaryingConstDouble); + Assert(Type::Equal(type, AtomicType::UniformDouble->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingDouble->GetAsConstType())); for (int j = 0; j < Count(); ++j) doubleVal[j] = f[j]; } @@ -4514,7 +5034,7 @@ ConstExpr::ConstExpr(const Type *t, bool b, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstBool); + Assert(Type::Equal(type, AtomicType::UniformBool->GetAsConstType())); boolVal[0] = b; } @@ -4523,8 +5043,8 @@ ConstExpr::ConstExpr(const Type *t, bool *b, SourcePos p) : Expr(p) { type = t; type = type->GetAsConstType(); - Assert(type == AtomicType::UniformConstBool || - type == AtomicType::VaryingConstBool); + Assert(Type::Equal(type, AtomicType::UniformBool->GetAsConstType()) || + Type::Equal(type, AtomicType::VaryingBool->GetAsConstType())); for (int j = 0; j < Count(); ++j) boolVal[j] = b[j]; } @@ -4992,7 +5512,8 @@ ConstExpr::GetConstant(const Type *type) const { Assert(Count() == 1); type = type->GetAsNonConstType(); - if (type == AtomicType::UniformBool || type == AtomicType::VaryingBool) { + if (Type::Equal(type, AtomicType::UniformBool) || + Type::Equal(type, AtomicType::VaryingBool)) { bool bv[ISPC_MAX_NVEC]; AsBool(bv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5000,7 +5521,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMBoolVector(bv); } - else if (type == AtomicType::UniformInt8 || type == AtomicType::VaryingInt8) { + else if (Type::Equal(type, AtomicType::UniformInt8) || + Type::Equal(type, AtomicType::VaryingInt8)) { int8_t iv[ISPC_MAX_NVEC]; AsInt8(iv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5008,8 +5530,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMInt8Vector(iv); } - else if (type == AtomicType::UniformUInt8 || type == AtomicType::VaryingUInt8 || - dynamic_cast(type) != NULL) { + else if (Type::Equal(type, AtomicType::UniformUInt8) || + Type::Equal(type, AtomicType::VaryingUInt8)) { uint8_t uiv[ISPC_MAX_NVEC]; AsUInt8(uiv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5017,7 +5539,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMUInt8Vector(uiv); } - else if (type == AtomicType::UniformInt16 || type == AtomicType::VaryingInt16) { + else if (Type::Equal(type, AtomicType::UniformInt16) || + Type::Equal(type, AtomicType::VaryingInt16)) { int16_t iv[ISPC_MAX_NVEC]; AsInt16(iv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5025,8 +5548,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMInt16Vector(iv); } - else if (type == AtomicType::UniformUInt16 || type == AtomicType::VaryingUInt16 || - dynamic_cast(type) != NULL) { + else if (Type::Equal(type, AtomicType::UniformUInt16) || + Type::Equal(type, AtomicType::VaryingUInt16)) { uint16_t uiv[ISPC_MAX_NVEC]; AsUInt16(uiv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5034,7 +5557,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMUInt16Vector(uiv); } - else if (type == AtomicType::UniformInt32 || type == AtomicType::VaryingInt32) { + else if (Type::Equal(type, AtomicType::UniformInt32) || + Type::Equal(type, AtomicType::VaryingInt32)) { int32_t iv[ISPC_MAX_NVEC]; AsInt32(iv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5042,7 +5566,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMInt32Vector(iv); } - else if (type == AtomicType::UniformUInt32 || type == AtomicType::VaryingUInt32 || + else if (Type::Equal(type, AtomicType::UniformUInt32) || + Type::Equal(type, AtomicType::VaryingUInt32) || dynamic_cast(type) != NULL) { uint32_t uiv[ISPC_MAX_NVEC]; AsUInt32(uiv, type->IsVaryingType()); @@ -5051,7 +5576,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMUInt32Vector(uiv); } - else if (type == AtomicType::UniformFloat || type == AtomicType::VaryingFloat) { + else if (Type::Equal(type, AtomicType::UniformFloat) || + Type::Equal(type, AtomicType::VaryingFloat)) { float fv[ISPC_MAX_NVEC]; AsFloat(fv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5059,7 +5585,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMFloatVector(fv); } - else if (type == AtomicType::UniformInt64 || type == AtomicType::VaryingInt64) { + else if (Type::Equal(type, AtomicType::UniformInt64) || + Type::Equal(type, AtomicType::VaryingInt64)) { int64_t iv[ISPC_MAX_NVEC]; AsInt64(iv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5067,7 +5594,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMInt64Vector(iv); } - else if (type == AtomicType::UniformUInt64 || type == AtomicType::VaryingUInt64) { + else if (Type::Equal(type, AtomicType::UniformUInt64) || + Type::Equal(type, AtomicType::VaryingUInt64)) { uint64_t uiv[ISPC_MAX_NVEC]; AsUInt64(uiv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5075,7 +5603,8 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMUInt64Vector(uiv); } - else if (type == AtomicType::UniformDouble || type == AtomicType::VaryingDouble) { + else if (Type::Equal(type, AtomicType::UniformDouble) || + Type::Equal(type, AtomicType::VaryingDouble)) { double dv[ISPC_MAX_NVEC]; AsDouble(dv, type->IsVaryingType()); if (type->IsUniformType()) @@ -5083,8 +5612,29 @@ ConstExpr::GetConstant(const Type *type) const { else return LLVMDoubleVector(dv); } + else if (dynamic_cast(type) != NULL) { + // The only time we should get here is if we have an integer '0' + // constant that should be turned into a NULL pointer of the + // appropriate type. + LLVM_TYPE_CONST llvm::Type *llvmType = type->LLVMType(g->ctx); + if (llvmType == NULL) { + Assert(m->errorCount > 0); + return NULL; + } + + int64_t iv[ISPC_MAX_NVEC]; + AsInt64(iv, type->IsVaryingType()); + for (int i = 0; i < Count(); ++i) + if (iv[i] != 0) + // We'll issue an error about this later--trying to assign + // a constant int to a pointer, without a typecast. + return NULL; + + return llvm::Constant::getNullValue(llvmType); + } else { - FATAL("unexpected type in ConstExpr::GetConstant()"); + Debug(pos, "Unable to handle type \"%s\" in ConstExpr::GetConstant().", + type->GetString().c_str()); return NULL; } } @@ -5719,8 +6269,8 @@ TypeCastExpr::GetValue(FunctionEmitContext *ctx) const { ctx->SetDebugPos(pos); const Type *toType = GetType(), *fromType = expr->GetType(); - if (!toType || !fromType || toType == AtomicType::Void || - fromType == AtomicType::Void) + if (!toType || !fromType || Type::Equal(toType, AtomicType::Void) || + Type::Equal(fromType, AtomicType::Void)) // an error should have been issued elsewhere in this case return NULL; @@ -5737,6 +6287,23 @@ TypeCastExpr::GetValue(FunctionEmitContext *ctx) const { if (value == NULL) return NULL; + if (fromPointerType->IsSlice() == false && + toPointerType->IsSlice() == true) { + // Convert from a non-slice pointer to a slice pointer by + // creating a slice pointer structure with zero offsets. + if (fromPointerType->IsUniformType()) + value = ctx->MakeSlicePointer(value, LLVMInt32(0)); + else + value = ctx->MakeSlicePointer(value, LLVMInt32Vector(0)); + + // FIXME: avoid error from unnecessary bitcast when all we + // need to do is the slice conversion and don't need to + // also do unif->varying conversions. But this is really + // ugly logic. + if (value->getType() == toType->LLVMType(g->ctx)) + return value; + } + if (fromType->IsUniformType() && toType->IsUniformType()) // bitcast to the actual pointer type return ctx->BitCastInst(value, toType->LLVMType(g->ctx)); @@ -5746,9 +6313,25 @@ TypeCastExpr::GetValue(FunctionEmitContext *ctx) const { return value; } else { + // Uniform -> varying pointer conversion Assert(fromType->IsUniformType() && toType->IsVaryingType()); - value = ctx->PtrToIntInst(value); - return ctx->SmearUniform(value); + if (fromPointerType->IsSlice()) { + // For slice pointers, we need to smear out both the + // pointer and the offset vector + Assert(toPointerType->IsSlice()); + llvm::Value *ptr = ctx->ExtractInst(value, 0); + llvm::Value *offset = ctx->ExtractInst(value, 1); + ptr = ctx->PtrToIntInst(ptr); + ptr = ctx->SmearUniform(ptr); + offset = ctx->SmearUniform(offset); + return ctx->MakeSlicePointer(ptr, offset); + } + else { + // Otherwise we just bitcast it to an int and smear it + // out to a vector + value = ctx->PtrToIntInst(value); + return ctx->SmearUniform(value); + } } } else { @@ -5987,15 +6570,14 @@ TypeCastExpr::TypeCheck() { expr, pos); return ::TypeCheck(tce); } - type = toType = type->ResolveUnboundVariability(Type::Varying); + type = toType = type->ResolveUnboundVariability(Variability::Varying); fromType = lDeconstifyType(fromType); toType = lDeconstifyType(toType); if (fromType->IsVaryingType() && toType->IsUniformType()) { - Error(pos, "Can't type cast from varying type \"%s\" to uniform " - "type \"%s\"", fromType->GetString().c_str(), - toType->GetString().c_str()); + Error(pos, "Can't type cast from type \"%s\" to type \"%s\"", + fromType->GetString().c_str(), toType->GetString().c_str()); return NULL; } @@ -6161,7 +6743,6 @@ TypeCastExpr::GetConstant(const Type *constType) const { // 1. Null pointers (NULL, 0) valued initializers, and // 2. Converting a uniform function pointer to a varying function // pointer of the same type. - Assert(Type::Equal(constType, type)); return expr->GetConstant(constType); } @@ -6268,6 +6849,9 @@ DereferenceExpr::GetValue(FunctionEmitContext *ctx) const { if (type == NULL) return NULL; + if (lVaryingStructHasUniformMember(type, pos)) + return NULL; + Symbol *baseSym = expr->GetBaseSymbol(); llvm::Value *mask = baseSym ? lMaskForSymbol(baseSym, ctx) : ctx->GetFullMask(); @@ -6454,7 +7038,7 @@ SizeOfExpr::SizeOfExpr(Expr *e, SourcePos p) SizeOfExpr::SizeOfExpr(const Type *t, SourcePos p) : Expr(p), expr(NULL), type(t) { if (type->HasUnboundVariability()) - type = type->ResolveUnboundVariability(Type::Varying); + type = type->ResolveUnboundVariability(Variability::Varying); } @@ -6620,7 +7204,7 @@ FunctionSymbolExpr::GetType() const { } return matchingFunc ? - new PointerType(matchingFunc->type, Type::Uniform, true) : NULL; + new PointerType(matchingFunc->type, Variability::Uniform, true) : NULL; } @@ -7152,8 +7736,6 @@ NewExpr::NewExpr(int typeQual, const Type *t, Expr *init, Expr *count, SourcePos tqPos, SourcePos p) : Expr(p) { allocType = t; - if (allocType != NULL && allocType->HasUnboundVariability()) - allocType = allocType->ResolveUnboundVariability(Type::Varying); initExpr = init; countExpr = count; @@ -7175,6 +7757,9 @@ NewExpr::NewExpr(int typeQual, const Type *t, Expr *init, Expr *count, // If no type qualifier is given before the 'new', treat it as a // varying new. isVarying = (typeQual == 0) || (typeQual & TYPEQUAL_VARYING); + + if (allocType != NULL && allocType->HasUnboundVariability()) + allocType = allocType->ResolveUnboundVariability(Variability::Uniform); } diff --git a/expr.h b/expr.h index 70224a7f..e0d1348c 100644 --- a/expr.h +++ b/expr.h @@ -299,7 +299,6 @@ public: llvm::Value *GetValue(FunctionEmitContext *ctx) const; llvm::Value *GetLValue(FunctionEmitContext *ctx) const; const Type *GetType() const; - const Type *GetLValueType() const; Symbol *GetBaseSymbol() const; void Print() const; Expr *Optimize(); diff --git a/func.cpp b/func.cpp index 26ea83ed..c1ca7ee6 100644 --- a/func.cpp +++ b/func.cpp @@ -355,7 +355,7 @@ Function::emitCode(FunctionEmitContext *ctx, llvm::Function *function, // issue a warning. Also need to warn if it's the entry block for // the function (in which case it will not have predeccesors but is // still reachable.) - if (type->GetReturnType() != AtomicType::Void && + if (Type::Equal(type->GetReturnType(), AtomicType::Void) == false && (pred_begin(ec.bblock) != pred_end(ec.bblock) || (ec.bblock == entryBBlock))) Warning(sym->pos, "Missing return statement in function returning \"%s\".", type->rType->GetString().c_str()); diff --git a/ispc.cpp b/ispc.cpp index 6729da92..49623be4 100644 --- a/ispc.cpp +++ b/ispc.cpp @@ -497,6 +497,7 @@ Opt::Opt() { disableMaskedStoreToStore = false; disableGatherScatterFlattening = false; disableUniformMemoryOptimizations = false; + disableCoalescing = false; } /////////////////////////////////////////////////////////////////////////// diff --git a/ispc.h b/ispc.h index 59c9140f..32999231 100644 --- a/ispc.h +++ b/ispc.h @@ -107,6 +107,7 @@ class ExprList; class Function; class FunctionType; class Module; +class PointerType; class Stmt; class Symbol; class SymbolTable; @@ -339,6 +340,10 @@ struct Opt { than gathers/scatters. This is likely only useful for measuring the impact of this optimization. */ bool disableUniformMemoryOptimizations; + + /** Disables optimizations that coalesce incoherent scalar memory + access from gathers into wider vector operations, when possible. */ + bool disableCoalescing; }; /** @brief This structure collects together a number of global variables. diff --git a/llvmutil.cpp b/llvmutil.cpp index e5c4785e..0f5bfd1b 100644 --- a/llvmutil.cpp +++ b/llvmutil.cpp @@ -469,6 +469,42 @@ LLVMBoolVector(const bool *bvec) { } +llvm::Constant * +LLVMIntAsType(int64_t val, LLVM_TYPE_CONST llvm::Type *type) { + LLVM_TYPE_CONST llvm::VectorType *vecType = + llvm::dyn_cast(type); + + if (vecType != NULL) { + llvm::Constant *v = llvm::ConstantInt::get(vecType->getElementType(), + val, true /* signed */); + std::vector vals; + for (int i = 0; i < (int)vecType->getNumElements(); ++i) + vals.push_back(v); + return llvm::ConstantVector::get(vals); + } + else + return llvm::ConstantInt::get(type, val, true /* signed */); +} + + +llvm::Constant * +LLVMUIntAsType(uint64_t val, LLVM_TYPE_CONST llvm::Type *type) { + LLVM_TYPE_CONST llvm::VectorType *vecType = + llvm::dyn_cast(type); + + if (vecType != NULL) { + llvm::Constant *v = llvm::ConstantInt::get(vecType->getElementType(), + val, false /* unsigned */); + std::vector vals; + for (int i = 0; i < (int)vecType->getNumElements(); ++i) + vals.push_back(v); + return llvm::ConstantVector::get(vals); + } + else + return llvm::ConstantInt::get(type, val, false /* unsigned */); +} + + /** Conservative test to see if two llvm::Values are equal. There are (potentially many) cases where the two values actually are equal but this will return false. However, if it does return true, the two @@ -585,7 +621,7 @@ LLVMFlattenInsertChain(llvm::InsertElementInst *ie, int vectorWidth, llvm::dyn_cast(insertBase); Assert(cv != NULL); Assert(iOffset < (int)cv->getNumOperands()); - elements[iOffset] = cv->getOperand(iOffset); + elements[iOffset] = cv->getOperand((int32_t)iOffset); } } } diff --git a/llvmutil.h b/llvmutil.h index a1257084..02857e34 100644 --- a/llvmutil.h +++ b/llvmutil.h @@ -173,6 +173,14 @@ extern llvm::Constant *LLVMFloatVector(float f); across all elements */ extern llvm::Constant *LLVMDoubleVector(double f); +/** Returns a constant integer or vector (according to the given type) of + the given signed integer value. */ +extern llvm::Constant *LLVMIntAsType(int64_t, LLVM_TYPE_CONST llvm::Type *t); + +/** Returns a constant integer or vector (according to the given type) of + the given unsigned integer value. */ +extern llvm::Constant *LLVMUIntAsType(uint64_t, LLVM_TYPE_CONST llvm::Type *t); + /** Returns an LLVM boolean vector based on the given array of values. The array should have g->target.vectorWidth elements. */ extern llvm::Constant *LLVMBoolVector(const bool *v); diff --git a/main.cpp b/main.cpp index 7b8c66d5..538414b6 100644 --- a/main.cpp +++ b/main.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2010-2011, Intel Corporation + Copyright (c) 2010-2012, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -141,16 +141,17 @@ devUsage(int ret) { printf(" [--fuzz-test]\t\t\tRandomly perturb program input to test error conditions\n"); printf(" [--fuzz-seed=]\t\tSeed value for RNG for fuzz testing\n"); printf(" [--opt=