From 60aae167525d54ac8a84a30fa7d825f63e9e994b Mon Sep 17 00:00:00 2001 From: Matt Pharr Date: Mon, 19 Mar 2012 11:57:04 -0700 Subject: [PATCH] Move check for linear vector to LLVMVectorIsLinear() function. --- llvmutil.cpp | 222 +++++++++++++++++++++++++++++++++++++++++++++++++++ llvmutil.h | 10 +++ opt.cpp | 206 +---------------------------------------------- 3 files changed, 233 insertions(+), 205 deletions(-) diff --git a/llvmutil.cpp b/llvmutil.cpp index a2eaad6f..1a065ac3 100644 --- a/llvmutil.cpp +++ b/llvmutil.cpp @@ -832,6 +832,228 @@ LLVMVectorValuesAllEqual(llvm::Value *v) { } +static bool +lVectorIsLinear(llvm::Value *v, int vectorLength, int stride, + std::vector &seenPhis); + +/** Given a vector of compile-time constant integer values, test to see if + they are a linear sequence of constant integers starting from an + arbirary value but then having a step of value "stride" between + elements. + */ +static bool +lVectorIsLinearConstantInts( +#ifdef LLVM_3_1svn + llvm::ConstantDataVector *cv, +#else + llvm::ConstantVector *cv, +#endif + int vectorLength, + int stride) { + // Flatten the vector out into the elements array + llvm::SmallVector elements; +#ifdef LLVM_3_1svn + for (int i = 0; i < (int)cv->getNumElements(); ++i) + elements.push_back(cv->getElementAsConstant(i)); +#else + cv->getVectorElements(elements); +#endif + Assert((int)elements.size() == vectorLength); + + llvm::ConstantInt *ci = llvm::dyn_cast(elements[0]); + if (ci == NULL) + // Not a vector of integers + return false; + + int64_t prevVal = ci->getSExtValue(); + + // For each element in the array, see if it is both a ConstantInt and + // if the difference between it and the value of the previous element + // is stride. If not, fail. + for (int i = 1; i < vectorLength; ++i) { + ci = llvm::dyn_cast(elements[i]); + if (ci == NULL) + return false; + + int64_t nextVal = ci->getSExtValue(); + if (prevVal + stride != nextVal) + return false; + + prevVal = nextVal; + } + return true; +} + + +/** Checks to see if (op0 * op1) is a linear vector where the result is a + vector with values that increase by stride. + */ +static bool +lCheckMulForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, + int stride, std::vector &seenPhis) { + // Is the first operand a constant integer value splatted across all of + // the lanes? +#ifdef LLVM_3_1svn + llvm::ConstantDataVector *cv = llvm::dyn_cast(op0); +#else + llvm::ConstantVector *cv = llvm::dyn_cast(op0); +#endif + if (cv == NULL) + return false; + + llvm::Constant *csplat = cv->getSplatValue(); + if (csplat == NULL) + return false; + + llvm::ConstantInt *splat = llvm::dyn_cast(csplat); + if (splat == NULL) + return false; + + // If the splat value doesn't evenly divide the stride we're looking + // for, there's no way that we can get the linear sequence we're + // looking or. + int64_t splatVal = splat->getSExtValue(); + if (splatVal == 0 || splatVal > stride || (stride % splatVal) != 0) + return false; + + // Check to see if the other operand is a linear vector with stride + // given by stride/splatVal. + return lVectorIsLinear(op1, vectorLength, (int)(stride / splatVal), + seenPhis); +} + + +static bool +lVectorIsLinear(llvm::Value *v, int vectorLength, int stride, + std::vector &seenPhis) { + // First try the easy case: if the values are all just constant + // integers and have the expected stride between them, then we're done. +#ifdef LLVM_3_1svn + llvm::ConstantDataVector *cv = llvm::dyn_cast(v); +#else + llvm::ConstantVector *cv = llvm::dyn_cast(v); +#endif + if (cv != NULL) + return lVectorIsLinearConstantInts(cv, vectorLength, stride); + + llvm::BinaryOperator *bop = llvm::dyn_cast(v); + if (bop != NULL) { + // FIXME: is it right to pass the seenPhis to the all equal check as well?? + llvm::Value *op0 = bop->getOperand(0), *op1 = bop->getOperand(1); + + if (bop->getOpcode() == llvm::Instruction::Add) { + // There are two cases to check if we have an add: + // + // programIndex + unif -> ascending linear seqeuence + // unif + programIndex -> ascending linear sequence + bool l0 = lVectorIsLinear(op0, vectorLength, stride, seenPhis); + bool e1 = lVectorValuesAllEqual(op1, vectorLength, seenPhis); + if (l0 && e1) + return true; + + bool e0 = lVectorValuesAllEqual(op0, vectorLength, seenPhis); + bool l1 = lVectorIsLinear(op1, vectorLength, stride, seenPhis); + return (e0 && l1); + } + else if (bop->getOpcode() == llvm::Instruction::Sub) + // For subtraction, we only match: + // programIndex - unif -> ascending linear seqeuence + return (lVectorIsLinear(bop->getOperand(0), vectorLength, + stride, seenPhis) && + lVectorValuesAllEqual(bop->getOperand(1), vectorLength, + seenPhis)); + else if (bop->getOpcode() == llvm::Instruction::Mul) { + // Multiplies are a bit trickier, so are handled in a separate + // function. + bool m0 = lCheckMulForLinear(op0, op1, vectorLength, stride, seenPhis); + if (m0) + return true; + bool m1 = lCheckMulForLinear(op1, op0, vectorLength, stride, seenPhis); + return m1; + } + else + return false; + } + + llvm::CastInst *ci = llvm::dyn_cast(v); + if (ci != NULL) + return lVectorIsLinear(ci->getOperand(0), vectorLength, + stride, seenPhis); + + if (llvm::isa(v) || llvm::isa(v)) + return false; + + llvm::PHINode *phi = llvm::dyn_cast(v); + if (phi != NULL) { + for (unsigned int i = 0; i < seenPhis.size(); ++i) + if (seenPhis[i] == phi) + return true; + + seenPhis.push_back(phi); + + unsigned int numIncoming = phi->getNumIncomingValues(); + // Check all of the incoming values: if all of them are all equal, + // then we're good. + for (unsigned int i = 0; i < numIncoming; ++i) { + if (!lVectorIsLinear(phi->getIncomingValue(i), vectorLength, stride, + seenPhis)) { + seenPhis.pop_back(); + return false; + } + } + + seenPhis.pop_back(); + return true; + } + + // TODO: is any reason to worry about these? + if (llvm::isa(v)) + return false; + + // TODO: we could also handle shuffles, but we haven't yet seen any + // cases where doing so would detect cases where actually have a linear + // vector. + llvm::ShuffleVectorInst *shuffle = llvm::dyn_cast(v); + if (shuffle != NULL) + return false; + +#if 0 + fprintf(stderr, "linear check: "); + v->dump(); + fprintf(stderr, "\n"); + llvm::Instruction *inst = llvm::dyn_cast(v); + if (inst) { + inst->getParent()->dump(); + fprintf(stderr, "\n"); + fprintf(stderr, "\n"); + } +#endif + + return false; +} + + +/** Given vector of integer-typed values, see if the elements of the array + have a step of 'stride' between their values. This function tries to + handle as many possibilities as possible, including things like all + elements equal to some non-constant value plus an integer offset, etc. +*/ +bool +LLVMVectorIsLinear(llvm::Value *v, int stride) { + LLVM_TYPE_CONST llvm::VectorType *vt = + llvm::dyn_cast(v->getType()); + Assert(vt != NULL); + int vectorLength = vt->getNumElements(); + + std::vector seenPhis; + bool linear = lVectorIsLinear(v, vectorLength, stride, seenPhis); + Debug(SourcePos(), "LLVMVectorIsLinear(%s) -> %s.", + v->getName().str().c_str(), linear ? "true" : "false"); + if (g->debugPrint) + LLVMDumpValue(v); + + return linear; +} static void diff --git a/llvmutil.h b/llvmutil.h index ab9fba82..96cdf079 100644 --- a/llvmutil.h +++ b/llvmutil.h @@ -230,6 +230,16 @@ extern llvm::Constant *LLVMMaskAllOff; return false for arrays where the values are actually all equal. */ extern bool LLVMVectorValuesAllEqual(llvm::Value *v); +/** Given vector of integer-typed values, this function returns true if it + can determine that the elements of the vector have a step of 'stride' + between their values and false otherwise. This function tries to + handle as many possibilities as possible, including things like all + elements equal to some non-constant value plus an integer offset, etc. + Needless to say (the halting problem and all that), it may return false + for some vectors that are in fact linear. + */ +extern bool LLVMVectorIsLinear(llvm::Value *v, int stride); + /** Given a vector-typed value v, if the vector is a vector with constant element values, this function extracts those element values into the ret[] array and returns the number of elements (i.e. the vector type's diff --git a/opt.cpp b/opt.cpp index 5d6c3dfd..558a4106 100644 --- a/opt.cpp +++ b/opt.cpp @@ -2267,208 +2267,6 @@ public: char GSToLoadStorePass::ID = 0; - -/** Given a vector of compile-time constant integer values, test to see if - they are a linear sequence of constant integers starting from an - arbirary value but then having a step of value "stride" between - elements. - */ -static bool -lVectorIsLinearConstantInts( -#ifdef LLVM_3_1svn - llvm::ConstantDataVector *cv, -#else - llvm::ConstantVector *cv, -#endif - int vectorLength, - int stride) { - // Flatten the vector out into the elements array - llvm::SmallVector elements; -#ifdef LLVM_3_1svn - for (int i = 0; i < (int)cv->getNumElements(); ++i) - elements.push_back(cv->getElementAsConstant(i)); -#else - cv->getVectorElements(elements); -#endif - Assert((int)elements.size() == vectorLength); - - llvm::ConstantInt *ci = llvm::dyn_cast(elements[0]); - if (ci == NULL) - // Not a vector of integers - return false; - - int64_t prevVal = ci->getSExtValue(); - - // For each element in the array, see if it is both a ConstantInt and - // if the difference between it and the value of the previous element - // is stride. If not, fail. - for (int i = 1; i < vectorLength; ++i) { - ci = llvm::dyn_cast(elements[i]); - if (ci == NULL) - return false; - - int64_t nextVal = ci->getSExtValue(); - if (prevVal + stride != nextVal) - return false; - - prevVal = nextVal; - } - return true; -} - - -static bool lVectorIsLinear(llvm::Value *v, int vectorLength, int stride, - std::vector &seenPhis); - -/** Checks to see if (op0 * op1) is a linear vector where the result is a - vector with values that increase by stride. - */ -static bool -lCheckMulForLinear(llvm::Value *op0, llvm::Value *op1, int vectorLength, - int stride, std::vector &seenPhis) { - // Is the first operand a constant integer value splatted across all of - // the lanes? -#ifdef LLVM_3_1svn - llvm::ConstantDataVector *cv = llvm::dyn_cast(op0); -#else - llvm::ConstantVector *cv = llvm::dyn_cast(op0); -#endif - if (cv == NULL) - return false; - - llvm::Constant *csplat = cv->getSplatValue(); - if (csplat == NULL) - return false; - - llvm::ConstantInt *splat = llvm::dyn_cast(csplat); - if (splat == NULL) - return false; - - // If the splat value doesn't evenly divide the stride we're looking - // for, there's no way that we can get the linear sequence we're - // looking or. - int64_t splatVal = splat->getSExtValue(); - if (splatVal == 0 || splatVal > stride || (stride % splatVal) != 0) - return false; - - // Check to see if the other operand is a linear vector with stride - // given by stride/splatVal. - return lVectorIsLinear(op1, vectorLength, (int)(stride / splatVal), - seenPhis); -} - - -/** Given vector of integer-typed values, see if the elements of the array - have a step of 'stride' between their values. This function tries to - handle as many possibilities as possible, including things like all - elements equal to some non-constant value plus an integer offset, etc. -*/ -static bool -lVectorIsLinear(llvm::Value *v, int vectorLength, int stride, - std::vector &seenPhis) { - // First try the easy case: if the values are all just constant - // integers and have the expected stride between them, then we're done. -#ifdef LLVM_3_1svn - llvm::ConstantDataVector *cv = llvm::dyn_cast(v); -#else - llvm::ConstantVector *cv = llvm::dyn_cast(v); -#endif - if (cv != NULL) - return lVectorIsLinearConstantInts(cv, vectorLength, stride); - - llvm::BinaryOperator *bop = llvm::dyn_cast(v); - if (bop != NULL) { - // FIXME: is it right to pass the seenPhis to the all equal check as well?? - llvm::Value *op0 = bop->getOperand(0), *op1 = bop->getOperand(1); - - if (bop->getOpcode() == llvm::Instruction::Add) - // There are two cases to check if we have an add: - // - // programIndex + unif -> ascending linear seqeuence - // unif + programIndex -> ascending linear sequence - return ((lVectorIsLinear(op0, vectorLength, stride, seenPhis) && - LLVMVectorValuesAllEqual(op1, vectorLength, seenPhis)) || - (lVectorIsLinear(op1, vectorLength, stride, seenPhis) && - LLVMVectorValuesAllEqual(op0, vectorLength, seenPhis))); - else if (bop->getOpcode() == llvm::Instruction::Sub) - // For subtraction, we only match: - // - // programIndex - unif -> ascending linear seqeuence - // - // In the future, we could also look for: - // unif - programIndex -> *descending* linear seqeuence - // And generate code for that as a vector load + shuffle. - return (lVectorIsLinear(bop->getOperand(0), vectorLength, - stride, seenPhis) && - LLVMVectorValuesAllEqual(bop->getOperand(1), vectorLength, - seenPhis)); - else if (bop->getOpcode() == llvm::Instruction::Mul) - // Multiplies are a bit trickier, so are handled in a separate - // function. - return (lCheckMulForLinear(op0, op1, vectorLength, stride, seenPhis) || - lCheckMulForLinear(op1, op0, vectorLength, stride, seenPhis)); - else - return false; - } - - llvm::CastInst *ci = llvm::dyn_cast(v); - if (ci != NULL) - return lVectorIsLinear(ci->getOperand(0), vectorLength, - stride, seenPhis); - - if (llvm::isa(v) || llvm::isa(v)) - return false; - - llvm::PHINode *phi = llvm::dyn_cast(v); - if (phi != NULL) { - for (unsigned int i = 0; i < seenPhis.size(); ++i) - if (seenPhis[i] == phi) - return true; - - seenPhis.push_back(phi); - - unsigned int numIncoming = phi->getNumIncomingValues(); - // Check all of the incoming values: if all of them are all equal, - // then we're good. - for (unsigned int i = 0; i < numIncoming; ++i) { - if (!lVectorIsLinear(phi->getIncomingValue(i), vectorLength, stride, - seenPhis)) { - seenPhis.pop_back(); - return false; - } - } - - seenPhis.pop_back(); - return true; - } - - // TODO: is any reason to worry about these? - if (llvm::isa(v)) - return false; - - // TODO: we could also handle shuffles, but we haven't yet seen any - // cases where doing so would detect cases where actually have a linear - // vector. - llvm::ShuffleVectorInst *shuffle = llvm::dyn_cast(v); - if (shuffle != NULL) - return false; - -#if 0 - fprintf(stderr, "linear check: "); - v->dump(); - fprintf(stderr, "\n"); - llvm::Instruction *inst = llvm::dyn_cast(v); - if (inst) { - inst->getParent()->dump(); - fprintf(stderr, "\n"); - fprintf(stderr, "\n"); - } -#endif - - return false; -} - - struct GatherImpInfo { GatherImpInfo(const char *pName, const char *lbName, const char *lmName, int a) @@ -2665,9 +2463,7 @@ GSToLoadStorePass::runOnBasicBlock(llvm::BasicBlock &bb) { else { int step = gatherInfo ? gatherInfo->align : scatterInfo->align; - std::vector seenPhis; - if (step > 0 && lVectorIsLinear(fullOffsets, g->target.vectorWidth, - step, seenPhis)) { + if (step > 0 && LLVMVectorIsLinear(fullOffsets, step)) { // We have a linear sequence of memory locations being accessed // starting with the location given by the offset from // offsetElements[0], with stride of 4 or 8 bytes (for 32 bit