aaron/ispc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #331 from jduprat/master

Browse Source 
New templated API for __setzero() __undef() and __smear()
This commit is contained in:
Matt Pharr
2012-07-18 16:39:38 -07:00
parent 6a410fc30e aecd6e0878
commit 658652a9ff
8 changed files with 163 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
cbackend.cpp
View File

@@ -523,7 +523,7 @@ static std::string CBEMangle(const std::string &S) {
std::string Result; std::string Result;
for (unsigned i = 0, e = S.size(); i != e; ++i) for (unsigned i = 0, e = S.size(); i != e; ++i)
if (isalnum(S[i]) || S[i] == '_') { if (isalnum(S[i]) || S[i] == '_' || S[i] == '<' || S[i] == '>') {
Result += S[i]; Result += S[i];
} else { } else {
Result += '_'; Result += '_';
@@ -1115,29 +1115,35 @@ bool CWriter::printCast(unsigned opc, llvm::Type *SrcTy, llvm::Type *DstTy) {
/** Construct the name of a function with the given base and returning a /** Construct the name of a function with the given base and returning a
vector of a given type. For example, if base is "foo" and matchType is vector of a given type, of the specified idth. For example, if base
i16, this will return the string "__foo_i16". is "foo" and matchType is i32 and width is 16, this will return the
string "__foo_i32<__vec16_i32>".
*/ */
static const char * static const char *
lGetTypedFunc(const char *base, llvm::Type *matchType) { lGetTypedFunc(const char *base, llvm::Type *matchType, int width) {
char buf[64]; static const char *ty_desc_str[] = {"f", "d", "i1", "i8", "i16", "i32", "i64"};
sprintf(buf, "__%s_", base); static const char *fn_desc_str[] = {"float", "double", "i1", "i8", "i16", "i32", "i64"};
enum {DESC_FLOAT, DESC_DOUBLE, DESC_I1, DESC_I8, DESC_I16, DESC_I32, DESC_I64} desc;
switch (matchType->getTypeID()) { switch (matchType->getTypeID()) {
case llvm::Type::FloatTyID: strcat(buf, "float"); break; case llvm::Type::FloatTyID: desc = DESC_FLOAT; break;
case llvm::Type::DoubleTyID: strcat(buf, "double"); break; case llvm::Type::DoubleTyID: desc = DESC_DOUBLE; break;
case llvm::Type::IntegerTyID: { case llvm::Type::IntegerTyID: {
switch (llvm::cast<llvm::IntegerType>(matchType)->getBitWidth()) { switch (llvm::cast<llvm::IntegerType>(matchType)->getBitWidth()) {
case 1: strcat(buf, "i1"); break; case 1: desc = DESC_I1; break;
case 8: strcat(buf, "i8"); break; case 8: desc = DESC_I8; break;
case 16: strcat(buf, "i16"); break; case 16: desc = DESC_I16; break;
case 32: strcat(buf, "i32"); break; case 32: desc = DESC_I32; break;
case 64: strcat(buf, "i64"); break; case 64: desc = DESC_I64; break;
default: return NULL; default: return NULL;
} }
break; break;
} }
default: return NULL; default: return NULL;
} }
char buf[64];
snprintf(buf, 64, "__%s_%s<__vec%d_%s>", base, fn_desc_str[desc], width, ty_desc_str[desc]);
return strdup(buf); return strdup(buf);
} }
@@ -1486,19 +1492,19 @@ void CWriter::printConstant(llvm::Constant *CPV, bool Static) {
if (llvm::isa<llvm::ConstantAggregateZero>(CPV)) { if (llvm::isa<llvm::ConstantAggregateZero>(CPV)) {
// All zeros; call the __setzero_* function. // All zeros; call the __setzero_* function.
const char *setZeroFunc = lGetTypedFunc("setzero", VT->getElementType()); const char *setZeroFunc = lGetTypedFunc("setzero", VT->getElementType(), vectorWidth);
assert(setZeroFunc != NULL); assert(setZeroFunc != NULL);
Out << setZeroFunc << "()"; Out << setZeroFunc << "()";
} }
else if (llvm::isa<llvm::UndefValue>(CPV)) { else if (llvm::isa<llvm::UndefValue>(CPV)) {
// Undefined value; call __undef_* so that we can potentially pass // Undefined value; call __undef_* so that we can potentially pass
// this information along.. // this information along..
const char *undefFunc = lGetTypedFunc("undef", VT->getElementType()); const char *undefFunc = lGetTypedFunc("undef", VT->getElementType(), vectorWidth);
assert(undefFunc != NULL); assert(undefFunc != NULL);
Out << undefFunc << "()"; Out << undefFunc << "()";
} }
else { else {
const char *smearFunc = lGetTypedFunc("smear", VT->getElementType()); const char *smearFunc = lGetTypedFunc("smear", VT->getElementType(), vectorWidth);
if (llvm::ConstantVector *CV = llvm::dyn_cast<llvm::ConstantVector>(CPV)) { if (llvm::ConstantVector *CV = llvm::dyn_cast<llvm::ConstantVector>(CPV)) {
llvm::Constant *splatValue = CV->getSplatValue(); llvm::Constant *splatValue = CV->getSplatValue();
@@ -1713,8 +1719,8 @@ void CWriter::printConstantWithCast(llvm::Constant* CPV, unsigned Opcode) {
std::string CWriter::GetValueName(const llvm::Value *Operand) { std::string CWriter::GetValueName(const llvm::Value *Operand) {
// Resolve potential alias. // Resolve potential alias.
if (const llvm::GlobalAlias *GA = llvm::dyn_cast<llvm::GlobalAlias>(Operand)) { if (const llvm::GlobalAlias *GA = llvm::dyn_cast<llvm::GlobalAlias>(Operand)) {
if (const llvm::Value *V = GA->resolveAliasedGlobal(false)) if (const llvm::Value *V = GA->resolveAliasedGlobal(false))
Operand = V; Operand = V;
} }
@@ -4362,12 +4368,11 @@ SmearCleanupPass::runOnBasicBlock(llvm::BasicBlock &bb) {
{ {
llvm::Type *matchType = toMatch->getType(); llvm::Type *matchType = toMatch->getType();
const char *smearFuncName = lGetTypedFunc("smear", matchType); const char *smearFuncName = lGetTypedFunc("smear", matchType, vectorWidth);
if (smearFuncName != NULL) { if (smearFuncName != NULL) {
llvm::Function *smearFunc = module->getFunction(smearFuncName); llvm::Function *smearFunc = module->getFunction(smearFuncName);
if (smearFunc == NULL) { if (smearFunc == NULL) {
// Declare the smar function if needed; it takes a single // Declare the smear function if needed; it takes a single
// scalar parameter and returns a vector of the same // scalar parameter and returns a vector of the same
// parameter type. // parameter type.
llvm::Constant *sf = llvm::Constant *sf =

19
examples/intrinsics/generic-16.h
View File

@@ -271,7 +271,8 @@ static FORCEINLINE TYPE NAME(TYPE a, int32_t b) { \
} }
#define SMEAR(VTYPE, NAME, STYPE) \ #define SMEAR(VTYPE, NAME, STYPE) \
static FORCEINLINE VTYPE __smear_##NAME(STYPE v) { \ template <class RetVecType> VTYPE __smear_##NAME(STYPE); \
template <> FORCEINLINE VTYPE __smear_##NAME<VTYPE>(STYPE v) { \
VTYPE ret; \ VTYPE ret; \
for (int i = 0; i < 16; ++i) \ for (int i = 0; i < 16; ++i) \
ret.v[i] = v; \ ret.v[i] = v; \
@@ -279,7 +280,8 @@ static FORCEINLINE VTYPE __smear_##NAME(STYPE v) { \
} }
#define SETZERO(VTYPE, NAME) \ #define SETZERO(VTYPE, NAME) \
static FORCEINLINE VTYPE __setzero_##NAME() { \ template <class RetVecType> VTYPE __setzero_##NAME(); \
template <> FORCEINLINE VTYPE __setzero_##NAME<VTYPE>() { \
VTYPE ret; \ VTYPE ret; \
for (int i = 0; i < 16; ++i) \ for (int i = 0; i < 16; ++i) \
ret.v[i] = 0; \ ret.v[i] = 0; \
@@ -287,7 +289,8 @@ static FORCEINLINE VTYPE __setzero_##NAME() { \
} }
#define UNDEF(VTYPE, NAME) \ #define UNDEF(VTYPE, NAME) \
static FORCEINLINE VTYPE __undef_##NAME() { \ template <class RetVecType> VTYPE __undef_##NAME(); \
template <> FORCEINLINE VTYPE __undef_##NAME<VTYPE>() { \
return VTYPE(); \ return VTYPE(); \
} }
@@ -416,18 +419,20 @@ template <int ALIGN> static FORCEINLINE void __store(__vec16_i1 *p, __vec16_i1 v
*ptr = v.v; *ptr = v.v;
} }
static FORCEINLINE __vec16_i1 __smear_i1(int v) { template <class RetVecType> __vec16_i1 __smear_i1(int i);
template <> FORCEINLINE __vec16_i1 __smear_i1<__vec16_i1>(int v) {
return __vec16_i1(v, v, v, v, v, v, v, v, return __vec16_i1(v, v, v, v, v, v, v, v,
v, v, v, v, v, v, v, v); v, v, v, v, v, v, v, v);
} }
static FORCEINLINE __vec16_i1 __setzero_i1() { template <class RetVecType> __vec16_i1 __setzero_i1();
template <> FORCEINLINE __vec16_i1 __setzero_i1<__vec16_i1>() {
return __vec16_i1(0, 0, 0, 0, 0, 0, 0, 0, return __vec16_i1(0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0); 0, 0, 0, 0, 0, 0, 0, 0);
} }
template <class RetVecType> __vec16_i1 __undef_i1();
static FORCEINLINE __vec16_i1 __undef_i1() { template <> FORCEINLINE __vec16_i1 __undef_i1<__vec16_i1>() {
return __vec16_i1(); return __vec16_i1();
} }

20
examples/intrinsics/generic-32.h
View File

@@ -336,15 +336,17 @@ static FORCEINLINE TYPE NAME(TYPE a, int32_t b) { \
} }
#define SMEAR(VTYPE, NAME, STYPE) \ #define SMEAR(VTYPE, NAME, STYPE) \
static FORCEINLINE VTYPE __smear_##NAME(STYPE v) { \ template <class RetVecType> VTYPE __smear_##NAME(STYPE); \
template <> FORCEINLINE VTYPE __smear_##NAME<VTYPE>(STYPE v) { \
VTYPE ret; \ VTYPE ret; \
for (int i = 0; i < 32; ++i) \ for (int i = 0; i < 32; ++i) \
ret.v[i] = v; \ ret.v[i] = v; \
return ret; \ return ret; \
} }
#define SETZERO(VTYPE, NAME) \ #define SETZERO(VTYPE, NAME) \
static FORCEINLINE VTYPE __setzero_##NAME() { \ template <class RetVecType> VTYPE __setzero_##NAME(); \
template <> FORCEINLINE VTYPE __setzero_##NAME<VTYPE>() { \
VTYPE ret; \ VTYPE ret; \
for (int i = 0; i < 32; ++i) \ for (int i = 0; i < 32; ++i) \
ret.v[i] = 0; \ ret.v[i] = 0; \
@@ -352,7 +354,8 @@ static FORCEINLINE VTYPE __setzero_##NAME() { \
} }
#define UNDEF(VTYPE, NAME) \ #define UNDEF(VTYPE, NAME) \
static FORCEINLINE VTYPE __undef_##NAME() { \ template <class RetVecType> VTYPE __undef_##NAME(); \
template <> FORCEINLINE VTYPE __undef_##NAME<VTYPE>() { \
return VTYPE(); \ return VTYPE(); \
} }
@@ -481,21 +484,24 @@ template <int ALIGN> static FORCEINLINE void __store(__vec32_i1 *p, __vec32_i1 v
*ptr = v.v; *ptr = v.v;
} }
static FORCEINLINE __vec32_i1 __smear_i1(int v) { template <class RetVecType> __vec32_i1 __smear_i1(int i);
template <> FORCEINLINE __vec32_i1 __smear_i1<__vec32_i1>(int v) {
return __vec32_i1(v, v, v, v, v, v, v, v, return __vec32_i1(v, v, v, v, v, v, v, v,
v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v,
v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v,
v, v, v, v, v, v, v, v); v, v, v, v, v, v, v, v);
} }
static FORCEINLINE __vec32_i1 __setzero_i1() { template <class RetVecType> __vec32_i1 __setzero_i1();
template <> FORCEINLINE __vec32_i1 __setzero_i1<__vec32_i1>() {
return __vec32_i1(0, 0, 0, 0, 0, 0, 0, 0, return __vec32_i1(0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0); 0, 0, 0, 0, 0, 0, 0, 0);
} }
static FORCEINLINE __vec32_i1 __undef_i1() { template <class RetVecType> __vec32_i1 __undef_i1();
template <> FORCEINLINE __vec32_i1 __undef_i1<__vec32_i1>() {
return __vec32_i1(); return __vec32_i1();
} }

18
examples/intrinsics/generic-64.h
View File

@@ -461,7 +461,8 @@ static FORCEINLINE TYPE NAME(TYPE a, int32_t b) { \
} }
#define SMEAR(VTYPE, NAME, STYPE) \ #define SMEAR(VTYPE, NAME, STYPE) \
static FORCEINLINE VTYPE __smear_##NAME(STYPE v) { \ template <class RetVecType> VTYPE __smear_##NAME(STYPE); \
template <> FORCEINLINE VTYPE __smear_##NAME<VTYPE>(STYPE v) { \
VTYPE ret; \ VTYPE ret; \
for (int i = 0; i < 64; ++i) \ for (int i = 0; i < 64; ++i) \
ret.v[i] = v; \ ret.v[i] = v; \
@@ -469,7 +470,8 @@ static FORCEINLINE VTYPE __smear_##NAME(STYPE v) { \
} }
#define SETZERO(VTYPE, NAME) \ #define SETZERO(VTYPE, NAME) \
static FORCEINLINE VTYPE __setzero_##NAME() { \ template <class RetVecType> VTYPE __setzero_##NAME(); \
template <> FORCEINLINE VTYPE __setzero_##NAME<VTYPE>() { \
VTYPE ret; \ VTYPE ret; \
for (int i = 0; i < 64; ++i) \ for (int i = 0; i < 64; ++i) \
ret.v[i] = 0; \ ret.v[i] = 0; \
@@ -477,7 +479,8 @@ static FORCEINLINE VTYPE __setzero_##NAME() { \
} }
#define UNDEF(VTYPE, NAME) \ #define UNDEF(VTYPE, NAME) \
static FORCEINLINE VTYPE __undef_##NAME(VTYPE retType) { \ template <class RetVecType> VTYPE __undef_##NAME(); \
template <> FORCEINLINE VTYPE __undef_##NAME<VTYPE>() { \
return VTYPE(); \ return VTYPE(); \
} }
@@ -606,7 +609,8 @@ template <int ALIGN> static FORCEINLINE void __store(__vec64_i1 *p, __vec64_i1 v
*ptr = v.v; *ptr = v.v;
} }
static FORCEINLINE __vec64_i1 __smear_i1(int v) { template <class RetVecType> __vec64_i1 __smear_i1(int i);
template <> FORCEINLINE __vec64_i1 __smear_i1<__vec64_i1>(int v) {
return __vec64_i1(v, v, v, v, v, v, v, v, return __vec64_i1(v, v, v, v, v, v, v, v,
v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v,
v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v,
@@ -617,7 +621,8 @@ static FORCEINLINE __vec64_i1 __smear_i1(int v) {
v, v, v, v, v, v, v, v); v, v, v, v, v, v, v, v);
} }
static FORCEINLINE __vec64_i1 __setzero_i1() { template <class RetVecType> __vec64_i1 __setzero_i1();
template <> FORCEINLINE __vec64_i1 __setzero_i1<__vec64_i1>() {
return __vec64_i1(0, 0, 0, 0, 0, 0, 0, 0, return __vec64_i1(0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -628,7 +633,8 @@ static FORCEINLINE __vec64_i1 __setzero_i1() {
0, 0, 0, 0, 0, 0, 0, 0); 0, 0, 0, 0, 0, 0, 0, 0);
} }
static FORCEINLINE __vec64_i1 __undef_i1() { template <class RetVecType> __vec64_i1 __undef_i1();
template <> FORCEINLINE __vec64_i1 __undef_i1<__vec64_i1>() {
return __vec64_i1(); return __vec64_i1();
} }

62
examples/intrinsics/knc.h
View File

@@ -477,15 +477,18 @@ template <int ALIGN> static FORCEINLINE void __store(__vec16_i1 *p, __vec16_i1 v
*ptr = v.m; *ptr = v.m;
} }
static FORCEINLINE __vec16_i1 __smear_i1(int i) { template <class RetVecType> __vec16_i1 __smear_i1(int i);
template <> static FORCEINLINE __vec16_i1 __smear_i1<__vec16_i1>(int i) {
return i?0xFFFF:0x0; return i?0xFFFF:0x0;
} }
static FORCEINLINE __vec16_i1 __setzero_i1() { template <class RetVecType> __vec16_i1 __setzero_i1();
template <> static FORCEINLINE __vec16_i1 __setzero_i1<__vec16_i1>() {
return 0; return 0;
} }
static FORCEINLINE __vec16_i1 __undef_i1() { template <class RetVecType> __vec16_i1 __undef_i1();
template <> static FORCEINLINE __vec16_i1 __undef_i1<__vec16_i1>() {
return __vec16_i1(); // FIXME? __mm512_undef_mask(); return __vec16_i1(); // FIXME? __mm512_undef_mask();
} }
@@ -744,15 +747,18 @@ static FORCEINLINE __vec16_i32 __select(bool cond, __vec16_i32 a, __vec16_i32 b)
static FORCEINLINE int32_t __extract_element(__vec16_i32 v, int index) { return ((int32_t *)&v)[index]; } static FORCEINLINE int32_t __extract_element(__vec16_i32 v, int index) { return ((int32_t *)&v)[index]; }
static FORCEINLINE void __insert_element(__vec16_i32 *v, int index, int32_t val) { ((int32_t *)v)[index] = val; } static FORCEINLINE void __insert_element(__vec16_i32 *v, int index, int32_t val) { ((int32_t *)v)[index] = val; }
static FORCEINLINE __vec16_i32 __smear_i32(int32_t i) { template <class RetVecType> __vec16_i32 __smear_i32(int32_t i);
template <> static FORCEINLINE __vec16_i32 __smear_i32<__vec16_i32>(int32_t i) {
return _mm512_set_1to16_epi32(i); return _mm512_set_1to16_epi32(i);
} }
static FORCEINLINE __vec16_i32 __setzero_i32() { template <class RetVecType> __vec16_i32 __setzero_i32();
template <> static FORCEINLINE __vec16_i32 __setzero_i32<__vec16_i32>() {
return _mm512_setzero_epi32(); return _mm512_setzero_epi32();
} }
static FORCEINLINE __vec16_i32 __undef_i32() { template <class RetVecType> __vec16_i32 __undef_i32();
template <> static FORCEINLINE __vec16_i32 __undef_i32<__vec16_i32>() {
return _mm512_undefined_epi32(); return _mm512_undefined_epi32();
} }
@@ -803,13 +809,22 @@ template <> static FORCEINLINE void __store<64>(__vec16_i32 *p, __vec16_i32 v) {
// int64 // int64
static FORCEINLINE __vec16_i64 __setzero_i64() { template <class RetVecType> __vec16_i64 __setzero_i64();
template <> static FORCEINLINE __vec16_i64 __setzero_i64<__vec16_i64>() {
__vec16_i64 ret; __vec16_i64 ret;
ret.v_lo = _mm512_setzero_epi32(); ret.v_lo = _mm512_setzero_epi32();
ret.v_hi = _mm512_setzero_epi32(); ret.v_hi = _mm512_setzero_epi32();
return ret; return ret;
} }
template <class RetVecType> __vec16_i64 __undef_i64();
template <> static FORCEINLINE __vec16_i64 __undef_i64<__vec16_i64>() {
__vec16_i64 ret;
ret.v_lo = _mm512_undefined_epi32();
ret.v_hi = _mm512_undefined_epi32();
return ret;
}
static FORCEINLINE __vec16_i64 __add(const __vec16_i64 &a, const __vec16_i64 &b) static FORCEINLINE __vec16_i64 __add(const __vec16_i64 &a, const __vec16_i64 &b)
{ {
__mmask16 carry = 0; __mmask16 carry = 0;
@@ -885,7 +900,8 @@ static FORCEINLINE int64_t __extract_element(const __vec16_i64 &v, int index)
return src[index+16] | (int64_t(src[index]) << 32); return src[index+16] | (int64_t(src[index]) << 32);
} }
static FORCEINLINE __vec16_i64 __smear_i64(const int64_t &l) { template <class RetVecType> __vec16_i64 __smear_i64(const int64_t &l);
template <> FORCEINLINE __vec16_i64 __smear_i64<__vec16_i64>(const int64_t &l) {
const int *i = (const int*)&l; const int *i = (const int*)&l;
return __vec16_i64(_mm512_set_1to16_epi32(i[0]), _mm512_set_1to16_epi32(i[1])); return __vec16_i64(_mm512_set_1to16_epi32(i[0]), _mm512_set_1to16_epi32(i[1]));
} }
@@ -897,12 +913,12 @@ LOAD_STORE(__vec16_i64, int64_t)
template <int ALIGN> static FORCEINLINE __vec16_i64 __load(const __vec16_i64 *p) { template <int ALIGN> static FORCEINLINE __vec16_i64 __load(const __vec16_i64 *p) {
__m512i v1; __vec16_i32 v1 = _mm512_undefined_epi32();
__m512i v2; __vec16_i32 v2 = _mm512_undefined_epi32();
v2 = _mm512_extloadunpackhi_epi32(v1, p, _MM_UPCONV_EPI32_NONE, _MM_HINT_NONE); v2 = _mm512_extloadunpackhi_epi32(v2, p, _MM_UPCONV_EPI32_NONE, _MM_HINT_NONE);
v2 = _mm512_extloadunpacklo_epi32(v1, p, _MM_UPCONV_EPI32_NONE, _MM_HINT_NONE); v2 = _mm512_extloadunpacklo_epi32(v2, p, _MM_UPCONV_EPI32_NONE, _MM_HINT_NONE);
v1 = _mm512_extloadunpackhi_epi32(v2, ((uint8_t*)p)+64, _MM_UPCONV_EPI32_NONE, _MM_HINT_NONE); v1 = _mm512_extloadunpackhi_epi32(v1, ((uint8_t*)p)+64, _MM_UPCONV_EPI32_NONE, _MM_HINT_NONE);
v1 = _mm512_extloadunpacklo_epi32(v2, ((uint8_t*)p)+64, _MM_UPCONV_EPI32_NONE, _MM_HINT_NONE); v1 = _mm512_extloadunpacklo_epi32(v1, ((uint8_t*)p)+64, _MM_UPCONV_EPI32_NONE, _MM_HINT_NONE);
__vec16_i64 ret; __vec16_i64 ret;
ret.v_hi = _mm512_mask_permutevar_epi32(ret.v_hi, 0xFF00, ret.v_hi = _mm512_mask_permutevar_epi32(ret.v_hi, 0xFF00,
@@ -1078,15 +1094,18 @@ static FORCEINLINE void __insert_element(__vec16_f *v, int index, float val) {
((float *)v)[index] = val; ((float *)v)[index] = val;
} }
static FORCEINLINE __vec16_f __smear_float(float f) { template <class RetVecType> __vec16_f __smear_float(float f);
template <> static FORCEINLINE __vec16_f __smear_float<__vec16_f>(float f) {
return _mm512_set_1to16_ps(f); return _mm512_set_1to16_ps(f);
} }
static FORCEINLINE __vec16_f __setzero_float() { template <class RetVecType> __vec16_f __setzero_float();
template <> static FORCEINLINE __vec16_f __setzero_float<__vec16_f>() {
return _mm512_setzero_ps(); return _mm512_setzero_ps();
} }
static FORCEINLINE __vec16_f __undef_float() { template <class RetVecType> __vec16_f __undef_float();
template <> static FORCEINLINE __vec16_f __undef_float<__vec16_f>() {
return _mm512_undefined_ps(); return _mm512_undefined_ps();
} }
@@ -1287,21 +1306,24 @@ static FORCEINLINE void __insert_element(__vec16_d *v, int index, double val) {
((double *)v)[index] = val; ((double *)v)[index] = val;
} }
static FORCEINLINE __vec16_d __smear_double(double d) { template <class RetVecType> __vec16_d __smear_double(double d);
template <> static FORCEINLINE __vec16_d __smear_double<__vec16_d>(double d) {
__vec16_d ret; __vec16_d ret;
ret.v1 = _mm512_extload_pd(&d, _MM_UPCONV_PD_NONE, _MM_BROADCAST_1X8, _MM_HINT_NONE); ret.v1 = _mm512_extload_pd(&d, _MM_UPCONV_PD_NONE, _MM_BROADCAST_1X8, _MM_HINT_NONE);
ret.v2 = ret.v1; ret.v2 = ret.v1;
return ret; return ret;
} }
static FORCEINLINE __vec16_d __setzero_double() { template <class RetVecType> __vec16_d __setzero_double();
template <> static FORCEINLINE __vec16_d __setzero_double<__vec16_d>() {
__vec16_d ret; __vec16_d ret;
ret.v1 = _mm512_setzero_pd(); ret.v1 = _mm512_setzero_pd();
ret.v2 = ret.v1; ret.v2 = ret.v1;
return ret; return ret;
} }
static FORCEINLINE __vec16_d __undef_double() { template <class RetVecType> __vec16_d __undef_double();
template <> static FORCEINLINE __vec16_d __undef_double<__vec16_d>() {
__vec16_d ret; __vec16_d ret;
ret.v1 = _mm512_undefined_pd(); ret.v1 = _mm512_undefined_pd();
ret.v2 = ret.v1; ret.v2 = ret.v1;

81
examples/intrinsics/sse4.h
View File

@@ -322,15 +322,18 @@ template <int ALIGN> static FORCEINLINE void __store(__vec4_i1 *p, __vec4_i1 val
_mm_storeu_ps((float *)(&p->v), value.v); _mm_storeu_ps((float *)(&p->v), value.v);
} }
static FORCEINLINE __vec4_i1 __smear_i1(int v) { template <class RetVecType> __vec4_i1 __smear_i1(int v);
template <> FORCEINLINE __vec4_i1 __smear_i1<__vec4_i1>(int v) {
return __vec4_i1(v, v, v, v); return __vec4_i1(v, v, v, v);
} }
static FORCEINLINE __vec4_i1 __setzero_i1() { template <class RetVecType> __vec4_i1 __setzero_i1();
template <> FORCEINLINE __vec4_i1 __setzero_i1<__vec4_i1>() {
return __vec4_i1(_mm_setzero_ps()); return __vec4_i1(_mm_setzero_ps());
} }
static FORCEINLINE __vec4_i1 __undef_i1() { template <class RetVecType> __vec4_i1 __undef_i1();
template <> FORCEINLINE __vec4_i1 __undef_i1<__vec4_i1>() {
return __vec4_i1(); return __vec4_i1();
} }
@@ -560,15 +563,18 @@ static FORCEINLINE void __insert_element(__vec4_i8 *v, int index, int8_t val) {
((int8_t *)v)[index] = val; ((int8_t *)v)[index] = val;
} }
static FORCEINLINE __vec4_i8 __smear_i8(int8_t v) { template <class RetVecType> __vec4_i8 __smear_i8(int8_t v);
template <> FORCEINLINE __vec4_i8 __smear_i8<__vec4_i8>(int8_t v) {
return _mm_set1_epi8(v); return _mm_set1_epi8(v);
} }
static FORCEINLINE __vec4_i8 __setzero_i8() { template <class RetVecType> __vec4_i8 __setzero_i8();
template <> FORCEINLINE __vec4_i8 __setzero_i8<__vec4_i8>() {
return _mm_set1_epi8(0); return _mm_set1_epi8(0);
} }
static FORCEINLINE __vec4_i8 __undef_i8() { template <class RetVecType> __vec4_i8 __undef_i8();
template <> FORCEINLINE __vec4_i8 __undef_i8<__vec4_i8>() {
return __vec4_i8(); return __vec4_i8();
} }
@@ -829,15 +835,18 @@ static FORCEINLINE void __insert_element(__vec4_i16 *v, int index, int16_t val)
((int16_t *)v)[index] = val; ((int16_t *)v)[index] = val;
} }
static FORCEINLINE __vec4_i16 __smear_i16(int16_t v) { template <class RetVecType> __vec4_i16 __smear_i16(int16_t v);
template <> FORCEINLINE __vec4_i16 __smear_i16<__vec4_i16>(int16_t v) {
return _mm_set1_epi16(v); return _mm_set1_epi16(v);
} }
static FORCEINLINE __vec4_i16 __setzero_i16() { template <class RetVecType> __vec4_i16 __setzero_i16();
template <> FORCEINLINE __vec4_i16 __setzero_i16<__vec4_i16>() {
return _mm_set1_epi16(0); return _mm_set1_epi16(0);
} }
static FORCEINLINE __vec4_i16 __undef_i16() { template <class RetVecType> __vec4_i16 __undef_i16();
template <> FORCEINLINE __vec4_i16 __undef_i16<__vec4_i16>() {
return __vec4_i16(); return __vec4_i16();
} }
@@ -1076,15 +1085,18 @@ static FORCEINLINE __vec4_i32 __select(__vec4_i1 mask, __vec4_i32 a, __vec4_i32
_mm_castsi128_ps(a.v), mask.v)); _mm_castsi128_ps(a.v), mask.v));
} }
static FORCEINLINE __vec4_i32 __smear_i32(int32_t v) { template <class RetVecType> __vec4_i32 __smear_i32(int32_t v);
template <> FORCEINLINE __vec4_i32 __smear_i32<__vec4_i32>(int32_t v) {
return _mm_set1_epi32(v); return _mm_set1_epi32(v);
} }
static FORCEINLINE __vec4_i32 __setzero_i32() { template <class RetVecType> __vec4_i32 __setzero_i32();
template <> FORCEINLINE __vec4_i32 __setzero_i32<__vec4_i32>() {
return _mm_castps_si128(_mm_setzero_ps()); return _mm_castps_si128(_mm_setzero_ps());
} }
static FORCEINLINE __vec4_i32 __undef_i32() { template <class RetVecType> __vec4_i32 __undef_i32();
template <> FORCEINLINE __vec4_i32 __undef_i32<__vec4_i32>() {
return __vec4_i32(); return __vec4_i32();
} }
@@ -1347,15 +1359,18 @@ static FORCEINLINE __vec4_i64 __select(__vec4_i1 mask, __vec4_i64 a, __vec4_i64
return __vec4_i64(_mm_castpd_si128(r0), _mm_castpd_si128(r1)); return __vec4_i64(_mm_castpd_si128(r0), _mm_castpd_si128(r1));
} }
static FORCEINLINE __vec4_i64 __smear_i64(int64_t v) { template <class RetVecType> __vec4_i64 __smear_i64(int64_t v);
template <> FORCEINLINE __vec4_i64 __smear_i64<__vec4_i64>(int64_t v) {
return __vec4_i64(v, v, v, v); return __vec4_i64(v, v, v, v);
} }
static FORCEINLINE __vec4_i64 __setzero_i64() { template <class RetVecType> __vec4_i64 __setzero_i64();
template <> FORCEINLINE __vec4_i64 __setzero_i64<__vec4_i64>() {
return __vec4_i64(0, 0, 0, 0); return __vec4_i64(0, 0, 0, 0);
} }
static FORCEINLINE __vec4_i64 __undef_i64() { template <class RetVecType> __vec4_i64 __undef_i64();
template <> FORCEINLINE __vec4_i64 __undef_i64<__vec4_i64>() {
return __vec4_i64(); return __vec4_i64();
} }
@@ -1465,15 +1480,18 @@ static FORCEINLINE __vec4_f __select(__vec4_i1 mask, __vec4_f a, __vec4_f b) {
return _mm_blendv_ps(b.v, a.v, mask.v); return _mm_blendv_ps(b.v, a.v, mask.v);
} }
static FORCEINLINE __vec4_f __smear_float(float v) { template <class RetVecType> __vec4_f __smear_float(float v);
template <> FORCEINLINE __vec4_f __smear_float<__vec4_f>(float v) {
return _mm_set1_ps(v); return _mm_set1_ps(v);
} }
static FORCEINLINE __vec4_f __setzero_float() { template <class RetVecType> __vec4_f __setzero_float();
template <> FORCEINLINE __vec4_f __setzero_float<__vec4_f>() {
return _mm_setzero_ps(); return _mm_setzero_ps();
} }
static FORCEINLINE __vec4_f __undef_float() { template <class RetVecType> __vec4_f __undef_float();
template <> FORCEINLINE __vec4_f __undef_float<__vec4_f>() {
return __vec4_f(); return __vec4_f();
} }
@@ -1614,15 +1632,18 @@ static FORCEINLINE __vec4_d __select(__vec4_i1 mask, __vec4_d a, __vec4_d b) {
return __vec4_d(r0, r1); return __vec4_d(r0, r1);
} }
static FORCEINLINE __vec4_d __smear_double(double v) { template <class RetVecType> __vec4_d __smear_double(double v);
template <> FORCEINLINE __vec4_d __smear_double<__vec4_d>(double v) {
return __vec4_d(_mm_set1_pd(v), _mm_set1_pd(v)); return __vec4_d(_mm_set1_pd(v), _mm_set1_pd(v));
} }
static FORCEINLINE __vec4_d __setzero_double() { template <class RetVecType> __vec4_d __setzero_double();
template <> FORCEINLINE __vec4_d __setzero_double<__vec4_d>() {
return __vec4_d(_mm_setzero_pd(), _mm_setzero_pd()); return __vec4_d(_mm_setzero_pd(), _mm_setzero_pd());
} }
static FORCEINLINE __vec4_d __undef_double() { template <class RetVecType> __vec4_d __undef_double();
template <> FORCEINLINE __vec4_d __undef_double<__vec4_d>() {
return __vec4_d(); return __vec4_d();
} }
@@ -1722,11 +1743,11 @@ static FORCEINLINE __vec4_i16 __cast_sext(__vec4_i16, __vec4_i8 val) {
} }
static FORCEINLINE __vec4_i8 __cast_sext(__vec4_i8, __vec4_i1 v) { static FORCEINLINE __vec4_i8 __cast_sext(__vec4_i8, __vec4_i1 v) {
return __select(v, __smear_i8(0xff), __setzero_i8()); return __select(v, __smear_i8<__vec4_i8>(0xff), __setzero_i8<__vec4_i8>());
} }
static FORCEINLINE __vec4_i16 __cast_sext(__vec4_i16, __vec4_i1 v) { static FORCEINLINE __vec4_i16 __cast_sext(__vec4_i16, __vec4_i1 v) {
return __select(v, __smear_i16(0xffff), __setzero_i16()); return __select(v, __smear_i16<__vec4_i16>(0xffff), __setzero_i16<__vec4_i16>());
} }
static FORCEINLINE __vec4_i32 __cast_sext(__vec4_i32, __vec4_i1 v) { static FORCEINLINE __vec4_i32 __cast_sext(__vec4_i32, __vec4_i1 v) {
@@ -1786,11 +1807,11 @@ static FORCEINLINE __vec4_i16 __cast_zext(__vec4_i16, __vec4_i8 val) {
} }
static FORCEINLINE __vec4_i8 __cast_zext(__vec4_i8, __vec4_i1 v) { static FORCEINLINE __vec4_i8 __cast_zext(__vec4_i8, __vec4_i1 v) {
return __select(v, __smear_i8(1), __setzero_i8()); return __select(v, __smear_i8<__vec4_i8>(1), __setzero_i8<__vec4_i8>());
} }
static FORCEINLINE __vec4_i16 __cast_zext(__vec4_i16, __vec4_i1 v) { static FORCEINLINE __vec4_i16 __cast_zext(__vec4_i16, __vec4_i1 v) {
return __select(v, __smear_i16(1), __setzero_i16()); return __select(v, __smear_i16<__vec4_i16>(1), __setzero_i16<__vec4_i16>());
} }
static FORCEINLINE __vec4_i32 __cast_zext(__vec4_i32, __vec4_i1 v) { static FORCEINLINE __vec4_i32 __cast_zext(__vec4_i32, __vec4_i1 v) {
@@ -1798,7 +1819,7 @@ static FORCEINLINE __vec4_i32 __cast_zext(__vec4_i32, __vec4_i1 v) {
} }
static FORCEINLINE __vec4_i64 __cast_zext(__vec4_i64, __vec4_i1 v) { static FORCEINLINE __vec4_i64 __cast_zext(__vec4_i64, __vec4_i1 v) {
return __select(v, __smear_i64(1), __setzero_i64()); return __select(v, __smear_i64<__vec4_i64>(1), __setzero_i64<__vec4_i64>());
} }
// truncations // truncations
@@ -1958,11 +1979,11 @@ static FORCEINLINE __vec4_d __cast_uitofp(__vec4_d, __vec4_i64 val) {
} }
static FORCEINLINE __vec4_f __cast_uitofp(__vec4_f, __vec4_i1 v) { static FORCEINLINE __vec4_f __cast_uitofp(__vec4_f, __vec4_i1 v) {
return __select(v, __smear_float(1.), __setzero_float()); return __select(v, __smear_float<__vec4_f>(1.), __setzero_float<__vec4_f>());
} }
static FORCEINLINE __vec4_d __cast_uitofp(__vec4_d, __vec4_i1 v) { static FORCEINLINE __vec4_d __cast_uitofp(__vec4_d, __vec4_i1 v) {
return __select(v, __smear_double(1.), __setzero_double()); return __select(v, __smear_double<__vec4_d>(1.), __setzero_double<__vec4_d>());
} }
// float/double to signed int // float/double to signed int
@@ -2897,7 +2918,7 @@ lGatherBaseOffsets32(RetVec, RetScalar, unsigned char *p, uint32_t scale,
RetScalar r[4]; RetScalar r[4];
#if 1 #if 1
// "Fast gather" trick... // "Fast gather" trick...
offsets = __select(mask, offsets, __setzero_i32()); offsets = __select(mask, offsets, __setzero_i32<__vec4_i32>());
int offset = scale * _mm_extract_epi32(offsets.v, 0); int offset = scale * _mm_extract_epi32(offsets.v, 0);
RetScalar *ptr = (RetScalar *)(p + offset); RetScalar *ptr = (RetScalar *)(p + offset);
@@ -2954,7 +2975,7 @@ lGatherBaseOffsets64(RetVec, RetScalar, unsigned char *p, uint32_t scale,
RetScalar r[4]; RetScalar r[4];
#if 1 #if 1
// "Fast gather" trick... // "Fast gather" trick...
offsets = __select(mask, offsets, __setzero_i64()); offsets = __select(mask, offsets, __setzero_i64<__vec4_i64>());
int64_t offset = scale * _mm_extract_epi64(offsets.v[0], 0); int64_t offset = scale * _mm_extract_epi64(offsets.v[0], 0);
RetScalar *ptr = (RetScalar *)(p + offset); RetScalar *ptr = (RetScalar *)(p + offset);

2
stmt.cpp
View File

@@ -2207,7 +2207,7 @@ ForeachUniqueStmt::EmitCode(FunctionEmitContext *ctx) const {
// lane's value of the varying expression is the same as the value // lane's value of the varying expression is the same as the value
// we've selected to process this time through--i.e.: // we've selected to process this time through--i.e.:
// oldMask & (smear(element) == exprValue) // oldMask & (smear(element) == exprValue)
llvm::Value *uniqueSmear = ctx->SmearUniform(uniqueValue, "unique_semar"); llvm::Value *uniqueSmear = ctx->SmearUniform(uniqueValue, "unique_smear");
llvm::Value *matchingLanes = NULL; llvm::Value *matchingLanes = NULL;
if (uniqueValue->getType()->isFloatingPointTy()) if (uniqueValue->getType()->isFloatingPointTy())
matchingLanes = matchingLanes =

8
type.cpp
View File

@@ -1060,12 +1060,12 @@ PointerType::Mangle() const {
return ""; return "";
} }
std::string ret = variability.MangleString() + std::string("<"); std::string ret = variability.MangleString() + std::string("_3C_"); // <
if (isSlice || isFrozen) ret += "-"; if (isSlice || isFrozen) ret += "-";
if (isSlice) ret += "s"; if (isSlice) ret += "s";
if (isFrozen) ret += "f"; if (isFrozen) ret += "f";
if (isSlice || isFrozen) ret += "-"; if (isSlice || isFrozen) ret += "-";
return ret + baseType->Mangle() + std::string(">"); return ret + baseType->Mangle() + std::string("_3E_"); // >
} }
@@ -1636,7 +1636,7 @@ std::string
VectorType::Mangle() const { VectorType::Mangle() const {
std::string s = base->Mangle(); std::string s = base->Mangle();
char buf[16]; char buf[16];
sprintf(buf, "<%d>", numElements); sprintf(buf, "_3C_%d_3E_", numElements); // "<%d>"
return s + std::string(buf); return s + std::string(buf);
} }
@@ -1789,7 +1789,7 @@ lMangleStructName(const std::string &name, Variability variability) {
n += buf; n += buf;
break; break;
default: default:
FATAL("Unexpected varaibility in lMangleStructName()"); FATAL("Unexpected variability in lMangleStructName()");
} }
// And stuff the name at the end.... // And stuff the name at the end....