From 7084432a0e4e67d496c0e28c0d4c3620906e6ecb Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 27 Nov 2014 13:07:58 +0400 Subject: [PATCH 01/15] Fix for max/min_varying_uint/int64 --- examples/intrinsics/knc.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index aa9bda32..f58e34a7 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -2386,7 +2386,37 @@ static FORCEINLINE __vec16_i32 __min_varying_int32 (__vec16_i32 v1, __vec16_i32 static FORCEINLINE __vec16_i32 __max_varying_uint32(__vec16_i32 v1, __vec16_i32 v2) { return _mm512_max_epu32(v1, v2); } static FORCEINLINE __vec16_i32 __min_varying_uint32(__vec16_i32 v1, __vec16_i32 v2) { return _mm512_min_epu32(v1, v2); } +static FORCEINLINE __vec16_i64 __max_varying_int64 (__vec16_i64 v1, __vec16_i64 v2) { + __vec16_i64 ret; + ret.v_hi = _mm512_max_epi32(v1.v_hi, v2.v_hi); + __vec16_i1 mask = _mm512_cmp_epi32_mask(ret.v_hi, v2.v_hi, _MM_CMPINT_EQ); + ret.v_lo = _mm512_mask_max_epi32(v1.v_lo, mask, v1.v_lo, v2.v_lo); + return ret; +} +static FORCEINLINE __vec16_i64 __min_varying_int64 (__vec16_i64 v1, __vec16_i64 v2) { + __vec16_i64 ret; + ret.v_hi = _mm512_min_epi32(v1.v_hi, v2.v_hi); + __vec16_i1 mask = _mm512_cmp_epi32_mask(ret.v_hi, v2.v_hi, _MM_CMPINT_EQ); + ret.v_lo = _mm512_mask_min_epi32(v1.v_lo, mask, v1.v_lo, v2.v_lo); + return ret; +} + +static FORCEINLINE __vec16_i64 __max_varying_uint64 (__vec16_i64 v1, __vec16_i64 v2) { + __vec16_i64 ret; + ret.v_hi = _mm512_max_epu32(v1.v_hi, v2.v_hi); + __vec16_i1 mask = _mm512_cmp_epu32_mask(ret.v_hi, v2.v_hi, _MM_CMPINT_EQ); + ret.v_lo = _mm512_mask_max_epu32(v1.v_lo, mask, v1.v_lo, v2.v_lo); + return ret; +} + +static FORCEINLINE __vec16_i64 __min_varying_uint64 (__vec16_i64 v1, __vec16_i64 v2) { + __vec16_i64 ret; + ret.v_hi = _mm512_min_epu32(v1.v_hi, v2.v_hi); + __vec16_i1 mask = _mm512_cmp_epu32_mask(ret.v_hi, v2.v_hi, _MM_CMPINT_EQ); + ret.v_lo = _mm512_mask_min_epu32(v1.v_lo, mask, v1.v_lo, v2.v_lo); + return ret; +} // sqrt/rsqrt/rcp From 0d9eceb668e36d563036ae4b6900a2ecad751e9a Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 27 Nov 2014 13:10:43 +0400 Subject: [PATCH 02/15] Fix for prefetch --- examples/intrinsics/knc.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index f58e34a7..c802718b 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -3104,21 +3104,21 @@ static FORCEINLINE int32_t __packed_store_active2(uint32_t *p, __vec16_i32 val, // prefetch /////////////////////////////////////////////////////////////////////////// -static FORCEINLINE void __prefetch_read_uniform_1(const char *p) { - _mm_prefetch(p, _MM_HINT_T0); // prefetch into L1$ +static FORCEINLINE void __prefetch_read_uniform_1(uint8_t *p) { + _mm_prefetch((const char*) p, _MM_HINT_T0); // prefetch into L1$ } -static FORCEINLINE void __prefetch_read_uniform_2(const char *p) { - _mm_prefetch(p, _MM_HINT_T1); // prefetch into L2$ +static FORCEINLINE void __prefetch_read_uniform_2(uint8_t *p) { + _mm_prefetch((const char*) p, _MM_HINT_T1); // prefetch into L2$ } -static FORCEINLINE void __prefetch_read_uniform_3(const char *p) { +static FORCEINLINE void __prefetch_read_uniform_3(uint8_t *p) { // There is no L3$ on KNC, but we prefetch into L2$ instead. - _mm_prefetch(p, _MM_HINT_T1); // prefetch into L2$ + _mm_prefetch((const char*) p, _MM_HINT_T1); // prefetch into L2$ } -static FORCEINLINE void __prefetch_read_uniform_nt(const char *p) { - _mm_prefetch(p, _MM_HINT_T2); // prefetch into L2$ with non-temporal hint +static FORCEINLINE void __prefetch_read_uniform_nt(uint8_t *p) { + _mm_prefetch((const char*) p, _MM_HINT_T2); // prefetch into L2$ with non-temporal hint // _mm_prefetch(p, _MM_HINT_NTA); // prefetch into L1$ with non-temporal hint } From 52eddcdcfd356669859c14d42e3a4432e6e330f0 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 27 Nov 2014 13:24:48 +0400 Subject: [PATCH 03/15] Some changes in max/min_varying_uint/int64 --- examples/intrinsics/knc.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index c802718b..64d7b448 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -2389,7 +2389,7 @@ static FORCEINLINE __vec16_i32 __min_varying_uint32(__vec16_i32 v1, __vec16_i32 static FORCEINLINE __vec16_i64 __max_varying_int64 (__vec16_i64 v1, __vec16_i64 v2) { __vec16_i64 ret; ret.v_hi = _mm512_max_epi32(v1.v_hi, v2.v_hi); - __vec16_i1 mask = _mm512_cmp_epi32_mask(ret.v_hi, v2.v_hi, _MM_CMPINT_EQ); + __vec16_i1 mask = _mm512_cmpeq_epi32_mask(ret.v_hi, v2.v_hi); ret.v_lo = _mm512_mask_max_epi32(v1.v_lo, mask, v1.v_lo, v2.v_lo); return ret; } @@ -2397,7 +2397,7 @@ static FORCEINLINE __vec16_i64 __max_varying_int64 (__vec16_i64 v1, __vec16_i64 static FORCEINLINE __vec16_i64 __min_varying_int64 (__vec16_i64 v1, __vec16_i64 v2) { __vec16_i64 ret; ret.v_hi = _mm512_min_epi32(v1.v_hi, v2.v_hi); - __vec16_i1 mask = _mm512_cmp_epi32_mask(ret.v_hi, v2.v_hi, _MM_CMPINT_EQ); + __vec16_i1 mask = _mm512_cmpeq_epi32_mask(ret.v_hi, v2.v_hi); ret.v_lo = _mm512_mask_min_epi32(v1.v_lo, mask, v1.v_lo, v2.v_lo); return ret; } @@ -2405,7 +2405,7 @@ static FORCEINLINE __vec16_i64 __min_varying_int64 (__vec16_i64 v1, __vec16_i64 static FORCEINLINE __vec16_i64 __max_varying_uint64 (__vec16_i64 v1, __vec16_i64 v2) { __vec16_i64 ret; ret.v_hi = _mm512_max_epu32(v1.v_hi, v2.v_hi); - __vec16_i1 mask = _mm512_cmp_epu32_mask(ret.v_hi, v2.v_hi, _MM_CMPINT_EQ); + __vec16_i1 mask = _mm512_cmpeq_epi32_mask(ret.v_hi, v2.v_hi); ret.v_lo = _mm512_mask_max_epu32(v1.v_lo, mask, v1.v_lo, v2.v_lo); return ret; } @@ -2413,7 +2413,7 @@ static FORCEINLINE __vec16_i64 __max_varying_uint64 (__vec16_i64 v1, __vec16_i64 static FORCEINLINE __vec16_i64 __min_varying_uint64 (__vec16_i64 v1, __vec16_i64 v2) { __vec16_i64 ret; ret.v_hi = _mm512_min_epu32(v1.v_hi, v2.v_hi); - __vec16_i1 mask = _mm512_cmp_epu32_mask(ret.v_hi, v2.v_hi, _MM_CMPINT_EQ); + __vec16_i1 mask = _mm512_cmpeq_epi32_mask(ret.v_hi, v2.v_hi); ret.v_lo = _mm512_mask_min_epu32(v1.v_lo, mask, v1.v_lo, v2.v_lo); return ret; } From 7e39ed115de24c7c51bd80e82dbfaff9f65260d8 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 27 Nov 2014 14:46:51 +0400 Subject: [PATCH 04/15] Fix for shuffle2_double --- examples/intrinsics/knc.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index 64d7b448..f1a5a136 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -1550,6 +1550,17 @@ static FORCEINLINE __vec16_d __broadcast_double(__vec16_d v, int index) { return ret; } +static FORCEINLINE __vec16_d __shuffle2_double(__vec16_d v0, __vec16_d v1, __vec16_i32 index) { + __vec16_d ret; + for (int i = 0; i < 16; ++i){ + if (__extract_element(index, i) < 16) + __insert_element(&ret, i, __extract_element(v0, __extract_element(index, i) & 0xF)); + else + __insert_element(&ret, i, __extract_element(v1, __extract_element(index, i) & 0xF)); + } + return ret; +} + template static FORCEINLINE __vec16_d __load(const __vec16_d *p) { __vec16_d ret; ret.v1 = _mm512_extloadunpacklo_pd(ret.v1, p, _MM_UPCONV_PD_NONE, _MM_HINT_NONE); From 307c825c4ebcbcf355f302ffac5d60ed89f33b57 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 27 Nov 2014 16:00:50 +0400 Subject: [PATCH 05/15] Fix for __shift_i32 --- examples/intrinsics/knc.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index f1a5a136..cfaba712 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -754,6 +754,16 @@ static FORCEINLINE __vec16_i32 __shuffle2_i32(__vec16_i32 v0, __vec16_i32 v1, __ return ret; } +static FORCEINLINE __vec16_i32 __shift_i32(__vec16_i32 v, int index) { + __vec16_i32 mod_index = _mm512_add_epi32(__ispc_stride1, __smear_i32<__vec16_i32>(index)); + __vec16_i1 mask_ge = _mm512_cmpge_epi32_mask (mod_index, __smear_i32<__vec16_i32>(0)); + __vec16_i1 mask_le = _mm512_cmple_epi32_mask (mod_index, __smear_i32<__vec16_i32>(0xF)); + __vec16_i1 mask = mask_ge & mask_le; + __vec16_i32 ret = __smear_i32<__vec16_i32>(0); + ret = _mm512_mask_permutevar_epi32(ret, mask, mod_index, v); + return ret; +} + template static FORCEINLINE __vec16_i32 __load(const __vec16_i32 *p) { #ifdef ISPC_FORCE_ALIGNED_MEMORY return _mm512_load_epi32(p); From 190d7957d4ef7af893450d0a5fadf1f257313b0d Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 27 Nov 2014 16:07:16 +0400 Subject: [PATCH 06/15] Fix for __shuffle2_float --- examples/intrinsics/knc.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index cfaba712..dc6dce43 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -1325,6 +1325,17 @@ static FORCEINLINE __vec16_f __shuffle_float(__vec16_f v, __vec16_i32 index) { return _mm512_castsi512_ps(_mm512_mask_permutevar_epi32(_mm512_castps_si512(v), 0xffff, index, _mm512_castps_si512(v))); } +static FORCEINLINE __vec16_f __shuffle2_float(__vec16_f v0, __vec16_f v1, __vec16_i32 index) { + __vec16_f ret; + for (int i = 0; i < 16; ++i){ + if (__extract_element(index, i) < 16) + __insert_element(&ret, i, __extract_element(v0, __extract_element(index, i) & 0xF)); + else + __insert_element(&ret, i, __extract_element(v1, __extract_element(index, i) & 0xF)); + } + return ret; +} + template static FORCEINLINE __vec16_f __load(const __vec16_f *p) { #ifdef ISPC_FORCE_ALIGNED_MEMORY return _mm512_load_ps(p); From 7e63862dff2259e50641fd2f91dbe5cec674b4b0 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 27 Nov 2014 17:35:21 +0400 Subject: [PATCH 07/15] __sqrt_varying_double --- examples/intrinsics/knc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index dc6dce43..c2f347fe 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -1397,6 +1397,10 @@ static FORCEINLINE __vec16_d __div(__vec16_d a, __vec16_d b) { return ret; } +static FORCEINLINE __vec16_d __sqrt_varying_double(__vec16_d v) { + return __vec16_d(_mm512_sqrt_pd(v.v1),_mm512_sqrt_pd(v.v2)); +} + static FORCEINLINE __vec16_i1 __equal_double(__vec16_d a, __vec16_d b) { __vec16_i1 ret1; __vec16_i1 ret2; From 70adb5d283ed15e403859d9e292d368a2281ba48 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 27 Nov 2014 19:00:38 +0400 Subject: [PATCH 08/15] Fix for unsigned cmp for int64 --- examples/intrinsics/knc.h | 61 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index c2f347fe..f8fb56aa 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -1028,6 +1028,67 @@ static FORCEINLINE __vec16_i1 __not_equal_i64_and_mask(const __vec16_i64 &a, con } +static FORCEINLINE __vec16_i1 __unsigned_less_equal_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_hi = __unsigned_less_equal_i32(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = __unsigned_less_equal_i32(a.v_lo, b.v_lo); + return ret_hi | ret_lo; +} + +static FORCEINLINE __vec16_i1 __unsigned_less_equal_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + __vec16_i1 ret_hi = __unsigned_less_equal_i32(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = __unsigned_less_equal_i32(a.v_lo, b.v_lo); + return (ret_hi | ret_lo) & m; +} + +static FORCEINLINE __vec16_i1 __unsigned_greater_equal_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_hi = __unsigned_greater_equal_i32(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = __unsigned_greater_equal_i32(a.v_lo, b.v_lo); + return ret_hi | ret_lo; +} + +static FORCEINLINE __vec16_i1 __unsigned_greater_equal_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + __vec16_i1 ret_hi = __unsigned_greater_equal_i32(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = __unsigned_greater_equal_i32(a.v_lo, b.v_lo); + return (ret_hi | ret_lo) & m; +} + +static FORCEINLINE __vec16_i1 __unsigned_less_than_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_hi = __unsigned_less_than_i32(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = __unsigned_less_than_i32(a.v_lo, b.v_lo); + return ret_hi | ret_lo; +} + +static FORCEINLINE __vec16_i1 __unsigned_less_than_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + __vec16_i1 ret_hi = __unsigned_less_than_i32(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = __unsigned_less_than_i32(a.v_lo, b.v_lo); + return (ret_hi | ret_lo) & m; +} + +static FORCEINLINE __vec16_i1 __unsigned_greater_than_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_hi = __unsigned_greater_than_i32(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = __unsigned_greater_than_i32(a.v_lo, b.v_lo); + return ret_hi | ret_lo; +} + +static FORCEINLINE __vec16_i1 __unsigned_greater_than_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + __vec16_i1 ret_hi = __unsigned_greater_than_i32(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = __unsigned_greater_than_i32(a.v_lo, b.v_lo); + return (ret_hi | ret_lo) & m; +} + + +static FORCEINLINE __vec16_i64 __select(__vec16_i1 mask, + __vec16_i64 a, __vec16_i64 b) { + __vec16_i64 ret; + ret.v_hi = _mm512_mask_mov_epi32(b.v_hi, mask, a.v_hi); + ret.v_lo = _mm512_mask_mov_epi32(b.v_lo, mask, a.v_lo); + return ret; +} + template static RetVecType __smear_i64(const int64_t &l); template <> FORCEINLINE __vec16_i64 __smear_i64<__vec16_i64>(const int64_t &l) { From 8d51e0620ec5fcc105c1daf8d3d3f97f4a5ebf12 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 4 Dec 2014 13:32:20 +0400 Subject: [PATCH 09/15] Fix for int64 comparison --- examples/intrinsics/knc.h | 112 ++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 36 deletions(-) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index f8fb56aa..ef0a9850 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -1028,56 +1028,96 @@ static FORCEINLINE __vec16_i1 __not_equal_i64_and_mask(const __vec16_i64 &a, con } -static FORCEINLINE __vec16_i1 __unsigned_less_equal_i64(__vec16_i64 a, __vec16_i64 b) { - __vec16_i1 ret_hi = __unsigned_less_equal_i32(a.v_hi, b.v_hi); - __vec16_i1 ret_lo = __unsigned_less_equal_i32(a.v_lo, b.v_lo); - return ret_hi | ret_lo; -} - -static FORCEINLINE __vec16_i1 __unsigned_less_equal_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) -{ - __vec16_i1 ret_hi = __unsigned_less_equal_i32(a.v_hi, b.v_hi); - __vec16_i1 ret_lo = __unsigned_less_equal_i32(a.v_lo, b.v_lo); - return (ret_hi | ret_lo) & m; -} - -static FORCEINLINE __vec16_i1 __unsigned_greater_equal_i64(__vec16_i64 a, __vec16_i64 b) { - __vec16_i1 ret_hi = __unsigned_greater_equal_i32(a.v_hi, b.v_hi); - __vec16_i1 ret_lo = __unsigned_greater_equal_i32(a.v_lo, b.v_lo); - return ret_hi | ret_lo; -} - -static FORCEINLINE __vec16_i1 __unsigned_greater_equal_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) -{ - __vec16_i1 ret_hi = __unsigned_greater_equal_i32(a.v_hi, b.v_hi); - __vec16_i1 ret_lo = __unsigned_greater_equal_i32(a.v_lo, b.v_lo); - return (ret_hi | ret_lo) & m; -} - static FORCEINLINE __vec16_i1 __unsigned_less_than_i64(__vec16_i64 a, __vec16_i64 b) { - __vec16_i1 ret_hi = __unsigned_less_than_i32(a.v_hi, b.v_hi); - __vec16_i1 ret_lo = __unsigned_less_than_i32(a.v_lo, b.v_lo); + __vec16_i1 ret_hi = _mm512_cmplt_epu32_mask(a.v_hi, b.v_hi); + __vec16_i1 hi_eq = _mm512_cmpeq_epu32_mask(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = _mm512_mask_cmplt_epu32_mask(hi_eq, a.v_lo, b.v_lo); return ret_hi | ret_lo; } static FORCEINLINE __vec16_i1 __unsigned_less_than_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) { - __vec16_i1 ret_hi = __unsigned_less_than_i32(a.v_hi, b.v_hi); - __vec16_i1 ret_lo = __unsigned_less_than_i32(a.v_lo, b.v_lo); - return (ret_hi | ret_lo) & m; + return __unsigned_less_than_i64(a, b) & m; } static FORCEINLINE __vec16_i1 __unsigned_greater_than_i64(__vec16_i64 a, __vec16_i64 b) { - __vec16_i1 ret_hi = __unsigned_greater_than_i32(a.v_hi, b.v_hi); - __vec16_i1 ret_lo = __unsigned_greater_than_i32(a.v_lo, b.v_lo); + __vec16_i1 ret_hi = _mm512_cmpgt_epu32_mask(a.v_hi, b.v_hi); + __vec16_i1 hi_eq = _mm512_cmpeq_epu32_mask(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = _mm512_mask_cmpgt_epu32_mask(hi_eq, a.v_lo, b.v_lo); return ret_hi | ret_lo; } static FORCEINLINE __vec16_i1 __unsigned_greater_than_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) { - __vec16_i1 ret_hi = __unsigned_greater_than_i32(a.v_hi, b.v_hi); - __vec16_i1 ret_lo = __unsigned_greater_than_i32(a.v_lo, b.v_lo); - return (ret_hi | ret_lo) & m; + return __unsigned_greater_than_i64(a, b) & m; +} + +static FORCEINLINE __vec16_i1 __unsigned_less_equal_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_s_cmp = __unsigned_less_than_i64(a, b); + __vec16_i1 ret_equal = __equal_i64(a, b); + return ret_equal | ret_s_cmp; +} + +static FORCEINLINE __vec16_i1 __unsigned_less_equal_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + return __unsigned_less_equal_i64(a, b) & m; +} + +static FORCEINLINE __vec16_i1 __unsigned_greater_equal_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_s_cmp = __unsigned_greater_than_i64(a, b); + __vec16_i1 ret_equal = __equal_i64(a, b); + return ret_equal | ret_s_cmp; +} + +static FORCEINLINE __vec16_i1 __unsigned_greater_equal_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + return __unsigned_greater_equal_i64(a, b) & m; +} + +static FORCEINLINE __vec16_i1 __signed_less_than_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_hi = _mm512_cmplt_epi32_mask(a.v_hi, b.v_hi); + __vec16_i1 hi_eq = _mm512_cmpeq_epi32_mask(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = _mm512_mask_cmplt_epu32_mask(hi_eq, a.v_lo, b.v_lo); + return ret_hi | ret_lo; +} + +static FORCEINLINE __vec16_i1 __signed_less_than_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + return __signed_less_than_i64(a, b) & m; +} + +static FORCEINLINE __vec16_i1 __signed_greater_than_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_hi = _mm512_cmpgt_epi32_mask(a.v_hi, b.v_hi); + __vec16_i1 hi_eq = _mm512_cmpeq_epi32_mask(a.v_hi, b.v_hi); + __vec16_i1 ret_lo = _mm512_mask_cmpgt_epu32_mask(hi_eq, a.v_lo, b.v_lo); + return ret_hi | ret_lo; +} + +static FORCEINLINE __vec16_i1 __signed_greater_than_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + return __signed_greater_than_i64(a, b) & m; +} + +static FORCEINLINE __vec16_i1 __signed_less_equal_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_s_cmp = __signed_less_than_i64(a, b); + __vec16_i1 ret_equal = __equal_i64(a, b); + return ret_equal | ret_s_cmp; +} + +static FORCEINLINE __vec16_i1 __signed_less_equal_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + return __signed_less_equal_i64(a, b) & m; +} + +static FORCEINLINE __vec16_i1 __signed_greater_equal_i64(__vec16_i64 a, __vec16_i64 b) { + __vec16_i1 ret_s_cmp = __signed_greater_than_i64(a, b); + __vec16_i1 ret_equal = __equal_i64(a, b); + return ret_equal | ret_s_cmp; +} + +static FORCEINLINE __vec16_i1 __signed_greater_equal_i64_and_mask(__vec16_i64 a, __vec16_i64 b, __vec16_i1 m) +{ + return __signed_greater_equal_i64(a, b) & m; } From df9f839ed458da596dfb26b2349be3d95ed04fdb Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 4 Dec 2014 13:45:47 +0400 Subject: [PATCH 10/15] Merge --- examples/intrinsics/knc.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index ef0a9850..2722d650 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -1121,15 +1121,6 @@ static FORCEINLINE __vec16_i1 __signed_greater_equal_i64_and_mask(__vec16_i64 a, } -static FORCEINLINE __vec16_i64 __select(__vec16_i1 mask, - __vec16_i64 a, __vec16_i64 b) { - __vec16_i64 ret; - ret.v_hi = _mm512_mask_mov_epi32(b.v_hi, mask, a.v_hi); - ret.v_lo = _mm512_mask_mov_epi32(b.v_lo, mask, a.v_lo); - return ret; -} - - template static RetVecType __smear_i64(const int64_t &l); template <> FORCEINLINE __vec16_i64 __smear_i64<__vec16_i64>(const int64_t &l) { const int *i = (const int*)&l; From b00575e95c0fd09c4cf5497dd7d3f22236f579fd Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 4 Dec 2014 18:59:40 +0400 Subject: [PATCH 11/15] Fix for __cast_zext(__vec16_i64, __vec16_i1) --- examples/intrinsics/knc.h | 45 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index 2722d650..afc935ed 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -1798,6 +1798,15 @@ static FORCEINLINE __vec16_i32 __cast_zext(const __vec16_i32 &, const __vec16_i1 return _mm512_extload_epi32(&val, _MM_UPCONV_EPI32_UINT16, _MM_BROADCAST32_NONE, _MM_HINT_NONE); } +static FORCEINLINE __vec16_i64 __cast_zext(const __vec16_i64 &, const __vec16_i1 &val) +{ + __vec16_i32 ret_hi = _mm512_setzero_epi32(); + __vec16_i32 ret_lo = _mm512_setzero_epi32(); + __vec16_i32 one = _mm512_set1_epi32(1); + _mm512_mask_mov_epi32(ret_lo, val, one); + return __vec16_i64 (ret_lo, ret_hi); +} + static FORCEINLINE __vec16_i64 __cast_zext(const __vec16_i64 &, const __vec16_i8 &val) { return __vec16_i64(__cast_zext(__vec16_i32(), val), _mm512_setzero_epi32()); @@ -1830,12 +1839,46 @@ static FORCEINLINE __vec16_f __cast_sitofp(__vec16_f, __vec16_i64 val) { __m512i tmp2; hilo2zmm(val, tmp1, tmp2); __vec16_f ret; +/* + ret[0] = (float)(((int64_t*)&tmp1)[0]); + ret[1] = (float)(((int64_t*)&tmp1)[1]); + ret[2] = (float)(((int64_t*)&tmp1)[2]); + ret[3] = (float)(((int64_t*)&tmp1)[3]); + ret[4] = (float)(((int64_t*)&tmp1)[4]); + ret[5] = (float)(((int64_t*)&tmp1)[5]); + ret[6] = (float)(((int64_t*)&tmp1)[6]); + ret[7] = (float)(((int64_t*)&tmp1)[7]); + + ret[8] = (float)(((int64_t*)&tmp2)[0]); + ret[9] = (float)(((int64_t*)&tmp2)[1]); + ret[10] = (float)(((int64_t*)&tmp2)[2]); + ret[11] = (float)(((int64_t*)&tmp2)[3]); + ret[12] = (float)(((int64_t*)&tmp2)[4]); + ret[13] = (float)(((int64_t*)&tmp2)[5]); + ret[14] = (float)(((int64_t*)&tmp2)[6]); + ret[15] = (float)(((int64_t*)&tmp2)[7]); +*/ + for (int i = 0; i < 8; i++) { - ((float*)&ret)[i] = (float)(((int64_t*)&tmp1)[i]); + ret[i] = (float)(((int64_t*)&tmp1)[i]); +// std::cout << "" << std::endl; } for (int i = 0; i < 8; i++) { ((float*)&ret)[i + 8] = (float)(((int64_t*)&tmp2)[i]); } + + for (int i = 0; i < 8; i++) { + int64_t t = ((int64_t*)&tmp1)[i]; + printf("%d: %llx, %lld, %f\n", i, t, t, (float)t); + } + for (int i = 0; i < 8; i++) { + int64_t t = ((int64_t*)&tmp2)[i]; + printf("%d: %llx, %lld, %f\n", i+8, t, t, (float)t); + } + for (int i = 0; i < 16; i++) { + float f = ((float*)&ret)[i]; + printf("%d-float: %f\n", i, f); + } return ret; } From ff9380b248e6f7448132225ed1860f2672bedb33 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 4 Dec 2014 19:30:30 +0400 Subject: [PATCH 12/15] Fix for count_leading_zeros --- examples/intrinsics/knc.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index afc935ed..42a97c45 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -2672,6 +2672,30 @@ static FORCEINLINE int64_t __count_trailing_zeros_i64(const __vec1_i64 mask) { return _mm_tzcnt_64(mask); } +static FORCEINLINE int32_t __count_leading_zeros_i32(__vec1_i32 mask) { + uint32_t n = 0; + if (mask == 0) + return 32; + while (1) { + if (mask < 0) break; + n ++; + mask <<= 1; + } + return n; +} + +static FORCEINLINE int64_t __count_leading_zeros_i64(__vec1_i64 mask) { + uint32_t n = 0; + if (mask == 0) + return 64; + while (1) { + if (mask < 0) break; + n ++; + mask <<= 1; + } + return n; +} + /////////////////////////////////////////////////////////////////////////// // reductions /////////////////////////////////////////////////////////////////////////// From 91866396efcfe50ca1f17d2c8a9bb8dbfa83b161 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 4 Dec 2014 19:40:30 +0400 Subject: [PATCH 13/15] Fix for half to float --- examples/intrinsics/knc.h | 135 +++++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 22 deletions(-) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index 42a97c45..77e10f54 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -1457,6 +1457,108 @@ template <> FORCEINLINE void __store<64>(__vec16_f *p, __vec16_f v) { } #endif +/////////////////////////////////////////////////////////////////////////// +// half<->float : this one passes the tests +// source : +// http://stackoverflow.com/questions/1659440/32-bit-to-16-bit-floating-point-conversion +/////////////////////////////////////////////////////////////////////////// +class Float16Compressor +{ + union Bits + { + float f; + int32_t si; + uint32_t ui; + }; + + static int const shift = 13; + static int const shiftSign = 16; + + static int32_t const infN = 0x7F800000; // flt32 infinity + static int32_t const maxN = 0x477FE000; // max flt16 normal as a flt32 + static int32_t const minN = 0x38800000; // min flt16 normal as a flt32 + static int32_t const signN = 0x80000000; // flt32 sign bit + + static int32_t const infC = infN >> shift; + static int32_t const nanN = (infC + 1) << shift; // minimum flt16 nan as a flt32 + static int32_t const maxC = maxN >> shift; + static int32_t const minC = minN >> shift; + static int32_t const signC = signN >> shiftSign; // flt16 sign bit + + static int32_t const mulN = 0x52000000; // (1 << 23) / minN + static int32_t const mulC = 0x33800000; // minN / (1 << (23 - shift)) + + static int32_t const subC = 0x003FF; // max flt32 subnormal down shifted + static int32_t const norC = 0x00400; // min flt32 normal down shifted + + static int32_t const maxD = infC - maxC - 1; + static int32_t const minD = minC - subC - 1; + + public: + + static uint16_t compress(float value) + { + Bits v, s; + v.f = value; + uint32_t sign = v.si & signN; + v.si ^= sign; + sign >>= shiftSign; // logical shift + s.si = mulN; + s.si = s.f * v.f; // correct subnormals + v.si ^= (s.si ^ v.si) & -(minN > v.si); + v.si ^= (infN ^ v.si) & -((infN > v.si) & (v.si > maxN)); + v.si ^= (nanN ^ v.si) & -((nanN > v.si) & (v.si > infN)); + v.ui >>= shift; // logical shift + v.si ^= ((v.si - maxD) ^ v.si) & -(v.si > maxC); + v.si ^= ((v.si - minD) ^ v.si) & -(v.si > subC); + return v.ui | sign; + } + + static float decompress(uint16_t value) + { + Bits v; + v.ui = value; + int32_t sign = v.si & signC; + v.si ^= sign; + sign <<= shiftSign; + v.si ^= ((v.si + minD) ^ v.si) & -(v.si > subC); + v.si ^= ((v.si + maxD) ^ v.si) & -(v.si > maxC); + Bits s; + s.si = mulC; + s.f *= v.si; + int32_t mask = -(norC > v.si); + v.si <<= shift; + v.si ^= (s.si ^ v.si) & mask; + v.si |= sign; + return v.f; + } +}; + +static FORCEINLINE float __half_to_float_uniform(int16_t h) +{ + return Float16Compressor::decompress(h); +} +static FORCEINLINE __vec16_f __half_to_float_varying(__vec16_i16 v) +{ + __vec16_f ret; + for (int i = 0; i < 16; ++i) + ret[i] = __half_to_float_uniform(v[i]); + return ret; +} + + +static FORCEINLINE int16_t __float_to_half_uniform(float f) +{ + return Float16Compressor::compress(f); +} +static FORCEINLINE __vec16_i16 __float_to_half_varying(__vec16_f v) +{ + __vec16_i16 ret; + for (int i = 0; i < 16; ++i) + ret[i] = __float_to_half_uniform(v[i]); + return ret; +} + /////////////////////////////////////////////////////////////////////////// // double /////////////////////////////////////////////////////////////////////////// @@ -1839,7 +1941,17 @@ static FORCEINLINE __vec16_f __cast_sitofp(__vec16_f, __vec16_i64 val) { __m512i tmp2; hilo2zmm(val, tmp1, tmp2); __vec16_f ret; + /* + // Cycles don't work. It seems that it is icc bug. + for (int i = 0; i < 8; i++) { + ret[i] = (float)(((int64_t*)&tmp1)[i]); + } + for (int i = 0; i < 8; i++) { + ((float*)&ret)[i + 8] = (float)(((int64_t*)&tmp2)[i]); + } +*/ + ret[0] = (float)(((int64_t*)&tmp1)[0]); ret[1] = (float)(((int64_t*)&tmp1)[1]); ret[2] = (float)(((int64_t*)&tmp1)[2]); @@ -1857,28 +1969,7 @@ static FORCEINLINE __vec16_f __cast_sitofp(__vec16_f, __vec16_i64 val) { ret[13] = (float)(((int64_t*)&tmp2)[5]); ret[14] = (float)(((int64_t*)&tmp2)[6]); ret[15] = (float)(((int64_t*)&tmp2)[7]); -*/ - for (int i = 0; i < 8; i++) { - ret[i] = (float)(((int64_t*)&tmp1)[i]); -// std::cout << "" << std::endl; - } - for (int i = 0; i < 8; i++) { - ((float*)&ret)[i + 8] = (float)(((int64_t*)&tmp2)[i]); - } - - for (int i = 0; i < 8; i++) { - int64_t t = ((int64_t*)&tmp1)[i]; - printf("%d: %llx, %lld, %f\n", i, t, t, (float)t); - } - for (int i = 0; i < 8; i++) { - int64_t t = ((int64_t*)&tmp2)[i]; - printf("%d: %llx, %lld, %f\n", i+8, t, t, (float)t); - } - for (int i = 0; i < 16; i++) { - float f = ((float*)&ret)[i]; - printf("%d-float: %f\n", i, f); - } return ret; } @@ -1947,7 +2038,7 @@ static FORCEINLINE __vec16_f __cast_uitofp(__vec16_f, __vec16_i64 val) { __m512i tmp2; hilo2zmm(val, tmp1, tmp2); __vec16_f ret; - // Cycles don't work. I don't know why. + // Cycles don't work. It seems that it is icc bug. /* for (int i = 0; i < 8; i++) { ((float*)&ret)[i] = ((float)(((uint64_t*)&tmp1)[i])); From 9022806fc0248654fc427290ea106b86bf7c5888 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 4 Dec 2014 19:53:23 +0400 Subject: [PATCH 14/15] Changes in cast --- examples/intrinsics/knc.h | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index 77e10f54..9ecb0454 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -2047,22 +2047,22 @@ static FORCEINLINE __vec16_f __cast_uitofp(__vec16_f, __vec16_i64 val) { ((float*)&ret)[i + 8] = ((float)(((uint64_t*)&tmp2)[i])); } */ - ((float*)&ret)[0] = ((float)(((uint64_t*)&tmp1)[0])); - ((float*)&ret)[1] = ((float)(((uint64_t*)&tmp1)[1])); - ((float*)&ret)[2] = ((float)(((uint64_t*)&tmp1)[2])); - ((float*)&ret)[3] = ((float)(((uint64_t*)&tmp1)[3])); - ((float*)&ret)[4] = ((float)(((uint64_t*)&tmp1)[4])); - ((float*)&ret)[5] = ((float)(((uint64_t*)&tmp1)[5])); - ((float*)&ret)[6] = ((float)(((uint64_t*)&tmp1)[6])); - ((float*)&ret)[7] = ((float)(((uint64_t*)&tmp1)[7])); - ((float*)&ret)[8] = ((float)(((uint64_t*)&tmp2)[0])); - ((float*)&ret)[9] = ((float)(((uint64_t*)&tmp2)[1])); - ((float*)&ret)[10] = ((float)(((uint64_t*)&tmp2)[2])); - ((float*)&ret)[11] = ((float)(((uint64_t*)&tmp2)[3])); - ((float*)&ret)[12] = ((float)(((uint64_t*)&tmp2)[4])); - ((float*)&ret)[13] = ((float)(((uint64_t*)&tmp2)[5])); - ((float*)&ret)[14] = ((float)(((uint64_t*)&tmp2)[6])); - ((float*)&ret)[15] = ((float)(((uint64_t*)&tmp2)[7])); + ret[0] = ((float)(((uint64_t*)&tmp1)[0])); + ret[1] = ((float)(((uint64_t*)&tmp1)[1])); + ret[2] = ((float)(((uint64_t*)&tmp1)[2])); + ret[3] = ((float)(((uint64_t*)&tmp1)[3])); + ret[4] = ((float)(((uint64_t*)&tmp1)[4])); + ret[5] = ((float)(((uint64_t*)&tmp1)[5])); + ret[6] = ((float)(((uint64_t*)&tmp1)[6])); + ret[7] = ((float)(((uint64_t*)&tmp1)[7])); + ret[8] = ((float)(((uint64_t*)&tmp2)[0])); + ret[9] = ((float)(((uint64_t*)&tmp2)[1])); + ret[10] = ((float)(((uint64_t*)&tmp2)[2])); + ret[11] = ((float)(((uint64_t*)&tmp2)[3])); + ret[12] = ((float)(((uint64_t*)&tmp2)[4])); + ret[13] = ((float)(((uint64_t*)&tmp2)[5])); + ret[14] = ((float)(((uint64_t*)&tmp2)[6])); + ret[15] = ((float)(((uint64_t*)&tmp2)[7])); return ret; } From 44ee5737a797ea4d7ec13bd7f90694a00258387d Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 4 Dec 2014 23:04:25 +0400 Subject: [PATCH 15/15] Changes in cast_zext --- examples/intrinsics/knc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/intrinsics/knc.h b/examples/intrinsics/knc.h index 9ecb0454..9fb5fd73 100644 --- a/examples/intrinsics/knc.h +++ b/examples/intrinsics/knc.h @@ -1905,7 +1905,7 @@ static FORCEINLINE __vec16_i64 __cast_zext(const __vec16_i64 &, const __vec16_i1 __vec16_i32 ret_hi = _mm512_setzero_epi32(); __vec16_i32 ret_lo = _mm512_setzero_epi32(); __vec16_i32 one = _mm512_set1_epi32(1); - _mm512_mask_mov_epi32(ret_lo, val, one); + ret_lo = _mm512_mask_mov_epi32(ret_lo, val, one); return __vec16_i64 (ret_lo, ret_hi); }