From 9a03cd359070e81f22c68c60d718e7e180721843 Mon Sep 17 00:00:00 2001 From: Vsevolod Livinskiy Date: Thu, 7 May 2015 11:36:48 +0300 Subject: [PATCH] [AVX512]: definitions through util.m4 was added --- builtins/target-avx512-common.ll | 97 +----- builtins/target-knl.ll | 512 +------------------------------ builtins/target-knl.ll_ | 34 -- 3 files changed, 14 insertions(+), 629 deletions(-) delete mode 100644 builtins/target-knl.ll_ diff --git a/builtins/target-avx512-common.ll b/builtins/target-avx512-common.ll index d402e927..daebe77b 100644 --- a/builtins/target-avx512-common.ll +++ b/builtins/target-avx512-common.ll @@ -29,8 +29,6 @@ ;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128-v16:16:16-v32:32:32-v4:128:128"; - define(`MASK',`i1') define(`HAVE_GATHER',`1') define(`HAVE_SCATTER',`1') @@ -40,7 +38,7 @@ include(`util.m4') stdlib_core() scans() reduce_equal(WIDTH) -rdrand_decls() +rdrand_definition() ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; broadcast/rotate/shuffle @@ -66,64 +64,15 @@ declare @__undef_i16() nounwind readnone declare @__undef_i32() nounwind readnone declare @__undef_i64() nounwind readnone -declare @__broadcast_float(, i32) nounwind readnone -declare @__broadcast_double(, i32) nounwind readnone -declare @__broadcast_i8(, i32) nounwind readnone -declare @__broadcast_i16(, i32) nounwind readnone -declare @__broadcast_i32(, i32) nounwind readnone -declare @__broadcast_i64(, i32) nounwind readnone +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; shuffle -declare @__rotate_i8(, i32) nounwind readnone -declare @__rotate_i16(, i32) nounwind readnone -declare @__rotate_float(, i32) nounwind readnone -declare @__rotate_i32(, i32) nounwind readnone -declare @__rotate_double(, i32) nounwind readnone -declare @__rotate_i64(, i32) nounwind readnone - -declare @__shift_i8(, i32) nounwind readnone -declare @__shift_i16(, i32) nounwind readnone -declare @__shift_float(, i32) nounwind readnone -declare @__shift_i32(, i32) nounwind readnone -declare @__shift_double(, i32) nounwind readnone -declare @__shift_i64(, i32) nounwind readnone - -declare @__shuffle_i8(, ) nounwind readnone -declare @__shuffle2_i8(, , - ) nounwind readnone -declare @__shuffle_i16(, ) nounwind readnone -declare @__shuffle2_i16(, , - ) nounwind readnone -declare @__shuffle_float(, - ) nounwind readnone -declare @__shuffle2_float(, , - ) nounwind readnone -declare @__shuffle_i32(, - ) nounwind readnone -declare @__shuffle2_i32(, , - ) nounwind readnone -declare @__shuffle_double(, - ) nounwind readnone -declare @__shuffle2_double(, - , ) nounwind readnone -declare @__shuffle_i64(, - ) nounwind readnone -declare @__shuffle2_i64(, , - ) nounwind readnone +define_shuffles() ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; aos/soa -declare void @__soa_to_aos3_float( %v0, %v1, - %v2, float * noalias %p) nounwind -declare void @__aos_to_soa3_float(float * noalias %p, * %out0, - * %out1, * %out2) nounwind -declare void @__soa_to_aos4_float( %v0, %v1, - %v2, %v3, - float * noalias %p) nounwind -declare void @__aos_to_soa4_float(float * noalias %p, * noalias %out0, - * noalias %out1, - * noalias %out2, - * noalias %out3) nounwind +aossoa() ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; half conversion routines @@ -157,16 +106,14 @@ declare @__ceil_varying_double() nounwind readn ;; min/max +int64minmax() + declare float @__max_uniform_float(float, float) nounwind readnone declare float @__min_uniform_float(float, float) nounwind readnone declare i32 @__min_uniform_int32(i32, i32) nounwind readnone declare i32 @__max_uniform_int32(i32, i32) nounwind readnone declare i32 @__min_uniform_uint32(i32, i32) nounwind readnone declare i32 @__max_uniform_uint32(i32, i32) nounwind readnone -declare i64 @__min_uniform_int64(i64, i64) nounwind readnone -declare i64 @__max_uniform_int64(i64, i64) nounwind readnone -declare i64 @__min_uniform_uint64(i64, i64) nounwind readnone -declare i64 @__max_uniform_uint64(i64, i64) nounwind readnone declare double @__min_uniform_double(double, double) nounwind readnone declare double @__max_uniform_double(double, double) nounwind readnone @@ -178,10 +125,6 @@ declare @__min_varying_int32(, ) nounwin declare @__max_varying_int32(, ) nounwind readnone declare @__min_varying_uint32(, ) nounwind readnone declare @__max_varying_uint32(, ) nounwind readnone -declare @__min_varying_int64(, ) nounwind readnone -declare @__max_varying_int64(, ) nounwind readnone -declare @__min_varying_uint64(, ) nounwind readnone -declare @__max_varying_uint64(, ) nounwind readnone declare @__min_varying_double(, ) nounwind readnone declare @__max_varying_double(, @@ -205,10 +148,7 @@ declare @__sqrt_varying_double() nounwind readn declare i32 @__popcnt_int32(i32) nounwind readnone declare i64 @__popcnt_int64(i64) nounwind readnone -declare i32 @__count_trailing_zeros_i32(i32) nounwind readnone -declare i64 @__count_trailing_zeros_i64(i64) nounwind readnone -declare i32 @__count_leading_zeros_i32(i32) nounwind readnone -declare i64 @__count_leading_zeros_i64(i64) nounwind readnone +ctlztz() ; FIXME: need either to wire these up to the 8-wide SVML entrypoints, ; or, use the macro to call the 4-wide ones twice with our 8-wide @@ -354,30 +294,13 @@ gather_scatter(float) gather_scatter(i64) gather_scatter(double) -declare i32 @__packed_load_active(i32 * nocapture, * nocapture, - ) nounwind -declare i32 @__packed_store_active(i32 * nocapture, %vals, - ) nounwind -declare i32 @__packed_store_active2(i32 * nocapture, %vals, - ) nounwind - +packed_load_and_store() ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; prefetch -declare void @__prefetch_read_uniform_1(i8 * nocapture) nounwind -declare void @__prefetch_read_uniform_2(i8 * nocapture) nounwind -declare void @__prefetch_read_uniform_3(i8 * nocapture) nounwind -declare void @__prefetch_read_uniform_nt(i8 * nocapture) nounwind +define_prefetches() -declare void @__prefetch_read_varying_1( %addr, %mask) nounwind -declare void @__prefetch_read_varying_1_native(i8 * %base, i32 %scale, %offsets, %mask) nounwind -declare void @__prefetch_read_varying_2( %addr, %mask) nounwind -declare void @__prefetch_read_varying_2_native(i8 * %base, i32 %scale, %offsets, %mask) nounwind -declare void @__prefetch_read_varying_3( %addr, %mask) nounwind -declare void @__prefetch_read_varying_3_native(i8 * %base, i32 %scale, %offsets, %mask) nounwind -declare void @__prefetch_read_varying_nt( %addr, %mask) nounwind -declare void @__prefetch_read_varying_nt_native(i8 * %base, i32 %scale, %offsets, %mask) nounwind ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; int8/int16 builtins diff --git a/builtins/target-knl.ll b/builtins/target-knl.ll index abc39885..65146f15 100644 --- a/builtins/target-knl.ll +++ b/builtins/target-knl.ll @@ -1,4 +1,4 @@ -;; Copyright (c) 2010-2013, Intel Corporation +;; Copyright (c) 2010-2014, Intel Corporation ;; All rights reserved. ;; ;; Redistribution and use in source and binary forms, with or without @@ -29,510 +29,6 @@ ;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -define(`HAVE_GATHER', `1') - -include(`target-avx-x2.ll') - -rdrand_definition() - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; int min/max - -declare <8 x i32> @llvm.x86.avx2.pmins.d(<8 x i32>, <8 x i32>) nounwind readonly -declare <8 x i32> @llvm.x86.avx2.pmaxs.d(<8 x i32>, <8 x i32>) nounwind readonly - -define <16 x i32> @__min_varying_int32(<16 x i32>, <16 x i32>) nounwind readonly alwaysinline { - binary8to16(m, i32, @llvm.x86.avx2.pmins.d, %0, %1) - ret <16 x i32> %m -} - -define <16 x i32> @__max_varying_int32(<16 x i32>, <16 x i32>) nounwind readonly alwaysinline { - binary8to16(m, i32, @llvm.x86.avx2.pmaxs.d, %0, %1) - ret <16 x i32> %m -} - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; unsigned int min/max - -declare <8 x i32> @llvm.x86.avx2.pminu.d(<8 x i32>, <8 x i32>) nounwind readonly -declare <8 x i32> @llvm.x86.avx2.pmaxu.d(<8 x i32>, <8 x i32>) nounwind readonly - -define <16 x i32> @__min_varying_uint32(<16 x i32>, <16 x i32>) nounwind readonly alwaysinline { - binary8to16(m, i32, @llvm.x86.avx2.pminu.d, %0, %1) - ret <16 x i32> %m -} - -define <16 x i32> @__max_varying_uint32(<16 x i32>, <16 x i32>) nounwind readonly alwaysinline { - binary8to16(m, i32, @llvm.x86.avx2.pmaxu.d, %0, %1) - ret <16 x i32> %m -} - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; float/half conversions - -declare <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16>) nounwind readnone -; 0 is round nearest even -declare <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float>, i32) nounwind readnone - -define <16 x float> @__half_to_float_varying(<16 x i16> %v) nounwind readnone { - %r_0 = shufflevector <16 x i16> %v, <16 x i16> undef, - <8 x i32> - %vr_0 = call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> %r_0) - %r_1 = shufflevector <16 x i16> %v, <16 x i16> undef, - <8 x i32> - %vr_1 = call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> %r_1) - %r = shufflevector <8 x float> %vr_0, <8 x float> %vr_1, - <16 x i32> - ret <16 x float> %r -} - -define <16 x i16> @__float_to_half_varying(<16 x float> %v) nounwind readnone { - %r_0 = shufflevector <16 x float> %v, <16 x float> undef, - <8 x i32> - %vr_0 = call <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %r_0, i32 0) - %r_1 = shufflevector <16 x float> %v, <16 x float> undef, - <8 x i32> - %vr_1 = call <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %r_1, i32 0) - %r = shufflevector <8 x i16> %vr_0, <8 x i16> %vr_1, - <16 x i32> - ret <16 x i16> %r -} - -define float @__half_to_float_uniform(i16 %v) nounwind readnone { - %v1 = bitcast i16 %v to <1 x i16> - %vv = shufflevector <1 x i16> %v1, <1 x i16> undef, - <8 x i32> - %rv = call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> %vv) - %r = extractelement <8 x float> %rv, i32 0 - ret float %r -} - -define i16 @__float_to_half_uniform(float %v) nounwind readnone { - %v1 = bitcast float %v to <1 x float> - %vv = shufflevector <1 x float> %v1, <1 x float> undef, - <8 x i32> - ; round to nearest even - %rv = call <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %vv, i32 0) - %r = extractelement <8 x i16> %rv, i32 0 - ret i16 %r -} - - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; gather - -declare void @llvm.trap() noreturn nounwind - -; $1: type -; $2: var base name -define(`extract_4s', ` - %$2_1 = shufflevector <16 x $1> %$2, <16 x $1> undef, <4 x i32> - %$2_2 = shufflevector <16 x $1> %$2, <16 x $1> undef, <4 x i32> - %$2_3 = shufflevector <16 x $1> %$2, <16 x $1> undef, <4 x i32> - %$2_4 = shufflevector <16 x $1> %$2, <16 x $1> undef, <4 x i32> -') - -; $1: type -; $2: var base name -define(`extract_8s', ` - %$2_1 = shufflevector <16 x $1> %$2, <16 x $1> undef, - <8 x i32> - %$2_2 = shufflevector <16 x $1> %$2, <16 x $1> undef, - <8 x i32> -') - -; $1: element type -; $2: ret name -; $3: v1 -; $4: v2 -define(`assemble_8s', ` - %$2 = shufflevector <8 x $1> %$3, <8 x $1> %$4, - <16 x i32> -') - -; $1: element type -; $2: ret name -; $3: v1 -; $4: v2 -; $5: v3 -; $6: v4 -define(`assemble_4s', ` - %$2_1 = shufflevector <4 x $1> %$3, <4 x $1> %$4, - <8 x i32> - %$2_2 = shufflevector <4 x $1> %$5, <4 x $1> %$6, - <8 x i32> - assemble_8s($1, $2, $2_1, $2_2) -') - - -gen_gather(i8) -gen_gather(i16) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; int32 gathers - -declare <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> %target, i8 * %ptr, - <8 x i32> %indices, <8 x i32> %mask, i8 %scale) readonly nounwind -declare <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> %target, i8 * %ptr, - <4 x i64> %indices, <4 x i32> %mask, i8 %scale) readonly nounwind - -define <16 x i32> @__gather_base_offsets32_i32(i8 * %ptr, i32 %scale, <16 x i32> %offsets, - <16 x i32> %vecmask) nounwind readonly alwaysinline { - %scale8 = trunc i32 %scale to i8 - extract_8s(i32, offsets) - extract_8s(i32, vecmask) - - %v1 = call <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> undef, i8 * %ptr, - <8 x i32> %offsets_1, <8 x i32> %vecmask_1, i8 %scale8) - %v2 = call <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> undef, i8 * %ptr, - <8 x i32> %offsets_2, <8 x i32> %vecmask_2, i8 %scale8) - - assemble_8s(i32, v, v1, v2) - - ret <16 x i32> %v -} - - -define <16 x i32> @__gather_base_offsets64_i32(i8 * %ptr, - i32 %scale, <16 x i64> %offsets, - <16 x i32> %vecmask) nounwind readonly alwaysinline { - %scale8 = trunc i32 %scale to i8 - - extract_4s(i32, vecmask) - extract_4s(i64, offsets) - - %v1 = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> undef, i8 * %ptr, - <4 x i64> %offsets_1, <4 x i32> %vecmask_1, i8 %scale8) - %v2 = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> undef, i8 * %ptr, - <4 x i64> %offsets_2, <4 x i32> %vecmask_2, i8 %scale8) - %v3 = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> undef, i8 * %ptr, - <4 x i64> %offsets_3, <4 x i32> %vecmask_3, i8 %scale8) - %v4 = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> undef, i8 * %ptr, - <4 x i64> %offsets_4, <4 x i32> %vecmask_4, i8 %scale8) - - assemble_4s(i32, v, v1, v2, v3, v4) - - ret <16 x i32> %v -} - - -define <16 x i32> @__gather32_i32(<16 x i32> %ptrs, - <16 x i32> %vecmask) nounwind readonly alwaysinline { - extract_8s(i32, ptrs) - extract_8s(i32, vecmask) - - %v1 = call <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> undef, i8 * null, - <8 x i32> %ptrs_1, <8 x i32> %vecmask_1, i8 1) - %v2 = call <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> undef, i8 * null, - <8 x i32> %ptrs_2, <8 x i32> %vecmask_2, i8 1) - - assemble_8s(i32, v, v1, v2) - - ret <16 x i32> %v -} - - -define <16 x i32> @__gather64_i32(<16 x i64> %ptrs, - <16 x i32> %vecmask) nounwind readonly alwaysinline { - extract_4s(i64, ptrs) - extract_4s(i32, vecmask) - - %v1 = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> undef, i8 * null, - <4 x i64> %ptrs_1, <4 x i32> %vecmask_1, i8 1) - %v2 = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> undef, i8 * null, - <4 x i64> %ptrs_2, <4 x i32> %vecmask_2, i8 1) - %v3 = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> undef, i8 * null, - <4 x i64> %ptrs_3, <4 x i32> %vecmask_3, i8 1) - %v4 = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> undef, i8 * null, - <4 x i64> %ptrs_4, <4 x i32> %vecmask_4, i8 1) - - assemble_4s(i32, v, v1, v2, v3, v4) - - ret <16 x i32> %v -} - - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; float gathers - -declare <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> %target, i8 * %ptr, - <8 x i32> %indices, <8 x float> %mask, i8 %scale8) readonly nounwind -declare <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> %target, i8 * %ptr, - <4 x i64> %indices, <4 x float> %mask, i8 %scale8) readonly nounwind - -define <16 x float> @__gather_base_offsets32_float(i8 * %ptr, - i32 %scale, <16 x i32> %offsets, - <16 x i32> %vecmask) nounwind readonly alwaysinline { - %scale8 = trunc i32 %scale to i8 - %mask = bitcast <16 x i32> %vecmask to <16 x float> - extract_8s(i32, offsets) - extract_8s(float, mask) - - %v1 = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> undef, i8 * %ptr, - <8 x i32> %offsets_1, <8 x float> %mask_1, i8 %scale8) - %v2 = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> undef, i8 * %ptr, - <8 x i32> %offsets_2, <8 x float> %mask_2, i8 %scale8) - - assemble_8s(float, v, v1, v2) - - ret <16 x float> %v -} - - -define <16 x float> @__gather_base_offsets64_float(i8 * %ptr, - i32 %scale, <16 x i64> %offsets, - <16 x i32> %vecmask) nounwind readonly alwaysinline { - %scale8 = trunc i32 %scale to i8 - %mask = bitcast <16 x i32> %vecmask to <16 x float> - extract_4s(i64, offsets) - extract_4s(float, mask) - - %v1 = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> undef, i8 * %ptr, - <4 x i64> %offsets_1, <4 x float> %mask_1, i8 %scale8) - %v2 = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> undef, i8 * %ptr, - <4 x i64> %offsets_2, <4 x float> %mask_2, i8 %scale8) - %v3 = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> undef, i8 * %ptr, - <4 x i64> %offsets_3, <4 x float> %mask_3, i8 %scale8) - %v4 = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> undef, i8 * %ptr, - <4 x i64> %offsets_4, <4 x float> %mask_4, i8 %scale8) - - assemble_4s(float, v, v1, v2, v3, v4) - - ret <16 x float> %v -} - - -define <16 x float> @__gather32_float(<16 x i32> %ptrs, - <16 x i32> %vecmask) nounwind readonly alwaysinline { - %mask = bitcast <16 x i32> %vecmask to <16 x float> - extract_8s(float, mask) - extract_8s(i32, ptrs) - - %v1 = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> undef, i8 * null, - <8 x i32> %ptrs_1, <8 x float> %mask_1, i8 1) - %v2 = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> undef, i8 * null, - <8 x i32> %ptrs_2, <8 x float> %mask_2, i8 1) - - assemble_8s(float, v, v1, v2) - - ret <16 x float> %v -} - - -define <16 x float> @__gather64_float(<16 x i64> %ptrs, - <16 x i32> %vecmask) nounwind readonly alwaysinline { - %mask = bitcast <16 x i32> %vecmask to <16 x float> - extract_4s(i64, ptrs) - extract_4s(float, mask) - - %v1 = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> undef, i8 * null, - <4 x i64> %ptrs_1, <4 x float> %mask_1, i8 1) - %v2 = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> undef, i8 * null, - <4 x i64> %ptrs_2, <4 x float> %mask_2, i8 1) - %v3 = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> undef, i8 * null, - <4 x i64> %ptrs_3, <4 x float> %mask_3, i8 1) - %v4 = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> undef, i8 * null, - <4 x i64> %ptrs_4, <4 x float> %mask_4, i8 1) - - assemble_4s(float, v, v1, v2, v3, v4) - - ret <16 x float> %v -} - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; int64 gathers - -declare <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> %target, i8 * %ptr, - <4 x i32> %indices, <4 x i64> %mask, i8 %scale) readonly nounwind -declare <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> %target, i8 * %ptr, - <4 x i64> %indices, <4 x i64> %mask, i8 %scale) readonly nounwind - -define <16 x i64> @__gather_base_offsets32_i64(i8 * %ptr, - i32 %scale, <16 x i32> %offsets, - <16 x i32> %mask32) nounwind readonly alwaysinline { - %scale8 = trunc i32 %scale to i8 - %vecmask = sext <16 x i32> %mask32 to <16 x i64> - extract_4s(i32, offsets) - extract_4s(i64, vecmask) - - %v1 = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> undef, i8 * %ptr, - <4 x i32> %offsets_1, <4 x i64> %vecmask_1, i8 %scale8) - %v2 = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> undef, i8 * %ptr, - <4 x i32> %offsets_2, <4 x i64> %vecmask_2, i8 %scale8) - %v3 = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> undef, i8 * %ptr, - <4 x i32> %offsets_3, <4 x i64> %vecmask_3, i8 %scale8) - %v4 = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> undef, i8 * %ptr, - <4 x i32> %offsets_4, <4 x i64> %vecmask_4, i8 %scale8) - - assemble_4s(i64, v, v1, v2, v3, v4) - - ret <16 x i64> %v -} - - -define <16 x i64> @__gather_base_offsets64_i64(i8 * %ptr, - i32 %scale, <16 x i64> %offsets, - <16 x i32> %mask32) nounwind readonly alwaysinline { - %scale8 = trunc i32 %scale to i8 - %vecmask = sext <16 x i32> %mask32 to <16 x i64> - extract_4s(i64, offsets) - extract_4s(i64, vecmask) - - %v1 = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> undef, i8 * %ptr, - <4 x i64> %offsets_1, <4 x i64> %vecmask_1, i8 %scale8) - %v2 = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> undef, i8 * %ptr, - <4 x i64> %offsets_2, <4 x i64> %vecmask_2, i8 %scale8) - %v3 = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> undef, i8 * %ptr, - <4 x i64> %offsets_3, <4 x i64> %vecmask_3, i8 %scale8) - %v4 = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> undef, i8 * %ptr, - <4 x i64> %offsets_4, <4 x i64> %vecmask_4, i8 %scale8) - - assemble_4s(i64, v, v1, v2, v3, v4) - - ret <16 x i64> %v -} - - -define <16 x i64> @__gather32_i64(<16 x i32> %ptrs, - <16 x i32> %mask32) nounwind readonly alwaysinline { - %vecmask = sext <16 x i32> %mask32 to <16 x i64> - extract_4s(i32, ptrs) - extract_4s(i64, vecmask) - - %v1 = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> undef, i8 * null, - <4 x i32> %ptrs_1, <4 x i64> %vecmask_1, i8 1) - %v2 = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> undef, i8 * null, - <4 x i32> %ptrs_2, <4 x i64> %vecmask_2, i8 1) - %v3 = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> undef, i8 * null, - <4 x i32> %ptrs_3, <4 x i64> %vecmask_3, i8 1) - %v4 = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> undef, i8 * null, - <4 x i32> %ptrs_4, <4 x i64> %vecmask_4, i8 1) - - assemble_4s(i64, v, v1, v2, v3, v4) - - ret <16 x i64> %v -} - -define <16 x i64> @__gather64_i64(<16 x i64> %ptrs, - <16 x i32> %mask32) nounwind readonly alwaysinline { - %vecmask = sext <16 x i32> %mask32 to <16 x i64> - extract_4s(i64, ptrs) - extract_4s(i64, vecmask) - - %v1 = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> undef, i8 * null, - <4 x i64> %ptrs_1, <4 x i64> %vecmask_1, i8 1) - %v2 = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> undef, i8 * null, - <4 x i64> %ptrs_2, <4 x i64> %vecmask_2, i8 1) - %v3 = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> undef, i8 * null, - <4 x i64> %ptrs_3, <4 x i64> %vecmask_3, i8 1) - %v4 = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> undef, i8 * null, - <4 x i64> %ptrs_4, <4 x i64> %vecmask_4, i8 1) - - assemble_4s(i64, v, v1, v2, v3, v4) - - ret <16 x i64> %v -} - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; double gathers - -declare <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> %target, i8 * %ptr, - <4 x i64> %indices, <4 x double> %mask, i8 %scale) readonly nounwind -declare <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> %target, i8 * %ptr, - <4 x i32> %indices, <4 x double> %mask, i8 %scale) readonly nounwind - -define <16 x double> @__gather_base_offsets32_double(i8 * %ptr, - i32 %scale, <16 x i32> %offsets, - <16 x i32> %mask32) nounwind readonly alwaysinline { - %scale8 = trunc i32 %scale to i8 - %vecmask64 = sext <16 x i32> %mask32 to <16 x i64> - %vecmask = bitcast <16 x i64> %vecmask64 to <16 x double> - extract_4s(i32, offsets) - extract_4s(double, vecmask) - - %v1 = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> undef, i8 * %ptr, - <4 x i32> %offsets_1, <4 x double> %vecmask_1, i8 %scale8) - %v2 = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> undef, i8 * %ptr, - <4 x i32> %offsets_2, <4 x double> %vecmask_2, i8 %scale8) - %v3 = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> undef, i8 * %ptr, - <4 x i32> %offsets_3, <4 x double> %vecmask_3, i8 %scale8) - %v4 = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> undef, i8 * %ptr, - <4 x i32> %offsets_4, <4 x double> %vecmask_4, i8 %scale8) - - assemble_4s(double, v, v1, v2, v3, v4) - - ret <16 x double> %v -} - - -define <16 x double> @__gather_base_offsets64_double(i8 * %ptr, - i32 %scale, <16 x i64> %offsets, - <16 x i32> %mask32) nounwind readonly alwaysinline { - %scale8 = trunc i32 %scale to i8 - %vecmask64 = sext <16 x i32> %mask32 to <16 x i64> - %vecmask = bitcast <16 x i64> %vecmask64 to <16 x double> - extract_4s(i64, offsets) - extract_4s(double, vecmask) - - %v1 = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> undef, i8 * %ptr, - <4 x i64> %offsets_1, <4 x double> %vecmask_1, i8 %scale8) - %v2 = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> undef, i8 * %ptr, - <4 x i64> %offsets_2, <4 x double> %vecmask_2, i8 %scale8) - %v3 = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> undef, i8 * %ptr, - <4 x i64> %offsets_3, <4 x double> %vecmask_3, i8 %scale8) - %v4 = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> undef, i8 * %ptr, - <4 x i64> %offsets_4, <4 x double> %vecmask_4, i8 %scale8) - - assemble_4s(double, v, v1, v2, v3, v4) - - ret <16 x double> %v -} - - -define <16 x double> @__gather32_double(<16 x i32> %ptrs, - <16 x i32> %mask32) nounwind readonly alwaysinline { - %vecmask64 = sext <16 x i32> %mask32 to <16 x i64> - %vecmask = bitcast <16 x i64> %vecmask64 to <16 x double> - extract_4s(i32, ptrs) - extract_4s(double, vecmask) - - %v1 = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> undef, i8 * null, - <4 x i32> %ptrs_1, <4 x double> %vecmask_1, i8 1) - %v2 = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> undef, i8 * null, - <4 x i32> %ptrs_2, <4 x double> %vecmask_2, i8 1) - %v3 = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> undef, i8 * null, - <4 x i32> %ptrs_3, <4 x double> %vecmask_3, i8 1) - %v4 = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> undef, i8 * null, - <4 x i32> %ptrs_4, <4 x double> %vecmask_4, i8 1) - - assemble_4s(double, v, v1, v2, v3, v4) - - ret <16 x double> %v -} - - -define <16 x double> @__gather64_double(<16 x i64> %ptrs, - <16 x i32> %mask32) nounwind readonly alwaysinline { - %vecmask64 = sext <16 x i32> %mask32 to <16 x i64> - %vecmask = bitcast <16 x i64> %vecmask64 to <16 x double> - extract_4s(i64, ptrs) - extract_4s(double, vecmask) - - %v1 = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> undef, i8 * null, - <4 x i64> %ptrs_1, <4 x double> %vecmask_1, i8 1) - %v2 = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> undef, i8 * null, - <4 x i64> %ptrs_2, <4 x double> %vecmask_2, i8 1) - %v3 = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> undef, i8 * null, - <4 x i64> %ptrs_3, <4 x double> %vecmask_3, i8 1) - %v4 = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> undef, i8 * null, - <4 x i64> %ptrs_4, <4 x double> %vecmask_4, i8 1) - - assemble_4s(double, v, v1, v2, v3, v4) - - ret <16 x double> %v -} +define(`WIDTH',`16') +include(`target-avx512-common.ll') +;;saturation_arithmetic_novec() diff --git a/builtins/target-knl.ll_ b/builtins/target-knl.ll_ deleted file mode 100644 index 8305dda7..00000000 --- a/builtins/target-knl.ll_ +++ /dev/null @@ -1,34 +0,0 @@ -;; Copyright (c) 2010-2014, Intel Corporation -;; All rights reserved. -;; -;; Redistribution and use in source and binary forms, with or without -;; modification, are permitted provided that the following conditions are -;; met: -;; -;; * Redistributions of source code must retain the above copyright -;; notice, this list of conditions and the following disclaimer. -;; -;; * Redistributions in binary form must reproduce the above copyright -;; notice, this list of conditions and the following disclaimer in the -;; documentation and/or other materials provided with the distribution. -;; -;; * Neither the name of Intel Corporation nor the names of its -;; contributors may be used to endorse or promote products derived from -;; this software without specific prior written permission. -;; -;; -;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -;; IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -;; TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -;; PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -;; OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -;; EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -;; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -define(`WIDTH',`16') -include(`target-avx512-common.ll') -saturation_arithmetic_novec()