python regex-based preprocessor proof of concept

Renaming AttributeSet to AttributeList to follow trunk changes.

.gitignore

@@ -4,11 +4,15 @@ depend
 ispc
 ispc_test
 ispc_ref
+llvm/
 objs
 docs/doxygen
 docs/*.html
 tests*/*cpp
 tests*/*run
+tests*/*.o
+tests_ispcpp/*.h
+tests_ispcpp/*pre*
 logs/
 notify_log.log
 alloy_results_*

LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2015, Intel Corporation
+Copyright (c) 2010-2016, Intel Corporation
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

Makefile

@@ -45,15 +45,15 @@ Location of LLVM files in your PATH is different than path in LLVM_HOME \n
 variable (or LLVM_HOME is not set). The most likely this means that you are \n
 using default LLVM installation on your system, which is very bad sign. \n
 Note, that ISPC uses LLVM optimizer and is highly dependent on it. We recommend \n
-using *patched* version of LLVM 3.4 or 3.5. Patches are availible in \n
+using *patched* version of LLVM 3.8. Patches are availible in \n
 llvm_patches folder. You can build LLVM manually, or run our scripts, which \n
 will do all the work for you. Do the following: \n
 1. Create a folder, where LLVM will reside and set LLVM_HOME variable to its \n
   path. \n
 2. Set ISPC_HOME variable to your ISPC location (probably current folder).
 3. Run alloy.py tool to checkout and build LLVM: \n
-  alloy.py -b --version=3.5 \n
+  alloy.py -b --version=3.8 \n
-4. Add $$LLVM_HOME/bin-3.5/bin path to your PATH. \n
+4. Add $$LLVM_HOME/bin-3.8/bin path to your PATH. \n
 ==============================================================================
 endef
 
@@ -202,7 +202,7 @@ HEADERS=ast.h builtins.h ctx.h decl.h expr.h func.h ispc.h llvmutil.h module.h \
 	opt.h stmt.h sym.h type.h util.h
 TARGETS=avx2-i64x4 avx11-i64x4 avx1-i64x4 avx1 avx1-x2 avx11 avx11-x2 avx2 avx2-x2 \
 	sse2 sse2-x2 sse4-8 sse4-16 sse4 sse4-x2 \
-	generic-4 generic-8 generic-16 generic-32 generic-64 generic-1 knl
+	generic-4 generic-8 generic-16 generic-32 generic-64 generic-1 knl skx
 ifneq ($(ARM_ENABLED), 0)
     TARGETS+=neon-32 neon-16 neon-8
 endif

alloy.py

@@ -33,6 +33,8 @@
 
 # // Author: Filippov Ilia
 
+import re
+
 def tail_and_save(file_in, file_out, tail = 100):    
     with open(file_in, 'r') as f_in:
         lines = f_in.readlines()[-tail:]
@@ -91,6 +93,7 @@ def check_LLVM(which_LLVM):
     return answer
 
 def try_do_LLVM(text, command, from_validation):
+    print_debug("Command line: "+command+"\n", True, alloy_build)
     if from_validation == True:
         text = text + "\n"
     print_debug("Trying to " + text, from_validation, alloy_build)
@@ -108,7 +111,77 @@ def try_do_LLVM(text, command, from_validation):
         error("can't " + text, 1)
     print_debug("DONE.\n", from_validation, alloy_build)
 
-def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra, from_validation, force, make, gcc_toolchain_path):
+def checkout_LLVM(component, use_git, version_LLVM, revision, target_dir, from_validation):
+    # Identify the component
+    GIT_REPO_BASE="http://llvm.org/git/"
+    #GIT_REPO_BASE="https://github.com/llvm-mirror/"
+    if component == "llvm":
+        SVN_REPO="http://llvm.org/svn/llvm-project/llvm/"
+        GIT_REPO=GIT_REPO_BASE+"llvm.git"
+    elif component == "clang":
+        SVN_REPO="http://llvm.org/svn/llvm-project/cfe/"
+        GIT_REPO=GIT_REPO_BASE+"clang.git"
+    elif component == "libcxx":
+        SVN_REPO="http://llvm.org/svn/llvm-project/libcxx/"
+        GIT_REPO=GIT_REPO_BASE+"libcxx.git"
+    elif component == "clang-tools-extra":
+        SVN_REPO="http://llvm.org/svn/llvm-project/clang-tools-extra/"
+        GIT_REPO=GIT_REPO_BASE+"clang-tools-extra.git"
+    elif component == "compiler-rt":
+        SVN_REPO="http://llvm.org/svn/llvm-project/compiler-rt/"
+        GIT_REPO=GIT_REPO_BASE+"compiler-rt.git"
+    else:
+        error("Trying to checkout unidentified component: " + component, 1)
+
+    # Identify the version
+    if  version_LLVM == "trunk":
+        SVN_PATH="trunk"
+        GIT_BRANCH="master"
+    elif  version_LLVM == "4_0":
+        SVN_PATH="branches/release_40"
+        GIT_BRANCH="release_40"
+    elif  version_LLVM == "3_9":
+        SVN_PATH="tags/RELEASE_390/final"
+        GIT_BRANCH="release_39"
+    elif  version_LLVM == "3_8":
+        SVN_PATH="tags/RELEASE_381/final"
+        GIT_BRANCH="release_38"
+    elif  version_LLVM == "3_7":
+        SVN_PATH="tags/RELEASE_370/final"
+        GIT_BRANCH="release_37"
+    elif  version_LLVM == "3_6":
+        SVN_PATH="tags/RELEASE_362/final"
+        GIT_BRANCH="release_36"
+    elif  version_LLVM == "3_5":
+        SVN_PATH="tags/RELEASE_351/final"
+        GIT_BRANCH="release_35"
+    elif  version_LLVM == "3_4":
+        SVN_PATH="tags/RELEASE_34/dot2-final"
+        GIT_BRANCH="release_34"
+    elif  version_LLVM == "3_3":
+        SVN_PATH="tags/RELEASE_33/final"
+        GIT_BRANCH="release_33"
+    elif  version_LLVM == "3_2":
+        SVN_PATH="tags/RELEASE_32/final"
+        GIT_BRANCH="release_32"
+    else:
+        error("Unsupported llvm version: " + version_LLVM, 1)
+
+    if use_git:
+        try_do_LLVM("clone "+component+" from "+GIT_REPO+" to "+target_dir+" ",
+                    "git clone "+GIT_REPO+" "+target_dir,
+                    from_validation)
+        if GIT_BRANCH != "master":
+            os.chdir(target_dir)
+            try_do_LLVM("switch to "+GIT_BRANCH+" branch ",
+                        "git checkout -b "+GIT_BRANCH+" remotes/origin/"+GIT_BRANCH, from_validation)
+            os.chdir("..")
+    else:
+        try_do_LLVM("load "+component+" from "+SVN_REPO+SVN_PATH+" ",
+                    "svn co "+revision+" "+SVN_REPO+SVN_PATH+" "+target_dir,
+                    from_validation)
+
+def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra, from_validation, force, make, gcc_toolchain_path, use_git):
     print_debug("Building LLVM. Version: " + version_LLVM + ". ", from_validation, alloy_build)
     if revision != "":
         print_debug("Revision: " + revision + ".\n", from_validation, alloy_build)
@@ -120,31 +193,10 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
     
     make_sure_dir_exists(llvm_home)
 
-    os.chdir(llvm_home)
     FOLDER_NAME=version_LLVM
-    if  version_LLVM == "trunk":
+    version_LLVM = re.sub('\.', '_', version_LLVM)
-        SVN_PATH="trunk"
+    
-    if  version_LLVM == "3.8":
+    os.chdir(llvm_home)
-        SVN_PATH="branches/release_38"
-        version_LLVM = "3_8"
-    if  version_LLVM == "3.7":
-        SVN_PATH="tags/RELEASE_370/final"
-        version_LLVM = "3_7"
-    if  version_LLVM == "3.6":
-        SVN_PATH="tags/RELEASE_362/final"
-        version_LLVM = "3_6"
-    if  version_LLVM == "3.5":
-        SVN_PATH="tags/RELEASE_351/final"
-        version_LLVM = "3_5"
-    if  version_LLVM == "3.4":
-        SVN_PATH="tags/RELEASE_34/dot2-final"
-        version_LLVM = "3_4"
-    if  version_LLVM == "3.3":
-        SVN_PATH="tags/RELEASE_33/final"
-        version_LLVM = "3_3"
-    if  version_LLVM == "3.2":
-        SVN_PATH="tags/RELEASE_32/final"
-        version_LLVM = "3_2"
     if revision != "":
         FOLDER_NAME = FOLDER_NAME + "_" + revision
         revision = "-" + revision
@@ -173,7 +225,7 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
             if os.path.exists(os.path.join(path, "xcrun")):
                 found_xcrun = True
         if found_xcrun:
-            mac_system_root = " --with-default-sysroot=`xcrun --show-sdk-path`"
+            mac_system_root = "`xcrun --show-sdk-path`"
         else:
             error("Can't find XCode (xcrun tool) - it's required on MacOS 10.9 and newer", 1)
 
@@ -184,13 +236,9 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
         llvm_home + "\n", from_validation, alloy_build)
     # load llvm
     if tarball == "":
-        try_do_LLVM("load LLVM from http://llvm.org/svn/llvm-project/llvm/" + SVN_PATH + " ",
+        checkout_LLVM("llvm", options.use_git, version_LLVM, revision, LLVM_SRC, from_validation)
-                    "svn co " + revision + " http://llvm.org/svn/llvm-project/llvm/" + SVN_PATH + " " + LLVM_SRC,
-                    from_validation)
         os.chdir(LLVM_SRC + "/tools")
-        try_do_LLVM("load clang from http://llvm.org/svn/llvm-project/cfe/" + SVN_PATH + " ",
+        checkout_LLVM("clang", options.use_git, version_LLVM, revision, "clang", from_validation)
-                    "svn co " + revision + " http://llvm.org/svn/llvm-project/cfe/" + SVN_PATH + " clang",
-                    from_validation)
         os.chdir("..")
         if current_OS == "MacOS" and int(current_OS_version.split(".")[0]) >= 13:
             # Starting with MacOS 10.9 Maverics, the system doesn't contain headers for standard C++ library and
@@ -202,19 +250,13 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
             # to the linker explicitly (either through command line or environment variables). So we are not doing it
             # currently to make the build process easier.
             os.chdir("projects")
-            try_do_LLVM("load libcxx http://llvm.org/svn/llvm-project/libcxx/" + SVN_PATH + " ",
+            checkout_LLVM("libcxx", options.use_git, version_LLVM, revision, "libcxx", from_validation)
-                    "svn co " + revision + " http://llvm.org/svn/llvm-project/libcxx/" + SVN_PATH + " libcxx",
-                    from_validation)
             os.chdir("..")
         if extra == True:
             os.chdir("tools/clang/tools")
-            try_do_LLVM("load extra clang extra tools ",
+            checkout_LLVM("clang-tools-extra", options.use_git, version_LLVM, revision, "extra", from_validation)
-                    "svn co " + revision + " http://llvm.org/svn/llvm-project/clang-tools-extra/" + SVN_PATH + " extra",
-                    from_validation)
             os.chdir("../../../projects")
-            try_do_LLVM("load extra clang compiler-rt ",
+            checkout_LLVM("compiler-rt", options.use_git, version_LLVM, revision, "compiler-rt", from_validation)
-                    "svn co " + revision + " http://llvm.org/svn/llvm-project/compiler-rt/" + SVN_PATH + " compiler-rt",
-                    from_validation)
             os.chdir("..")
     else:
         tar = tarball.split(" ")
@@ -241,14 +283,14 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
     os.makedirs(LLVM_BUILD)
     os.makedirs(LLVM_BIN)
     selfbuild_compiler = ""
+    LLVM_configure_capable = ["3_2", "3_3", "3_4", "3_5", "3_6", "3_7"]
     if selfbuild:
         print_debug("Making selfbuild and use folders " + LLVM_BUILD_selfbuild + " and " +
             LLVM_BIN_selfbuild + "\n", from_validation, alloy_build)
         os.makedirs(LLVM_BUILD_selfbuild)
         os.makedirs(LLVM_BIN_selfbuild)
         os.chdir(LLVM_BUILD_selfbuild)
-        if  version_LLVM == "trunk":
+        if  version_LLVM not in LLVM_configure_capable:
-            # TODO: mac_root
             try_do_LLVM("configure release version for selfbuild ",
                     "cmake -G Unix\ Makefiles" + " -DCMAKE_EXPORT_COMPILE_COMMANDS=ON" +
                     "  -DCMAKE_INSTALL_PREFIX=" + llvm_home + "/" + LLVM_BIN_selfbuild +
@@ -257,6 +299,7 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
                     (("  -DGCC_INSTALL_PREFIX=" + gcc_toolchain_path) if gcc_toolchain_path != "" else "") +
                     (("  -DCMAKE_C_COMPILER=" + gcc_toolchain_path+"/bin/gcc") if gcc_toolchain_path != "" else "") +
                     (("  -DCMAKE_CXX_COMPILER=" + gcc_toolchain_path+"/bin/g++") if gcc_toolchain_path != "" else "") +
+                    (("  -DDEFAULT_SYSROOT=" + mac_system_root) if mac_system_root != "" else "") +
                     "  -DLLVM_TARGETS_TO_BUILD=NVPTX\;X86" +
                     " ../" + LLVM_SRC,
                     from_validation)
@@ -268,7 +311,7 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
                         LLVM_BIN_selfbuild + " --enable-optimized" +
                         " --enable-targets=x86,x86_64,nvptx" +
                         ((" --with-gcc-toolchain=" + gcc_toolchain_path) if gcc_toolchain_path != "" else "") +
-                        mac_system_root,
+                        ((" --with-default-sysroot=" + mac_system_root) if mac_system_root != "" else ""),
                         from_validation)
             selfbuild_compiler = ("CC=" +llvm_home+ "/" + LLVM_BIN_selfbuild + "/bin/clang " +
                                   "CXX="+llvm_home+ "/" + LLVM_BIN_selfbuild + "/bin/clang++ ")
@@ -283,8 +326,7 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
     os.chdir(LLVM_BUILD)
     if debug == False:
         if current_OS != "Windows":
-            if  version_LLVM == "trunk":
+            if  version_LLVM not in LLVM_configure_capable:
-                # TODO: mac_root
                 try_do_LLVM("configure release version ",
                         "cmake -G Unix\ Makefiles" + " -DCMAKE_EXPORT_COMPILE_COMMANDS=ON" +
                         selfbuild_compiler +
@@ -294,6 +336,7 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
                         (("  -DGCC_INSTALL_PREFIX=" + gcc_toolchain_path) if gcc_toolchain_path != "" else "") +
                         (("  -DCMAKE_C_COMPILER=" + gcc_toolchain_path+"/bin/gcc") if gcc_toolchain_path != "" and selfbuild_compiler == "" else "") +
                         (("  -DCMAKE_CXX_COMPILER=" + gcc_toolchain_path+"/bin/g++") if gcc_toolchain_path != "" and selfbuild_compiler == "" else "") +
+                        (("  -DDEFAULT_SYSROOT=" + mac_system_root) if mac_system_root != "" else "") +
                         "  -DLLVM_TARGETS_TO_BUILD=NVPTX\;X86" +
                         " ../" + LLVM_SRC,
                         from_validation)
@@ -303,7 +346,7 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
                         LLVM_BIN + " --enable-optimized" +
                         " --enable-targets=x86,x86_64,nvptx" +
                         ((" --with-gcc-toolchain=" + gcc_toolchain_path) if gcc_toolchain_path != "" else "") +
-                        mac_system_root,
+                        ((" --with-default-sysroot=" + mac_system_root) if mac_system_root != "" else ""),
                         from_validation)
         else:
             try_do_LLVM("configure release version ",
@@ -311,8 +354,7 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
                     '" -DLLVM_LIT_TOOLS_DIR="C:\\gnuwin32\\bin" ..\\' + LLVM_SRC,
                     from_validation)
     else:
-        if  version_LLVM == "trunk":
+        if  version_LLVM not in LLVM_configure_capable:
-            # TODO: mac_root
             try_do_LLVM("configure debug version ",
                     "cmake -G Unix\ Makefiles" + " -DCMAKE_EXPORT_COMPILE_COMMANDS=ON" +
                     selfbuild_compiler +
@@ -322,6 +364,7 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
                     (("  -DGCC_INSTALL_PREFIX=" + gcc_toolchain_path) if gcc_toolchain_path != "" else "") +
                     (("  -DCMAKE_C_COMPILER=" + gcc_toolchain_path+"/bin/gcc") if gcc_toolchain_path != "" and selfbuild_compiler == "" else "") +
                     (("  -DCMAKE_CXX_COMPILER=" + gcc_toolchain_path+"/bin/g++") if gcc_toolchain_path != "" and selfbuild_compiler == "" else "") +
+                    (("  -DDEFAULT_SYSROOT=" + mac_system_root) if mac_system_root != "" else "") +
                     "  -DLLVM_TARGETS_TO_BUILD=NVPTX\;X86" +
                     " ../" + LLVM_SRC,
                     from_validation)
@@ -331,7 +374,7 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
                         " --enable-debug-runtime --enable-debug-symbols --enable-keep-symbols" +
                         " --enable-targets=x86,x86_64,nvptx" +
                         ((" --with-gcc-toolchain=" + gcc_toolchain_path) if gcc_toolchain_path != "" else "") +
-                        mac_system_root,
+                        ((" --with-default-sysroot=" + mac_system_root) if mac_system_root != "" else ""),
                         from_validation)
     # building llvm
     if current_OS != "Windows":
@@ -343,14 +386,16 @@ def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, extra,
 
 
 def unsupported_llvm_targets(LLVM_VERSION):
-    prohibited_list = {"3.2":["avx512knl-i32x16"],
+    prohibited_list = {"3.2":["avx512knl-i32x16", "avx512skx-i32x16"],
-                       "3.3":["avx512knl-i32x16"],
+                       "3.3":["avx512knl-i32x16", "avx512skx-i32x16"],
-                       "3.4":["avx512knl-i32x16"],
+                       "3.4":["avx512knl-i32x16", "avx512skx-i32x16"],
-                       "3.5":["avx512knl-i32x16"],
+                       "3.5":["avx512knl-i32x16", "avx512skx-i32x16"],
-                       "3.6":["avx512knl-i32x16"],
+                       "3.6":["avx512knl-i32x16", "avx512skx-i32x16"],
-                       "3.7":[],
+                       "3.7":["avx512skx-i32x16"],
                        "3.8":[],
                        "3.9":[],
+                       "4.0":[],
+                       "5.0":[],
                        "trunk":[]}   
     return prohibited_list[LLVM_VERSION]
 
@@ -377,9 +422,10 @@ def check_targets():
     AVX11 = ["avx1.1-i32x8","avx1.1-i32x16","avx1.1-i64x4"]
     AVX2  = ["avx2-i32x8",  "avx2-i32x16",  "avx2-i64x4"]
     KNL   = ["knl-generic", "avx512knl-i32x16"]
+    SKX   = ["avx512skx-i32x16"]
 
     targets = [["AVX2", AVX2, False], ["AVX1.1", AVX11, False], ["AVX", AVX, False], ["SSE4", SSE4, False], 
-               ["SSE2", SSE2, False], ["KNL", KNL, False]]
+               ["SSE2", SSE2, False], ["KNL", KNL, False], ["SKX", SKX, False]]
     f_lines = take_lines("check_isa.exe", "first")
     for i in range(0,5):
         if targets[i][0] in f_lines:
@@ -403,6 +449,8 @@ def check_targets():
     # here we have SDE
     f_lines = take_lines(sde_exists + " -help", "all")
     for i in range(0,len(f_lines)):
+        if targets[6][2] == False and "skx" in f_lines[i]:
+            answer_sde = answer_sde + [["-skx", "avx512skx-i32x16"]]
         if targets[5][2] == False and "knl" in f_lines[i]:
             answer_sde = answer_sde + [["-knl", "knl-generic"], ["-knl", "avx512knl-i32x16"]]
         if targets[3][2] == False and "wsm" in f_lines[i]:
@@ -472,8 +520,12 @@ def build_ispc(version_LLVM, make):
             temp = "3_7"
         if version_LLVM == "3.8":
             temp = "3_8"
-        if version_LLVM == "trunk":
+        if version_LLVM == "3.9":
             temp = "3_9"
+        if version_LLVM == "4.0":
+            temp = "4_0"
+        if version_LLVM == "trunk":
+            temp = "5_0"
         os.environ["LLVM_VERSION"] = "LLVM_" + temp
         try_do_LLVM("clean ISPC for building", "msbuild ispc.vcxproj /t:clean", True)
         try_do_LLVM("build ISPC with LLVM version " + version_LLVM + " ", "msbuild ispc.vcxproj /V:m /p:Platform=Win32 /p:Configuration=Release /t:rebuild", True)
@@ -613,7 +665,7 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
             archs.append("x86-64")
         if "native" in only:
             sde_targets_t = []
-        for i in ["3.2", "3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "trunk"]:
+        for i in ["3.2", "3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0", "trunk"]:
             if i in only:
                 LLVM.append(i)
         if "current" in only:
@@ -671,7 +723,7 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
         gen_archs = ["x86-64"]
         need_LLVM = check_LLVM(LLVM)
         for i in range(0,len(need_LLVM)):
-            build_LLVM(need_LLVM[i], "", "", "", False, False, False, True, False, make, options.gcc_toolchain_path)
+            build_LLVM(need_LLVM[i], "", "", "", False, False, False, True, False, make, options.gcc_toolchain_path, False)
 # begin validation run for stabitily
         common.remove_if_exists(stability.in_file)
         R = [[[],[]],[[],[]],[[],[]],[[],[]]]
@@ -727,7 +779,10 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
                 # *always* specify default values for global variables on each loop iteration
                 stability.wrapexe = ""
                 stability.compiler_exe = None
-
+                # choosing right compiler for a given target
+                # sometimes clang++ is not avaluable. if --ispc-build-compiler = gcc we will pass in g++ compiler
+                if options.ispc_build_compiler == "gcc":
+                    stability.compiler_exe = "g++"
                 if ("knc-generic" in stability.target) or ("knl-generic" in stability.target):
                     stability.compiler_exe = "icpc"
                 stability.wrapexe = get_sde() + " " + sde_targets[j][0] + " -- "
@@ -782,7 +837,7 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
 # prepare newest LLVM
         need_LLVM = check_LLVM([newest_LLVM])
         if len(need_LLVM) != 0:
-            build_LLVM(need_LLVM[0], "", "", "", False, False, False, True, False, make, options.gcc_toolchain_path)
+            build_LLVM(need_LLVM[0], "", "", "", False, False, False, True, False, make, options.gcc_toolchain_path, options.use_git)
         if perf_llvm == False:
             # prepare reference point. build both test and reference compilers
             try_do_LLVM("apply git", "git branch", True)
@@ -896,7 +951,7 @@ def Main():
         if os.environ.get("SMTP_ISPC") == None:
             error("you have no SMTP_ISPC in your environment for option notify", 1)
     if options.only != "":
-        test_only_r = " 3.2 3.3 3.4 3.5 3.6 3.7 3.8 trunk current build stability performance x86 x86-64 x86_64 -O0 -O2 native debug nodebug "
+        test_only_r = " 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 4.0 trunk current build stability performance x86 x86-64 x86_64 -O0 -O2 native debug nodebug "
         test_only = options.only.split(" ")
         for iterator in test_only:
             if not (" " + iterator + " " in test_only_r):
@@ -922,11 +977,14 @@ def Main():
     if options.perf_llvm == True:
         if options.branch == "master":
             options.branch = "trunk"
+    if options.use_git and options.revision != "":
+        error("--revision is not supported with --git", 1)
+
     try:
         start_time = time.time()
         if options.build_llvm:
             build_LLVM(options.version, options.revision, options.folder, options.tarball,
-                    options.debug, options.selfbuild, options.extra, False, options.force, make, options.gcc_toolchain_path)
+                    options.debug, options.selfbuild, options.extra, False, options.force, make, options.gcc_toolchain_path, options.use_git)
         if options.validation_run:
             validation_run(options.only, options.only_targets, options.branch,
                     options.number_for_performance, options.notify, options.update, int(options.speed),
@@ -1006,13 +1064,13 @@ if __name__ == '__main__':
     llvm_group = OptionGroup(parser, "Options for building LLVM",
                     "These options must be used with -b option.")
     llvm_group.add_option('--version', dest='version',
-        help='version of llvm to build: 3.2 3.3 3.4 3.5 3.6 3.7 3.8 trunk. Default: trunk', default="trunk")
+        help='version of llvm to build: 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 4.0 trunk. Default: trunk', default="trunk")
     llvm_group.add_option('--with-gcc-toolchain', dest='gcc_toolchain_path',
          help='GCC install dir to use when building clang. It is important to set when ' +
          'you have alternative gcc installation. Note that otherwise gcc from standard ' +
          'location will be used, not from your PATH', default="")
     llvm_group.add_option('--revision', dest='revision',
-        help='revision of llvm to build in format r172870', default="")
+        help='revision of llvm to build in format r172870 (not supported with --git)', default="")
     llvm_group.add_option('--debug', dest='debug',
         help='debug build of LLVM?', default=False, action="store_true")
     llvm_group.add_option('--folder', dest='folder',
@@ -1025,6 +1083,8 @@ if __name__ == '__main__':
         help='rebuild LLVM', default=False, action='store_true')
     llvm_group.add_option('--extra', dest='extra',
         help='load extra clang tools', default=False, action='store_true')
+    llvm_group.add_option('--git', dest='use_git',
+        help='use git llvm repository instead of svn', default=False, action='store_true')
     parser.add_option_group(llvm_group)
     # options for activity "validation run"
     run_group = OptionGroup(parser, "Options for validation run",
@@ -1047,7 +1107,7 @@ if __name__ == '__main__':
     run_group.add_option('--only', dest='only',
         help='set types of tests. Possible values:\n' + 
             '-O0, -O2, x86, x86-64, stability (test only stability), performance (test only performance),\n' +
-            'build (only build with different LLVM), 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, trunk, native (do not use SDE),\n' +
+            'build (only build with different LLVM), 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 4.0, trunk, native (do not use SDE),\n' +
             'current (do not rebuild ISPC), debug (only with debug info), nodebug (only without debug info, default).',
             default="")
     run_group.add_option('--perf_LLVM', dest='perf_llvm',

builtins.cpp

@@ -72,7 +72,11 @@
 #include <llvm/Target/TargetMachine.h>
 #include <llvm/ADT/Triple.h>
 #include <llvm/Support/MemoryBuffer.h>
-#include <llvm/Bitcode/ReaderWriter.h>
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
+    #include <llvm/Bitcode/ReaderWriter.h>
+#else
+    #include <llvm/Bitcode/BitcodeReader.h>
+#endif
 
 extern int yyparse();
 struct yy_buffer_state;
@@ -445,6 +449,8 @@ lSetInternalFunctions(llvm::Module *module) {
         "__broadcast_i32",
         "__broadcast_i64",
         "__broadcast_i8",
+        "__cast_mask_to_i1",
+        "__cast_mask_to_i16",
         "__ceil_uniform_double",
         "__ceil_uniform_float",
         "__ceil_varying_double",
@@ -482,6 +488,8 @@ lSetInternalFunctions(llvm::Module *module) {
         "__extract_float",
         "__extract_double",
 //#endif /* ISPC_NVPTX_ENABLED */
+        "__extract_mask_low",
+        "__extract_mask_hi",
         "__fastmath",
         "__float_to_half_uniform",
         "__float_to_half_varying",
@@ -796,7 +804,13 @@ AddBitcodeToModule(const unsigned char *bitcode, int length,
     llvm::MemoryBufferRef bcBuf = llvm::MemoryBuffer::getMemBuffer(sb)->getMemBufferRef();
 #endif
 
-#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 // LLVM 3.7+
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_4_0 // LLVM 4.0+
+    llvm::Expected<std::unique_ptr<llvm::Module>> ModuleOrErr = llvm::parseBitcodeFile(bcBuf, *g->ctx);
+    if (!ModuleOrErr) {
+        Error(SourcePos(), "Error parsing stdlib bitcode: %s", toString(ModuleOrErr.takeError()).c_str());
+    } else {
+        llvm::Module *bcModule = ModuleOrErr.get().release();
+#elif ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 // LLVM 3.7+
     llvm::ErrorOr<std::unique_ptr<llvm::Module>> ModuleOrErr = llvm::parseBitcodeFile(bcBuf, *g->ctx);
     if (std::error_code EC = ModuleOrErr.getError())
         Error(SourcePos(), "Error parsing stdlib bitcode: %s", EC.message().c_str());
@@ -985,7 +999,7 @@ lDefineConstantInt(const char *name, int val, llvm::Module *module,
                                                diType,
                                                true /* static */,
                                                sym_const_storagePtr);
-#else // LLVM 3.7+
+#elif ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 && ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // LLVM 3.7 - 3.9
     llvm::Constant *sym_const_storagePtr = llvm::dyn_cast<llvm::Constant>(sym->storagePtr);
     Assert(sym_const_storagePtr);
     m->diBuilder->createGlobalVariable(
@@ -997,6 +1011,17 @@ lDefineConstantInt(const char *name, int val, llvm::Module *module,
               diType,
               true /* static */,
               sym_const_storagePtr);
+#else // LLVM 4.0+
+        llvm::GlobalVariable *sym_GV_storagePtr = llvm::dyn_cast<llvm::GlobalVariable>(sym->storagePtr);
+        llvm::DIGlobalVariableExpression *var = m->diBuilder->createGlobalVariableExpression(
+                                              file,
+                                              name,
+                                              name,
+                                              file,
+                                              0 /* line */,
+                                              diType,
+                                              true /* static */);
+        sym_GV_storagePtr->addDebugInfo(var);
 #endif
 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_6
         Assert(var.Verify());
@@ -1083,7 +1108,7 @@ lDefineProgramIndex(llvm::Module *module, SymbolTable *symbolTable) {
                                                diType,
                                                false /* static */,
                                                sym->storagePtr);
-#else // LLVM 3.7+
+#elif ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 && ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // LLVM 3.7 - 3.9
         llvm::Constant *sym_const_storagePtr = llvm::dyn_cast<llvm::Constant>(sym->storagePtr);
         Assert(sym_const_storagePtr);
         m->diBuilder->createGlobalVariable(
@@ -1095,6 +1120,17 @@ lDefineProgramIndex(llvm::Module *module, SymbolTable *symbolTable) {
                                                diType,
                                                false /* static */,
                                                sym_const_storagePtr);
+#else // LLVM 4.0+
+        llvm::GlobalVariable *sym_GV_storagePtr = llvm::dyn_cast<llvm::GlobalVariable>(sym->storagePtr);
+        llvm::DIGlobalVariableExpression *var = m->diBuilder->createGlobalVariableExpression(
+                                              file,
+                                              sym->name.c_str(),
+                                              sym->name.c_str(),
+                                              file,
+                                              0 /* line */,
+                                              diType,
+                                              false /* static */);
+        sym_GV_storagePtr->addDebugInfo(var);
 #endif
 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_6
         Assert(var.Verify());
@@ -1377,6 +1413,23 @@ DefineStdlib(SymbolTable *symbolTable, llvm::LLVMContext *ctx, llvm::Module *mod
         }
         break;
     }
+#endif
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // LLVM 3.8+
+    case Target::SKX_AVX512: {
+        switch (g->target->getVectorWidth()) {
+        case 16:
+            if (runtime32) {
+                EXPORT_MODULE(builtins_bitcode_skx_32bit);
+            }
+            else {
+                EXPORT_MODULE(builtins_bitcode_skx_64bit);
+            }
+            break;
+        default:
+            FATAL("logic error in DefineStdlib");
+        }
+        break;
+    }
 #endif
     case Target::GENERIC: {
         switch (g->target->getVectorWidth()) {

builtins/dispatch.ll

@@ -151,6 +151,10 @@ define(`PTR_OP_ARGS',
          LLVM_VERSION, LLVM_3_8,
     ``$1 , $1 *'',
          LLVM_VERSION, LLVM_3_9,
+    ``$1 , $1 *'',
+         LLVM_VERSION, LLVM_4_0,
+    ``$1 , $1 *'',
+         LLVM_VERSION, LLVM_5_0,
     ``$1 , $1 *'',
     ``$1 *''
   )

builtins/target-avx512-common.ll

@@ -29,7 +29,7 @@
 ;;   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 ;;   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.  
 
-define(`MASK',`i1')
+define(`MASK',`i8')
 define(`HAVE_GATHER',`1')
 define(`HAVE_SCATTER',`1')
 
@@ -50,6 +50,33 @@ define_shuffles()
 
 aossoa()
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Stub for mask conversion. LLVM's intrinsics want i1 mask, but we use i8
+
+define <WIDTH x i1> @__cast_mask_to_i1 (<WIDTH x MASK> %mask) alwaysinline {
+  %mask_vec_i1 = icmp ne <WIDTH x MASK> %mask, const_vector(MASK, 0)
+  ret <WIDTH x i1> %mask_vec_i1
+}
+
+define i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask) alwaysinline {
+  %mask_i1 = call <WIDTH x i1> @__cast_mask_to_i1 (<WIDTH x MASK> %mask)
+  %mask_i16 = bitcast <WIDTH x i1> %mask_i1 to i16
+  ret i16 %mask_i16
+}
+
+define i8 @__extract_mask_low (<WIDTH x MASK> %mask) alwaysinline {
+  %mask_i16 = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
+  %mask_low = trunc i16 %mask_i16 to i8 
+  ret i8 %mask_low
+}
+
+define i8 @__extract_mask_hi (<WIDTH x MASK> %mask) alwaysinline {
+  %mask_i16 = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
+  %mask_shifted = lshr i16 %mask_i16, 8
+  %mask_hi = trunc i16 %mask_shifted to i8
+  ret i8 %mask_hi
+}
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; half conversion routines
 
@@ -511,13 +538,6 @@ define float @__rsqrt_uniform_float(float) nounwind readonly alwaysinline {
   ret float %half_scale
 }
 
-declare <16 x float> @llvm.x86.avx512.rsqrt28.ps(<16 x float>, <16 x float>, i16, i32) nounwind readnone
-
-define <16 x float> @__rsqrt_varying_float(<16 x float> %v) nounwind readonly alwaysinline {
-  %res = call <16 x float> @llvm.x86.avx512.rsqrt28.ps(<16 x float> %v, <16 x float> undef, i16 -1, i32 8)
-  ret <16 x float> %res  
-}
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; rcp
 
@@ -538,13 +558,6 @@ define float @__rcp_uniform_float(float) nounwind readonly alwaysinline {
   ret float %iv_mul
 }
 
-declare <16 x float> @llvm.x86.avx512.rcp28.ps(<16 x float>, <16 x float>, i16, i32) nounwind readnone
-
-define <16 x float> @__rcp_varying_float(<16 x float>) nounwind readonly alwaysinline {
-  %res = call <16 x float> @llvm.x86.avx512.rcp28.ps(<16 x float> %0, <16 x float> undef, i16 -1, i32 8)
-  ret <16 x float> %res
-}
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; sqrt
 
@@ -604,40 +617,42 @@ define i64 @__popcnt_int64(i64) nounwind readonly alwaysinline {
 }
 ctlztz()
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; TODO: should we use masked versions of SVML functions?
-; FIXME: need either to wire these up to the 8-wide SVML entrypoints,
-; or, use the macro to call the 4-wide ones twice with our 8-wide
-; vectors...
-
 ;; svml
 
 include(`svml.m4')
-svml_stubs(float,f,WIDTH)
+svml_declare(float,f16,16)
-svml_stubs(double,d,WIDTH)
+svml_define(float,f16,16,f)
+
+;; double precision
+svml_declare(double,8,8)
+svml_define_x(double,8,8,d,16)
+
+
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; reductions
 
-define i64 @__movmsk(<WIDTH x i1>) nounwind readnone alwaysinline {
+define i64 @__movmsk(<WIDTH x MASK> %mask) nounwind readnone alwaysinline {
-  %intmask = bitcast <WIDTH x i1> %0 to i16
+  %intmask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
   %res = zext i16 %intmask to i64
   ret i64 %res
 }
 
-define i1 @__any(<WIDTH x i1>) nounwind readnone alwaysinline {
+define i1 @__any(<WIDTH x MASK> %mask) nounwind readnone alwaysinline {
-  %intmask = bitcast <WIDTH x i1> %0 to i16
+  %intmask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
   %res = icmp ne i16 %intmask, 0
   ret i1 %res
 }
 
-define i1 @__all(<WIDTH x i1>) nounwind readnone alwaysinline {
+define i1 @__all(<WIDTH x MASK> %mask) nounwind readnone alwaysinline {
-  %intmask = bitcast <WIDTH x i1> %0 to i16
+  %intmask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
   %res = icmp eq i16 %intmask, 65535
   ret i1 %res
 }
 
-define i1 @__none(<WIDTH x i1>) nounwind readnone alwaysinline {
+define i1 @__none(<WIDTH x MASK> %mask) nounwind readnone alwaysinline {
-  %intmask = bitcast <WIDTH x i1> %0 to i16
+  %intmask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
   %res = icmp eq i16 %intmask, 0
   ret i1 %res
 }
@@ -813,19 +828,16 @@ masked_load(i8,  1)
 masked_load(i16, 2)
 
 declare <16 x i32> @llvm.x86.avx512.mask.loadu.d.512(i8*, <16 x i32>, i16)
-define <16 x i32> @__masked_load_i32(i8 * %ptr, <16 x i1> %mask) nounwind alwaysinline {
+define <16 x i32> @__masked_load_i32(i8 * %ptr, <WIDTH x MASK> %mask) nounwind alwaysinline {
-  %mask_i16 = bitcast <16 x i1> %mask to i16
+  %mask_i16 = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
   %res = call <16 x i32> @llvm.x86.avx512.mask.loadu.d.512(i8* %ptr, <16 x i32> zeroinitializer, i16 %mask_i16)
   ret <16 x i32> %res
 }
 
 declare <8 x i64> @llvm.x86.avx512.mask.loadu.q.512(i8*, <8 x i64>, i8)
-define <16 x i64> @__masked_load_i64(i8 * %ptr, <16 x i1> %mask) nounwind alwaysinline {
+define <16 x i64> @__masked_load_i64(i8 * %ptr, <WIDTH x MASK> %mask) nounwind alwaysinline {
-  %mask_i16 = bitcast <16 x i1> %mask to i16
+  %mask_lo_i8 = call i8 @__extract_mask_low (<WIDTH x MASK> %mask) 
-  %mask_lo_i8 = trunc i16 %mask_i16 to i8
+  %mask_hi_i8 = call i8 @__extract_mask_hi (<WIDTH x MASK> %mask)
-  %mask_hi = shufflevector <16 x i1> %mask, <16 x i1> undef,
-                           <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-  %mask_hi_i8 = bitcast <8 x i1> %mask_hi to i8
   
   %ptr_d = bitcast i8* %ptr to <16 x i64>*
   %ptr_hi = getelementptr PTR_OP_ARGS(`<16 x i64>') %ptr_d, i32 0, i32 8
@@ -842,19 +854,16 @@ define <16 x i64> @__masked_load_i64(i8 * %ptr, <16 x i1> %mask) nounwind always
 
 
 declare <16 x float> @llvm.x86.avx512.mask.loadu.ps.512(i8*, <16 x float>, i16)
-define <16 x float> @__masked_load_float(i8 * %ptr, <16 x i1> %mask) readonly alwaysinline {
+define <16 x float> @__masked_load_float(i8 * %ptr, <WIDTH x MASK> %mask) readonly alwaysinline {
-  %mask_i16 = bitcast <16 x i1> %mask to i16
+  %mask_i16 = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
   %res = call <16 x float> @llvm.x86.avx512.mask.loadu.ps.512(i8* %ptr, <16 x float> zeroinitializer, i16 %mask_i16)
   ret <16 x float> %res
 }
 
 declare <8 x double> @llvm.x86.avx512.mask.loadu.pd.512(i8*, <8 x double>, i8)
-define <16 x double> @__masked_load_double(i8 * %ptr, <16 x i1> %mask) readonly alwaysinline {
+define <16 x double> @__masked_load_double(i8 * %ptr, <WIDTH x MASK> %mask) readonly alwaysinline {
-  %mask_i16 = bitcast <16 x i1> %mask to i16
+  %mask_lo_i8 = call i8 @__extract_mask_low (<WIDTH x MASK> %mask)
-  %mask_lo_i8 = trunc i16 %mask_i16 to i8
+  %mask_hi_i8 = call i8 @__extract_mask_hi (<WIDTH x MASK> %mask)
-  %mask_hi = shufflevector <16 x i1> %mask, <16 x i1> undef,
-                           <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-  %mask_hi_i8 = bitcast <8 x i1> %mask_hi to i8
 
   %ptr_d = bitcast i8* %ptr to <16 x double>*
   %ptr_hi = getelementptr PTR_OP_ARGS(`<16 x double>') %ptr_d, i32 0, i32 8
@@ -874,20 +883,17 @@ gen_masked_store(i8) ; llvm.x86.sse2.storeu.dq
 gen_masked_store(i16)
 
 declare void @llvm.x86.avx512.mask.storeu.d.512(i8*, <16 x i32>, i16)
-define void @__masked_store_i32(<16 x i32>* nocapture, <16 x i32> %v, <16 x i1> %mask) nounwind alwaysinline {
+define void @__masked_store_i32(<16 x i32>* nocapture, <16 x i32> %v, <WIDTH x MASK> %mask) nounwind alwaysinline {
-  %mask_i16 = bitcast <16 x i1> %mask to i16
+  %mask_i16 = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
   %ptr_i8 = bitcast <16 x i32>* %0 to i8*
   call void @llvm.x86.avx512.mask.storeu.d.512(i8* %ptr_i8, <16 x i32> %v, i16 %mask_i16)
   ret void
 }
 
 declare void @llvm.x86.avx512.mask.storeu.q.512(i8*, <8 x i64>, i8)
-define void @__masked_store_i64(<16 x i64>* nocapture, <16 x i64> %v, <16 x i1> %mask) nounwind alwaysinline {
+define void @__masked_store_i64(<16 x i64>* nocapture, <16 x i64> %v, <WIDTH x MASK> %mask) nounwind alwaysinline {
-  %mask_i16 = bitcast <16 x i1> %mask to i16
+  %mask_lo_i8 = call i8 @__extract_mask_low (<WIDTH x MASK> %mask)
-  %mask_lo_i8 = trunc i16 %mask_i16 to i8
+  %mask_hi_i8 = call i8 @__extract_mask_hi (<WIDTH x MASK> %mask)
-  %mask_hi = shufflevector <16 x i1> %mask, <16 x i1> undef,
-                           <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-  %mask_hi_i8 = bitcast <8 x i1> %mask_hi to i8
 
   %ptr_i8 = bitcast <16 x i64>* %0 to i8*
   %ptr_lo = getelementptr PTR_OP_ARGS(`<16 x i64>') %0, i32 0, i32 8
@@ -904,20 +910,18 @@ define void @__masked_store_i64(<16 x i64>* nocapture, <16 x i64> %v, <16 x i1>
 }
 
 declare void @llvm.x86.avx512.mask.storeu.ps.512(i8*, <16 x float>, i16 )
-define void @__masked_store_float(<16 x float>* nocapture, <16 x float> %v, <16 x i1> %mask) nounwind alwaysinline {
+define void @__masked_store_float(<16 x float>* nocapture, <16 x float> %v, <WIDTH x MASK> %mask) nounwind alwaysinline {
-  %mask_i16 = bitcast <16 x i1> %mask to i16
+  %mask_i16 = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %mask)
+
   %ptr_i8 = bitcast <16 x float>* %0 to i8*
   call void @llvm.x86.avx512.mask.storeu.ps.512(i8* %ptr_i8, <16 x float> %v, i16 %mask_i16)
   ret void
 }
 
 declare void @llvm.x86.avx512.mask.storeu.pd.512(i8*, <8 x double>, i8)
-define void @__masked_store_double(<16 x double>* nocapture, <16 x double> %v, <16 x i1> %mask) nounwind alwaysinline {
+define void @__masked_store_double(<16 x double>* nocapture, <16 x double> %v, <WIDTH x MASK> %mask) nounwind alwaysinline {
-  %mask_i16 = bitcast <16 x i1> %mask to i16
+  %mask_lo_i8 = call i8 @__extract_mask_low (<WIDTH x MASK> %mask)
-  %mask_lo_i8 = trunc i16 %mask_i16 to i8
+  %mask_hi_i8 = call i8 @__extract_mask_hi (<WIDTH x MASK> %mask)
-  %mask_hi = shufflevector <16 x i1> %mask, <16 x i1> undef,
-                           <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-  %mask_hi_i8 = bitcast <8 x i1> %mask_hi to i8
 
   %ptr_i8 = bitcast <16 x double>* %0 to i8*
   %ptr_lo = getelementptr PTR_OP_ARGS(`<16 x double>') %0, i32 0, i32 8
@@ -934,49 +938,55 @@ define void @__masked_store_double(<16 x double>* nocapture, <16 x double> %v, <
 }
 
 define void @__masked_store_blend_i8(<16 x i8>* nocapture, <16 x i8>, 
-                                     <16 x i1>) nounwind alwaysinline {
+                                     <WIDTH x MASK>) nounwind alwaysinline {
   %v = load PTR_OP_ARGS(`<16 x i8> ')  %0
-  %v1 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %v
+  %mask_vec_i1 = call <WIDTH x i1> @__cast_mask_to_i1 (<WIDTH x MASK> %2)
+  %v1 = select <WIDTH x i1> %mask_vec_i1, <16 x i8> %1, <16 x i8> %v
   store <16 x i8> %v1, <16 x i8> * %0
   ret void
 }
 
 define void @__masked_store_blend_i16(<16 x i16>* nocapture, <16 x i16>, 
-                                      <16 x i1>) nounwind alwaysinline {
+                                      <WIDTH x MASK>) nounwind alwaysinline {
   %v = load PTR_OP_ARGS(`<16 x i16> ')  %0
-  %v1 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %v
+  %mask_vec_i1 = call <WIDTH x i1> @__cast_mask_to_i1 (<WIDTH x MASK> %2)
+  %v1 = select <WIDTH x i1> %mask_vec_i1, <16 x i16> %1, <16 x i16> %v
   store <16 x i16> %v1, <16 x i16> * %0
   ret void
 }
 
 define void @__masked_store_blend_i32(<16 x i32>* nocapture, <16 x i32>, 
-                                      <16 x i1>) nounwind alwaysinline {
+                                      <WIDTH x MASK>) nounwind alwaysinline {
   %v = load PTR_OP_ARGS(`<16 x i32> ')  %0
-  %v1 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %v
+  %mask_vec_i1 = call <WIDTH x i1> @__cast_mask_to_i1 (<WIDTH x MASK> %2)
+  %v1 = select <WIDTH x i1> %mask_vec_i1, <16 x i32> %1, <16 x i32> %v
   store <16 x i32> %v1, <16 x i32> * %0
   ret void
 }
 
 define void @__masked_store_blend_float(<16 x float>* nocapture, <16 x float>, 
-                                        <16 x i1>) nounwind alwaysinline {
+                                        <WIDTH x MASK>) nounwind alwaysinline {
   %v = load PTR_OP_ARGS(`<16 x float> ')  %0
-  %v1 = select <16 x i1> %2, <16 x float> %1, <16 x float> %v
+  %mask_vec_i1 = call <WIDTH x i1> @__cast_mask_to_i1 (<WIDTH x MASK> %2)
+  %v1 = select <WIDTH x i1> %mask_vec_i1, <16 x float> %1, <16 x float> %v
   store <16 x float> %v1, <16 x float> * %0
   ret void
 }
 
 define void @__masked_store_blend_i64(<16 x i64>* nocapture,
-                            <16 x i64>, <16 x i1>) nounwind alwaysinline {
+                            <16 x i64>, <WIDTH x MASK>) nounwind alwaysinline {
   %v = load PTR_OP_ARGS(`<16 x i64> ')  %0
-  %v1 = select <16 x i1> %2, <16 x i64> %1, <16 x i64> %v
+  %mask_vec_i1 = call <WIDTH x i1> @__cast_mask_to_i1 (<WIDTH x MASK> %2)
+  %v1 = select <WIDTH x i1> %mask_vec_i1, <16 x i64> %1, <16 x i64> %v
   store <16 x i64> %v1, <16 x i64> * %0
   ret void
 }
 
 define void @__masked_store_blend_double(<16 x double>* nocapture,
-                            <16 x double>, <16 x i1>) nounwind alwaysinline {
+                            <16 x double>, <WIDTH x MASK>) nounwind alwaysinline {
   %v = load PTR_OP_ARGS(`<16 x double> ')  %0
-  %v1 = select <16 x i1> %2, <16 x double> %1, <16 x double> %v
+  %mask_vec_i1 = call <WIDTH x i1> @__cast_mask_to_i1 (<WIDTH x MASK> %2)
+  %v1 = select <WIDTH x i1> %mask_vec_i1, <16 x double> %1, <16 x double> %v
   store <16 x double> %v1, <16 x double> * %0
   ret void
 }
@@ -993,16 +1003,16 @@ gen_gather(i16)
 ;; gather - i32
 declare <16 x i32> @llvm.x86.avx512.gather.dpi.512(<16 x i32>, i8*, <16 x i32>, i16, i32)
 define <16 x i32> 
-@__gather_base_offsets32_i32(i8 * %ptr, i32 %offset_scale, <16 x i32> %offsets, <16 x i1> %vecmask) nounwind readonly alwaysinline {
+@__gather_base_offsets32_i32(i8 * %ptr, i32 %offset_scale, <16 x i32> %offsets, <WIDTH x MASK> %vecmask) nounwind readonly alwaysinline {
-  %mask = bitcast <16 x i1> %vecmask to i16
+  %mask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %vecmask)
   %res =  call <16 x i32> @llvm.x86.avx512.gather.dpi.512 (<16 x i32> undef, i8* %ptr, <16 x i32> %offsets, i16 %mask, i32 %offset_scale)
   ret <16 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.gather.qpi.512 (<8 x i32>, i8*, <8 x i64>, i8, i32)
 define <16 x i32> 
-@__gather_base_offsets64_i32(i8 * %ptr, i32 %offset_scale, <16 x i64> %offsets, <16 x i1> %vecmask) nounwind readonly alwaysinline {
+@__gather_base_offsets64_i32(i8 * %ptr, i32 %offset_scale, <16 x i64> %offsets, <WIDTH x MASK> %vecmask) nounwind readonly alwaysinline {
-  %scalarMask = bitcast <16 x i1> %vecmask to i16 
+  %scalarMask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %vecmask) 
   %scalarMask1 = trunc i16 %scalarMask to i8 
   %scalarMask2Tmp = lshr i16 %scalarMask, 8
   %scalarMask2 = trunc i16  %scalarMask2Tmp to i8 
@@ -1015,14 +1025,14 @@ define <16 x i32>
 }
 
 define <16 x i32> 
-@__gather32_i32(<16 x i32> %ptrs, <16 x i1> %vecmask) nounwind readonly alwaysinline {
+@__gather32_i32(<16 x i32> %ptrs, <WIDTH x MASK> %vecmask) nounwind readonly alwaysinline {
-  %res = call <16 x i32> @__gather_base_offsets32_i32(i8 * zeroinitializer, i32 1, <16 x i32> %ptrs, <16 x i1> %vecmask)
+  %res = call <16 x i32> @__gather_base_offsets32_i32(i8 * zeroinitializer, i32 1, <16 x i32> %ptrs, <WIDTH x MASK> %vecmask)
   ret <16 x i32> %res
 }
 
 define <16 x i32> 
-@__gather64_i32(<16x i64> %ptrs, <16 x i1> %vecmask) nounwind readonly alwaysinline {
+@__gather64_i32(<16x i64> %ptrs, <WIDTH x MASK> %vecmask) nounwind readonly alwaysinline {
-  %res = call <16 x i32> @__gather_base_offsets64_i32(i8 * zeroinitializer, i32 1, <16 x i64> %ptrs, <16 x i1> %vecmask)
+  %res = call <16 x i32> @__gather_base_offsets64_i32(i8 * zeroinitializer, i32 1, <16 x i64> %ptrs, <WIDTH x MASK> %vecmask)
   ret <16 x i32> %res
 }
 
@@ -1032,16 +1042,16 @@ gen_gather(i64)
 ;; gather - float
 declare <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float>, i8*, <16 x i32>, i16, i32)
 define <16 x float>
-@__gather_base_offsets32_float(i8 * %ptr, i32 %offset_scale, <16 x i32> %offsets, <16 x i1> %vecmask) nounwind readonly alwaysinline {
+@__gather_base_offsets32_float(i8 * %ptr, i32 %offset_scale, <16 x i32> %offsets, <WIDTH x MASK> %vecmask) nounwind readonly alwaysinline {
-  %mask = bitcast <16 x i1> %vecmask to i16
+  %mask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %vecmask)
   %res = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> undef, i8* %ptr, <16 x i32>%offsets, i16 %mask, i32 %offset_scale)
   ret <16 x float> %res
 }
 
 declare <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float>, i8*, <8 x i64>, i8, i32)
 define <16 x float>
-@__gather_base_offsets64_float(i8 * %ptr, i32 %offset_scale, <16 x i64> %offsets, <16 x i1> %vecmask) nounwind readonly alwaysinline {
+@__gather_base_offsets64_float(i8 * %ptr, i32 %offset_scale, <16 x i64> %offsets, <WIDTH x MASK> %vecmask) nounwind readonly alwaysinline {
-  %mask = bitcast <16 x i1> %vecmask to i16
+  %mask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %vecmask)
   %mask_shifted = lshr i16 %mask, 8
   %mask_lo = trunc i16 %mask to i8 
   %mask_hi = trunc i16 %mask_shifted to i8 
@@ -1054,14 +1064,14 @@ define <16 x float>
 }
 
 define <16 x float> 
-@__gather32_float(<16 x i32> %ptrs, <16 x i1> %vecmask) nounwind readonly alwaysinline {
+@__gather32_float(<16 x i32> %ptrs, <WIDTH x MASK> %vecmask) nounwind readonly alwaysinline {
-  %res = call <16 x float> @__gather_base_offsets32_float(i8 * zeroinitializer, i32 1, <16 x i32> %ptrs, <16 x i1> %vecmask)
+  %res = call <16 x float> @__gather_base_offsets32_float(i8 * zeroinitializer, i32 1, <16 x i32> %ptrs, <WIDTH x MASK> %vecmask)
   ret <16 x float> %res
 }
 
 define <16 x float> 
-@__gather64_float(<16 x i64> %ptrs,  <16 x i1> %vecmask) nounwind readonly alwaysinline {
+@__gather64_float(<16 x i64> %ptrs,  <WIDTH x MASK> %vecmask) nounwind readonly alwaysinline {
-  %res = call <16 x float> @__gather_base_offsets64_float(i8 * zeroinitializer, i32 1, <16 x i64> %ptrs, <16 x i1> %vecmask)
+  %res = call <16 x float> @__gather_base_offsets64_float(i8 * zeroinitializer, i32 1, <16 x i64> %ptrs, <WIDTH x MASK> %vecmask)
   ret <16 x float> %res
 }
 
@@ -1071,16 +1081,16 @@ gen_gather(double)
 
 define(`scatterbo32_64', `
 define void @__scatter_base_offsets32_$1(i8* %ptr, i32 %scale, <WIDTH x i32> %offsets,
-                                         <WIDTH x $1> %vals, <WIDTH x i1> %mask) nounwind {
+                                         <WIDTH x $1> %vals, <WIDTH x MASK> %mask) nounwind {
   call void @__scatter_factored_base_offsets32_$1(i8* %ptr, <16 x i32> %offsets,
-      i32 %scale, <16 x i32> zeroinitializer, <16 x $1> %vals, <WIDTH x i1> %mask)
+      i32 %scale, <16 x i32> zeroinitializer, <16 x $1> %vals, <WIDTH x MASK> %mask)
   ret void
 }
 
 define void @__scatter_base_offsets64_$1(i8* %ptr, i32 %scale, <WIDTH x i64> %offsets,
-                                         <WIDTH x $1> %vals, <WIDTH x i1> %mask) nounwind {
+                                         <WIDTH x $1> %vals, <WIDTH x MASK> %mask) nounwind {
   call void @__scatter_factored_base_offsets64_$1(i8* %ptr, <16 x i64> %offsets,
-      i32 %scale, <16 x i64> zeroinitializer, <16 x $1> %vals, <WIDTH x i1> %mask)
+      i32 %scale, <16 x i64> zeroinitializer, <16 x $1> %vals, <WIDTH x MASK> %mask)
   ret void
 } 
 ')
@@ -1096,16 +1106,16 @@ gen_scatter(i16)
 ;; scatter - i32
 declare void @llvm.x86.avx512.scatter.dpi.512 (i8*, i16, <16 x i32>, <16 x i32>, i32)
 define void 
-@__scatter_base_offsets32_i32(i8* %ptr, i32 %offset_scale, <16 x i32> %offsets, <16 x i32> %vals, <16 x i1> %vecmask) nounwind {
+@__scatter_base_offsets32_i32(i8* %ptr, i32 %offset_scale, <16 x i32> %offsets, <16 x i32> %vals, <WIDTH x MASK> %vecmask) nounwind {
-  %mask = bitcast <16 x i1> %vecmask to i16
+  %mask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %vecmask)
   call void @llvm.x86.avx512.scatter.dpi.512 (i8* %ptr, i16 %mask, <16 x i32> %offsets, <16 x i32> %vals, i32 %offset_scale)
   ret void
 }
 
 declare void @llvm.x86.avx512.scatter.qpi.512 (i8*, i8, <8 x i64>, <8 x i32>, i32)
 define void 
-@__scatter_base_offsets64_i32(i8* %ptr, i32 %offset_scale, <16 x i64> %offsets, <16 x i32> %vals, <16 x i1> %vecmask) nounwind {
+@__scatter_base_offsets64_i32(i8* %ptr, i32 %offset_scale, <16 x i64> %offsets, <16 x i32> %vals, <WIDTH x MASK> %vecmask) nounwind {
-  %mask = bitcast <16 x i1> %vecmask to i16
+  %mask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %vecmask)
   %mask_shifted = lshr i16 %mask, 8
   %mask_lo = trunc i16 %mask to i8 
   %mask_hi = trunc i16 %mask_shifted to i8 
@@ -1119,14 +1129,14 @@ define void
 } 
 
 define void 
-@__scatter32_i32(<16 x i32> %ptrs, <16 x i32> %values, <16 x i1> %vecmask) nounwind alwaysinline {
+@__scatter32_i32(<16 x i32> %ptrs, <16 x i32> %values, <WIDTH x MASK> %vecmask) nounwind alwaysinline {
-  call void @__scatter_base_offsets32_i32(i8 * zeroinitializer, i32 1, <16 x i32> %ptrs, <16 x i32> %values, <16 x i1> %vecmask)
+  call void @__scatter_base_offsets32_i32(i8 * zeroinitializer, i32 1, <16 x i32> %ptrs, <16 x i32> %values, <WIDTH x MASK> %vecmask)
   ret void
 }
 
 define void 
-@__scatter64_i32(<16 x i64> %ptrs, <16 x i32> %values, <16 x i1> %vecmask) nounwind alwaysinline {
+@__scatter64_i32(<16 x i64> %ptrs, <16 x i32> %values, <WIDTH x MASK> %vecmask) nounwind alwaysinline {
-  call void @__scatter_base_offsets64_i32(i8 * zeroinitializer, i32 1, <16 x i64> %ptrs, <16 x i32> %values, <16 x i1> %vecmask)
+  call void @__scatter_base_offsets64_i32(i8 * zeroinitializer, i32 1, <16 x i64> %ptrs, <16 x i32> %values, <WIDTH x MASK> %vecmask)
   ret void
 }
 
@@ -1137,16 +1147,16 @@ gen_scatter(i64)
 ;; scatter - float
 declare void @llvm.x86.avx512.scatter.dps.512 (i8*, i16, <16 x i32>, <16 x float>, i32)
 define void 
-@__scatter_base_offsets32_float(i8* %ptr, i32 %offset_scale, <16 x i32> %offsets, <16 x float> %vals, <16 x i1> %vecmask) nounwind {
+@__scatter_base_offsets32_float(i8* %ptr, i32 %offset_scale, <16 x i32> %offsets, <16 x float> %vals, <WIDTH x MASK> %vecmask) nounwind {
-  %mask = bitcast <16 x i1> %vecmask to i16
+  %mask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %vecmask)
   call void @llvm.x86.avx512.scatter.dps.512 (i8* %ptr, i16 %mask, <16 x i32> %offsets, <16 x float> %vals, i32 %offset_scale)
   ret void
 }
 
 declare void @llvm.x86.avx512.scatter.qps.512 (i8*, i8, <8 x i64>, <8 x float>, i32)
 define void 
-@__scatter_base_offsets64_float(i8* %ptr, i32 %offset_scale, <16 x i64> %offsets, <16 x float> %vals, <16 x i1> %vecmask) nounwind {
+@__scatter_base_offsets64_float(i8* %ptr, i32 %offset_scale, <16 x i64> %offsets, <16 x float> %vals, <WIDTH x MASK> %vecmask) nounwind {
-  %mask = bitcast <16 x i1> %vecmask to i16
+  %mask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %vecmask)
   %mask_shifted = lshr i16 %mask, 8
   %mask_lo = trunc i16 %mask to i8 
   %mask_hi = trunc i16 %mask_shifted to i8 
@@ -1160,14 +1170,14 @@ define void
 } 
 
 define void 
-@__scatter32_float(<16 x i32> %ptrs, <16 x float> %values, <16 x i1> %vecmask) nounwind alwaysinline {
+@__scatter32_float(<16 x i32> %ptrs, <16 x float> %values, <WIDTH x MASK> %vecmask) nounwind alwaysinline {
-  call void @__scatter_base_offsets32_float(i8 * zeroinitializer, i32 1, <16 x i32> %ptrs, <16 x float> %values, <16 x i1> %vecmask)
+  call void @__scatter_base_offsets32_float(i8 * zeroinitializer, i32 1, <16 x i32> %ptrs, <16 x float> %values, <WIDTH x MASK> %vecmask)
   ret void
 }
 
 define void 
-@__scatter64_float(<16 x i64> %ptrs, <16 x float> %values, <16 x i1> %vecmask) nounwind alwaysinline {
+@__scatter64_float(<16 x i64> %ptrs, <16 x float> %values, <WIDTH x MASK> %vecmask) nounwind alwaysinline {
-  call void @__scatter_base_offsets64_float(i8 * zeroinitializer, i32 1, <16 x i64> %ptrs, <16 x float> %values, <16 x i1> %vecmask)
+  call void @__scatter_base_offsets64_float(i8 * zeroinitializer, i32 1, <16 x i64> %ptrs, <16 x float> %values, <WIDTH x MASK> %vecmask)
   ret void
 }
 
@@ -1181,10 +1191,10 @@ gen_scatter(double)
 declare <16 x i32> @llvm.x86.avx512.mask.expand.load.d.512(i8* %addr, <16 x i32> %data, i16 %mask)
 
 define i32 @__packed_load_active(i32 * %startptr, <16 x i32> * %val_ptr,
-                                 <16 x i1> %full_mask) nounwind alwaysinline {
+                                 <WIDTH x MASK> %full_mask) nounwind alwaysinline {
   %addr = bitcast i32* %startptr to i8*
   %data = load PTR_OP_ARGS(`<16 x i32> ') %val_ptr
-  %mask = bitcast <16 x i1> %full_mask to i16
+  %mask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %full_mask)
   %store_val = call <16 x i32> @llvm.x86.avx512.mask.expand.load.d.512(i8* %addr, <16 x i32> %data, i16 %mask)
   store <16 x i32> %store_val, <16 x i32> * %val_ptr
   %mask_i32 = zext i16 %mask to i32
@@ -1195,9 +1205,9 @@ define i32 @__packed_load_active(i32 * %startptr, <16 x i32> * %val_ptr,
 declare void @llvm.x86.avx512.mask.compress.store.d.512(i8* %addr, <16 x i32> %data, i16 %mask)
 
 define i32 @__packed_store_active(i32 * %startptr, <16 x i32> %vals,
-                                   <16 x i1> %full_mask) nounwind alwaysinline {
+                                   <WIDTH x MASK> %full_mask) nounwind alwaysinline {
   %addr = bitcast i32* %startptr to i8*
-  %mask = bitcast <16 x i1> %full_mask to i16
+  %mask = call i16 @__cast_mask_to_i16 (<WIDTH x MASK> %full_mask)
   call void @llvm.x86.avx512.mask.compress.store.d.512(i8* %addr, <16 x i32> %vals, i16 %mask)
   %mask_i32 = zext i16 %mask to i32
   %res = call i32 @llvm.ctpop.i32(i32 %mask_i32)
@@ -1205,9 +1215,9 @@ define i32 @__packed_store_active(i32 * %startptr, <16 x i32> %vals,
 }
 
 define i32 @__packed_store_active2(i32 * %startptr, <16 x i32> %vals,
-                                   <16 x i1> %full_mask) nounwind alwaysinline {
+                                   <WIDTH x MASK> %full_mask) nounwind alwaysinline {
   %res = call i32 @__packed_store_active(i32 * %startptr, <16 x i32> %vals,
-                                   <16 x i1> %full_mask)
+                                   <WIDTH x MASK> %full_mask)
   ret i32 %res
 }
 

builtins/target-knl.ll

@@ -31,14 +31,44 @@
 
 define(`WIDTH',`16')
 
-
 ifelse(LLVM_VERSION, LLVM_3_7,
     `include(`target-avx512-common.ll')',
          LLVM_VERSION, LLVM_3_8,
     `include(`target-avx512-common.ll')',
          LLVM_VERSION, LLVM_3_9,
+    `include(`target-avx512-common.ll')',
+         LLVM_VERSION, LLVM_4_0,
+    `include(`target-avx512-common.ll')',
+         LLVM_VERSION, LLVM_5_0,
     `include(`target-avx512-common.ll')'
   )
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; rcp, rsqrt
+
+define(`rcp_rsqrt_varying_float_knl',`
+declare <16 x float> @llvm.x86.avx512.rcp28.ps(<16 x float>, <16 x float>, i16, i32) nounwind readnone
+define <16 x float> @__rcp_varying_float(<16 x float>) nounwind readonly alwaysinline {
+  %res = call <16 x float> @llvm.x86.avx512.rcp28.ps(<16 x float> %0, <16 x float> undef, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.rsqrt28.ps(<16 x float>, <16 x float>, i16, i32) nounwind readnone
+define <16 x float> @__rsqrt_varying_float(<16 x float> %v) nounwind readonly alwaysinline {
+  %res = call <16 x float> @llvm.x86.avx512.rsqrt28.ps(<16 x float> %v, <16 x float> undef, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+')
+
+ifelse(LLVM_VERSION, LLVM_3_7,
+    rcp_rsqrt_varying_float_knl(),
+         LLVM_VERSION, LLVM_3_8,
+    rcp_rsqrt_varying_float_knl(),
+         LLVM_VERSION, LLVM_3_9,
+    rcp_rsqrt_varying_float_knl(),
+         LLVM_VERSION, LLVM_4_0,
+    rcp_rsqrt_varying_float_knl(),
+         LLVM_VERSION, LLVM_5_0,
+    rcp_rsqrt_varying_float_knl()
+  )
 
 ;;saturation_arithmetic_novec()

builtins/target-neon-16.ll

@@ -42,12 +42,12 @@ include(`target-neon-common.ll')
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; half conversion routines
 
-define <8 x float> @__half_to_float_varying(<8 x i16> %v) nounwind readnone {
+define <8 x float> @__half_to_float_varying(<8 x i16> %v) nounwind readnone alwaysinline {
   unary4to8conv(r, i16, float, @llvm.arm.neon.vcvthf2fp, %v)
   ret <8 x float> %r
 }
 
-define <8 x i16> @__float_to_half_varying(<8 x float> %v) nounwind readnone {
+define <8 x i16> @__float_to_half_varying(<8 x float> %v) nounwind readnone alwaysinline {
   unary4to8conv(r, float, i16, @llvm.arm.neon.vcvtfp2hf, %v)
   ret <8 x i16> %r
 }
@@ -115,13 +115,13 @@ declare <4 x float> @llvm.arm.neon.vmins.v4f32(<4 x float>, <4 x float>) nounwin
 declare <4 x float> @llvm.arm.neon.vmaxs.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
 define <WIDTH x float> @__max_varying_float(<WIDTH x float>,
-                                            <WIDTH x float>) nounwind readnone {
+                                            <WIDTH x float>) nounwind readnone alwaysinline {
   binary4to8(r, float, @llvm.arm.neon.vmaxs.v4f32, %0, %1)
   ret <WIDTH x float> %r
 }
 
 define <WIDTH x float> @__min_varying_float(<WIDTH x float>,
-                                            <WIDTH x float>) nounwind readnone {
+                                            <WIDTH x float>) nounwind readnone alwaysinline {
   binary4to8(r, float, @llvm.arm.neon.vmins.v4f32, %0, %1)
   ret <WIDTH x float> %r
 }
@@ -131,22 +131,22 @@ declare <4 x i32> @llvm.arm.neon.vminu.v4i32(<4 x i32>, <4 x i32>) nounwind read
 declare <4 x i32> @llvm.arm.neon.vmaxs.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 declare <4 x i32> @llvm.arm.neon.vmaxu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 
-define <WIDTH x i32> @__min_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__min_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   binary4to8(r, i32, @llvm.arm.neon.vmins.v4i32, %0, %1)
   ret <WIDTH x i32> %r
 }
 
-define <WIDTH x i32> @__max_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__max_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   binary4to8(r, i32, @llvm.arm.neon.vmaxs.v4i32, %0, %1)
   ret <WIDTH x i32> %r
 }
 
-define <WIDTH x i32> @__min_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__min_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   binary4to8(r, i32, @llvm.arm.neon.vminu.v4i32, %0, %1)
   ret <WIDTH x i32> %r
 }
 
-define <WIDTH x i32> @__max_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__max_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   binary4to8(r, i32, @llvm.arm.neon.vmaxu.v4i32, %0, %1)
   ret <WIDTH x i32> %r
 }
@@ -156,7 +156,7 @@ define <WIDTH x i32> @__max_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwin
 declare <4 x float> @llvm.arm.neon.vrecpe.v4f32(<4 x float>) nounwind readnone
 declare <4 x float> @llvm.arm.neon.vrecps.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
-define <WIDTH x float> @__rcp_varying_float(<WIDTH x float> %d) nounwind readnone {
+define <WIDTH x float> @__rcp_varying_float(<WIDTH x float> %d) nounwind readnone alwaysinline {
   unary4to8(x0, float, @llvm.arm.neon.vrecpe.v4f32, %d)
   binary4to8(x0_nr, float, @llvm.arm.neon.vrecps.v4f32, %d, %x0)
   %x1 = fmul <WIDTH x float> %x0, %x0_nr
@@ -168,7 +168,7 @@ define <WIDTH x float> @__rcp_varying_float(<WIDTH x float> %d) nounwind readnon
 declare <4 x float> @llvm.arm.neon.vrsqrte.v4f32(<4 x float>) nounwind readnone
 declare <4 x float> @llvm.arm.neon.vrsqrts.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
-define <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %d) nounwind readnone {
+define <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %d) nounwind readnone alwaysinline {
   unary4to8(x0, float, @llvm.arm.neon.vrsqrte.v4f32, %d)
   %x0_2 = fmul <WIDTH x float> %x0, %x0
   binary4to8(x0_nr, float, @llvm.arm.neon.vrsqrts.v4f32, %d, %x0_2)
@@ -179,7 +179,7 @@ define <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %d) nounwind readn
   ret <WIDTH x float> %x2
 }
 
-define float @__rsqrt_uniform_float(float) nounwind readnone {
+define float @__rsqrt_uniform_float(float) nounwind readnone alwaysinline {
   %v1 = bitcast float %0 to <1 x float>
   %vs = shufflevector <1 x float> %v1, <1 x float> undef,
           <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef,
@@ -189,7 +189,7 @@ define float @__rsqrt_uniform_float(float) nounwind readnone {
   ret float %r
 }
 
-define float @__rcp_uniform_float(float) nounwind readnone {
+define float @__rcp_uniform_float(float) nounwind readnone alwaysinline {
   %v1 = bitcast float %0 to <1 x float>
   %vs = shufflevector <1 x float> %v1, <1 x float> undef,
           <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef,
@@ -201,7 +201,7 @@ define float @__rcp_uniform_float(float) nounwind readnone {
 
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
 
-define <WIDTH x float> @__sqrt_varying_float(<WIDTH x float>) nounwind readnone {
+define <WIDTH x float> @__sqrt_varying_float(<WIDTH x float>) nounwind readnone alwaysinline {
   unary4to8(result, float, @llvm.sqrt.v4f32, %0)
 ;; this returns nan for v=0, which is undesirable..
 ;;  %rsqrt = call <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %0)
@@ -211,7 +211,7 @@ define <WIDTH x float> @__sqrt_varying_float(<WIDTH x float>) nounwind readnone
 
 declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
 
-define <WIDTH x double> @__sqrt_varying_double(<WIDTH x double>) nounwind readnone {
+define <WIDTH x double> @__sqrt_varying_double(<WIDTH x double>) nounwind readnone alwaysinline {
   unary4to8(r, double, @llvm.sqrt.v4f64, %0)
   ret <WIDTH x double> %r
 }
@@ -219,7 +219,7 @@ define <WIDTH x double> @__sqrt_varying_double(<WIDTH x double>) nounwind readno
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; reductions
 
-define i64 @__movmsk(<WIDTH x MASK>) nounwind readnone {
+define i64 @__movmsk(<WIDTH x MASK>) nounwind readnone alwaysinline {
   %and_mask = and <WIDTH x i16> %0,
     <i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128>
   %v4 = call <4 x i32> @llvm.arm.neon.vpaddlu.v4i32.v8i16(<8 x i16> %and_mask)
@@ -288,48 +288,48 @@ define(`neon_reduce', `
 
 declare <2 x float> @llvm.arm.neon.vpadd.v2f32(<2 x float>, <2 x float>) nounwind readnone
 
-define internal float @add_f32(float, float) {
+define internal float @add_f32(float, float) nounwind readnone alwaysinline {
   %r = fadd float %0, %1
   ret float %r
 }
 
-define internal <WIDTH x float> @__add_varying_float(<WIDTH x float>, <WIDTH x float>) {
+define internal <WIDTH x float> @__add_varying_float(<WIDTH x float>, <WIDTH x float>) nounwind readnone alwaysinline {
   %r = fadd <WIDTH x float> %0, %1
   ret <WIDTH x float> %r
 }
 
-define float @__reduce_add_float(<WIDTH x float>) nounwind readnone {
+define float @__reduce_add_float(<WIDTH x float>) nounwind readnone alwaysinline {
   neon_reduce(float, @__add_varying_float, @llvm.arm.neon.vpadd.v2f32, @add_f32)
 }
 
 declare <2 x float> @llvm.arm.neon.vpmins.v2f32(<2 x float>, <2 x float>) nounwind readnone
 
-define internal float @min_f32(float, float) {
+define internal float @min_f32(float, float) nounwind readnone alwaysinline {
   %cmp = fcmp olt float %0, %1
   %r = select i1 %cmp, float %0, float %1
   ret float %r
 }
 
-define float @__reduce_min_float(<WIDTH x float>) nounwind readnone {
+define float @__reduce_min_float(<WIDTH x float>) nounwind readnone alwaysinline {
   neon_reduce(float, @__min_varying_float, @llvm.arm.neon.vpmins.v2f32, @min_f32)
 }
 
 declare <2 x float> @llvm.arm.neon.vpmaxs.v2f32(<2 x float>, <2 x float>) nounwind readnone
 
-define internal float @max_f32(float, float) {
+define internal float @max_f32(float, float) nounwind readnone alwaysinline {
   %cmp = fcmp ugt float %0, %1
   %r = select i1 %cmp, float %0, float %1
   ret float %r
 }
 
-define float @__reduce_max_float(<WIDTH x float>) nounwind readnone {
+define float @__reduce_max_float(<WIDTH x float>) nounwind readnone alwaysinline {
   neon_reduce(float, @__max_varying_float, @llvm.arm.neon.vpmaxs.v2f32, @max_f32)
 }
 
 declare <4 x i16> @llvm.arm.neon.vpaddls.v4i16.v8i8(<8 x i8>) nounwind readnone
 declare <2 x i32> @llvm.arm.neon.vpaddlu.v2i32.v4i16(<4 x i16>) nounwind readnone
 
-define i16 @__reduce_add_int8(<WIDTH x i8>) nounwind readnone {
+define i16 @__reduce_add_int8(<WIDTH x i8>) nounwind readnone alwaysinline {
   %a16 = call <4 x i16> @llvm.arm.neon.vpaddls.v4i16.v8i8(<8 x i8> %0)
   %a32 = call <2 x i32> @llvm.arm.neon.vpaddlu.v2i32.v4i16(<4 x i16> %a16)
   %a0 = extractelement <2 x i32> %a32, i32 0
@@ -341,7 +341,7 @@ define i16 @__reduce_add_int8(<WIDTH x i8>) nounwind readnone {
 
 declare <4 x i32> @llvm.arm.neon.vpaddlu.v4i32.v8i16(<WIDTH x i16>)
 
-define i64 @__reduce_add_int16(<WIDTH x i16>) nounwind readnone {
+define i64 @__reduce_add_int16(<WIDTH x i16>) nounwind readnone alwaysinline {
   %a1 = call <4 x i32> @llvm.arm.neon.vpaddlu.v4i32.v8i16(<WIDTH x i16> %0)
   %a2 = call <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32> %a1)
   %aa = extractelement <2 x i64> %a2, i32 0
@@ -352,7 +352,7 @@ define i64 @__reduce_add_int16(<WIDTH x i16>) nounwind readnone {
 
 declare <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32>) nounwind readnone
 
-define i64 @__reduce_add_int32(<WIDTH x i32>) nounwind readnone {
+define i64 @__reduce_add_int32(<WIDTH x i32>) nounwind readnone alwaysinline {
   v8tov4(i32, %0, %va, %vb)
   %pa = call <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32> %va)
   %pb = call <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32> %vb)
@@ -365,53 +365,53 @@ define i64 @__reduce_add_int32(<WIDTH x i32>) nounwind readnone {
 
 declare <2 x i32> @llvm.arm.neon.vpmins.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @min_si32(i32, i32) {
+define internal i32 @min_si32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp slt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_min_int32(<WIDTH x i32>) nounwind readnone {
+define i32 @__reduce_min_int32(<WIDTH x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @__min_varying_int32, @llvm.arm.neon.vpmins.v2i32, @min_si32)
 }
 
 declare <2 x i32> @llvm.arm.neon.vpmaxs.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @max_si32(i32, i32) {
+define internal i32 @max_si32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp sgt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_max_int32(<WIDTH x i32>) nounwind readnone {
+define i32 @__reduce_max_int32(<WIDTH x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @__max_varying_int32, @llvm.arm.neon.vpmaxs.v2i32, @max_si32)
 }
 
 declare <2 x i32> @llvm.arm.neon.vpminu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @min_ui32(i32, i32) {
+define internal i32 @min_ui32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp ult i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_min_uint32(<WIDTH x i32>) nounwind readnone {
+define i32 @__reduce_min_uint32(<WIDTH x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @__min_varying_uint32, @llvm.arm.neon.vpmins.v2i32, @min_ui32)
 }
 
 declare <2 x i32> @llvm.arm.neon.vpmaxu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @max_ui32(i32, i32) {
+define internal i32 @max_ui32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp ugt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_max_uint32(<WIDTH x i32>) nounwind readnone {
+define i32 @__reduce_max_uint32(<WIDTH x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @__max_varying_uint32, @llvm.arm.neon.vpmaxs.v2i32, @max_ui32)
 }
 
-define double @__reduce_add_double(<WIDTH x double>) nounwind readnone {
+define double @__reduce_add_double(<WIDTH x double>) nounwind readnone alwaysinline {
   v8tov2(double, %0, %v0, %v1, %v2, %v3)
   %v01 = fadd <2 x double> %v0, %v1
   %v23 = fadd <2 x double> %v2, %v3
@@ -422,15 +422,15 @@ define double @__reduce_add_double(<WIDTH x double>) nounwind readnone {
   ret double %m
 }
 
-define double @__reduce_min_double(<WIDTH x double>) nounwind readnone {
+define double @__reduce_min_double(<WIDTH x double>) nounwind readnone alwaysinline {
   reduce8(double, @__min_varying_double, @__min_uniform_double)
 }
 
-define double @__reduce_max_double(<WIDTH x double>) nounwind readnone {
+define double @__reduce_max_double(<WIDTH x double>) nounwind readnone alwaysinline {
   reduce8(double, @__max_varying_double, @__max_uniform_double)
 }
 
-define i64 @__reduce_add_int64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_add_int64(<WIDTH x i64>) nounwind readnone alwaysinline {
   v8tov2(i64, %0, %v0, %v1, %v2, %v3)
   %v01 = add <2 x i64> %v0, %v1
   %v23 = add <2 x i64> %v2, %v3
@@ -441,19 +441,19 @@ define i64 @__reduce_add_int64(<WIDTH x i64>) nounwind readnone {
   ret i64 %m
 }
 
-define i64 @__reduce_min_int64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_min_int64(<WIDTH x i64>) nounwind readnone alwaysinline {
   reduce8(i64, @__min_varying_int64, @__min_uniform_int64)
 }
 
-define i64 @__reduce_max_int64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_max_int64(<WIDTH x i64>) nounwind readnone alwaysinline {
   reduce8(i64, @__max_varying_int64, @__max_uniform_int64)
 }
 
-define i64 @__reduce_min_uint64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_min_uint64(<WIDTH x i64>) nounwind readnone alwaysinline {
   reduce8(i64, @__min_varying_uint64, @__min_uniform_uint64)
 }
 
-define i64 @__reduce_max_uint64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_max_uint64(<WIDTH x i64>) nounwind readnone alwaysinline {
   reduce8(i64, @__max_varying_uint64, @__max_uniform_uint64)
 }
 
@@ -462,56 +462,56 @@ define i64 @__reduce_max_uint64(<WIDTH x i64>) nounwind readnone {
 
 declare <8 x i8> @llvm.arm.neon.vrhaddu.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
 
-define <8 x i8> @__avg_up_uint8(<8 x i8>, <8 x i8>) nounwind readnone {
+define <8 x i8> @__avg_up_uint8(<8 x i8>, <8 x i8>) nounwind readnone alwaysinline {
   %r = call <8 x i8> @llvm.arm.neon.vrhaddu.v8i8(<8 x i8> %0, <8 x i8> %1)
   ret <8 x i8> %r
 }
 
 declare <8 x i8> @llvm.arm.neon.vrhadds.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
 
-define <8 x i8> @__avg_up_int8(<8 x i8>, <8 x i8>) nounwind readnone {
+define <8 x i8> @__avg_up_int8(<8 x i8>, <8 x i8>) nounwind readnone alwaysinline {
   %r = call <8 x i8> @llvm.arm.neon.vrhadds.v8i8(<8 x i8> %0, <8 x i8> %1)
   ret <8 x i8> %r
 }
 
 declare <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
 
-define <8 x i8> @__avg_down_uint8(<8 x i8>, <8 x i8>) nounwind readnone {
+define <8 x i8> @__avg_down_uint8(<8 x i8>, <8 x i8>) nounwind readnone alwaysinline {
   %r = call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %0, <8 x i8> %1)
   ret <8 x i8> %r
 }
 
 declare <8 x i8> @llvm.arm.neon.vhadds.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
 
-define <8 x i8> @__avg_down_int8(<8 x i8>, <8 x i8>) nounwind readnone {
+define <8 x i8> @__avg_down_int8(<8 x i8>, <8 x i8>) nounwind readnone alwaysinline {
   %r = call <8 x i8> @llvm.arm.neon.vhadds.v8i8(<8 x i8> %0, <8 x i8> %1)
   ret <8 x i8> %r
 }
 
 declare <8 x i16> @llvm.arm.neon.vrhaddu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
-define <8 x i16> @__avg_up_uint16(<8 x i16>, <8 x i16>) nounwind readnone {
+define <8 x i16> @__avg_up_uint16(<8 x i16>, <8 x i16>) nounwind readnone alwaysinline {
   %r = call <8 x i16> @llvm.arm.neon.vrhaddu.v8i16(<8 x i16> %0, <8 x i16> %1)
   ret <8 x i16> %r
 }
 
 declare <8 x i16> @llvm.arm.neon.vrhadds.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
-define <8 x i16> @__avg_up_int16(<8 x i16>, <8 x i16>) nounwind readnone {
+define <8 x i16> @__avg_up_int16(<8 x i16>, <8 x i16>) nounwind readnone alwaysinline {
   %r = call <8 x i16> @llvm.arm.neon.vrhadds.v8i16(<8 x i16> %0, <8 x i16> %1)
   ret <8 x i16> %r
 }
 
 declare <8 x i16> @llvm.arm.neon.vhaddu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
-define <8 x i16> @__avg_down_uint16(<8 x i16>, <8 x i16>) nounwind readnone {
+define <8 x i16> @__avg_down_uint16(<8 x i16>, <8 x i16>) nounwind readnone alwaysinline {
   %r = call <8 x i16> @llvm.arm.neon.vhaddu.v8i16(<8 x i16> %0, <8 x i16> %1)
   ret <8 x i16> %r
 }
 
 declare <8 x i16> @llvm.arm.neon.vhadds.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
-define <8 x i16> @__avg_down_int16(<8 x i16>, <8 x i16>) nounwind readnone {
+define <8 x i16> @__avg_down_int16(<8 x i16>, <8 x i16>) nounwind readnone alwaysinline {
   %r = call <8 x i16> @llvm.arm.neon.vhadds.v8i16(<8 x i16> %0, <8 x i16> %1)
   ret <8 x i16> %r
 }

builtins/target-neon-32.ll

@@ -43,12 +43,12 @@ include(`target-neon-common.ll')
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; half conversion routines
 
-define <4 x float> @__half_to_float_varying(<4 x i16> %v) nounwind readnone {
+define <4 x float> @__half_to_float_varying(<4 x i16> %v) nounwind readnone alwaysinline {
   %r = call <4 x float> @llvm.arm.neon.vcvthf2fp(<4 x i16> %v)
   ret <4 x float> %r
 }
 
-define <4 x i16> @__float_to_half_varying(<4 x float> %v) nounwind readnone {
+define <4 x i16> @__float_to_half_varying(<4 x float> %v) nounwind readnone alwaysinline {
   %r = call <4 x i16> @llvm.arm.neon.vcvtfp2hf(<4 x float> %v)
   ret <4 x i16> %r
 }
@@ -106,13 +106,13 @@ declare <4 x float> @llvm.arm.neon.vmins.v4f32(<4 x float>, <4 x float>) nounwin
 declare <4 x float> @llvm.arm.neon.vmaxs.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
 define <WIDTH x float> @__max_varying_float(<WIDTH x float>,
-                                            <WIDTH x float>) nounwind readnone {
+                                            <WIDTH x float>) nounwind readnone alwaysinline {
   %r = call <4 x float> @llvm.arm.neon.vmaxs.v4f32(<4 x float> %0, <4 x float> %1)
   ret <WIDTH x float> %r
 }
 
 define <WIDTH x float> @__min_varying_float(<WIDTH x float>,
-                                            <WIDTH x float>) nounwind readnone {
+                                            <WIDTH x float>) nounwind readnone alwaysinline {
   %r = call <4 x float> @llvm.arm.neon.vmins.v4f32(<4 x float> %0, <4 x float> %1)
   ret <WIDTH x float> %r
 }
@@ -122,22 +122,22 @@ declare <4 x i32> @llvm.arm.neon.vminu.v4i32(<4 x i32>, <4 x i32>) nounwind read
 declare <4 x i32> @llvm.arm.neon.vmaxs.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 declare <4 x i32> @llvm.arm.neon.vmaxu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 
-define <WIDTH x i32> @__min_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__min_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   %r = call <4 x i32> @llvm.arm.neon.vmins.v4i32(<4 x i32> %0, <4 x i32> %1)
   ret <4 x i32> %r
 }
 
-define <WIDTH x i32> @__max_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__max_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   %r = call <4 x i32> @llvm.arm.neon.vmaxs.v4i32(<4 x i32> %0, <4 x i32> %1)
   ret <4 x i32> %r
 }
 
-define <WIDTH x i32> @__min_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__min_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   %r = call <4 x i32> @llvm.arm.neon.vminu.v4i32(<4 x i32> %0, <4 x i32> %1)
   ret <4 x i32> %r
 }
 
-define <WIDTH x i32> @__max_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__max_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   %r = call <4 x i32> @llvm.arm.neon.vmaxu.v4i32(<4 x i32> %0, <4 x i32> %1)
   ret <4 x i32> %r
 }
@@ -147,7 +147,7 @@ define <WIDTH x i32> @__max_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwin
 declare <4 x float> @llvm.arm.neon.vrecpe.v4f32(<4 x float>) nounwind readnone
 declare <4 x float> @llvm.arm.neon.vrecps.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
-define <WIDTH x float> @__rcp_varying_float(<WIDTH x float> %d) nounwind readnone {
+define <WIDTH x float> @__rcp_varying_float(<WIDTH x float> %d) nounwind readnone alwaysinline {
   %x0 = call <4 x float> @llvm.arm.neon.vrecpe.v4f32(<4 x float> %d)
   %x0_nr = call <4 x float> @llvm.arm.neon.vrecps.v4f32(<4 x float> %d, <4 x float> %x0)
   %x1 = fmul <4 x float> %x0, %x0_nr
@@ -159,7 +159,7 @@ define <WIDTH x float> @__rcp_varying_float(<WIDTH x float> %d) nounwind readnon
 declare <4 x float> @llvm.arm.neon.vrsqrte.v4f32(<4 x float>) nounwind readnone
 declare <4 x float> @llvm.arm.neon.vrsqrts.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
-define <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %d) nounwind readnone {
+define <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %d) nounwind readnone alwaysinline {
   %x0 = call <4 x float> @llvm.arm.neon.vrsqrte.v4f32(<4 x float> %d)
   %x0_2 = fmul <4 x float> %x0, %x0
   %x0_nr = call <4 x float> @llvm.arm.neon.vrsqrts.v4f32(<4 x float> %d, <4 x float> %x0_2)
@@ -170,7 +170,7 @@ define <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %d) nounwind readn
   ret <4 x float> %x2
 }
 
-define float @__rsqrt_uniform_float(float) nounwind readnone {
+define float @__rsqrt_uniform_float(float) nounwind readnone alwaysinline {
   %v1 = bitcast float %0 to <1 x float>
   %vs = shufflevector <1 x float> %v1, <1 x float> undef,
           <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
@@ -179,7 +179,7 @@ define float @__rsqrt_uniform_float(float) nounwind readnone {
   ret float %r
 }
 
-define float @__rcp_uniform_float(float) nounwind readnone {
+define float @__rcp_uniform_float(float) nounwind readnone alwaysinline {
   %v1 = bitcast float %0 to <1 x float>
   %vs = shufflevector <1 x float> %v1, <1 x float> undef,
           <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
@@ -190,7 +190,7 @@ define float @__rcp_uniform_float(float) nounwind readnone {
 
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
 
-define <WIDTH x float> @__sqrt_varying_float(<WIDTH x float>) nounwind readnone {
+define <WIDTH x float> @__sqrt_varying_float(<WIDTH x float>) nounwind readnone alwaysinline {
   %result = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %0)
 ;; this returns nan for v=0, which is undesirable..
 ;;  %rsqrt = call <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %0)
@@ -200,7 +200,7 @@ define <WIDTH x float> @__sqrt_varying_float(<WIDTH x float>) nounwind readnone
 
 declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
 
-define <WIDTH x double> @__sqrt_varying_double(<WIDTH x double>) nounwind readnone {
+define <WIDTH x double> @__sqrt_varying_double(<WIDTH x double>) nounwind readnone alwaysinline {
   %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %0)
   ret <4 x double> %r
 }
@@ -208,7 +208,7 @@ define <WIDTH x double> @__sqrt_varying_double(<WIDTH x double>) nounwind readno
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; reductions
 
-define i64 @__movmsk(<4 x MASK>) nounwind readnone {
+define i64 @__movmsk(<4 x MASK>) nounwind readnone alwaysinline {
   %and_mask = and <4 x MASK> %0, <MASK 1, MASK 2, MASK 4, MASK 8>
   %v01 = shufflevector <4 x i32> %and_mask, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
   %v23 = shufflevector <4 x i32> %and_mask, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -264,42 +264,42 @@ define(`neon_reduce', `
 
 declare <2 x float> @llvm.arm.neon.vpadd.v2f32(<2 x float>, <2 x float>) nounwind readnone
 
-define internal float @add_f32(float, float) {
+define internal float @add_f32(float, float) nounwind readnone alwaysinline {
   %r = fadd float %0, %1
   ret float %r
 }
 
-define float @__reduce_add_float(<4 x float>) nounwind readnone {
+define float @__reduce_add_float(<4 x float>) nounwind readnone alwaysinline {
   neon_reduce(float, @llvm.arm.neon.vpadd.v2f32, @add_f32)
 }
 
 declare <2 x float> @llvm.arm.neon.vpmins.v2f32(<2 x float>, <2 x float>) nounwind readnone
 
-define internal float @min_f32(float, float) {
+define internal float @min_f32(float, float) nounwind readnone alwaysinline {
   %cmp = fcmp olt float %0, %1
   %r = select i1 %cmp, float %0, float %1
   ret float %r
 }
 
-define float @__reduce_min_float(<4 x float>) nounwind readnone {
+define float @__reduce_min_float(<4 x float>) nounwind readnone alwaysinline {
   neon_reduce(float, @llvm.arm.neon.vpmins.v2f32, @min_f32)
 }
 
 declare <2 x float> @llvm.arm.neon.vpmaxs.v2f32(<2 x float>, <2 x float>) nounwind readnone
 
-define internal float @max_f32(float, float) {
+define internal float @max_f32(float, float) nounwind readnone alwaysinline {
   %cmp = fcmp ugt float %0, %1
   %r = select i1 %cmp, float %0, float %1
   ret float %r
 }
 
-define float @__reduce_max_float(<4 x float>) nounwind readnone {
+define float @__reduce_max_float(<4 x float>) nounwind readnone alwaysinline {
   neon_reduce(float, @llvm.arm.neon.vpmaxs.v2f32, @max_f32)
 }
 
 declare <4 x i16> @llvm.arm.neon.vpaddls.v4i16.v8i8(<8 x i8>) nounwind readnone
 
-define i16 @__reduce_add_int8(<WIDTH x i8>) nounwind readnone {
+define i16 @__reduce_add_int8(<WIDTH x i8>) nounwind readnone alwaysinline {
   %v8 = shufflevector <4 x i8> %0, <4 x i8> zeroinitializer,
            <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
   %a16 = call <4 x i16> @llvm.arm.neon.vpaddls.v4i16.v8i8(<8 x i8> %v8)
@@ -313,7 +313,7 @@ define i16 @__reduce_add_int8(<WIDTH x i8>) nounwind readnone {
 
 declare <2 x i32> @llvm.arm.neon.vpaddlu.v2i32.v4i16(<4 x i16>) nounwind readnone
 
-define i32 @__reduce_add_int16(<WIDTH x i16>) nounwind readnone {
+define i32 @__reduce_add_int16(<WIDTH x i16>) nounwind readnone alwaysinline {
   %a32 = call <2 x i32> @llvm.arm.neon.vpaddlu.v2i32.v4i16(<4 x i16> %0)
   %a0 = extractelement <2 x i32> %a32, i32 0
   %a1 = extractelement <2 x i32> %a32, i32 1
@@ -323,7 +323,7 @@ define i32 @__reduce_add_int16(<WIDTH x i16>) nounwind readnone {
 
 declare <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32>) nounwind readnone
 
-define i64 @__reduce_add_int32(<WIDTH x i32>) nounwind readnone {
+define i64 @__reduce_add_int32(<WIDTH x i32>) nounwind readnone alwaysinline {
   %a64 = call <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32> %0)
   %a0 = extractelement <2 x i64> %a64, i32 0
   %a1 = extractelement <2 x i64> %a64, i32 1
@@ -333,53 +333,53 @@ define i64 @__reduce_add_int32(<WIDTH x i32>) nounwind readnone {
 
 declare <2 x i32> @llvm.arm.neon.vpmins.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @min_si32(i32, i32) {
+define internal i32 @min_si32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp slt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_min_int32(<4 x i32>) nounwind readnone {
+define i32 @__reduce_min_int32(<4 x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @llvm.arm.neon.vpmins.v2i32, @min_si32)
 }
 
 declare <2 x i32> @llvm.arm.neon.vpmaxs.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @max_si32(i32, i32) {
+define internal i32 @max_si32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp sgt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_max_int32(<4 x i32>) nounwind readnone {
+define i32 @__reduce_max_int32(<4 x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @llvm.arm.neon.vpmaxs.v2i32, @max_si32)
 }
 
 declare <2 x i32> @llvm.arm.neon.vpminu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @min_ui32(i32, i32) {
+define internal i32 @min_ui32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp ult i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_min_uint32(<4 x i32>) nounwind readnone {
+define i32 @__reduce_min_uint32(<4 x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @llvm.arm.neon.vpmins.v2i32, @min_ui32)
 }
 
 declare <2 x i32> @llvm.arm.neon.vpmaxu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @max_ui32(i32, i32) {
+define internal i32 @max_ui32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp ugt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_max_uint32(<4 x i32>) nounwind readnone {
+define i32 @__reduce_max_uint32(<4 x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @llvm.arm.neon.vpmaxs.v2i32, @max_ui32)
 }
 
-define double @__reduce_add_double(<4 x double>) nounwind readnone {
+define double @__reduce_add_double(<4 x double>) nounwind readnone alwaysinline {
   %v0 = shufflevector <4 x double> %0, <4 x double> undef,
                       <2 x i32> <i32 0, i32 1>
   %v1 = shufflevector <4 x double> %0, <4 x double> undef,
@@ -391,15 +391,15 @@ define double @__reduce_add_double(<4 x double>) nounwind readnone {
   ret double %m
 }
 
-define double @__reduce_min_double(<4 x double>) nounwind readnone {
+define double @__reduce_min_double(<4 x double>) nounwind readnone alwaysinline {
   reduce4(double, @__min_varying_double, @__min_uniform_double)
 }
 
-define double @__reduce_max_double(<4 x double>) nounwind readnone {
+define double @__reduce_max_double(<4 x double>) nounwind readnone alwaysinline {
   reduce4(double, @__max_varying_double, @__max_uniform_double)
 }
 
-define i64 @__reduce_add_int64(<4 x i64>) nounwind readnone {
+define i64 @__reduce_add_int64(<4 x i64>) nounwind readnone alwaysinline {
   %v0 = shufflevector <4 x i64> %0, <4 x i64> undef,
                       <2 x i32> <i32 0, i32 1>
   %v1 = shufflevector <4 x i64> %0, <4 x i64> undef,
@@ -411,19 +411,19 @@ define i64 @__reduce_add_int64(<4 x i64>) nounwind readnone {
   ret i64 %m
 }
 
-define i64 @__reduce_min_int64(<4 x i64>) nounwind readnone {
+define i64 @__reduce_min_int64(<4 x i64>) nounwind readnone alwaysinline {
   reduce4(i64, @__min_varying_int64, @__min_uniform_int64)
 }
 
-define i64 @__reduce_max_int64(<4 x i64>) nounwind readnone {
+define i64 @__reduce_max_int64(<4 x i64>) nounwind readnone alwaysinline {
   reduce4(i64, @__max_varying_int64, @__max_uniform_int64)
 }
 
-define i64 @__reduce_min_uint64(<4 x i64>) nounwind readnone {
+define i64 @__reduce_min_uint64(<4 x i64>) nounwind readnone alwaysinline {
   reduce4(i64, @__min_varying_uint64, @__min_uniform_uint64)
 }
 
-define i64 @__reduce_max_uint64(<4 x i64>) nounwind readnone {
+define i64 @__reduce_max_uint64(<4 x i64>) nounwind readnone alwaysinline {
   reduce4(i64, @__max_varying_uint64, @__max_uniform_uint64)
 }
 
@@ -432,56 +432,56 @@ define i64 @__reduce_max_uint64(<4 x i64>) nounwind readnone {
 
 declare <4 x i8> @llvm.arm.neon.vrhaddu.v4i8(<4 x i8>, <4 x i8>) nounwind readnone
 
-define <4 x i8> @__avg_up_uint8(<4 x i8>, <4 x i8>) nounwind readnone {
+define <4 x i8> @__avg_up_uint8(<4 x i8>, <4 x i8>) nounwind readnone alwaysinline {
   %r = call <4 x i8> @llvm.arm.neon.vrhaddu.v4i8(<4 x i8> %0, <4 x i8> %1)
   ret <4 x i8> %r
 }
 
 declare <4 x i8> @llvm.arm.neon.vrhadds.v4i8(<4 x i8>, <4 x i8>) nounwind readnone
 
-define <4 x i8> @__avg_up_int8(<4 x i8>, <4 x i8>) nounwind readnone {
+define <4 x i8> @__avg_up_int8(<4 x i8>, <4 x i8>) nounwind readnone alwaysinline {
   %r = call <4 x i8> @llvm.arm.neon.vrhadds.v4i8(<4 x i8> %0, <4 x i8> %1)
   ret <4 x i8> %r
 }
 
 declare <4 x i8> @llvm.arm.neon.vhaddu.v4i8(<4 x i8>, <4 x i8>) nounwind readnone
 
-define <4 x i8> @__avg_down_uint8(<4 x i8>, <4 x i8>) nounwind readnone {
+define <4 x i8> @__avg_down_uint8(<4 x i8>, <4 x i8>) nounwind readnone alwaysinline {
   %r = call <4 x i8> @llvm.arm.neon.vhaddu.v4i8(<4 x i8> %0, <4 x i8> %1)
   ret <4 x i8> %r
 }
 
 declare <4 x i8> @llvm.arm.neon.vhadds.v4i8(<4 x i8>, <4 x i8>) nounwind readnone
 
-define <4 x i8> @__avg_down_int8(<4 x i8>, <4 x i8>) nounwind readnone {
+define <4 x i8> @__avg_down_int8(<4 x i8>, <4 x i8>) nounwind readnone alwaysinline {
   %r = call <4 x i8> @llvm.arm.neon.vhadds.v4i8(<4 x i8> %0, <4 x i8> %1)
   ret <4 x i8> %r
 }
 
 declare <4 x i16> @llvm.arm.neon.vrhaddu.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
 
-define <4 x i16> @__avg_up_uint16(<4 x i16>, <4 x i16>) nounwind readnone {
+define <4 x i16> @__avg_up_uint16(<4 x i16>, <4 x i16>) nounwind readnone alwaysinline {
   %r = call <4 x i16> @llvm.arm.neon.vrhaddu.v4i16(<4 x i16> %0, <4 x i16> %1)
   ret <4 x i16> %r
 }
 
 declare <4 x i16> @llvm.arm.neon.vrhadds.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
 
-define <4 x i16> @__avg_up_int16(<4 x i16>, <4 x i16>) nounwind readnone {
+define <4 x i16> @__avg_up_int16(<4 x i16>, <4 x i16>) nounwind readnone alwaysinline {
   %r = call <4 x i16> @llvm.arm.neon.vrhadds.v4i16(<4 x i16> %0, <4 x i16> %1)
   ret <4 x i16> %r
 }
 
 declare <4 x i16> @llvm.arm.neon.vhaddu.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
 
-define <4 x i16> @__avg_down_uint16(<4 x i16>, <4 x i16>) nounwind readnone {
+define <4 x i16> @__avg_down_uint16(<4 x i16>, <4 x i16>) nounwind readnone alwaysinline {
   %r = call <4 x i16> @llvm.arm.neon.vhaddu.v4i16(<4 x i16> %0, <4 x i16> %1)
   ret <4 x i16> %r
 }
 
 declare <4 x i16> @llvm.arm.neon.vhadds.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
 
-define <4 x i16> @__avg_down_int16(<4 x i16>, <4 x i16>) nounwind readnone {
+define <4 x i16> @__avg_down_int16(<4 x i16>, <4 x i16>) nounwind readnone alwaysinline {
   %r = call <4 x i16> @llvm.arm.neon.vhadds.v4i16(<4 x i16> %0, <4 x i16> %1)
   ret <4 x i16> %r
 }

builtins/target-neon-8.ll

@@ -42,12 +42,12 @@ include(`target-neon-common.ll')
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; half conversion routines
 
-define <16 x float> @__half_to_float_varying(<16 x i16> %v) nounwind readnone {
+define <16 x float> @__half_to_float_varying(<16 x i16> %v) nounwind readnone alwaysinline {
   unary4to16conv(r, i16, float, @llvm.arm.neon.vcvthf2fp, %v)
   ret <16 x float> %r
 }
 
-define <16 x i16> @__float_to_half_varying(<16 x float> %v) nounwind readnone {
+define <16 x i16> @__float_to_half_varying(<16 x float> %v) nounwind readnone alwaysinline {
   unary4to16conv(r, float, i16, @llvm.arm.neon.vcvtfp2hf, %v)
   ret <16 x i16> %r
 }
@@ -125,13 +125,13 @@ declare <4 x float> @llvm.arm.neon.vmins.v4f32(<4 x float>, <4 x float>) nounwin
 declare <4 x float> @llvm.arm.neon.vmaxs.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
 define <WIDTH x float> @__max_varying_float(<WIDTH x float>,
-                                            <WIDTH x float>) nounwind readnone {
+                                            <WIDTH x float>) nounwind readnone alwaysinline {
   binary4to16(r, float, @llvm.arm.neon.vmaxs.v4f32, %0, %1)
   ret <WIDTH x float> %r
 }
 
 define <WIDTH x float> @__min_varying_float(<WIDTH x float>,
-                                            <WIDTH x float>) nounwind readnone {
+                                            <WIDTH x float>) nounwind readnone alwaysinline {
   binary4to16(r, float, @llvm.arm.neon.vmins.v4f32, %0, %1)
   ret <WIDTH x float> %r
 }
@@ -141,22 +141,22 @@ declare <4 x i32> @llvm.arm.neon.vminu.v4i32(<4 x i32>, <4 x i32>) nounwind read
 declare <4 x i32> @llvm.arm.neon.vmaxs.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 declare <4 x i32> @llvm.arm.neon.vmaxu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 
-define <WIDTH x i32> @__min_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__min_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   binary4to16(r, i32, @llvm.arm.neon.vmins.v4i32, %0, %1)
   ret <WIDTH x i32> %r
 }
 
-define <WIDTH x i32> @__max_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__max_varying_int32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   binary4to16(r, i32, @llvm.arm.neon.vmaxs.v4i32, %0, %1)
   ret <WIDTH x i32> %r
 }
 
-define <WIDTH x i32> @__min_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__min_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   binary4to16(r, i32, @llvm.arm.neon.vminu.v4i32, %0, %1)
   ret <WIDTH x i32> %r
 }
 
-define <WIDTH x i32> @__max_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone {
+define <WIDTH x i32> @__max_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwind readnone alwaysinline {
   binary4to16(r, i32, @llvm.arm.neon.vmaxu.v4i32, %0, %1)
   ret <WIDTH x i32> %r
 }
@@ -166,7 +166,7 @@ define <WIDTH x i32> @__max_varying_uint32(<WIDTH x i32>, <WIDTH x i32>) nounwin
 declare <4 x float> @llvm.arm.neon.vrecpe.v4f32(<4 x float>) nounwind readnone
 declare <4 x float> @llvm.arm.neon.vrecps.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
-define <WIDTH x float> @__rcp_varying_float(<WIDTH x float> %d) nounwind readnone {
+define <WIDTH x float> @__rcp_varying_float(<WIDTH x float> %d) nounwind readnone alwaysinline {
   unary4to16(x0, float, @llvm.arm.neon.vrecpe.v4f32, %d)
   binary4to16(x0_nr, float, @llvm.arm.neon.vrecps.v4f32, %d, %x0)
   %x1 = fmul <WIDTH x float> %x0, %x0_nr
@@ -178,7 +178,7 @@ define <WIDTH x float> @__rcp_varying_float(<WIDTH x float> %d) nounwind readnon
 declare <4 x float> @llvm.arm.neon.vrsqrte.v4f32(<4 x float>) nounwind readnone
 declare <4 x float> @llvm.arm.neon.vrsqrts.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
-define <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %d) nounwind readnone {
+define <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %d) nounwind readnone alwaysinline {
   unary4to16(x0, float, @llvm.arm.neon.vrsqrte.v4f32, %d)
   %x0_2 = fmul <WIDTH x float> %x0, %x0
   binary4to16(x0_nr, float, @llvm.arm.neon.vrsqrts.v4f32, %d, %x0_2)
@@ -189,7 +189,7 @@ define <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %d) nounwind readn
   ret <WIDTH x float> %x2
 }
 
-define float @__rsqrt_uniform_float(float) nounwind readnone {
+define float @__rsqrt_uniform_float(float) nounwind readnone alwaysinline {
   %v1 = bitcast float %0 to <1 x float>
   %vs = shufflevector <1 x float> %v1, <1 x float> undef,
           <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef,
@@ -201,7 +201,7 @@ define float @__rsqrt_uniform_float(float) nounwind readnone {
   ret float %r
 }
 
-define float @__rcp_uniform_float(float) nounwind readnone {
+define float @__rcp_uniform_float(float) nounwind readnone alwaysinline {
   %v1 = bitcast float %0 to <1 x float>
   %vs = shufflevector <1 x float> %v1, <1 x float> undef,
           <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef,
@@ -215,7 +215,7 @@ define float @__rcp_uniform_float(float) nounwind readnone {
 
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
 
-define <WIDTH x float> @__sqrt_varying_float(<WIDTH x float>) nounwind readnone {
+define <WIDTH x float> @__sqrt_varying_float(<WIDTH x float>) nounwind readnone alwaysinline {
   unary4to16(result, float, @llvm.sqrt.v4f32, %0)
 ;; this returns nan for v=0, which is undesirable..
 ;;  %rsqrt = call <WIDTH x float> @__rsqrt_varying_float(<WIDTH x float> %0)
@@ -225,7 +225,7 @@ define <WIDTH x float> @__sqrt_varying_float(<WIDTH x float>) nounwind readnone
 
 declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
 
-define <WIDTH x double> @__sqrt_varying_double(<WIDTH x double>) nounwind readnone {
+define <WIDTH x double> @__sqrt_varying_double(<WIDTH x double>) nounwind readnone alwaysinline {
   unary4to16(r, double, @llvm.sqrt.v4f64, %0)
   ret <WIDTH x double> %r
 }
@@ -233,7 +233,7 @@ define <WIDTH x double> @__sqrt_varying_double(<WIDTH x double>) nounwind readno
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; reductions
 
-define i64 @__movmsk(<WIDTH x MASK>) nounwind readnone {
+define i64 @__movmsk(<WIDTH x MASK>) nounwind readnone alwaysinline {
   %and_mask = and <WIDTH x i8> %0,
     <i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128,
      i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128>
@@ -327,41 +327,41 @@ define(`neon_reduce', `
 
 declare <2 x float> @llvm.arm.neon.vpadd.v2f32(<2 x float>, <2 x float>) nounwind readnone
 
-define internal float @add_f32(float, float) {
+define internal float @add_f32(float, float) nounwind readnone alwaysinline {
   %r = fadd float %0, %1
   ret float %r
 }
 
-define internal <WIDTH x float> @__add_varying_float(<WIDTH x float>, <WIDTH x float>) {
+define internal <WIDTH x float> @__add_varying_float(<WIDTH x float>, <WIDTH x float>) nounwind readnone alwaysinline {
   %r = fadd <WIDTH x float> %0, %1
   ret <WIDTH x float> %r
 }
 
-define float @__reduce_add_float(<WIDTH x float>) nounwind readnone {
+define float @__reduce_add_float(<WIDTH x float>) nounwind readnone alwaysinline {
   neon_reduce(float, @__add_varying_float, @llvm.arm.neon.vpadd.v2f32, @add_f32)
 }
 
 declare <2 x float> @llvm.arm.neon.vpmins.v2f32(<2 x float>, <2 x float>) nounwind readnone
 
-define internal float @min_f32(float, float) {
+define internal float @min_f32(float, float) nounwind readnone alwaysinline {
   %cmp = fcmp olt float %0, %1
   %r = select i1 %cmp, float %0, float %1
   ret float %r
 }
 
-define float @__reduce_min_float(<WIDTH x float>) nounwind readnone {
+define float @__reduce_min_float(<WIDTH x float>) nounwind readnone alwaysinline {
   neon_reduce(float, @__min_varying_float, @llvm.arm.neon.vpmins.v2f32, @min_f32)
 }
 
 declare <2 x float> @llvm.arm.neon.vpmaxs.v2f32(<2 x float>, <2 x float>) nounwind readnone
 
-define internal float @max_f32(float, float) {
+define internal float @max_f32(float, float) nounwind readnone alwaysinline {
   %cmp = fcmp ugt float %0, %1
   %r = select i1 %cmp, float %0, float %1
   ret float %r
 }
 
-define float @__reduce_max_float(<WIDTH x float>) nounwind readnone {
+define float @__reduce_max_float(<WIDTH x float>) nounwind readnone alwaysinline {
   neon_reduce(float, @__max_varying_float, @llvm.arm.neon.vpmaxs.v2f32, @max_f32)
 }
 
@@ -369,7 +369,7 @@ declare <8 x i16> @llvm.arm.neon.vpaddlu.v8i16.v16i8(<16 x i8>) nounwind readnon
 declare <4 x i32> @llvm.arm.neon.vpaddlu.v4i32.v8i16(<8 x i16>) nounwind readnone
 declare <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32>) nounwind readnone
 
-define i64 @__reduce_add_int8(<WIDTH x i8>) nounwind readnone {
+define i64 @__reduce_add_int8(<WIDTH x i8>) nounwind readnone alwaysinline {
   %a16 = call <8 x i16> @llvm.arm.neon.vpaddlu.v8i16.v16i8(<16 x i8> %0)
   %a32 = call <4 x i32> @llvm.arm.neon.vpaddlu.v4i32.v8i16(<8 x i16> %a16)
   %a64 = call <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32> %a32)
@@ -379,7 +379,7 @@ define i64 @__reduce_add_int8(<WIDTH x i8>) nounwind readnone {
   ret i64 %r
 }
 
-define i64 @__reduce_add_int16(<WIDTH x i16>) nounwind readnone {
+define i64 @__reduce_add_int16(<WIDTH x i16>) nounwind readnone alwaysinline {
   v16tov8(i16, %0, %va, %vb)
   %a32 = call <4 x i32> @llvm.arm.neon.vpaddlu.v4i32.v8i16(<8 x i16> %va)
   %b32 = call <4 x i32> @llvm.arm.neon.vpaddlu.v4i32.v8i16(<8 x i16> %vb)
@@ -392,7 +392,7 @@ define i64 @__reduce_add_int16(<WIDTH x i16>) nounwind readnone {
   ret i64 %r
 }
 
-define i64 @__reduce_add_int32(<WIDTH x i32>) nounwind readnone {
+define i64 @__reduce_add_int32(<WIDTH x i32>) nounwind readnone alwaysinline {
   v16tov4(i32, %0, %va, %vb, %vc, %vd)
   %a64 = call <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32> %va)
   %b64 = call <2 x i64> @llvm.arm.neon.vpaddlu.v2i64.v4i32(<4 x i32> %vb)
@@ -409,101 +409,101 @@ define i64 @__reduce_add_int32(<WIDTH x i32>) nounwind readnone {
 
 declare <2 x i32> @llvm.arm.neon.vpmins.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @min_si32(i32, i32) {
+define internal i32 @min_si32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp slt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_min_int32(<WIDTH x i32>) nounwind readnone {
+define i32 @__reduce_min_int32(<WIDTH x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @__min_varying_int32, @llvm.arm.neon.vpmins.v2i32, @min_si32)
 }
 
 declare <2 x i32> @llvm.arm.neon.vpmaxs.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @max_si32(i32, i32) {
+define internal i32 @max_si32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp sgt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_max_int32(<WIDTH x i32>) nounwind readnone {
+define i32 @__reduce_max_int32(<WIDTH x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @__max_varying_int32, @llvm.arm.neon.vpmaxs.v2i32, @max_si32)
 }
 
 declare <2 x i32> @llvm.arm.neon.vpminu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @min_ui32(i32, i32) {
+define internal i32 @min_ui32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp ult i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_min_uint32(<WIDTH x i32>) nounwind readnone {
+define i32 @__reduce_min_uint32(<WIDTH x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @__min_varying_uint32, @llvm.arm.neon.vpmins.v2i32, @min_ui32)
 }
 
 declare <2 x i32> @llvm.arm.neon.vpmaxu.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
-define internal i32 @max_ui32(i32, i32) {
+define internal i32 @max_ui32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp ugt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__reduce_max_uint32(<WIDTH x i32>) nounwind readnone {
+define i32 @__reduce_max_uint32(<WIDTH x i32>) nounwind readnone alwaysinline {
   neon_reduce(i32, @__max_varying_uint32, @llvm.arm.neon.vpmaxs.v2i32, @max_ui32)
 }
 
-define internal double @__add_uniform_double(double, double) {
+define internal double @__add_uniform_double(double, double) nounwind readnone alwaysinline {
   %r = fadd double %0, %1
   ret double %r
 }
 
-define internal <WIDTH x double> @__add_varying_double(<WIDTH x double>, <WIDTH x double>) {
+define internal <WIDTH x double> @__add_varying_double(<WIDTH x double>, <WIDTH x double>) nounwind readnone alwaysinline {
   %r = fadd <WIDTH x double> %0, %1
   ret <WIDTH x double> %r
 }
 
-define double @__reduce_add_double(<WIDTH x double>) nounwind readnone {
+define double @__reduce_add_double(<WIDTH x double>) nounwind readnone alwaysinline {
   reduce16(double, @__add_varying_double, @__add_uniform_double)
 }
 
-define double @__reduce_min_double(<WIDTH x double>) nounwind readnone {
+define double @__reduce_min_double(<WIDTH x double>) nounwind readnone alwaysinline {
   reduce16(double, @__min_varying_double, @__min_uniform_double)
 }
 
-define double @__reduce_max_double(<WIDTH x double>) nounwind readnone {
+define double @__reduce_max_double(<WIDTH x double>) nounwind readnone alwaysinline {
   reduce16(double, @__max_varying_double, @__max_uniform_double)
 }
 
-define internal i64 @__add_uniform_int64(i64, i64) {
+define internal i64 @__add_uniform_int64(i64, i64) nounwind readnone alwaysinline {
   %r = add i64 %0, %1
   ret i64 %r
 }
 
-define internal <WIDTH x i64> @__add_varying_int64(<WIDTH x i64>, <WIDTH x i64>) {
+define internal <WIDTH x i64> @__add_varying_int64(<WIDTH x i64>, <WIDTH x i64>) nounwind readnone alwaysinline {
   %r = add <WIDTH x i64> %0, %1
   ret <WIDTH x i64> %r
 }
 
-define i64 @__reduce_add_int64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_add_int64(<WIDTH x i64>) nounwind readnone alwaysinline {
   reduce16(i64, @__add_varying_int64, @__add_uniform_int64)
 }
 
-define i64 @__reduce_min_int64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_min_int64(<WIDTH x i64>) nounwind readnone alwaysinline {
   reduce16(i64, @__min_varying_int64, @__min_uniform_int64)
 }
 
-define i64 @__reduce_max_int64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_max_int64(<WIDTH x i64>) nounwind readnone alwaysinline {
   reduce16(i64, @__max_varying_int64, @__max_uniform_int64)
 }
 
-define i64 @__reduce_min_uint64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_min_uint64(<WIDTH x i64>) nounwind readnone alwaysinline {
   reduce16(i64, @__min_varying_uint64, @__min_uniform_uint64)
 }
 
-define i64 @__reduce_max_uint64(<WIDTH x i64>) nounwind readnone {
+define i64 @__reduce_max_uint64(<WIDTH x i64>) nounwind readnone alwaysinline {
   reduce16(i64, @__max_varying_uint64, @__max_uniform_uint64)
 }
 
@@ -512,35 +512,35 @@ define i64 @__reduce_max_uint64(<WIDTH x i64>) nounwind readnone {
 
 declare <16 x i8> @llvm.arm.neon.vrhaddu.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
-define <16 x i8> @__avg_up_uint8(<16 x i8>, <16 x i8>) nounwind readnone {
+define <16 x i8> @__avg_up_uint8(<16 x i8>, <16 x i8>) nounwind readnone alwaysinline {
   %r = call <16 x i8> @llvm.arm.neon.vrhaddu.v16i8(<16 x i8> %0, <16 x i8> %1)
   ret <16 x i8> %r
 }
 
 declare <16 x i8> @llvm.arm.neon.vrhadds.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
-define <16 x i8> @__avg_up_int8(<16 x i8>, <16 x i8>) nounwind readnone {
+define <16 x i8> @__avg_up_int8(<16 x i8>, <16 x i8>) nounwind readnone alwaysinline {
   %r = call <16 x i8> @llvm.arm.neon.vrhadds.v16i8(<16 x i8> %0, <16 x i8> %1)
   ret <16 x i8> %r
 }
 
 declare <16 x i8> @llvm.arm.neon.vhaddu.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
-define <16 x i8> @__avg_down_uint8(<16 x i8>, <16 x i8>) nounwind readnone {
+define <16 x i8> @__avg_down_uint8(<16 x i8>, <16 x i8>) nounwind readnone alwaysinline {
   %r = call <16 x i8> @llvm.arm.neon.vhaddu.v16i8(<16 x i8> %0, <16 x i8> %1)
   ret <16 x i8> %r
 }
 
 declare <16 x i8> @llvm.arm.neon.vhadds.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
-define <16 x i8> @__avg_down_int8(<16 x i8>, <16 x i8>) nounwind readnone {
+define <16 x i8> @__avg_down_int8(<16 x i8>, <16 x i8>) nounwind readnone alwaysinline {
   %r = call <16 x i8> @llvm.arm.neon.vhadds.v16i8(<16 x i8> %0, <16 x i8> %1)
   ret <16 x i8> %r
 }
 
 declare <8 x i16> @llvm.arm.neon.vrhaddu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
-define <16 x i16> @__avg_up_uint16(<16 x i16>, <16 x i16>) nounwind readnone {
+define <16 x i16> @__avg_up_uint16(<16 x i16>, <16 x i16>) nounwind readnone alwaysinline {
   v16tov8(i16, %0, %a0, %b0)
   v16tov8(i16, %1, %a1, %b1)
   %r0 = call <8 x i16> @llvm.arm.neon.vrhaddu.v8i16(<8 x i16> %a0, <8 x i16> %a1)
@@ -551,7 +551,7 @@ define <16 x i16> @__avg_up_uint16(<16 x i16>, <16 x i16>) nounwind readnone {
 
 declare <8 x i16> @llvm.arm.neon.vrhadds.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
-define <16 x i16> @__avg_up_int16(<16 x i16>, <16 x i16>) nounwind readnone {
+define <16 x i16> @__avg_up_int16(<16 x i16>, <16 x i16>) nounwind readnone alwaysinline {
   v16tov8(i16, %0, %a0, %b0)
   v16tov8(i16, %1, %a1, %b1)
   %r0 = call <8 x i16> @llvm.arm.neon.vrhadds.v8i16(<8 x i16> %a0, <8 x i16> %a1)
@@ -562,7 +562,7 @@ define <16 x i16> @__avg_up_int16(<16 x i16>, <16 x i16>) nounwind readnone {
 
 declare <8 x i16> @llvm.arm.neon.vhaddu.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
-define <16 x i16> @__avg_down_uint16(<16 x i16>, <16 x i16>) nounwind readnone {
+define <16 x i16> @__avg_down_uint16(<16 x i16>, <16 x i16>) nounwind readnone alwaysinline {
   v16tov8(i16, %0, %a0, %b0)
   v16tov8(i16, %1, %a1, %b1)
   %r0 = call <8 x i16> @llvm.arm.neon.vhaddu.v8i16(<8 x i16> %a0, <8 x i16> %a1)
@@ -573,7 +573,7 @@ define <16 x i16> @__avg_down_uint16(<16 x i16>, <16 x i16>) nounwind readnone {
 
 declare <8 x i16> @llvm.arm.neon.vhadds.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
-define <16 x i16> @__avg_down_int16(<16 x i16>, <16 x i16>) nounwind readnone {
+define <16 x i16> @__avg_down_int16(<16 x i16>, <16 x i16>) nounwind readnone alwaysinline {
   v16tov8(i16, %0, %a0, %b0)
   v16tov8(i16, %1, %a1, %b1)
   %r0 = call <8 x i16> @llvm.arm.neon.vhadds.v8i16(<8 x i16> %a0, <8 x i16> %a1)

builtins/target-neon-common.ll

@@ -49,7 +49,7 @@ ctlztz()
 declare <4 x i16> @llvm.arm.neon.vcvtfp2hf(<4 x float>) nounwind readnone
 declare <4 x float> @llvm.arm.neon.vcvthf2fp(<4 x i16>) nounwind readnone
 
-define float @__half_to_float_uniform(i16 %v) nounwind readnone {
+define float @__half_to_float_uniform(i16 %v) nounwind readnone alwaysinline {
   %v1 = bitcast i16 %v to <1 x i16>
   %vec = shufflevector <1 x i16> %v1, <1 x i16> undef, 
            <4 x i32> <i32 0, i32 0, i32 0, i32 0>
@@ -58,7 +58,7 @@ define float @__half_to_float_uniform(i16 %v) nounwind readnone {
   ret float %r
 }
 
-define i16 @__float_to_half_uniform(float %v) nounwind readnone {
+define i16 @__float_to_half_uniform(float %v) nounwind readnone alwaysinline {
   %v1 = bitcast float %v to <1 x float>
   %vec = shufflevector <1 x float> %v1, <1 x float> undef, 
            <4 x i32> <i32 0, i32 0, i32 0, i32 0>
@@ -70,7 +70,14 @@ define i16 @__float_to_half_uniform(float %v) nounwind readnone {
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; math
 
-define void @__fastmath() nounwind {
+declare i32 @llvm.arm.get.fpscr() nounwind
+declare void @llvm.arm.set.fpscr(i32) nounwind
+
+define void @__fastmath() nounwind alwaysinline {
+  %x = call i32 @llvm.arm.get.fpscr()
+  ; Turn on FTZ (bit 24) and default NaN (bit 25)
+  %y = or i32 %x, 50331648
+  call void @llvm.arm.set.fpscr(i32 %y)
   ret void
 }
 
@@ -120,111 +127,111 @@ declare double @__ceil_uniform_double(double) nounwind readnone
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; min/max
 
-define float @__max_uniform_float(float, float) nounwind readnone {
+define float @__max_uniform_float(float, float) nounwind readnone alwaysinline {
   %cmp = fcmp ugt float %0, %1
   %r = select i1 %cmp, float %0, float %1
   ret float %r
 }
 
-define float @__min_uniform_float(float, float) nounwind readnone {
+define float @__min_uniform_float(float, float) nounwind readnone alwaysinline {
   %cmp = fcmp ult float %0, %1
   %r = select i1 %cmp, float %0, float %1
   ret float %r
 }
 
-define i32 @__min_uniform_int32(i32, i32) nounwind readnone {
+define i32 @__min_uniform_int32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp slt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__max_uniform_int32(i32, i32) nounwind readnone {
+define i32 @__max_uniform_int32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp sgt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__min_uniform_uint32(i32, i32) nounwind readnone {
+define i32 @__min_uniform_uint32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp ult i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i32 @__max_uniform_uint32(i32, i32) nounwind readnone {
+define i32 @__max_uniform_uint32(i32, i32) nounwind readnone alwaysinline {
   %cmp = icmp ugt i32 %0, %1
   %r = select i1 %cmp, i32 %0, i32 %1
   ret i32 %r
 }
 
-define i64 @__min_uniform_int64(i64, i64) nounwind readnone {
+define i64 @__min_uniform_int64(i64, i64) nounwind readnone alwaysinline {
   %cmp = icmp slt i64 %0, %1
   %r = select i1 %cmp, i64 %0, i64 %1
   ret i64 %r
 }
 
-define i64 @__max_uniform_int64(i64, i64) nounwind readnone {
+define i64 @__max_uniform_int64(i64, i64) nounwind readnone alwaysinline {
   %cmp = icmp sgt i64 %0, %1
   %r = select i1 %cmp, i64 %0, i64 %1
   ret i64 %r
 }
 
-define i64 @__min_uniform_uint64(i64, i64) nounwind readnone {
+define i64 @__min_uniform_uint64(i64, i64) nounwind readnone alwaysinline {
   %cmp = icmp ult i64 %0, %1
   %r = select i1 %cmp, i64 %0, i64 %1
   ret i64 %r
 }
 
-define i64 @__max_uniform_uint64(i64, i64) nounwind readnone {
+define i64 @__max_uniform_uint64(i64, i64) nounwind readnone alwaysinline {
   %cmp = icmp ugt i64 %0, %1
   %r = select i1 %cmp, i64 %0, i64 %1
   ret i64 %r
 }
 
-define double @__min_uniform_double(double, double) nounwind readnone {
+define double @__min_uniform_double(double, double) nounwind readnone alwaysinline {
   %cmp = fcmp olt double %0, %1
   %r = select i1 %cmp, double %0, double %1
   ret double %r
 }
 
-define double @__max_uniform_double(double, double) nounwind readnone {
+define double @__max_uniform_double(double, double) nounwind readnone alwaysinline {
   %cmp = fcmp ogt double %0, %1
   %r = select i1 %cmp, double %0, double %1
   ret double %r
 }
 
-define <WIDTH x i64> @__min_varying_int64(<WIDTH x i64>, <WIDTH x i64>) nounwind readnone {
+define <WIDTH x i64> @__min_varying_int64(<WIDTH x i64>, <WIDTH x i64>) nounwind readnone alwaysinline {
   %m = icmp slt <WIDTH x i64> %0, %1
   %r = select <WIDTH x i1> %m, <WIDTH x i64> %0, <WIDTH x i64> %1
   ret <WIDTH x i64> %r
 }
 
-define <WIDTH x i64> @__max_varying_int64(<WIDTH x i64>, <WIDTH x i64>) nounwind readnone {
+define <WIDTH x i64> @__max_varying_int64(<WIDTH x i64>, <WIDTH x i64>) nounwind readnone alwaysinline {
   %m = icmp sgt <WIDTH x i64> %0, %1
   %r = select <WIDTH x i1> %m, <WIDTH x i64> %0, <WIDTH x i64> %1
   ret <WIDTH x i64> %r
 }
 
-define <WIDTH x i64> @__min_varying_uint64(<WIDTH x i64>, <WIDTH x i64>) nounwind readnone {
+define <WIDTH x i64> @__min_varying_uint64(<WIDTH x i64>, <WIDTH x i64>) nounwind readnone alwaysinline {
   %m = icmp ult <WIDTH x i64> %0, %1
   %r = select <WIDTH x i1> %m, <WIDTH x i64> %0, <WIDTH x i64> %1
   ret <WIDTH x i64> %r
 }
 
-define <WIDTH x i64> @__max_varying_uint64(<WIDTH x i64>, <WIDTH x i64>) nounwind readnone {
+define <WIDTH x i64> @__max_varying_uint64(<WIDTH x i64>, <WIDTH x i64>) nounwind readnone alwaysinline {
   %m = icmp ugt <WIDTH x i64> %0, %1
   %r = select <WIDTH x i1> %m, <WIDTH x i64> %0, <WIDTH x i64> %1
   ret <WIDTH x i64> %r
 }
 
 define <WIDTH x double> @__min_varying_double(<WIDTH x double>,
-                                              <WIDTH x double>) nounwind readnone {
+                                              <WIDTH x double>) nounwind readnone alwaysinline {
   %m = fcmp olt <WIDTH x double> %0, %1
   %r = select <WIDTH x i1> %m, <WIDTH x double> %0, <WIDTH x double> %1
   ret <WIDTH x double> %r
 }
 
 define <WIDTH x double> @__max_varying_double(<WIDTH x double>,
-                                              <WIDTH x double>) nounwind readnone {
+                                              <WIDTH x double>) nounwind readnone alwaysinline {
   %m = fcmp ogt <WIDTH x double> %0, %1
   %r = select <WIDTH x i1> %m, <WIDTH x double> %0, <WIDTH x double> %1
   ret <WIDTH x double> %r
@@ -234,14 +241,14 @@ define <WIDTH x double> @__max_varying_double(<WIDTH x double>,
 
 declare float @llvm.sqrt.f32(float)
 
-define float @__sqrt_uniform_float(float) nounwind readnone {
+define float @__sqrt_uniform_float(float) nounwind readnone alwaysinline {
   %r = call float @llvm.sqrt.f32(float %0)
   ret float %r
 }
 
 declare double @llvm.sqrt.f64(double)
 
-define double @__sqrt_uniform_double(double) nounwind readnone {
+define double @__sqrt_uniform_double(double) nounwind readnone alwaysinline {
   %r = call double @llvm.sqrt.f64(double %0)
   ret double %r
 }
@@ -251,12 +258,12 @@ define double @__sqrt_uniform_double(double) nounwind readnone {
 declare i32 @llvm.ctpop.i32(i32) nounwind readnone
 declare i64 @llvm.ctpop.i64(i64) nounwind readnone
 
-define i32 @__popcnt_int32(i32) nounwind readnone {
+define i32 @__popcnt_int32(i32) nounwind readnone alwaysinline {
   %v = call i32 @llvm.ctpop.i32(i32 %0)
   ret i32 %v
 }
 
-define i64 @__popcnt_int64(i64) nounwind readnone {
+define i64 @__popcnt_int64(i64) nounwind readnone alwaysinline {
   %v = call i64 @llvm.ctpop.i64(i64 %0)
   ret i64 %v
 }

builtins/target-skx.ll

@@ -0,0 +1,94 @@
+;;  Copyright (c) 2016, Intel Corporation
+;;  All rights reserved.
+;;
+;;  Redistribution and use in source and binary forms, with or without
+;;  modification, are permitted provided that the following conditions are
+;;  met:
+;;
+;;    * Redistributions of source code must retain the above copyright
+;;      notice, this list of conditions and the following disclaimer.
+;;
+;;    * Redistributions in binary form must reproduce the above copyright
+;;      notice, this list of conditions and the following disclaimer in the
+;;      documentation and/or other materials provided with the distribution.
+;;
+;;    * Neither the name of Intel Corporation nor the names of its
+;;      contributors may be used to endorse or promote products derived from
+;;      this software without specific prior written permission.
+;;
+;;
+;;   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+;;   IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+;;   TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+;;   PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+;;   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+;;   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+;;   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+;;   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+;;   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+;;   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+;;   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+define(`WIDTH',`16')
+
+
+ifelse(LLVM_VERSION, LLVM_3_8,
+    `include(`target-avx512-common.ll')',
+         LLVM_VERSION, LLVM_3_9,
+    `include(`target-avx512-common.ll')',
+         LLVM_VERSION, LLVM_4_0,
+    `include(`target-avx512-common.ll')',
+         LLVM_VERSION, LLVM_5_0,
+    `include(`target-avx512-common.ll')'
+  )
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; rcp, rsqrt
+
+define(`rcp_rsqrt_varying_float_skx',`
+declare <16 x float> @llvm.x86.avx512.rcp14.ps.512(<16 x float>, <16 x float>, i16) nounwind readnone
+define <16 x float> @__rcp_varying_float(<16 x float>) nounwind readonly alwaysinline {
+  %call = call <16 x float> @llvm.x86.avx512.rcp14.ps.512(<16 x float> %0, <16 x float> undef, i16 -1)
+  ;; do one Newton-Raphson iteration to improve precision
+  ;;  float iv = __rcp_v(v);
+  ;;  return iv * (2. - v * iv);
+  %v_iv = fmul <16 x float> %0`,' %call
+  %two_minus = fsub <16 x float> <float 2.`,' float 2.`,' float 2.`,' float 2.`,'
+                                  float 2.`,' float 2.`,' float 2.`,' float 2.`,'
+                                  float 2.`,' float 2.`,' float 2.`,' float 2.`,'
+                                  float 2.`,' float 2.`,' float 2.`,' float 2.>`,' %v_iv
+  %iv_mul = fmul <16 x float> %call`,'  %two_minus
+  ret <16 x float> %iv_mul
+}
+declare <16 x float> @llvm.x86.avx512.rsqrt14.ps.512(<16 x float>`,'  <16 x float>`,'  i16) nounwind readnone
+define <16 x float> @__rsqrt_varying_float(<16 x float> %v) nounwind readonly alwaysinline {
+  %is = call <16 x float> @llvm.x86.avx512.rsqrt14.ps.512(<16 x float> %v`,'  <16 x float> undef`,'  i16 -1)
+  ; Newton-Raphson iteration to improve precision
+  ;  float is = __rsqrt_v(v);
+  ;  return 0.5 * is * (3. - (v * is) * is);
+  %v_is = fmul <16 x float> %v`,'  %is
+  %v_is_is = fmul <16 x float> %v_is`,'  %is
+  %three_sub = fsub <16 x float> <float 3.`,' float 3.`,' float 3.`,' float 3.`,'
+                                  float 3.`,' float 3.`,' float 3.`,' float 3.`,'
+                                  float 3.`,' float 3.`,' float 3.`,' float 3.`,'
+                                  float 3.`,' float 3.`,' float 3.`,' float 3.>`,' %v_is_is
+  %is_mul = fmul <16 x float> %is`,'  %three_sub
+  %half_scale = fmul <16 x float> <float 0.5`,' float 0.5`,' float 0.5`,' float 0.5`,'
+                                   float 0.5`,' float 0.5`,' float 0.5`,' float 0.5`,'
+                                   float 0.5`,' float 0.5`,' float 0.5`,' float 0.5`,'
+                                   float 0.5`,' float 0.5`,' float 0.5`,' float 0.5>`,' %is_mul
+  ret <16 x float> %half_scale
+}
+')
+
+ifelse(LLVM_VERSION, LLVM_3_8,
+    rcp_rsqrt_varying_float_skx(),
+         LLVM_VERSION, LLVM_3_9,
+    rcp_rsqrt_varying_float_skx(),
+         LLVM_VERSION, LLVM_4_0,
+    rcp_rsqrt_varying_float_skx(),
+         LLVM_VERSION, LLVM_5_0,
+    rcp_rsqrt_varying_float_skx()
+  )
+
+;;saturation_arithmetic_novec()

builtins/util-nvptx.m4

@@ -54,9 +54,13 @@ define(`MASK_HIGH_BIT_ON',
 define(`PTR_OP_ARGS',
   ifelse(LLVM_VERSION, LLVM_3_7,
     ``$1 , $1 *'',
-  ifelse(LLVM_VERSION, LLVM_3_8,
+         LLVM_VERSION, LLVM_3_8,
     ``$1 , $1 *'',
-  ifelse(LLVM_VERSION, LLVM_3_9,
+         LLVM_VERSION, LLVM_3_9,
+    ``$1 , $1 *'',
+         LLVM_VERSION, LLVM_4_0,
+    ``$1 , $1 *'',
+         LLVM_VERSION, LLVM_5_0,
     ``$1 , $1 *'',
     ``$1 *''
   )

builtins/util.m4

@@ -57,6 +57,10 @@ define(`PTR_OP_ARGS',
          LLVM_VERSION, LLVM_3_8,
     ``$1 , $1 *'',
          LLVM_VERSION, LLVM_3_9,
+    ``$1 , $1 *'',
+         LLVM_VERSION, LLVM_4_0,
+    ``$1 , $1 *'',
+         LLVM_VERSION, LLVM_5_0,
     ``$1 , $1 *'',
     ``$1 *''
   )
@@ -69,6 +73,10 @@ define(`MdORi64',
     ``i64'',
     LLVM_VERSION, LLVM_3_9,
     ``i64'',
+    LLVM_VERSION, LLVM_4_0,
+    ``i64'',
+    LLVM_VERSION, LLVM_5_0,
+    ``i64'',
     ``double''
   )
 )
@@ -78,6 +86,10 @@ define(`MfORi32',
     ``i32'',
     LLVM_VERSION, LLVM_3_9,
     ``i32'',
+    LLVM_VERSION, LLVM_4_0,
+    ``i32'',
+    LLVM_VERSION, LLVM_5_0,
+    ``i32'',
     ``float''
   )
 )
@@ -1586,6 +1598,12 @@ define <$1 x $2> @__atomic_compare_exchange_$3_global($2* %ptr, <$1 x $2> %cmp,
   ',LLVM_VERSION,LLVM_3_9,`
     %r_LANE_ID_t = cmpxchg $2 * %ptr, $2 %cmp_LANE_ID, $2 %val_LANE_ID seq_cst seq_cst
     %r_LANE_ID = extractvalue { $2, i1 } %r_LANE_ID_t, 0
+  ',LLVM_VERSION,LLVM_4_0,`
+    %r_LANE_ID_t = cmpxchg $2 * %ptr, $2 %cmp_LANE_ID, $2 %val_LANE_ID seq_cst seq_cst
+    %r_LANE_ID = extractvalue { $2, i1 } %r_LANE_ID_t, 0
+  ',LLVM_VERSION,LLVM_5_0,`
+    %r_LANE_ID_t = cmpxchg $2 * %ptr, $2 %cmp_LANE_ID, $2 %val_LANE_ID seq_cst seq_cst
+    %r_LANE_ID = extractvalue { $2, i1 } %r_LANE_ID_t, 0
   ',`
     %r_LANE_ID = cmpxchg $2 * %ptr, $2 %cmp_LANE_ID, $2 %val_LANE_ID seq_cst
   ')
@@ -1614,6 +1632,12 @@ define $2 @__atomic_compare_exchange_uniform_$3_global($2* %ptr, $2 %cmp,
   ',LLVM_VERSION,LLVM_3_9,`
    %r_t = cmpxchg $2 * %ptr, $2 %cmp, $2 %val seq_cst seq_cst
    %r = extractvalue { $2, i1 } %r_t, 0
+  ',LLVM_VERSION,LLVM_4_0,`
+   %r_t = cmpxchg $2 * %ptr, $2 %cmp, $2 %val seq_cst seq_cst
+   %r = extractvalue { $2, i1 } %r_t, 0
+  ',LLVM_VERSION,LLVM_5_0,`
+   %r_t = cmpxchg $2 * %ptr, $2 %cmp, $2 %val seq_cst seq_cst
+   %r = extractvalue { $2, i1 } %r_t, 0
   ',`
    %r = cmpxchg $2 * %ptr, $2 %cmp, $2 %val seq_cst
   ')

cbackend.cpp

@@ -462,7 +462,11 @@ namespace {
       VectorConstantIndex = 0;
     }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // <= 3.9
     virtual const char *getPassName() const { return "C backend"; }
+#else // LLVM 4.0+
+    virtual llvm::StringRef getPassName() const { return "C backend"; }
+#endif
 
     void getAnalysisUsage(llvm::AnalysisUsage &AU) const {
 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_6 // <= 3.6
@@ -525,8 +529,10 @@ namespace {
                            bool IgnoreName = false,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
                            const llvm::AttrListPtr &PAL = llvm::AttrListPtr()
-#else // LLVM 3.3+
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
                            const llvm::AttributeSet &PAL = llvm::AttributeSet()
+#else // LLVM 5.0+
+                           const llvm::AttributeList &PAL = llvm::AttributeList()
 #endif
                                  );
     llvm::raw_ostream &printSimpleType(llvm::raw_ostream &Out, llvm::Type *Ty,
@@ -536,8 +542,10 @@ namespace {
     void printStructReturnPointerFunctionType(llvm::raw_ostream &Out,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
                                               const llvm::AttrListPtr &PAL,
-#else // LLVM 3.3+
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
                                               const llvm::AttributeSet &PAL,
+#else // LLVM 5.0+
+                                              const llvm::AttributeList &PAL,
 #endif
                                               llvm::PointerType *Ty);
 
@@ -782,8 +790,10 @@ std::string CWriter::getArrayName(llvm::ArrayType *AT) {
 void CWriter::printStructReturnPointerFunctionType(llvm::raw_ostream &Out,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
                                                    const llvm::AttrListPtr &PAL,
-#else // LLVM 3.3+
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
                                                    const llvm::AttributeSet &PAL,
+#else // LLVM 5.0+
+                                                   const llvm::AttributeList &PAL,
 #endif
                                                    llvm::PointerType *TheTy) {
   llvm::FunctionType *FTy = llvm::cast<llvm::FunctionType>(TheTy->getElementType());
@@ -801,8 +811,10 @@ void CWriter::printStructReturnPointerFunctionType(llvm::raw_ostream &Out,
     llvm::Type *ArgTy = *I;
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
     if (PAL.getParamAttributes(Idx).hasAttribute(llvm::Attributes::ByVal)) {
-#else // LLVM 3.3+
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
         if (PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::ByVal)) {
+#else // LLVM 5.0+
+        if (PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::ByVal)) {
 #endif
       assert(ArgTy->isPointerTy());
       ArgTy = llvm::cast<llvm::PointerType>(ArgTy)->getElementType();
@@ -810,8 +822,10 @@ void CWriter::printStructReturnPointerFunctionType(llvm::raw_ostream &Out,
     printType(FunctionInnards, ArgTy,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
               PAL.getParamAttributes(Idx).hasAttribute(llvm::Attributes::SExt),
-#else // LLVM 3.3+
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
               PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::SExt),
+#else // LLVM 5.0+
+              PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::SExt),
 #endif
               "");
     PrintedType = true;
@@ -827,8 +841,10 @@ void CWriter::printStructReturnPointerFunctionType(llvm::raw_ostream &Out,
   printType(Out, RetTy,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
             PAL.getParamAttributes(0).hasAttribute(llvm::Attributes::SExt),
-#else // LLVM 3.3+
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
             PAL.getParamAttributes(0).hasAttribute(llvm::AttributeSet::ReturnIndex, llvm::Attribute::SExt),
+#else // LLVM 5.0+
+            PAL.getParamAttributes(0).hasAttribute(llvm::AttributeList::ReturnIndex, llvm::Attribute::SExt),
 #endif
             FunctionInnards.str());
 }
@@ -925,8 +941,10 @@ llvm::raw_ostream &CWriter::printType(llvm::raw_ostream &Out, llvm::Type *Ty,
                                 bool IgnoreName,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
                                 const llvm::AttrListPtr &PAL
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
                                 const llvm::AttributeSet &PAL
+#else // LLVM 5.0+
+                                const llvm::AttributeList &PAL
 #endif
                                       ) {
 
@@ -947,8 +965,10 @@ llvm::raw_ostream &CWriter::printType(llvm::raw_ostream &Out, llvm::Type *Ty,
       llvm::Type *ArgTy = *I;
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
       if (PAL.getParamAttributes(Idx).hasAttribute(llvm::Attributes::ByVal)) {
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
           if (PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::ByVal)) {
+#else // LLVM 5.0+
+          if (PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::ByVal)) {
 #endif
         assert(ArgTy->isPointerTy());
         ArgTy = llvm::cast<llvm::PointerType>(ArgTy)->getElementType();
@@ -958,8 +978,10 @@ llvm::raw_ostream &CWriter::printType(llvm::raw_ostream &Out, llvm::Type *Ty,
       printType(FunctionInnards, ArgTy,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
                 PAL.getParamAttributes(Idx).hasAttribute(llvm::Attributes::SExt),
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
                 PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::SExt),
+#else // LLVM 5.0+
+                PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::SExt),
 #endif
                 "");
       ++Idx;
@@ -975,8 +997,10 @@ llvm::raw_ostream &CWriter::printType(llvm::raw_ostream &Out, llvm::Type *Ty,
     printType(Out, FTy->getReturnType(),
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
               PAL.getParamAttributes(0).hasAttribute(llvm::Attributes::SExt),
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
               PAL.getParamAttributes(0).hasAttribute(llvm::AttributeSet::ReturnIndex, llvm::Attribute::SExt),
+#else // LLVM 5.0+
+              PAL.getParamAttributes(0).hasAttribute(llvm::AttributeList::ReturnIndex, llvm::Attribute::SExt),
 #endif
               FunctionInnards.str());
     return Out;
@@ -1180,6 +1204,7 @@ void CWriter::printConstantDataSequential(llvm::ConstantDataSequential *CDS,
 
 static inline std::string ftostr(const llvm::APFloat& V) {
   std::string Buf;
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
   if (&V.getSemantics() == &llvm::APFloat::IEEEdouble) {
     llvm::raw_string_ostream(Buf) << V.convertToDouble();
     return Buf;
@@ -1187,6 +1212,15 @@ static inline std::string ftostr(const llvm::APFloat& V) {
     llvm::raw_string_ostream(Buf) << (double)V.convertToFloat();
     return Buf;
   }
+#else // LLVM 4.0+
+  if (&V.getSemantics() == &llvm::APFloat::IEEEdouble()) {
+    llvm::raw_string_ostream(Buf) << V.convertToDouble();
+    return Buf;
+  } else if (&V.getSemantics() == &llvm::APFloat::IEEEsingle()) {
+    llvm::raw_string_ostream(Buf) << (double)V.convertToFloat();
+    return Buf;
+  }
+#endif
   return "<unknown format in ftostr>"; // error
 }
 
@@ -1206,7 +1240,11 @@ static bool isFPCSafeToPrint(const llvm::ConstantFP *CFP) {
     return false;
   llvm::APFloat APF = llvm::APFloat(CFP->getValueAPF());  // copy
   if (CFP->getType() == llvm::Type::getFloatTy(CFP->getContext()))
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // <= 3.9
     APF.convert(llvm::APFloat::IEEEdouble, llvm::APFloat::rmNearestTiesToEven, &ignored);
+#else // LLVM 4.0+
+    APF.convert(llvm::APFloat::IEEEdouble(), llvm::APFloat::rmNearestTiesToEven, &ignored);
+#endif
 #if HAVE_PRINTF_A && ENABLE_CBE_PRINTF_A
   char Buffer[100];
   sprintf(Buffer, "%a", APF.convertToDouble());
@@ -1637,11 +1675,15 @@ void CWriter::printConstant(llvm::Constant *CPV, bool Static) {
         // useful.
         llvm::APFloat Tmp = FPC->getValueAPF();
         bool LosesInfo;
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // <= 3.9
         Tmp.convert(llvm::APFloat::IEEEdouble, llvm::APFloat::rmTowardZero, &LosesInfo);
+#else // LLVM 4.0+
+        Tmp.convert(llvm::APFloat::IEEEdouble(), llvm::APFloat::rmTowardZero, &LosesInfo);
+#endif
         V = Tmp.convertToDouble();
       }
 
-      if (isnan(V)) {
+      if (std::isnan(V)) {
         // The value is NaN
 
         // FIXME the actual NaN bits should be emitted.
@@ -1665,7 +1707,7 @@ void CWriter::printConstant(llvm::Constant *CPV, bool Static) {
         else
           Out << "LLVM_NAN" << (Val == QuietNaN ? "" : "S") << "(\""
               << Buffer << "\") /*nan*/ ";
-      } else if (isinf(V)) {
+      } else if (std::isinf(V)) {
         // The value is Inf
         if (V < 0) Out << '-';
         Out << "LLVM_INF" <<
@@ -2337,7 +2379,7 @@ static SpecialGlobalClass getGlobalVariableClass(const llvm::GlobalVariable *GV)
 
   // Otherwise, if it is other metadata, don't print it.  This catches things
   // like debug information.
-#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_5 /* LLVM 3.5+ */
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_5 && ISPC_LLVM_VERSION <= ISPC_LLVM_3_8 /* LLVM 3.5-3.8 */
   // Here we compare char *
   if (!strcmp(GV->getSection(), "llvm.metadata"))
 #else
@@ -3142,8 +3184,10 @@ void CWriter::printFunctionSignature(const llvm::Function *F, bool Prototype) {
   llvm::FunctionType *FT = llvm::cast<llvm::FunctionType>(F->getFunctionType());
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
   const llvm::AttrListPtr &PAL = F->getAttributes();
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
   const llvm::AttributeSet &PAL = F->getAttributes();
+#else // LLVM 5.0+
+  const llvm::AttributeList &PAL = F->getAttributes();
 #endif
 
   std::string tstr;
@@ -3180,8 +3224,10 @@ void CWriter::printFunctionSignature(const llvm::Function *F, bool Prototype) {
         llvm::Type *ArgTy = I->getType();
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
         if (PAL.getParamAttributes(Idx).hasAttribute(llvm::Attributes::ByVal)) {
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
             if (PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::ByVal)) {
+#else // LLVM 5.0+
+            if (PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::ByVal)) {
 #endif
           ArgTy = llvm::cast<llvm::PointerType>(ArgTy)->getElementType();
 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_7 /* 3.2, 3.3, 3.4, 3.5, 3.6, 3.7 */
@@ -3193,8 +3239,10 @@ void CWriter::printFunctionSignature(const llvm::Function *F, bool Prototype) {
         printType(FunctionInnards, ArgTy,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
                   PAL.getParamAttributes(Idx).hasAttribute(llvm::Attributes::SExt),
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
                   PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::SExt),
+#else // LLVM 5.0+
+                  PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::SExt),
 #endif
                   ArgName);
         PrintedArg = true;
@@ -3219,8 +3267,10 @@ void CWriter::printFunctionSignature(const llvm::Function *F, bool Prototype) {
       llvm::Type *ArgTy = *I;
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
       if (PAL.getParamAttributes(Idx).hasAttribute(llvm::Attributes::ByVal)) {
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
           if (PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::ByVal)) {
+#else // LLVM 5.0+
+          if (PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::ByVal)) {
 #endif
         assert(ArgTy->isPointerTy());
         ArgTy = llvm::cast<llvm::PointerType>(ArgTy)->getElementType();
@@ -3228,8 +3278,10 @@ void CWriter::printFunctionSignature(const llvm::Function *F, bool Prototype) {
       printType(FunctionInnards, ArgTy,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
                 PAL.getParamAttributes(Idx).hasAttribute(llvm::Attributes::SExt)
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
                 PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::SExt)
+#else // LLVM 5.0+
+                PAL.getParamAttributes(Idx).hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::SExt)
 #endif
                 );
       PrintedArg = true;
@@ -3265,8 +3317,10 @@ void CWriter::printFunctionSignature(const llvm::Function *F, bool Prototype) {
   printType(Out, RetTy,
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
             PAL.getParamAttributes(0).hasAttribute(llvm::Attributes::SExt),
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
             PAL.getParamAttributes(0).hasAttribute(llvm::AttributeSet::ReturnIndex, llvm::Attribute::SExt),
+#else // LLVM 5.0+
+            PAL.getParamAttributes(0).hasAttribute(llvm::AttributeList::ReturnIndex, llvm::Attribute::SExt),
 #endif
             FunctionInnards.str());
 }
@@ -3343,7 +3397,11 @@ void CWriter::printFunction(llvm::Function &F) {
   // print the basic blocks
   for (llvm::Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
     if (llvm::Loop *L = LI->getLoopFor(&*BB)) {
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_9 // LLVM 3.9+
+      if (L->getHeader()->getIterator() == BB && L->getParentLoop() == 0)
+#else
       if (L->getHeader() == BB && L->getParentLoop() == 0)
+#endif
         printLoop(L);
     } else {
       printBasicBlock(&*BB);
@@ -4197,8 +4255,10 @@ void CWriter::visitCallInst(llvm::CallInst &I) {
   // parameter instead of passing it to the call.
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
   const llvm::AttrListPtr &PAL = I.getAttributes();
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
   const llvm::AttributeSet &PAL = I.getAttributes();
+#else // LLVM 5.0+
+  const llvm::AttributeList &PAL = I.getAttributes();
 #endif
 
   bool hasByVal = I.hasByValArgument();
@@ -4310,8 +4370,10 @@ void CWriter::visitCallInst(llvm::CallInst &I) {
       printType(Out, FTy->getParamType(ArgNo),
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2
                 PAL.getParamAttributes(ArgNo+1).hasAttribute(llvm::Attributes::SExt)
-#else /* LLVM 3.3+ */
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
                 PAL.getParamAttributes(ArgNo+1).hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::SExt)
+#else // LLVM 5.0+
+                PAL.getParamAttributes(ArgNo+1).hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::SExt)
 #endif
                 );
       Out << ')';
@@ -4373,7 +4435,7 @@ bool CWriter::visitBuiltinCall(llvm::CallInst &I, llvm::Intrinsic::ID ID,
     if (I.getParent()->getParent()->arg_empty())
       Out << "vararg_dummy_arg";
     else
-      writeOperand(&*(--I.getParent()->getParent()->arg_end()));
+      writeOperand(&*(std::prev(I.getParent()->getParent()->arg_end())));
     Out << ')';
     return true;
   case llvm::Intrinsic::vaend:
@@ -4548,7 +4610,11 @@ void CWriter::printGEPExpression(llvm::Value *Ptr, llvm::gep_type_iterator I,
   llvm::VectorType *LastIndexIsVector = 0;
   {
     for (llvm::gep_type_iterator TmpI = I; TmpI != E; ++TmpI)
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
       LastIndexIsVector = llvm::dyn_cast<llvm::VectorType>(*TmpI);
+#else // LLVM 4.0+
+    LastIndexIsVector = llvm::dyn_cast<llvm::VectorType>(TmpI.getIndexedType());
+#endif
   }
 
   Out << "(";
@@ -4577,7 +4643,11 @@ void CWriter::printGEPExpression(llvm::Value *Ptr, llvm::gep_type_iterator I,
     // exposed, like a global, avoid emitting (&foo)[0], just emit foo instead.
     if (isAddressExposed(Ptr)) {
       writeOperandInternal(Ptr, Static);
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     } else if (I != E && (*I)->isStructTy()) {
+#else // LLVM 4.0+
+    } else if (I != E && I.isStruct()) {
+#endif
       // If we didn't already emit the first operand, see if we can print it as
       // P->f instead of "P[0].f"
       writeOperand(Ptr);
@@ -4592,13 +4662,18 @@ void CWriter::printGEPExpression(llvm::Value *Ptr, llvm::gep_type_iterator I,
   }
 
   for (; I != E; ++I) {
-    if ((*I)->isStructTy()) {
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
+    llvm::Type *type = *I;
+#else // LLVM 4.0+
+    llvm::Type *type = I.getIndexedType();
+#endif
+    if (type->isStructTy()) {
       Out << ".field" << llvm::cast<llvm::ConstantInt>(I.getOperand())->getZExtValue();
-    } else if ((*I)->isArrayTy()) {
+    } else if (type->isArrayTy()) {
       Out << ".array[";
       writeOperandWithCast(I.getOperand(), llvm::Instruction::GetElementPtr);
       Out << ']';
-    } else if (!(*I)->isVectorTy()) {
+    } else if (!type->isVectorTy()) {
       Out << '[';
       writeOperandWithCast(I.getOperand(), llvm::Instruction::GetElementPtr);
       Out << ']';
@@ -4897,7 +4972,11 @@ public:
     SmearCleanupPass(llvm::Module *m, int width)
         : BasicBlockPass(ID) { module = m; vectorWidth = width; }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // <= 3.9
     const char *getPassName() const { return "Smear Cleanup Pass"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Smear Cleanup Pass"; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 
     static char ID;
@@ -5105,7 +5184,11 @@ public:
     AndCmpCleanupPass()
         : BasicBlockPass(ID) { }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // <= 3.9
     const char *getPassName() const { return "AndCmp Cleanup Pass"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "AndCmp Cleanup Pass"; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 
     static char ID;
@@ -5247,7 +5330,11 @@ public:
 #endif
     }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // <= 3.9
     const char *getPassName() const { return "MaskOps Cleanup Pass"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "MaskOps Cleanup Pass"; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 
 private:

contrib/ispc.vim

@@ -1,7 +1,7 @@
 " Vim syntax file
 " Language:	ISPC
 " Maintainer:	Andreas Wendleder <andreas.wendleder@gmail.com>
-" Last Change:	2011 Aug 3
+" Last Change:	2016 May 04
 
 " Quit when a syntax file was already loaded
 if exists("b:current_syntax")
@@ -13,11 +13,11 @@ runtime! syntax/c.vim
 unlet b:current_syntax
 
 " New keywords
-syn keyword	ispcStatement	cbreak ccontinue creturn launch print reference soa sync task
+syn keyword	ispcStatement	cbreak ccontinue creturn launch print reference soa sync
 syn keyword	ispcConditional	cif
-syn keyword	ispcRepeat	cdo cfor cwhile
+syn keyword	ispcRepeat	cdo cfor cwhile foreach foreach_tiled foreach_unique foreach_active
-syn keyword	ispcBuiltin	programCount programIndex	
+syn keyword	ispcBuiltin	programCount programIndex taskCount taskCount0 taskCount1 taskCount3 taskIndex taskIndex0 taskIndex1 taskIndex2
-syn keyword	ispcType	export uniform varying int8 int16 int32 int64
+syn keyword	ispcType	export uniform varying int8 int16 int32 int64 task new delete
 syn keyword	ispcOperator	operator
 
 "double precision floating point number, with dot, optional exponent

ctx.cpp

@@ -385,10 +385,14 @@ FunctionEmitContext::FunctionEmitContext(Function *func, Symbol *funSym,
         llvm::DISubroutineType *diSubprogramType_n =
             llvm::cast<llvm::DISubroutineType>(getDICompositeType(diSubprogramType));
         int flags = llvm::DINode::FlagPrototyped;
-#else /* LLVM 3.8+ */
+#elif ISPC_LLVM_VERSION == ISPC_LLVM_3_8 || ISPC_LLVM_VERSION == ISPC_LLVM_3_9 /* LLVM 3.8, 3.9 */
         Assert(llvm::isa<llvm::DISubroutineType>(diSubprogramType));
         llvm::DISubroutineType *diSubprogramType_n = llvm::cast<llvm::DISubroutineType>(diSubprogramType);
         int flags = llvm::DINode::FlagPrototyped;
+#else /* LLVM 4.0+ */
+        Assert(llvm::isa<llvm::DISubroutineType>(diSubprogramType));
+        llvm::DISubroutineType *diSubprogramType_n = llvm::cast<llvm::DISubroutineType>(diSubprogramType);
+        llvm::DINode::DIFlags flags = llvm::DINode::FlagPrototyped;
 
 #endif
 
@@ -417,7 +421,7 @@ FunctionEmitContext::FunctionEmitContext(Function *func, Symbol *funSym,
                                          isStatic,           true, /* is defn */
                                          firstLine,          flags,
                                          isOptimized,        llvmFunction);
-#else /* LLVM 3.8+ */
+#elif ISPC_LLVM_VERSION == ISPC_LLVM_3_8 || ISPC_LLVM_VERSION == ISPC_LLVM_3_9 /* LLVM 3.8, 3.9 */
         diSubprogram =
             m->diBuilder->createFunction(diFile /* scope */, funSym->name,
                                          mangledName,        diFile,
@@ -425,6 +429,16 @@ FunctionEmitContext::FunctionEmitContext(Function *func, Symbol *funSym,
                                          isStatic,           true, /* is defn */
                                          firstLine,          flags,
                                          isOptimized);
+        llvmFunction->setSubprogram(diSubprogram);
+#else /* LLVM 4.0+ */
+        diSubprogram =
+            m->diBuilder->createFunction(diFile /* scope */, funSym->name,
+                                         mangledName,        diFile,
+                                         firstLine,          diSubprogramType_n,
+                                         isStatic,           true, /* is defn */
+                                         firstLine,          flags,
+                                         isOptimized);
+        llvmFunction->setSubprogram(diSubprogram);
 #endif
 
         /* And start a scope representing the initial function scope */
@@ -1820,7 +1834,11 @@ FunctionEmitContext::EmitFunctionParameterDebugInfo(Symbol *sym, int argNum) {
     if (m->diBuilder == NULL)
         return;
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     int flags = 0;
+#else // LLVM 4.0+
+    llvm::DINode::DIFlags flags = llvm::DINode::FlagZero;
+#endif
 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_6 /* 3.2, 3.3, 3.4, 3.5, 3.6 */
     llvm::DIScope scope = diSubprogram;
     llvm::DIType diType = sym->type->GetDIType(scope);

docs/ReleaseNotes.txt

@@ -1,3 +1,11 @@
+=== v1.9.1 === (8 July 2016)
+
+An ISPC update with new native AVX512 target for future Xeon CPUs and
+improvements for debugging, including new switch --dwarf-version to support
+debugging on old systems.
+
+The release is based on patched LLVM 3.8.
+
 === v1.9.0 === (12 Feb 2016)
 
 An ISPC release with AVX512 (KNL flavor) support and a number of bug fixes,

docs/ispc.rst

@@ -53,6 +53,7 @@ Contents:
   + `Updating ISPC Programs For Changes In ISPC 1.7.0`_
   + `Updating ISPC Programs For Changes In ISPC 1.8.2`_
   + `Updating ISPC Programs For Changes In ISPC 1.9.0`_
+  + `Updating ISPC Programs For Changes In ISPC 1.9.1`_
 
 * `Getting Started with ISPC`_
 
@@ -345,6 +346,12 @@ Updating ISPC Programs For Changes In ISPC 1.9.0
 The release doesn't contains language changes, which may affect compatibility with
 older versions. It introduces new AVX512 target: avx512knl-i32x16.
 
+Updating ISPC Programs For Changes In ISPC 1.9.1
+------------------------------------------------
+
+The release doesn't contains language changes, which may affect compatibility with
+older versions. It introduces new AVX512 target: avx512skx-i32x16.
+
 
 Getting Started with ISPC
 =========================
@@ -578,6 +585,7 @@ avx, avx1    AVX (2010-2011 era Intel CPUs)
 avx1.1       AVX 1.1 (2012 era "Ivybridge" Intel CPUs)
 avx2         AVX 2 target (2013- Intel "Haswell" CPUs)
 avx512knl    AVX 512 target (Xeon Phi chips codename Knights Landing)
+avx512skx    AVX 512 target (future Xeon CPUs)
 neon         ARM NEON
 sse2         SSE2 (early 2000s era x86 CPUs)
 sse4         SSE4 (generally 2008-2010 Intel CPUs)
@@ -807,7 +815,7 @@ preprocessor runs:
   * - ISPC
     - 1
     - Detecting that the ``ispc`` compiler is processing the file
-  * - ISPC_TARGET_{NEON_8,NEON_16,NEON_32,SSE2,SSE4,AVX,AVX11,AVX2,AVX512KNL,GENERIC}
+  * - ISPC_TARGET_{NEON_8, NEON_16, NEON_32, SSE2, SSE4, AVX, AVX11, AVX2, AVX512KNL, AVX512SKX, GENERIC}
     - 1
     - One of these will be set, depending on the compilation target.
   * - ISPC_POINTER_SIZE

docs/news.rst

@@ -2,6 +2,16 @@
 ispc News
 =========
 
+ispc 1.9.1 is Released
+----------------------
+
+An ``ispc`` release with new native AVX512 target for future Xeon CPUs and
+improvements for debugging. Release is based on patched version LLVM 3.8 backend.
+
+For more details, please check `Release Notes`_.
+
+.. _Release Notes: https://github.com/ispc/ispc/blob/master/docs/ReleaseNotes.txt
+
 ispc 1.9.0 is Released
 ----------------------
 

doxygen.cfg

@@ -31,7 +31,7 @@ PROJECT_NAME           = "Intel SPMD Program Compiler"
 # This could be handy for archiving the generated documentation or
 # if some version control system is used.
 
-PROJECT_NUMBER         = 1.9.0
+PROJECT_NUMBER         = 1.9.2dev
 
 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
 # base path where the generated documentation will be put.

examples/aobench/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=ao
 CPP_SRC=ao.cpp ao_serial.cpp
 ISPC_SRC=ao.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x4,avx1-i32x8,avx2-i32x8
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x4,avx1-i32x8,avx2-i32x8,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 
 include ../common.mk

examples/common.mk

@@ -35,6 +35,12 @@ ifeq ($(ARCH),x86)
     ifneq (,$(findstring avx2,$(ISPC_IA_TARGETS)))
       ISPC_OBJS+=$(addprefix objs/, $(ISPC_SRC:.ispc=)_ispc_avx2.o)
     endif
+    ifneq (,$(findstring avx512knl,$(ISPC_IA_TARGETS)))
+      ISPC_OBJS+=$(addprefix objs/, $(ISPC_SRC:.ispc=)_ispc_avx512knl.o)
+    endif
+    ifneq (,$(findstring avx512skx,$(ISPC_IA_TARGETS)))
+      ISPC_OBJS+=$(addprefix objs/, $(ISPC_SRC:.ispc=)_ispc_avx512skx.o)
+    endif
   endif
   ISPC_TARGETS=$(ISPC_IA_TARGETS)
   ARCH_BIT:=$(shell getconf LONG_BIT)
@@ -86,7 +92,7 @@ objs/%.o: ../%.cpp dirs
 
 objs/$(EXAMPLE).o: objs/$(EXAMPLE)_ispc.h dirs
 
-objs/%_ispc.h objs/%_ispc.o objs/%_ispc_sse2.o objs/%_ispc_sse4.o objs/%_ispc_avx.o objs/%_ispc_avx11.o objs/%_ispc_avx2.o: %.ispc dirs
+objs/%_ispc.h objs/%_ispc.o objs/%_ispc_sse2.o objs/%_ispc_sse4.o objs/%_ispc_avx.o objs/%_ispc_avx11.o objs/%_ispc_avx2.o objs/%_ispc_avx512knl.o objs/%_ispc_avx512skx.o : %.ispc dirs
 	$(ISPC) $(ISPC_FLAGS) --target=$(ISPC_TARGETS) $< -o objs/$*_ispc.o -h objs/$*_ispc.h
 
 objs/$(ISPC_SRC:.ispc=)_sse4.cpp: $(ISPC_SRC)

examples/deferred/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=deferred_shading
 CPP_SRC=common.cpp main.cpp dynamic_c.cpp dynamic_cilk.cpp
 ISPC_SRC=kernels.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 ISPC_FLAGS=--opt=fast-math
 

examples/gmres/Makefile

@@ -3,7 +3,7 @@ EXAMPLE=gmres
 CPP_SRC=algorithm.cpp main.cpp matrix.cpp
 CC_SRC=mmio.c
 ISPC_SRC=matrix.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 
 include ../common.mk

examples/mandelbrot/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=mandelbrot
 CPP_SRC=mandelbrot.cpp mandelbrot_serial.cpp
 ISPC_SRC=mandelbrot.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 
 include ../common.mk

examples/mandelbrot_tasks/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=mandelbrot_tasks
 CPP_SRC=mandelbrot_tasks.cpp mandelbrot_tasks_serial.cpp
 ISPC_SRC=mandelbrot_tasks.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 
 include ../common.mk

examples/noise/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=noise
 CPP_SRC=noise.cpp noise_serial.cpp
 ISPC_SRC=noise.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x4,avx1-i32x16,avx2-i32x16
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x4,avx1-i32x16,avx2-i32x16,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 
 include ../common.mk

examples/options/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=options
 CPP_SRC=options.cpp options_serial.cpp
 ISPC_SRC=options.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 
 include ../common.mk

examples/perfbench/Makefile

@@ -1,5 +1,5 @@
 
-EXAMPLE=perbench
+EXAMPLE=perfbench
 CPP_SRC=perfbench.cpp perfbench_serial.cpp
 ISPC_SRC=perfbench.ispc
 ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x4,avx1-i32x8,avx2-i32x8

examples/rt/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=rt
 CPP_SRC=rt.cpp rt_serial.cpp
 ISPC_SRC=rt.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x8,avx2-i32x8
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x8,avx2-i32x8,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 
 include ../common.mk

examples/sort/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=sort
 CPP_SRC=sort.cpp sort_serial.cpp
 ISPC_SRC=sort.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x8,avx2-i32x8
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x8,avx2-i32x8,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 #ISPC_FLAGS=-DDEBUG
 

examples/stencil/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=stencil
 CPP_SRC=stencil.cpp stencil_serial.cpp
 ISPC_SRC=stencil.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x16,avx2-i32x16,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 
 include ../common.mk

examples/tasksys.cpp

@@ -47,6 +47,7 @@
     - Cilk Plus (ISPC_USE_CILK)
     - TBB (ISPC_USE_TBB_TASK_GROUP, ISPC_USE_TBB_PARALLEL_FOR)
     - OpenMP (ISPC_USE_OMP)
+    - HPX (ISPC_USE_HPX)
 
   The task system implementation can be selected at compile time, by defining 
   the appropriate preprocessor symbol on the command line (for e.g.: -D ISPC_USE_TBB).
@@ -69,13 +70,20 @@
   the machine, but less so when there are other tasks that need running on the machine.
 
 #define ISPC_USE_CREW
+#define ISPC_USE_HPX
+  The HPX model requires the HPX runtime environment to be set up. This can be
+  done manually, e.g. with hpx::init, or by including hpx/hpx_main.hpp which
+  uses the main() function as entry point and sets up the runtime system.
+  Number of threads can be specified as commandline parameter with
+  --hpx:threads, use "all" to spawn one thread per processing unit.
 
 */
 
 #if !(defined ISPC_USE_CONCRT          || defined ISPC_USE_GCD              || \
       defined ISPC_USE_PTHREADS        || defined ISPC_USE_PTHREADS_FULLY_SUBSCRIBED || \
       defined ISPC_USE_TBB_TASK_GROUP  || defined ISPC_USE_TBB_PARALLEL_FOR || \
-      defined ISPC_USE_OMP             || defined ISPC_USE_CILK             )
+      defined ISPC_USE_OMP             || defined ISPC_USE_CILK             || \
+      defined ISPC_USE_HPX)
 
     // If no task model chosen from the compiler cmdline, pick a reasonable default
     #if defined(_WIN32) || defined(_WIN64)
@@ -157,6 +165,10 @@
 #ifdef ISPC_USE_OMP
   #include <omp.h>
 #endif // ISPC_USE_OMP
+#ifdef ISPC_USE_HPX
+#include <hpx/include/async.hpp>
+#include <hpx/lcos/wait_all.hpp>
+#endif // ISPC_USE_HPX
 #ifdef ISPC_IS_LINUX
   #include <malloc.h>
 #endif // ISPC_IS_LINUX
@@ -505,6 +517,20 @@ private:
 
 #endif // ISPC_USE_TBB_TASK_GROUP
 
+#ifdef ISPC_USE_HPX
+
+class TaskGroup : public TaskGroupBase {
+public:
+    void Launch(int baseIndex, int count);
+    void Sync();
+private:
+    std::vector<hpx::future<void>> futures;
+};
+
+#endif // ISPC_USE_HPX
+
+///////////////////////////////////////////////////////////////////////////
+
 ///////////////////////////////////////////////////////////////////////////
 // Grand Central Dispatch
 
@@ -1048,6 +1074,33 @@ TaskGroup::Sync() {
 
 #endif // ISPC_USE_TBB_TASK_GROUP
 
+///////////////////////////////////////////////////////////////////////////
+// ISPC_USE_HPX
+
+#ifdef ISPC_USE_HPX
+
+static void
+InitTaskSystem() {
+}
+
+inline void
+TaskGroup::Launch(int baseIndex, int count) {
+    for (int i = 0; i < count; ++i) {
+        TaskInfo *ti = GetTaskInfo(baseIndex + i);
+        int threadIndex = i;
+        int threadCount = count;
+        futures.push_back(hpx::async(ti->func, ti->data, threadIndex, threadCount, ti->taskIndex, ti->taskCount(),
+            ti->taskIndex0(), ti->taskIndex1(), ti->taskIndex2(),
+            ti->taskCount0(), ti->taskCount1(), ti->taskCount2()));
+    }
+}
+
+inline void
+TaskGroup::Sync() {
+    hpx::wait_all(futures);
+    futures.clear();
+}
+#endif
 ///////////////////////////////////////////////////////////////////////////
 
 #ifndef ISPC_USE_PTHREADS_FULLY_SUBSCRIBED

examples/volume_rendering/Makefile

@@ -2,7 +2,7 @@
 EXAMPLE=volume
 CPP_SRC=volume.cpp volume_serial.cpp
 ISPC_SRC=volume.ispc
-ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x8,avx2-i32x8
+ISPC_IA_TARGETS=sse2-i32x4,sse4-i32x8,avx1-i32x8,avx2-i32x8,avx512knl-i32x16,avx512skx-i32x16
 ISPC_ARM_TARGETS=neon
 
 include ../common.mk

fail_db.txt

@@ -121,6 +121,8 @@
 ./tests/test-143.ispc runfail  x86-64     generic-16   Linux LLVM 3.7 clang++3.4 -O0 *
 ./tests/test-143.ispc runfail  x86-64     generic-16   Linux LLVM 3.8 clang++3.4 -O2 *
 ./tests/test-143.ispc runfail  x86-64     generic-16   Linux LLVM 3.8 clang++3.4 -O0 *
+./tests/test-143.ispc runfail  x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/test-143.ispc runfail  x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
 ./tests/masked-scatter-struct.ispc runfail  x86-64      generic-4   Linux LLVM 3.5 clang++3.4 -O2 *
 ./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.5 clang++3.4 -O2 *
 ./tests/ptr-15.ispc runfail  x86-64      generic-4   Linux LLVM 3.3 clang++3.4 -O0 *
@@ -135,6 +137,8 @@
 ./tests/ptr-15.ispc runfail  x86-64     generic-16   Linux LLVM 3.7 clang++3.4 -O0 *
 ./tests/ptr-15.ispc runfail  x86-64      generic-4   Linux LLVM 3.8 clang++3.4 -O0 *
 ./tests/ptr-15.ispc runfail  x86-64     generic-16   Linux LLVM 3.8 clang++3.4 -O0 *
+./tests/ptr-15.ispc runfail  x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/ptr-15.ispc runfail  x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
 ./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.5 clang++3.4 -O0 *
 .\tests\foreach-double-1.ispc runfail     x86     avx2-i32x8 Windows LLVM 3.5         cl -O2 *
 .\tests\foreach-double-1.ispc runfail     x86    avx2-i32x16 Windows LLVM 3.5         cl -O2 *
@@ -151,6 +155,8 @@
 ./tests/ptr-22.ispc runfail  x86-64     generic-16   Linux LLVM 3.7 clang++3.4 -O0 *
 ./tests/ptr-22.ispc runfail  x86-64      generic-4   Linux LLVM 3.8 clang++3.4 -O0 *
 ./tests/ptr-22.ispc runfail  x86-64     generic-16   Linux LLVM 3.8 clang++3.4 -O0 *
+./tests/ptr-22.ispc runfail  x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/ptr-22.ispc runfail  x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
 ./tests/masked-scatter-struct.ispc runfail  x86-64      generic-4   Linux LLVM 3.6 clang++3.4 -O2 *
 ./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.6 clang++3.4 -O2 *
 ./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.6 clang++3.4 -O0 *
@@ -160,10 +166,15 @@
 ./tests/ptr-19.ispc runfail  x86-64     generic-16   Linux LLVM 3.6 clang++3.4 -O0 *
 ./tests/ptr-19.ispc runfail  x86-64     generic-16   Linux LLVM 3.7 clang++3.4 -O0 *
 ./tests/ptr-19.ispc runfail  x86-64     generic-16   Linux LLVM 3.8 clang++3.4 -O0 *
+./tests/ptr-19.ispc runfail  x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
 ./tests/masked-scatter-struct.ispc runfail  x86-64      generic-4   Linux LLVM 3.7 clang++3.4 -O2 *
 ./tests/masked-scatter-struct.ispc runfail  x86-64      generic-4   Linux LLVM 3.8 clang++3.4 -O2 *
 ./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.7 clang++3.4 -O2 *
 ./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.7 clang++3.4 -O0 *
+./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.8 clang++3.4 -O2 *
+./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.8 clang++3.4 -O0 *
+./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/short-vec-8.ispc compfail  x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
 
 % [KNC-generic]: vec16-i64 memory layout bugs. Need to rewrite knc.h to fix that
 ./tests/soa-18.ispc runfail  x86-64 knc-generic   Linux LLVM 3.4   icpc13.1 -O0 *
@@ -379,4 +390,36 @@
 ./tests/short-vec-7.ispc compfail     x86-64      generic-4   Linux LLVM 3.8 clang++3.4 -O2 *
 ./tests/short-vec-7.ispc compfail     x86-64     generic-16   Linux LLVM 3.8 clang++3.4 -O0 *
 ./tests/short-vec-7.ispc compfail     x86-64     generic-16   Linux LLVM 3.8 clang++3.4 -O2 *
+./tests/pointer-decl-craziness-1.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/pointer-decl-craziness-1.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/pointer-decl-craziness-1.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/pointer-decl-craziness-1.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/pointer-decl-craziness-3.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/pointer-decl-craziness-3.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/pointer-decl-craziness-3.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/pointer-decl-craziness-3.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/pointer-decl-craziness.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/pointer-decl-craziness.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/pointer-decl-craziness.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/pointer-decl-craziness.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/ptr-23.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/ptr-23.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/ptr-23.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/ptr-23.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/ref-6.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/ref-6.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/ref-6.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/ref-6.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/scatter-vector.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/scatter-vector.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/scatter-vector.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/scatter-vector.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/short-vec-6.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/short-vec-6.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/short-vec-6.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/short-vec-6.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/short-vec-7.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/short-vec-7.ispc compfail     x86-64      generic-4   Linux LLVM 3.9 clang++3.4 -O2 *
+./tests/short-vec-7.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O0 *
+./tests/short-vec-7.ispc compfail     x86-64     generic-16   Linux LLVM 3.9 clang++3.4 -O2 *
 % End of anonymous namespace bug

func.cpp

@@ -420,8 +420,10 @@ Function::emitCode(FunctionEmitContext *ctx, llvm::Function *function,
             (
 #if ISPC_LLVM_VERSION == ISPC_LLVM_3_2 // 3.2
               (function->getFnAttributes().hasAttribute(llvm::Attributes::AlwaysInline) == false)
-#else // LLVM 3.3+
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
               (function->getAttributes().getFnAttributes().hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::AlwaysInline) == false)
+#else // LLVM 5.0+
+              (function->getAttributes().getFnAttributes().hasAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::AlwaysInline) == false)
 #endif
              &&
              costEstimate > CHECK_MASK_AT_FUNCTION_START_COST);

ispc.cpp

@@ -165,7 +165,7 @@ lGetSystemISA() {
             (info2[1] & (1 << 28)) != 0 && // AVX512 CDI
             (info2[1] & (1 << 30)) != 0 && // AVX512 BW
             (info2[1] & (1 << 31)) != 0) { // AVX512 VL
-            return "skx";
+            return "avx512skx-i32x16";
         }
         else if ((info2[1] & (1 << 26)) != 0 && // AVX512 PF
                  (info2[1] & (1 << 27)) != 0 && // AVX512 ER
@@ -239,10 +239,24 @@ typedef enum {
 #endif
 
 #if ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 // LLVM 3.7+
-    // KNL. Supports AVX512.
+    // Knights Landing - Xeon Phi.
+    // Supports AVX-512F: All the key AVX-512 features: masking, broadcast... ;
+    //          AVX-512CDI: Conflict Detection;
+    //          AVX-512ERI & PRI: 28-bit precision RCP, RSQRT and EXP transcendentals,
+    //                            new prefetch instructions.
     CPU_KNL,
 #endif
 
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // LLVM 3.8+
+    // Skylake Xeon.
+    // Supports AVX-512F: All the key AVX-512 features: masking, broadcast... ;
+    //          AVX-512CDI: Conflict Detection;
+    //          AVX-512VL: Vector Length Orthogonality;
+    //          AVX-512DQ: New HPC ISA (vs AVX512F);
+    //          AVX-512BW: Byte and Word Support.
+    CPU_SKX,
+#endif
+
 #if ISPC_LLVM_VERSION >= ISPC_LLVM_3_4 // LLVM 3.4+
     // Late Atom-like design. Supports SSE 4.2 + POPCNT/LZCNT.
     CPU_Silvermont,
@@ -273,14 +287,14 @@ private:
     std::vector<std::vector<std::string> > names;
     std::vector<std::set<CPUtype> > compat;
 
-    std::set<CPUtype> Set(CPUtype type, ...) {
+    std::set<CPUtype> Set(int type, ...) {
         std::set<CPUtype> retn;
         va_list args;
 
-        retn.insert(type);
+        retn.insert((CPUtype)type);
         va_start(args, type);
-        while ((type = (CPUtype)va_arg(args, int)) != CPU_None)
+        while ((type = va_arg(args, int)) != CPU_None)
-            retn.insert(type);
+            retn.insert((CPUtype)type);
         va_end(args);
 
         return retn;
@@ -327,6 +341,10 @@ public:
          names[CPU_KNL].push_back("knl");
 #endif
 
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // LLVM 3.8+
+         names[CPU_SKX].push_back("skx");
+#endif
+
 #ifdef ISPC_ARM_ENABLED
         names[CPU_CortexA15].push_back("cortex-a15");
 
@@ -353,6 +371,13 @@ public:
                                       CPU_Haswell, CPU_Broadwell, CPU_None);
 #endif
 
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // LLVM 3.8+
+        compat[CPU_SKX]         = Set(CPU_SKX, CPU_Bonnell, CPU_Penryn,
+                                      CPU_Core2, CPU_Nehalem, CPU_Silvermont,
+                                      CPU_SandyBridge, CPU_IvyBridge,
+                                      CPU_Haswell, CPU_Broadwell, CPU_None);
+#endif
+
 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_5 // LLVM 3.2, 3.3, 3.4 or 3.5
         #define CPU_Broadwell CPU_Haswell
 #else /* LLVM 3.6+ */
@@ -513,6 +538,12 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic, boo
                 break;
 #endif
 
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // LLVM 3.8+
+            case CPU_SKX:
+                isa = "avx512skx-i32x16";
+                break;
+#endif
+
 #if ISPC_LLVM_VERSION >= ISPC_LLVM_3_6
             case CPU_Broadwell:
 #endif
@@ -903,7 +934,7 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic, boo
         // ?? this->m_dataTypeWidth = 32;
         this->m_vectorWidth = 16;
         this->m_maskingIsFree = true;
-        this->m_maskBitCount = 1;
+        this->m_maskBitCount = 8;
         this->m_hasHalf = true;
         this->m_hasRand = true;
         this->m_hasGather = this->m_hasScatter = true;
@@ -915,7 +946,26 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic, boo
         CPUfromISA = CPU_KNL;
     }
 #endif
-
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // LLVM 3.8+
+    else if (!strcasecmp(isa, "avx512skx-i32x16")) {
+        this->m_isa = Target::SKX_AVX512;
+        this->m_nativeVectorWidth = 16;
+        this->m_nativeVectorAlignment = 64;
+        // ?? this->m_dataTypeWidth = 32;
+        this->m_vectorWidth = 16;
+        this->m_maskingIsFree = true;
+        this->m_maskBitCount = 8;
+        this->m_hasHalf = true;
+        this->m_hasRand = true;
+        this->m_hasGather = this->m_hasScatter = true;
+        this->m_hasTranscendentals = false;
+        // For MIC it is set to true due to performance reasons. The option should be tested.
+        this->m_hasTrigonometry = false;
+        this->m_hasRsqrtd = this->m_hasRcpd = false;
+        this->m_hasVecPrefetch = false;
+        CPUfromISA = CPU_SKX;
+    }
+#endif
 #ifdef ISPC_ARM_ENABLED
     else if (!strcasecmp(isa, "neon-i8x16")) {
         this->m_isa = Target::NEON8;
@@ -1013,8 +1063,15 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic, boo
         // Create TargetMachine
         std::string triple = GetTripleString();
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_8
         llvm::Reloc::Model relocModel = m_generatePIC ? llvm::Reloc::PIC_ :
             llvm::Reloc::Default;
+#else
+        llvm::Optional<llvm::Reloc::Model> relocModel;
+        if (m_generatePIC) {
+          relocModel = llvm::Reloc::PIC_;
+        }
+#endif
         std::string featuresString = m_attributes;
         llvm::TargetOptions options;
 #ifdef ISPC_ARM_ENABLED
@@ -1045,6 +1102,14 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic, boo
 #else /* LLVM 3.7+ */
         m_targetMachine->Options.MCOptions.AsmVerbose = true;
 #endif
+
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_5
+        // Change default version of generated DWARF.
+        if (g->generateDWARFVersion != 0) {
+            m_targetMachine->Options.MCOptions.DwarfVersion = g->generateDWARFVersion;
+        }
+#endif
+
         // Initialize TargetData/DataLayout in 3 steps.
         // 1. Get default data layout first
         std::string dl_string;
@@ -1093,11 +1158,19 @@ Target::Target(const char *arch, const char *cpu, const char *isa, bool pic, boo
 #endif
             attrBuilder.addAttribute("target-cpu", this->m_cpu);
             attrBuilder.addAttribute("target-features", this->m_attributes);
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
             this->m_tf_attributes = new llvm::AttributeSet(
                 llvm::AttributeSet::get(
                     *g->ctx,
                     llvm::AttributeSet::FunctionIndex,
                     attrBuilder));
+#else // LLVM 5.0+
+            this->m_tf_attributes = new llvm::AttributeList(
+                llvm::AttributeList::get(
+                    *g->ctx,
+                    llvm::AttributeList::FunctionIndex,
+                    attrBuilder));
+#endif
         }
 #endif
 
@@ -1144,9 +1217,12 @@ Target::SupportedTargets() {
         "avx2-i32x8, avx2-i32x16, avx2-i64x4, "
 #if ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 // LLVM 3.7+
         "avx512knl-i32x16, "
+#endif
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // LLVM 3.8+
+        "avx512skx-i32x16, "
 #endif
         "generic-x1, generic-x4, generic-x8, generic-x16, "
-        "generic-x32, generic-x64, *-generic-x16, "
+        "generic-x32, generic-x64, *-generic-x16"
 #ifdef ISPC_ARM_ENABLED
         ", neon-i8x16, neon-i16x8, neon-i32x4"
 #endif
@@ -1219,8 +1295,10 @@ Target::ISAToString(ISA isa) {
     case Target::KNL_AVX512:
         return "avx512knl";
 #endif
-    case Target::SKX:
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // LLVM 3.8+
-        return "skx";
+    case Target::SKX_AVX512:
+        return "avx512skx";
+#endif
     case Target::GENERIC:
         return "generic";
 #ifdef ISPC_NVPTX_ENABLED
@@ -1267,8 +1345,10 @@ Target::ISAToTargetString(ISA isa) {
     case Target::KNL_AVX512:
         return "avx512knl-i32x16";
 #endif
-    case Target::SKX:
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // LLVM 3.8+
-        return "avx2";
+    case Target::SKX_AVX512:
+        return "avx512skx-i32x16";
+#endif
     case Target::GENERIC:
         return "generic-4";
 #ifdef ISPC_NVPTX_ENABLED
@@ -1405,7 +1485,11 @@ Target::StructOffset(llvm::Type *type, int element,
 void Target::markFuncWithTargetAttr(llvm::Function* func) {
 #if ISPC_LLVM_VERSION >= ISPC_LLVM_3_3
     if (m_tf_attributes) {
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
         func->addAttributes(llvm::AttributeSet::FunctionIndex, *m_tf_attributes);
+#else // LLVM 5.0+
+        func->addAttributes(llvm::AttributeList::FunctionIndex, *m_tf_attributes);
+#endif
     }
 #endif
 }
@@ -1455,6 +1539,9 @@ Globals::Globals() {
     emitPerfWarnings = true;
     emitInstrumentation = false;
     generateDebuggingSymbols = false;
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_5
+    generateDWARFVersion = 0;
+#endif
     enableFuzzTest = false;
     fuzzTestSeed = -1;
     mangleFunctionsWithTarget = false;

ispc.h

@@ -41,7 +41,7 @@
 #include "ispc_version.h"
 
 #if ISPC_LLVM_VERSION < OLDEST_SUPPORTED_LLVM || ISPC_LLVM_VERSION > LATEST_SUPPORTED_LLVM
-#error "Only LLVM 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8 and 3.9 development branch are supported"
+#error "Only LLVM 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 4.0 and 5.0 development branch are supported"
 #endif
 
 #if defined(_WIN32) || defined(_WIN64)
@@ -72,7 +72,11 @@
 
 // Forward declarations of a number of widely-used LLVM types
 namespace llvm {
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
     class AttributeSet;
+#else // LLVM 5.0+
+    class AttributeList;
+#endif
     class BasicBlock;
     class Constant;
     class ConstantValue;
@@ -86,13 +90,11 @@ namespace llvm {
     class TargetMachine;
     class Type;
     class Value;
-#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_6
     class DIFile;
     class DIType;
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_6
     class DIDescriptor;
 #else // LLVM 3.7+
-    class DIFile;
-    class DIType;
     class DIScope;
 #endif
 }
@@ -193,7 +195,7 @@ public:
         AVX11          = 3,
         AVX2           = 4,
         KNL_AVX512     = 5,
-        SKX            = 6,
+        SKX_AVX512     = 6,
         GENERIC        = 7,
 #ifdef ISPC_NVPTX_ENABLED
         NVPTX,
@@ -348,7 +350,11 @@ private:
     /** Target-specific LLVM attribute, which has to be attached to every
         function to ensure that it is generated for correct target architecture.
         This is requirement was introduced in LLVM 3.3 */
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_4_0
     llvm::AttributeSet* m_tf_attributes;
+#else // LLVM 5.0+
+    llvm::AttributeList* m_tf_attributes;
+#endif
 #endif
 
     /** Native vector width of the vector instruction set.  Note that this
@@ -606,6 +612,16 @@ struct Globals {
         program in its output. */
     bool generateDebuggingSymbols;
 
+    /** Require generation of DWARF of certain version (2, 3, 4). For
+        default version, this field is set to 0. */
+    // Hint: to verify dwarf version in the object file, run on Linux:
+    // readelf --debug-dump=info object.o | grep -A 2 'Compilation Unit @'
+    // on Mac:
+    // xcrun dwarfdump -r0 object.o
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_5
+    int generateDWARFVersion;
+#endif
+
     /** If true, function names are mangled by appending the target ISA and
         vector width to them. */
     bool mangleFunctionsWithTarget;

ispc.vcxproj

@@ -38,6 +38,8 @@
     <ClCompile Include="$(Configuration)\gen-bitcode-avx2-i64x4-64bit.cpp" />
     <ClCompile Include="$(Configuration)\gen-bitcode-knl-32bit.cpp" />
     <ClCompile Include="$(Configuration)\gen-bitcode-knl-64bit.cpp" />
+    <ClCompile Include="$(Configuration)\gen-bitcode-skx-32bit.cpp" />
+    <ClCompile Include="$(Configuration)\gen-bitcode-skx-64bit.cpp" />
     <ClCompile Include="$(Configuration)\gen-bitcode-c-32.cpp" />
     <ClCompile Include="$(Configuration)\gen-bitcode-c-64.cpp" />
     <ClCompile Include="$(Configuration)\gen-bitcode-dispatch.cpp" />
@@ -291,6 +293,16 @@
       <Message>Building gen-bitcode-knl-32bit.cpp and gen-bitcode-knl-64bit.cpp</Message>
     </CustomBuild>
   </ItemGroup>
+  <ItemGroup>
+    <CustomBuild Include="builtins\target-skx.ll">
+      <FileType>Document</FileType>
+      <Command>m4 -Ibuiltins/ -DLLVM_VERSION=%LLVM_VERSION% -DBUILD_OS=WINDOWS -DRUNTIME=32 builtins/target-skx.ll | python bitcode2cpp.py builtins\target-skx.ll 32bit &gt; $(Configuration)/gen-bitcode-skx-32bit.cpp;
+               m4 -Ibuiltins/ -DLLVM_VERSION=%LLVM_VERSION% -DBUILD_OS=WINDOWS -DRUNTIME=64 builtins/target-skx.ll | python bitcode2cpp.py builtins\target-skx.ll 64bit &gt; $(Configuration)/gen-bitcode-skx-64bit.cpp</Command>
+      <Outputs>$(Configuration)/gen-bitcode-skx-32bit.cpp; $(Configuration)/gen-bitcode-skx-64bit.cpp</Outputs>
+      <AdditionalInputs>builtins\util.m4;builtins\svml.m4;builtins\target-avx-common.ll;builtins\target-avx.ll</AdditionalInputs>
+      <Message>Building gen-bitcode-skx-32bit.cpp and gen-bitcode-skx-64bit.cpp</Message>
+    </CustomBuild>
+  </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="builtins\target-generic-1.ll">
       <FileType>Document</FileType>

ispc_version.h

@@ -38,7 +38,7 @@
 #ifndef ISPC_VERSION_H
 #define ISPC_VERSION_H
 
-#define ISPC_VERSION "1.9.0"
+#define ISPC_VERSION "1.9.2dev"
 #include "llvm/Config/llvm-config.h"
 
 #define ISPC_LLVM_VERSION ( LLVM_VERSION_MAJOR * 10000 + LLVM_VERSION_MINOR * 100 )
@@ -51,9 +51,11 @@
 #define ISPC_LLVM_3_7 30700
 #define ISPC_LLVM_3_8 30800
 #define ISPC_LLVM_3_9 30900
+#define ISPC_LLVM_4_0 40000
+#define ISPC_LLVM_5_0 50000
 
 #define OLDEST_SUPPORTED_LLVM ISPC_LLVM_3_2
-#define LATEST_SUPPORTED_LLVM ISPC_LLVM_3_9
+#define LATEST_SUPPORTED_LLVM ISPC_LLVM_5_0
 
 #ifdef __ispc__xstr
 #undef __ispc__xstr

lex.ll

@@ -637,14 +637,14 @@ lParseInteger(bool dotdotdot) {
             // No u or l suffix
             // If we're compiling to an 8-bit mask target and the constant
             // fits into 8 bits, return an 8-bit int.
-            if (g->target->getMaskBitCount() == 8) {
+            if (g->target->getDataTypeWidth() == 8) {
                 if (yylval.intVal <= 0x7fULL)
                     return TOKEN_INT8_CONSTANT;
                 else if (yylval.intVal <= 0xffULL)
                     return TOKEN_UINT8_CONSTANT;
             }
             // And similarly for 16-bit masks and constants
-            if (g->target->getMaskBitCount() == 16) {
+            if (g->target->getDataTypeWidth() == 16) {
                 if (yylval.intVal <= 0x7fffULL)
                     return TOKEN_INT16_CONSTANT;
                 else if (yylval.intVal <= 0xffffULL)

llvm_patches/3_8_skx_patch_pack.patch

@@ -0,0 +1,248 @@
+--- lib/Target/X86/X86ISelLowering.cpp	2016-04-07 01:11:55.018960678 +0300
++++ lib/Target/X86/X86ISelLowering.cpp	2016-04-07 01:13:57.643965706 +0300
+@@ -1413,9 +1413,6 @@
+     setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v8i1,  Custom);
+     setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v16i1, Custom);
+     if (Subtarget->hasDQI()) {
+-      setOperationAction(ISD::TRUNCATE,         MVT::v2i1, Custom);
+-      setOperationAction(ISD::TRUNCATE,         MVT::v4i1, Custom);
+-
+       setOperationAction(ISD::SINT_TO_FP,       MVT::v8i64, Legal);
+       setOperationAction(ISD::UINT_TO_FP,       MVT::v8i64, Legal);
+       setOperationAction(ISD::FP_TO_SINT,       MVT::v8i64, Legal);
+@@ -1709,6 +1706,8 @@
+     addRegisterClass(MVT::v4i1,   &X86::VK4RegClass);
+     addRegisterClass(MVT::v2i1,   &X86::VK2RegClass);
+ 
++    setOperationAction(ISD::TRUNCATE,           MVT::v2i1, Custom);
++    setOperationAction(ISD::TRUNCATE,           MVT::v4i1, Custom);
+     setOperationAction(ISD::SETCC,              MVT::v4i1, Custom);
+     setOperationAction(ISD::SETCC,              MVT::v2i1, Custom);
+     setOperationAction(ISD::CONCAT_VECTORS,     MVT::v4i1, Custom);
+@@ -11737,10 +11736,15 @@
+   }
+ 
+   unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
+-  const TargetRegisterClass* rc = getRegClassFor(VecVT);
+-  if (!Subtarget->hasDQI() && (VecVT.getVectorNumElements() <= 8))
+-    rc = getRegClassFor(MVT::v16i1);
+-  unsigned MaxSift = rc->getSize()*8 - 1;
++  if (!Subtarget->hasDQI() && (VecVT.getVectorNumElements() <= 8)) {
++    // Use kshiftlw/rw instruction.
++    VecVT = MVT::v16i1;
++    Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VecVT,
++                      DAG.getUNDEF(VecVT),
++                      Vec,
++                      DAG.getIntPtrConstant(0, dl));
++  }
++  unsigned MaxSift = VecVT.getVectorNumElements() - 1;
+   Vec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, Vec,
+                     DAG.getConstant(MaxSift - IdxVal, dl, MVT::i8));
+   Vec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, Vec,
+--- lib/Target/X86/X86InstrAVX512.td	2016-04-07 01:11:55.020960678 +0300
++++ lib/Target/X86/X86InstrAVX512.td	2016-04-07 01:12:30.680962140 +0300
+@@ -2043,9 +2043,6 @@
+                VEX, PD, VEX_W;
+   defm KMOVD : avx512_mask_mov_gpr<0x92, 0x93, "kmovd", VK32, GR32>,
+                VEX, XD;
+-}
+-
+-let Predicates = [HasBWI] in {
+   defm KMOVQ : avx512_mask_mov<0x90, 0x90, 0x91, "kmovq", VK64, v64i1, i64mem>,
+                VEX, PS, VEX_W;
+   defm KMOVQ : avx512_mask_mov_gpr<0x92, 0x93, "kmovq", VK64, GR64>,
+@@ -2085,8 +2082,27 @@
+             (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK4:$src, VK8))>;
+   def : Pat<(store VK2:$src, addr:$dst),
+             (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK2:$src, VK8))>;
++  def : Pat<(store VK1:$src, addr:$dst),
++            (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK1:$src, VK8))>;
+ }
+ let Predicates = [HasAVX512, NoDQI] in {
++  def : Pat<(store VK1:$src, addr:$dst),
++            (MOV8mr addr:$dst,
++             (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)),
++              sub_8bit))>;
++  def : Pat<(store VK2:$src, addr:$dst),
++            (MOV8mr addr:$dst,
++             (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK2:$src, VK16)),
++              sub_8bit))>;
++  def : Pat<(store VK4:$src, addr:$dst),
++            (MOV8mr addr:$dst,
++             (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK4:$src, VK16)),
++              sub_8bit))>;  
++  def : Pat<(store VK8:$src, addr:$dst),
++            (MOV8mr addr:$dst,
++             (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)),
++              sub_8bit))>;
++  
+   def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst),
+             (KMOVWmk addr:$dst, (COPY_TO_REGCLASS VK8:$src, VK16))>;
+   def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))),
+@@ -2166,6 +2182,17 @@
+ def : Pat<(v64i1 (scalar_to_vector VK1:$src)),
+           (COPY_TO_REGCLASS VK1:$src, VK64)>;
+ 
++def : Pat<(store (i1 -1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
++def : Pat<(store (i1  1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
++def : Pat<(store (i1  0), addr:$dst), (MOV8mi addr:$dst, (i8 0))>;
++
++def truncstorei1 : PatFrag<(ops node:$val, node:$ptr),
++                           (truncstore node:$val, node:$ptr), [{
++  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i1;
++}]>;
++
++def : Pat<(truncstorei1 GR8:$src, addr:$dst),
++          (MOV8mr addr:$dst, GR8:$src)>;
+ 
+ // With AVX-512 only, 8-bit mask is promoted to 16-bit mask.
+ let Predicates = [HasAVX512, NoDQI] in {
+@@ -6540,28 +6567,6 @@
+ def v16i1sextv16i32  : PatLeaf<(v16i32 (X86vsrai VR512:$src, (i8 31)))>;
+ def v8i1sextv8i64  : PatLeaf<(v8i64 (X86vsrai VR512:$src, (i8 63)))>;
+ 
+-def : Pat<(store (i1 -1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
+-def : Pat<(store (i1  1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
+-def : Pat<(store (i1  0), addr:$dst), (MOV8mi addr:$dst, (i8 0))>;
+-
+-def : Pat<(store VK1:$src, addr:$dst),
+-          (MOV8mr addr:$dst,
+-           (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)),
+-            sub_8bit))>, Requires<[HasAVX512, NoDQI]>;
+-
+-def : Pat<(store VK8:$src, addr:$dst),
+-          (MOV8mr addr:$dst,
+-           (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)),
+-            sub_8bit))>, Requires<[HasAVX512, NoDQI]>;
+-
+-def truncstorei1 : PatFrag<(ops node:$val, node:$ptr),
+-                           (truncstore node:$val, node:$ptr), [{
+-  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i1;
+-}]>;
+-
+-def : Pat<(truncstorei1 GR8:$src, addr:$dst),
+-          (MOV8mr addr:$dst, GR8:$src)>;
+-
+ multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > {
+ def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
+                   !strconcat(OpcodeStr##Vec.Suffix, "\t{$src, $dst|$dst, $src}"),
+--- lib/Target/X86/X86InstrInfo.cpp	2016-04-07 01:11:55.016960678 +0300
++++ lib/Target/X86/X86InstrInfo.cpp	2016-04-07 01:13:00.255963353 +0300
+@@ -4286,12 +4286,14 @@
+   return 0;
+ }
+ 
++static bool isMaskRegClass(const TargetRegisterClass *RC) {
++  // All KMASK RegClasses hold the same k registers, can be tested against anyone.
++  return X86::VK16RegClass.hasSubClassEq(RC);
++}
++
+ static bool MaskRegClassContains(unsigned Reg) {
+-  return X86::VK8RegClass.contains(Reg) ||
+-         X86::VK16RegClass.contains(Reg) ||
+-         X86::VK32RegClass.contains(Reg) ||
+-         X86::VK64RegClass.contains(Reg) ||
+-         X86::VK1RegClass.contains(Reg);
++  // All KMASK RegClasses hold the same k registers, can be tested against anyone.
++  return X86::VK16RegClass.contains(Reg);
+ }
+ 
+ static bool GRRegClassContains(unsigned Reg) {
+@@ -4493,15 +4495,28 @@
+   llvm_unreachable("Cannot emit physreg copy instruction");
+ }
+ 
++static unsigned getLoadStoreMaskRegOpcode(const TargetRegisterClass *RC,
++                                          bool load) {
++  switch (RC->getSize()) {
++  default:
++    llvm_unreachable("Unknown spill size");
++  case 2:
++    return load ? X86::KMOVWkm : X86::KMOVWmk;
++  case 4:
++    return load ? X86::KMOVDkm : X86::KMOVDmk;
++  case 8:
++    return load ? X86::KMOVQkm : X86::KMOVQmk;
++  }
++}
++
+ static unsigned getLoadStoreRegOpcode(unsigned Reg,
+                                       const TargetRegisterClass *RC,
+                                       bool isStackAligned,
+                                       const X86Subtarget &STI,
+                                       bool load) {
+   if (STI.hasAVX512()) {
+-    if (X86::VK8RegClass.hasSubClassEq(RC)  ||
+-      X86::VK16RegClass.hasSubClassEq(RC))
+-      return load ? X86::KMOVWkm : X86::KMOVWmk;
++    if (isMaskRegClass(RC))
++      return getLoadStoreMaskRegOpcode(RC, load);
+     if (RC->getSize() == 4 && X86::FR32XRegClass.hasSubClassEq(RC))
+       return load ? X86::VMOVSSZrm : X86::VMOVSSZmr;
+     if (RC->getSize() == 8 && X86::FR64XRegClass.hasSubClassEq(RC))
+--- lib/Target/X86/X86InstrInfo.td	2016-04-07 01:11:55.018960678 +0300
++++ lib/Target/X86/X86InstrInfo.td	2016-04-07 01:14:17.400966516 +0300
+@@ -728,6 +728,8 @@
+ 
+ //===----------------------------------------------------------------------===//
+ // X86 Instruction Predicate Definitions.
++def TruePredicate : Predicate<"true">;
++
+ def HasCMov      : Predicate<"Subtarget->hasCMov()">;
+ def NoCMov       : Predicate<"!Subtarget->hasCMov()">;
+ 
+--- lib/Target/X86/X86InstrSSE.td	2016-04-07 01:11:55.014960678 +0300
++++ lib/Target/X86/X86InstrSSE.td	2016-04-07 01:14:18.172966548 +0300
+@@ -4273,17 +4273,17 @@
+ //===---------------------------------------------------------------------===//
+ 
+ defm PCMPEQB : PDI_binop_all<0x74, "pcmpeqb", X86pcmpeq, v16i8, v32i8,
+-                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
++                             SSE_INTALU_ITINS_P, 1, TruePredicate>;
+ defm PCMPEQW : PDI_binop_all<0x75, "pcmpeqw", X86pcmpeq, v8i16, v16i16,
+-                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
++                             SSE_INTALU_ITINS_P, 1, TruePredicate>;
+ defm PCMPEQD : PDI_binop_all<0x76, "pcmpeqd", X86pcmpeq, v4i32, v8i32,
+-                             SSE_INTALU_ITINS_P, 1, NoVLX>;
++                             SSE_INTALU_ITINS_P, 1, TruePredicate>;
+ defm PCMPGTB : PDI_binop_all<0x64, "pcmpgtb", X86pcmpgt, v16i8, v32i8,
+-                             SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>;
++                             SSE_INTALU_ITINS_P, 0, TruePredicate>;
+ defm PCMPGTW : PDI_binop_all<0x65, "pcmpgtw", X86pcmpgt, v8i16, v16i16,
+-                             SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>;
++                             SSE_INTALU_ITINS_P, 0, TruePredicate>;
+ defm PCMPGTD : PDI_binop_all<0x66, "pcmpgtd", X86pcmpgt, v4i32, v8i32,
+-                             SSE_INTALU_ITINS_P, 0, NoVLX>;
++                             SSE_INTALU_ITINS_P, 0, TruePredicate>;
+ 
+ //===---------------------------------------------------------------------===//
+ // SSE2 - Packed Integer Shuffle Instructions
+--- lib/Target/X86/X86RegisterInfo.td	2016-04-07 01:11:55.018960678 +0300
++++ lib/Target/X86/X86RegisterInfo.td	2016-04-07 01:13:01.037963385 +0300
+@@ -477,18 +477,18 @@
+                            256, (sequence "YMM%u", 0, 31)>;
+ 
+ // Mask registers
+-def VK1     : RegisterClass<"X86", [i1],    8,  (sequence "K%u", 0, 7)> {let Size = 8;}
+-def VK2     : RegisterClass<"X86", [v2i1],  8,  (add VK1)> {let Size = 8;}
+-def VK4     : RegisterClass<"X86", [v4i1],  8,  (add VK2)> {let Size = 8;}
+-def VK8     : RegisterClass<"X86", [v8i1],  8,  (add VK4)> {let Size = 8;}
++def VK1     : RegisterClass<"X86", [i1],    16,  (sequence "K%u", 0, 7)> {let Size = 16;}
++def VK2     : RegisterClass<"X86", [v2i1],  16,  (add VK1)> {let Size = 16;}
++def VK4     : RegisterClass<"X86", [v4i1],  16,  (add VK2)> {let Size = 16;}
++def VK8     : RegisterClass<"X86", [v8i1],  16,  (add VK4)> {let Size = 16;}
+ def VK16    : RegisterClass<"X86", [v16i1], 16, (add VK8)> {let Size = 16;}
+ def VK32    : RegisterClass<"X86", [v32i1], 32, (add VK16)> {let Size = 32;}
+ def VK64    : RegisterClass<"X86", [v64i1], 64, (add VK32)> {let Size = 64;}
+ 
+-def VK1WM   : RegisterClass<"X86", [i1],    8,  (sub VK1, K0)> {let Size = 8;}
+-def VK2WM   : RegisterClass<"X86", [v2i1],  8,  (sub VK2, K0)> {let Size = 8;}
+-def VK4WM   : RegisterClass<"X86", [v4i1],  8,  (sub VK4, K0)> {let Size = 8;}
+-def VK8WM   : RegisterClass<"X86", [v8i1],  8,  (sub VK8, K0)> {let Size = 8;}
++def VK1WM   : RegisterClass<"X86", [i1],    16,  (sub VK1, K0)> {let Size = 16;}
++def VK2WM   : RegisterClass<"X86", [v2i1],  16,  (sub VK2, K0)> {let Size = 16;}
++def VK4WM   : RegisterClass<"X86", [v4i1],  16,  (sub VK4, K0)> {let Size = 16;}
++def VK8WM   : RegisterClass<"X86", [v8i1],  16,  (sub VK8, K0)> {let Size = 16;}
+ def VK16WM  : RegisterClass<"X86", [v16i1], 16, (add VK8WM)>   {let Size = 16;}
+ def VK32WM  : RegisterClass<"X86", [v32i1], 32, (add VK16WM)> {let Size = 32;}
+ def VK64WM  : RegisterClass<"X86", [v64i1], 64, (add VK32WM)> {let Size = 64;}

llvmutil.h

@@ -51,7 +51,11 @@
   #include <llvm/IR/Constants.h>
 #endif
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
 #define PTYPE(p) (llvm::cast<llvm::SequentialType>((p)->getType()->getScalarType())->getElementType())
+#else // LLVM 4.0+
+#define PTYPE(p) (llvm::cast<llvm::PointerType>((p)->getType()->getScalarType())->getElementType())
+#endif
 
 namespace llvm {
     class PHINode;

main.cpp

@@ -103,13 +103,16 @@ usage(int ret) {
     printf("    [--dev-stub <filename>]\t\tEmit device-side offload stub functions to file\n");
 #ifdef ISPC_IS_WINDOWS
     printf("    [--dllexport]\t\t\tMake non-static functions DLL exported.  Windows only.\n");
+#endif
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_5
+    printf("    [--dwarf-version={2,3,4}]\t\tGenerate source-level debug information with given DWARF version (triggers -g)\n");
 #endif
     printf("    [--emit-asm]\t\t\tGenerate assembly language file as output\n");
     printf("    [--emit-c++]\t\t\tEmit a C++ source file as output\n");
     printf("    [--emit-llvm]\t\t\tEmit LLVM bitode file as output\n");
     printf("    [--emit-obj]\t\t\tGenerate object file file as output (default)\n");
     printf("    [--force-alignment=<value>]\t\tForce alignment in memory allocations routine to be <value>\n");
-    printf("    [-g]\t\t\t\tGenerate debugging information\n");
+    printf("    [-g]\t\t\t\tGenerate source-level debug information\n");
     printf("    [--help]\t\t\t\tPrint help\n");
     printf("    [--help-dev]\t\t\tPrint help for developer options\n");
     printf("    [--host-stub <filename>]\t\tEmit host-side offload stub functions to file\n");
@@ -121,12 +124,12 @@ usage(int ret) {
     printf("        fast\t\t\t\tUse high-performance but lower-accuracy math functions\n");
     printf("        svml\t\t\t\tUse the Intel(r) SVML math libraries\n");
     printf("        system\t\t\t\tUse the system's math library (*may be quite slow*)\n");
-    printf("    [-MMM <filename>\t\t\t\tWrite #include dependencies to given file.\n");
+    printf("    [-MMM <filename>\t\t\tWrite #include dependencies to given file.\n");
     printf("    [--no-omit-frame-pointer]\t\tDisable frame pointer omission. It may be useful for profiling\n");
     printf("    [--nostdlib]\t\t\tDon't make the ispc standard library available\n");
     printf("    [--nocpp]\t\t\t\tDon't run the C preprocessor\n");
     printf("    [-o <name>/--outfile=<name>]\tOutput filename (may be \"-\" for standard output)\n");
-    printf("    [-O0/-O(1/2/3)]\t\t\t\tSet optimization level (off or on). Optimizations are on by default.\n");
+    printf("    [-O0/-O(1/2/3)]\t\t\tSet optimization level (off or on). Optimizations are on by default.\n");
     printf("    [--opt=<option>]\t\t\tSet optimization option\n");
     printf("        disable-assertions\t\tRemove assertion statements from final code.\n");
     printf("        disable-fma\t\t\tDisable 'fused multiply-add' instructions (on targets that support them)\n");
@@ -386,6 +389,19 @@ int main(int Argc, char *Argv[]) {
 #ifdef ISPC_IS_WINDOWS
         else if (!strcmp(argv[i], "--dllexport"))
             g->dllExport = true;
+#endif
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_5
+        else if (!strncmp(argv[i], "--dwarf-version=", 16)) {
+            int val = atoi(argv[i] + 16);
+            if (2 <= val && val <=4) {
+                g->generateDebuggingSymbols = true;
+                g->generateDWARFVersion=val;
+            } else {
+                fprintf(stderr, "Invalid value for DWARF version: \"%s\" -- "
+                        "only 2, 3 and 4 are allowed.\n", argv[i]+16);
+                usage(1);
+            }
+        }
 #endif
         else if (!strcmp(argv[i], "--print-target"))
             g->printTarget = true;

module.cpp

@@ -124,10 +124,15 @@
 #include <clang/Frontend/TextDiagnosticPrinter.h>
 #include <clang/Frontend/Utils.h>
 #include <clang/Basic/TargetInfo.h>
+#include <clang/Lex/PreprocessorOptions.h>
 #include <llvm/Support/ToolOutputFile.h>
 #include <llvm/Support/Host.h>
 #include <llvm/Support/raw_ostream.h>
-#include <llvm/Bitcode/ReaderWriter.h>
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
+    #include <llvm/Bitcode/ReaderWriter.h>
+#else
+    #include <llvm/Bitcode/BitcodeWriter.h>
+#endif
 
 /*! list of files encountered by the parser. this allows emitting of
     the module file's dependencies via the -MMM option */
@@ -167,6 +172,11 @@ lDeclareSizeAndPtrIntTypes(SymbolTable *symbolTable) {
  */
 static void
 lStripUnusedDebugInfo(llvm::Module *module) {
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_9
+    // In LLVM 3.9 Global DCE is much more efficient than the LLVM 3.8's one.
+    // So, the fruitfulness of this function is negligible.
+    return;
+#else
     if (g->generateDebuggingSymbols == false)
         return;
 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_5 // <= 3.5
@@ -368,6 +378,7 @@ lStripUnusedDebugInfo(llvm::Module *module) {
     }
     for (int i = 0; i < (int)toErase.size(); ++i)
         module->eraseNamedMetadata(toErase[i]);
+#endif
 }
 
 
@@ -420,9 +431,7 @@ Module::Module(const char *fn) {
             sprintf(producerString, "ispc version %s (built on %s)",
                     ISPC_VERSION, __DATE__);
 #endif
-#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_4 // LLVM 3.4+
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_3
-            diCompileUnit = 
-#endif // LLVM_3_4+            
             diBuilder->createCompileUnit(llvm::dwarf::DW_LANG_C99,  /* lang */
                                          name,  /* filename */
                                          directory, /* directory */
@@ -430,6 +439,25 @@ Module::Module(const char *fn) {
                                          g->opt.level > 0 /* is optimized */,
                                          "-g", /* command line args */
                                          0 /* run time version */);
+#elif ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // LLVM 3.4-3.9
+            diCompileUnit =
+            diBuilder->createCompileUnit(llvm::dwarf::DW_LANG_C99,  /* lang */
+                                         name,  /* filename */
+                                         directory, /* directory */
+                                         producerString, /* producer */
+                                         g->opt.level > 0 /* is optimized */,
+                                         "-g", /* command line args */
+                                         0 /* run time version */);
+#elif ISPC_LLVM_VERSION >= ISPC_LLVM_4_0 // LLVM 4.0+
+            auto srcFile = diBuilder->createFile(name, directory);
+            diCompileUnit =
+            diBuilder->createCompileUnit(llvm::dwarf::DW_LANG_C99,  /* lang */
+                                         srcFile,  /* filename */
+                                         producerString, /* producer */
+                                         g->opt.level > 0 /* is optimized */,
+                                         "-g", /* command line args */
+                                         0 /* run time version */);
+#endif
         }
     }
     else
@@ -728,7 +756,7 @@ Module::AddGlobalVariable(const std::string &name, const Type *type, Expr *initE
                                             sym->type->GetDIType(file),
                                             (sym->storageClass == SC_STATIC),
                                             sym_const_storagePtr);
-#else // LLVM 3.7+
+#elif ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 && ISPC_LLVM_VERSION <= ISPC_LLVM_3_9 // LLVM 3.7 - 3.9
         llvm::DIFile *file = pos.GetDIFile();
         //llvm::MDFile *file = pos.GetDIFile();
         llvm::Constant *sym_const_storagePtr = llvm::dyn_cast<llvm::Constant>(sym->storagePtr);
@@ -742,6 +770,20 @@ Module::AddGlobalVariable(const std::string &name, const Type *type, Expr *initE
                                             sym->type->GetDIType(file),
                                             (sym->storageClass == SC_STATIC),
                                             sym_const_storagePtr);
+#else // LLVM 4.0+
+        llvm::DIFile *file = pos.GetDIFile();
+        //llvm::MDFile *file = pos.GetDIFile();
+        llvm::GlobalVariable *sym_GV_storagePtr = llvm::dyn_cast<llvm::GlobalVariable>(sym->storagePtr);
+        Assert(sym_GV_storagePtr);
+        llvm::DIGlobalVariableExpression *var = diBuilder->createGlobalVariableExpression(
+                                            file,
+                                            name,
+                                            name,
+                                            file,
+                                            pos.first_line,
+                                            sym->type->GetDIType(file),
+                                            (sym->storageClass == SC_STATIC));
+        sym_GV_storagePtr->addDebugInfo(var);
 #endif
 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_6
         Assert(var.Verify());
@@ -2220,9 +2262,9 @@ Module::writeHeader(const char *fn) {
 
     if (g->emitInstrumentation) {
         fprintf(f, "#define ISPC_INSTRUMENTATION 1\n");
-        fprintf(f, "extern \"C\" {\n");
+        fprintf(f, "#if defined(__cplusplus) && (! defined(__ISPC_NO_EXTERN_C) || !__ISPC_NO_EXTERN_C )\nextern \"C\" {\n#endif // __cplusplus\n");
         fprintf(f, "  void ISPCInstrument(const char *fn, const char *note, int line, uint64_t mask);\n");
-        fprintf(f, "}\n");
+        fprintf(f, "#if defined(__cplusplus) && (! defined(__ISPC_NO_EXTERN_C) || !__ISPC_NO_EXTERN_C )\n} /* end extern C */\n#endif // __cplusplus\n");
     }
 
     // end namespace
@@ -2333,9 +2375,9 @@ Module::writeDispatchHeader(DispatchHeaderInfo *DHI) {
 
       if (g->emitInstrumentation) {
         fprintf(f, "#define ISPC_INSTRUMENTATION 1\n");
-        fprintf(f, "extern \"C\" {\n");
+        fprintf(f, "#if defined(__cplusplus) && (! defined(__ISPC_NO_EXTERN_C) || !__ISPC_NO_EXTERN_C )\nextern \"C\" {\n#endif // __cplusplus\n");
         fprintf(f, "  void ISPCInstrument(const char *fn, const char *note, int line, uint64_t mask);\n");
-        fprintf(f, "}\n");
+        fprintf(f, "#if defined(__cplusplus) && (! defined(__ISPC_NO_EXTERN_C) || !__ISPC_NO_EXTERN_C )\n} /* end extern C */\n#endif // __cplusplus\n");
       }
 
       // end namespace
@@ -2809,10 +2851,8 @@ lCreateDispatchFunction(llvm::Module *module, llvm::Function *setISAFunc,
             !g->target->getTreatGenericAsSmth().empty()) {
             if (g->target->getTreatGenericAsSmth() == "knl_generic")
                 dispatchNum = Target::KNL_AVX512;
-            else if (g->target->getTreatGenericAsSmth() == "skx_generic")
-                dispatchNum = Target::SKX;
             else {
-                Error(SourcePos(), "*-generic target can be called only with knl or skx");
+                Error(SourcePos(), "*-generic target can be called only with knl");
                 exit(1);
             }
         }

opt.cpp

@@ -112,6 +112,10 @@
   #include <llvm/Analysis/BasicAliasAnalysis.h>
   #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
 #endif
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_9 // LLVM 3.9+
+  #include "llvm/Transforms/IPO/FunctionAttrs.h"
+  #include "llvm/Transforms/Scalar/GVN.h"
+#endif
 #include <llvm/Analysis/Passes.h>
 #include <llvm/Support/raw_ostream.h>
 #include <llvm/Support/Dwarf.h>
@@ -499,8 +503,13 @@ DebugPassManager::add(llvm::Pass * P, int stage = -1) {
         if (g->debug_stages.find(number) != g->debug_stages.end()) {
             // adding dump of LLVM IR after optimization
             char buf[100];
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
             sprintf(buf, "\n\n*****LLVM IR after phase %d: %s*****\n\n",
                 number, P->getPassName());
+#else // LLVM 4.0+
+            sprintf(buf, "\n\n*****LLVM IR after phase %d: %s*****\n\n",
+                number, P->getPassName().data());
+#endif
             PM.add(CreateDebugPass(buf));
         }
 
@@ -672,7 +681,10 @@ Optimize(llvm::Module *module, int optLevel) {
         optPM.add(llvm::createInstructionCombiningPass());
         optPM.add(llvm::createCFGSimplificationPass());
         optPM.add(llvm::createPruneEHPass());
-#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_8 // 3.8+
+#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_9 // 3.9+
+        optPM.add(llvm::createPostOrderFunctionAttrsLegacyPass());
+        optPM.add(llvm::createReversePostOrderFunctionAttrsPass());
+#elif ISPC_LLVM_VERSION == ISPC_LLVM_3_8 // 3.8
         optPM.add(llvm::createPostOrderFunctionAttrsPass());
         optPM.add(llvm::createReversePostOrderFunctionAttrsPass());
 #else // 3.7 and earlier
@@ -936,7 +948,11 @@ class IntrinsicsOpt : public llvm::BasicBlockPass {
 public:
     IntrinsicsOpt() : BasicBlockPass(ID) {};
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Intrinsics Cleanup Optimization"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Intrinsics Cleanup Optimization"; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 
     static char ID;
@@ -1250,7 +1266,11 @@ public:
     InstructionSimplifyPass()
         : BasicBlockPass(ID) { }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Vector Select Optimization"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Vector Select Optimization"; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 
     static char ID;
@@ -1419,7 +1439,11 @@ public:
     static char ID;
     ImproveMemoryOpsPass() : BasicBlockPass(ID) { }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Improve Memory Ops"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Improve Memory Ops"; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 };
 
@@ -3271,7 +3295,11 @@ public:
     static char ID;
     GatherCoalescePass() : BasicBlockPass(ID) { }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Gather Coalescing"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Gather Coalescing"; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 };
 
@@ -4329,7 +4357,11 @@ public:
     static char ID;
     ReplacePseudoMemoryOpsPass() : BasicBlockPass(ID) { }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Replace Pseudo Memory Ops"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Replace Pseudo Memory Ops"; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 };
 
@@ -4698,7 +4730,11 @@ public:
         isLastTry = last;
     }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Resolve \"is compile time constant\""; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Resolve \"is compile time constant\""; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 
     bool isLastTry;
@@ -4793,7 +4829,11 @@ public:
         sprintf(str_output, "%s", output);
     }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Dump LLVM IR"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Dump LLVM IR"; }
+#endif
     bool runOnModule(llvm::Module &m);
 
 private:
@@ -4839,7 +4879,11 @@ public:
         AU.setPreservesCFG();
     }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Make internal funcs \"static\""; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Make internal funcs \"static\""; }
+#endif
     bool runOnModule(llvm::Module &m);
 };
 
@@ -4946,7 +4990,11 @@ class PeepholePass : public llvm::BasicBlockPass {
 public:
     PeepholePass();
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Peephole Optimizations"; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Peephole Optimizations"; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 
     static char ID;
@@ -5347,7 +5395,11 @@ public:
     ReplaceStdlibShiftPass() : BasicBlockPass(ID) {
     }
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Resolve \"replace extract insert chains\""; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Resolve \"replace extract insert chains\""; }
+#endif
     bool runOnBasicBlock(llvm::BasicBlock &BB);
 
 };
@@ -5446,7 +5498,11 @@ public:
     static char ID;
     FixBooleanSelectPass() :FunctionPass(ID) {}
 
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
     const char *getPassName() const { return "Resolve \"replace extract insert chains\""; }
+#else // LLVM 4.0+
+    llvm::StringRef getPassName() const { return "Resolve \"replace extract insert chains\""; }
+#endif
     bool runOnFunction(llvm::Function &F);
 
 private:

perf.py

@@ -339,7 +339,8 @@ def perf(options1, args):
     if options.perf_target != "":
         test_only_r = " sse2-i32x4 sse2-i32x8 sse4-i32x4 sse4-i32x8 sse4-i16x8 \
                         sse4-i8x16 avx1-i32x4 avx1-i32x8 avx1-i32x16 avx1-i64x4 avx1.1-i32x8 \
-                        avx1.1-i32x16 avx1.1-i64x4 avx2-i32x8 avx2-i32x16 avx2-i64x4 "
+                        avx1.1-i32x16 avx1.1-i64x4 avx2-i32x8 avx2-i32x16 avx2-i64x4 \
+                        avx512knl-i32x16 avx512skx-i32x16 "
         test_only = options.perf_target.split(",")
         for iterator in test_only:
             if not (" " + iterator + " " in test_only_r):

prepro.py

@@ -0,0 +1,189 @@
+import re
+import sys
+
+def floating2float(function, idx):
+    typ = 'floating<' + str(idx) + '>'
+    return function.replace(typ, 'float')
+
+def floating2double(function, idx):
+    typ = 'floating<' + str(idx) + '>'
+    return function.replace(typ, 'double')
+
+def number2float(function, idx):
+    typ = 'number<' + str(idx) + '>'
+    return function.replace(typ, 'float')
+
+def number2double(function, idx):
+    typ = 'number<' + str(idx) + '>'
+    return function.replace(typ, 'double')
+
+def number2int(function, idx):
+    typ = 'number<' + str(idx) + '>'
+    return function.replace(typ, 'int')
+
+def number2long(function, idx):
+    typ = 'number<' + str(idx) + '>'
+    return function.replace(typ, 'long')
+
+def number2short(function, idx):
+    typ = 'number<' + str(idx) + '>'
+    return function.replace(typ, 'short')
+
+def number2char(function, idx):
+    typ = 'number<' + str(idx) + '>'
+    return function.replace(typ, 'char')
+
+def integer2int(function, idx):
+    typ = 'integer<' + str(idx) + '>'
+    return function.replace(typ, 'int')
+
+def integer2long(function, idx):
+    typ = 'integer<' + str(idx) + '>'
+    return function.replace(typ, 'long')
+
+def integer2short(function, idx):
+    typ = 'integer<' + str(idx) + '>'
+    return function.replace(typ, 'short')
+
+def integer2char(function, idx):
+    typ = 'integer<' + str(idx) + '>'
+    return function.replace(typ, 'char')
+
+def strip_types(fun, i):
+    return fun.replace('export', '').replace('void', '').replace('float', '') \
+            .replace('double', '').replace('char','').replace('short','') \
+            .replace('int','').replace('long','').replace('[]','') \
+            .replace('*','').replace('(','_' + str(i) + '(') \
+            .replace('uniform','')
+
+if __name__ == '__main__':
+    if (len(sys.argv) != 2):
+        print 'usage:\n\tpython ',
+        print sys.argv[0],
+        print '[file.ispc]'
+        exit(1)
+
+    f = open(sys.argv[1], 'r')
+    function = f.read()
+    fun_def = re.findall(r'export [^{]*\)', function)[0]
+    print fun_def
+    f.close()
+
+    floating = 0
+    number = 0
+    integer = 0
+
+    while ('floating<' + str(floating) + '>' in function):
+        floating += 1
+
+    while ('number<' + str(number) + '>' in function):
+        number += 1
+
+    while ('integer<' + str(integer) + '>' in function):
+        integer += 1
+
+    for i in range(len(re.findall(r'floating[\s+]', function))):
+        function = function.replace('floating ',
+                                    'floating<' + str(floating) + '> ', 1)
+        fun_def = fun_def.replace('floating ',
+                                  'floating<' + str(floating) + '> ', 1)
+        floating += 1
+
+    for i in range(len(re.findall(r'number[\s+]', function))):
+        function = function.replace('number ',
+                                    'number<' + str(number) + '> ', 1)
+        fun_def = fun_def.replace('number ',
+                                  'number<' + str(number) + '> ', 1)
+        number += 1
+
+    for i in range(len(re.findall(r'integer[\s+]', function))):
+        function = function.replace('integer ',
+                                    'integer<' + str(integer) + '> ', 1)
+        fun_def = fun_def.replace('integer ',
+                                  'integer<' + str(integer) + '> ', 1)
+        integer += 1
+
+    floats = [[(fun_def, function)]]
+
+    for i in range(floating):
+        floats.append([])
+        for (h, f) in floats[i]:
+            floats[i+1].append((floating2float(h, i),
+                                floating2float(f, i)
+                               ))
+            floats[i+1].append((floating2double(h, i),
+                                floating2double(f, i)
+                               ))
+
+    numbers = [[]]
+
+    for f in floats[floating]:
+        numbers[0].append(f)
+
+    for i in range(number):
+        numbers.append([])
+        for (h, f) in numbers[i]:
+            numbers[i+1].append((number2float(h, i),
+                                 number2float(f, i)
+                               ))
+            numbers[i+1].append((number2double(h, i),
+                                 number2double(f, i)
+                               ))
+            numbers[i+1].append((number2int(h, i),
+                                 number2int(f, i)
+                               ))
+            numbers[i+1].append((number2long(h, i),
+                                 number2long(f, i)
+                               ))
+            numbers[i+1].append((number2short(h, i),
+                                 number2short(f, i)
+                               ))
+            numbers[i+1].append((number2char(h, i),
+                                 number2char(f, i)
+                               ))
+
+    integers = [[]]
+
+    for f in numbers[number]:
+        integers[0].append(f)
+
+    for i in range(integer):
+        integers.append([])
+        for (h, f) in integers[i]:
+            integers[i+1].append((integer2int(h, i),
+                                 integer2int(f, i)
+                                ))
+            integers[i+1].append((integer2long(h, i),
+                                 integer2long(f, i)
+                                ))
+            integers[i+1].append((integer2short(h, i),
+                                 integer2short(f, i)
+                                ))
+            integers[i+1].append((integer2char(h, i),
+                                 integer2char(f, i)
+                                ))
+
+    o = open(sys.argv[1] + '.pre.ispc', 'w')
+    hdr = open(sys.argv[1] + '.h', 'w')
+
+    hdr.write('#include "' + '.'.join(sys.argv[1].split('.')[:-1]) + '.h"\n\n')
+    hdr.write('#ifndef _' + sys.argv[1].upper().replace('.', '_') + '_H_\n')
+    hdr.write('#define _' + sys.argv[1].upper().replace('.', '_') + '_H_\n')
+    hdr.write('namespace ispc {\n')
+
+    fun_name = re.findall(r'[a-zA-Z_][a-zA-Z0-9_]*\(', function)[0][:-1]
+
+    i = 0
+    for (h, f) in integers[integer]:
+        o.write(f.replace(fun_name, fun_name+'_'+str(i), 1))
+        hdr.write(h.replace('uniform','').replace('export','') \
+                  + '\n{\n\treturn ' + strip_types(h, i) + ';\n}\n\n')
+
+        i += 1
+
+    o.close()
+    hdr.write('}\n')
+    hdr.write('#endif\n')
+    hdr.close()
+
+    exit(0)

run_tests.py

@@ -270,9 +270,6 @@ def run_test(testname):
                 elif (options.target == "knl-generic"):
                     cc_cmd = "%s -O2 -I. %s %s test_static.cpp -DTEST_SIG=%d %s -o %s" % \
                          (options.compiler_exe, gcc_arch, "-xMIC-AVX512", match, obj_name, exe_name)
-                elif (options.target == "avx512knl-i32x16"):
-                    cc_cmd = "%s -O2 -I. %s %s test_static.cpp -DTEST_SIG=%d %s -o %s" % \
-                         (options.compiler_exe, gcc_arch, "-march=knl", match, obj_name, exe_name)
                 else:
                     cc_cmd = "%s -O2 -I. %s %s test_static.cpp -DTEST_SIG=%d %s -o %s" % \
                          (options.compiler_exe, gcc_arch, gcc_isa, match, obj_name, exe_name)                    

stmt.cpp

@@ -1988,7 +1988,13 @@ ForeachStmt::EmitCode(FunctionEmitContext *ctx) const {
         }
 
         ctx->StoreInst(LLVMFalse, stepIndexAfterMaskedBodyPtr);
-        ctx->BranchInst(bbMaskedBody);
+
+        // check to see if counter != end, otherwise, the next step is not necessary
+        llvm::Value *counter = ctx->LoadInst(uniformCounterPtrs[nDims-1], "counter");
+        llvm::Value *atEnd =
+            ctx->CmpInst(llvm::Instruction::ICmp, llvm::CmpInst::ICMP_NE,
+                         counter, endVals[nDims-1], "at_end");
+        ctx->BranchInst(bbMaskedBody, bbReset[nDims-1], atEnd);
     }
 
     ///////////////////////////////////////////////////////////////////////////

tests/foreach-32.ispc

@@ -0,0 +1,29 @@
+export uniform int width() { return programCount; }
+
+export void f_f(uniform float RET[], uniform float aFOO[]) {
+#define A_BEGIN 11
+#define A_END   14
+#define B_BEGIN 28
+#define B_END   31
+#define C_BEGIN 0
+#define C_END   8
+    uniform int t = 0;
+
+    foreach_tiled (i = A_BEGIN ... A_END, j = B_BEGIN ... B_END, k = C_BEGIN ... C_END) {
+        t++;
+    }
+
+    //the comparison with the expected number of iterations
+    if (programCount == 4)
+        RET[programIndex] = t - 24;
+    else if (programCount == 8)
+        RET[programIndex] = t - 16;
+    else if (programCount == 16)
+        RET[programIndex] = t - 8;
+    else
+        RET[programIndex] = t; //this case is still unknown, error in general
+}
+
+export void result(uniform float RET[]) {
+    RET[programIndex] = 0;
+}

tests/masked-scatter-struct.ispc

@@ -5,8 +5,8 @@ struct Foo { float x; float y; };
 
 export void f_fu(uniform float ret[], uniform float aa[], uniform float b) {
     float a = aa[programIndex];
-    uniform Foo foo[programCount];
+    uniform Foo foo[programCount+1];
-    for (uniform int i = 0; i < programCount; ++i) {
+    for (uniform int i = 0; i < programCount+1; ++i) {
         foo[i].x = i;
         foo[i].y = -1234 + i;
     }

tests/masked-scatter-vector.ispc

@@ -5,8 +5,8 @@ typedef int<3> int3;
 
 export void f_fu(uniform float ret[], uniform float aa[], uniform float b) {
     float a = aa[programIndex];
-    uniform int3 array[programCount];
+    uniform int3 array[programCount+1];
-    for (uniform int i = 0; i < programCount + 5 - b; ++i) {
+    for (uniform int i = 0; i < programCount + 6 - b; ++i) {
         for (uniform int j = 0; j < 3; ++j)
             array[i][j] = i+100*j;
     }

tests_ispcpp/hello.cpp

@@ -0,0 +1,23 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "hello.ispc.h"
+
+int main() {
+    float A[100];
+    float B[100];
+    double result[100];
+
+    for (int i=0; i<100; i++) {
+        A[i] = 100 - i;
+        B[i] = i*i;
+    }
+
+    ispc::saxpy(100, 3.1415926535, (float*)&A, (float*)&B, (double*)&result);
+
+    for (int i=0; i<100; i++) {
+        printf("%.6f\n", result[i]);
+    }
+
+    return 0;
+}

tests_ispcpp/hello.ispc

@@ -0,0 +1,11 @@
+export void saxpy(uniform int N,
+                  uniform floating<0> scale,
+                  uniform floating<1> X[],
+                  uniform floating<1> Y[],
+                  uniform floating<2> result[])
+{
+    foreach (i = 0 ... N) {
+        floating<2> tmp = scale * X[i] + Y[i];
+        result[i] = tmp;
+    }
+}

type.cpp

@@ -541,6 +541,8 @@ llvm::DIType *AtomicType::GetDIType(llvm::DIScope *scope) const {
 #else //LLVM 3.7++
             return NULL;
 #endif
+
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
         case TYPE_BOOL:
             return m->diBuilder->createBasicType("bool", 32 /* size */, 32 /* align */,
                                                  llvm::dwarf::DW_ATE_unsigned);
@@ -585,6 +587,53 @@ llvm::DIType *AtomicType::GetDIType(llvm::DIScope *scope) const {
             return m->diBuilder->createBasicType("uint64", 64 /* size */, 64 /* align */,
                                                  llvm::dwarf::DW_ATE_unsigned);
             break;
+#else // LLVM 4.0+
+        case TYPE_BOOL:
+            return m->diBuilder->createBasicType("bool", 32 /* size */,
+                                                 llvm::dwarf::DW_ATE_unsigned);
+            break;
+        case TYPE_INT8:
+            return m->diBuilder->createBasicType("int8", 8 /* size */,
+                                                 llvm::dwarf::DW_ATE_signed);
+            break;
+        case TYPE_UINT8:
+            return m->diBuilder->createBasicType("uint8", 8 /* size */,
+                                                 llvm::dwarf::DW_ATE_unsigned);
+            break;
+        case TYPE_INT16:
+            return m->diBuilder->createBasicType("int16", 16 /* size */,
+                                                 llvm::dwarf::DW_ATE_signed);
+            break;
+        case TYPE_UINT16:
+            return m->diBuilder->createBasicType("uint16", 16 /* size */,
+                                                 llvm::dwarf::DW_ATE_unsigned);
+            break;
+        case TYPE_INT32:
+            return m->diBuilder->createBasicType("int32", 32 /* size */,
+                                                 llvm::dwarf::DW_ATE_signed);
+            break;
+        case TYPE_UINT32:
+            return m->diBuilder->createBasicType("uint32", 32 /* size */,
+                                                 llvm::dwarf::DW_ATE_unsigned);
+            break;
+        case TYPE_FLOAT:
+            return m->diBuilder->createBasicType("float", 32 /* size */,
+                                                 llvm::dwarf::DW_ATE_float);
+            break;
+        case TYPE_DOUBLE:
+            return m->diBuilder->createBasicType("double", 64 /* size */,
+                                                 llvm::dwarf::DW_ATE_float);
+            break;
+        case TYPE_INT64:
+            return m->diBuilder->createBasicType("int64", 64 /* size */,
+                                                 llvm::dwarf::DW_ATE_signed);
+            break;
+        case TYPE_UINT64:
+            return m->diBuilder->createBasicType("uint64", 64 /* size */,
+                                                 llvm::dwarf::DW_ATE_unsigned);
+            break;
+#endif
+
         default:
             FATAL("unhandled basic type in AtomicType::GetDIType()");
 #if ISPC_LLVM_VERSION <= ISPC_LLVM_3_6
@@ -2307,9 +2356,15 @@ llvm::DIType *StructType::GetDIType(llvm::DIScope *scope) const {
         llvm::DIFile *diFile = elementPositions[i].GetDIFile();
         llvm::DIDerivedType *fieldType =
 #endif
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
             m->diBuilder->createMemberType(scope, elementNames[i], diFile,
                                            line, eltSize, eltAlign,
                                            currentSize, 0, eltType);
+#else // LLVM 4.0+
+            m->diBuilder->createMemberType(scope, elementNames[i], diFile,
+                                           line, eltSize, eltAlign,
+                                           currentSize, llvm::DINode::FlagZero, eltType);
+#endif
         elementLLVMTypes.push_back(fieldType);
 
         currentSize += eltSize;
@@ -2334,7 +2389,11 @@ llvm::DIType *StructType::GetDIType(llvm::DIScope *scope) const {
         pos.first_line, // Line number
         currentSize,    // Size in bits
         align,          // Alignment in bits
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
         0,              // Flags
+#else // LLVM 4.0+
+        llvm::DINode::FlagZero, // Flags
+#endif
 #if ISPC_LLVM_VERSION >= ISPC_LLVM_3_3 && ISPC_LLVM_VERSION <= ISPC_LLVM_3_6
         llvm::DIType(), // DerivedFrom
 #elif ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 // LLVM 3.7++
@@ -2584,7 +2643,11 @@ llvm::DIType *UndefinedStructType::GetDIType(llvm::DIScope *scope) const {
         pos.first_line, // Line number
         0,              // Size
         0,              // Align
+#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_9
         0,              // Flags
+#else // LLVM 4.0+
+        llvm::DINode::FlagZero, // Flags
+#endif
 #if ISPC_LLVM_VERSION >= ISPC_LLVM_3_3 && ISPC_LLVM_VERSION <= ISPC_LLVM_3_6
         llvm::DIType(), // DerivedFrom
 #elif ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 // LLVM 3.7+