diff --git a/alloy.py b/alloy.py new file mode 100755 index 00000000..67f534ca --- /dev/null +++ b/alloy.py @@ -0,0 +1,600 @@ +#!/usr/bin/python +# +# Copyright (c) 2013, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# // Author: Filippov Ilia + +def attach_mail_file(msg, filename, name): + if os.path.exists(filename): + fp = open(filename, "rb") + to_attach = MIMEBase("application", "octet-stream") + to_attach.set_payload(fp.read()) + encode_base64(to_attach) + to_attach.add_header("Content-Disposition", "attachment", filename=name) + fp.close() + msg.attach(to_attach) + +def setting_paths(llvm, ispc, sde): + if llvm != "": + os.environ["LLVM_HOME"]=llvm + if ispc != "": + os.environ["ISPC_HOME"]=ispc + if sde != "": + os.environ["SDE_HOME"]=sde + +def check_LLVM(which_LLVM): + answer = [] + if which_LLVM[0] == " ": + return answer + p = os.environ["LLVM_HOME"] + for i in range(0,len(which_LLVM)): + if not os.path.exists(p + os.sep + "bin-" + which_LLVM[i] + os.sep + "bin"): + answer.append(which_LLVM[i]) + return answer + +def try_do_LLVM(text, command, from_validation): + if from_validation == True: + text = text + "\n" + print_debug("Trying to " + text, from_validation, alloy_build) + if os.system(command + " >> " + alloy_build + " 2>> " + alloy_build) != 0: + print_debug("ERROR.\n", from_validation, alloy_build) + error("can't " + text, 1) + print_debug("DONE.\n", from_validation, alloy_build) + +def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, from_validation, force): + print_debug("Building LLVM. Version: " + version_LLVM + ". ", from_validation, alloy_build) + if revision != "": + print_debug("Revision: " + revision + ".\n", from_validation, alloy_build) + else: + print_debug("\n", from_validation, alloy_build) + # Here we understand what and where do we want to build + current_path = os.getcwd() + llvm_home = os.environ["LLVM_HOME"] + os.chdir(llvm_home) + FOLDER_NAME=version_LLVM + if version_LLVM == "head": + SVN_PATH="trunk" + if version_LLVM == "3.3": + SVN_PATH="tags/RELEASE_33/final" + version_LLVM = "3_3" + if version_LLVM == "3.2": + SVN_PATH="tags/RELEASE_32/final" + version_LLVM = "3_2" + if version_LLVM == "3.1": + SVN_PATH="tags/RELEASE_31/final" + version_LLVM = "3_1" + if revision != "": + FOLDER_NAME = FOLDER_NAME + "_" + revision + revision = "-" + revision + if folder == "": + folder = FOLDER_NAME + LLVM_SRC="llvm-" + folder + LLVM_BUILD="build-" + folder + LLVM_BIN="bin-" + folder + if os.path.exists(LLVM_BIN) and not force: + print_debug("You have folder " + LLVM_BIN + ". If you want to rebuild use --force\n", False, "") + exit(0) + LLVM_BUILD_selfbuild = LLVM_BUILD + "_temp" + LLVM_BIN_selfbuild = LLVM_BIN + "_temp" + common.remove_if_exists(LLVM_SRC) + common.remove_if_exists(LLVM_BUILD) + common.remove_if_exists(LLVM_BIN) + if selfbuild: + common.remove_if_exists(LLVM_BUILD_selfbuild) + common.remove_if_exists(LLVM_BIN_selfbuild) + MAKE = "gmake" + print_debug("Using folders: " + LLVM_SRC + " " + LLVM_BUILD + " " + LLVM_BIN + " in " + + llvm_home + "\n", from_validation, alloy_build) + # load llvm + if tarball == "": + try_do_LLVM("load LLVM from http://llvm.org/svn/llvm-project/llvm/" + SVN_PATH + " ", + "svn co " + revision + " http://llvm.org/svn/llvm-project/llvm/" + SVN_PATH + " " + LLVM_SRC, + from_validation) + os.chdir(LLVM_SRC + "/tools") + try_do_LLVM("load clang from http://llvm.org/svn/llvm-project/cfe/" + SVN_PATH + " ", + "svn co " + revision + " http://llvm.org/svn/llvm-project/cfe/" + SVN_PATH + " clang", + from_validation) + os.chdir("../") + else: + tar = tarball.split(" ") + os.makedirs(LLVM_SRC) + os.chdir(LLVM_SRC) + try_do_LLVM("untar LLVM from " + tar[0] + " ", + "tar -xvzf " + tar[0] + " --strip-components 1", from_validation) + os.chdir("./tools") + os.makedirs("clang") + os.chdir("./clang") + try_do_LLVM("untar clang from " + tar[1] + " ", + "tar -xvzf " + tar[1] + " --strip-components 1", from_validation) + os.chdir("../../") + # paching llvm + patches = glob.glob(os.environ["ISPC_HOME"] + "/llvm_patches/*.*") + for patch in patches: + if version_LLVM in os.path.basename(patch): + try_do_LLVM("patch LLVM with patch" + patch + " ", "patch -p0 < " + patch, from_validation) + os.chdir("../") + # configuring llvm, build first part of selfbuild + os.makedirs(LLVM_BUILD) + os.makedirs(LLVM_BIN) + selfbuild_compiler = "" + if selfbuild: + print_debug("Making selfbuild and use folders " + LLVM_BUILD_selfbuild + " and " + + LLVM_BIN_selfbuild + "\n", from_validation, alloy_build) + os.makedirs(LLVM_BUILD_selfbuild) + os.makedirs(LLVM_BIN_selfbuild) + os.chdir(LLVM_BUILD_selfbuild) + try_do_LLVM("configure release version for selfbuild ", + "../" + LLVM_SRC + "/configure --prefix=" + llvm_home + "/" + + LLVM_BIN_selfbuild + " --enable-optimized", + from_validation) + try_do_LLVM("build release version for selfbuild ", + MAKE + " -j32", from_validation) + try_do_LLVM("install release version for selfbuild ", + MAKE + " install", + from_validation) + os.chdir("../") + selfbuild_compiler = " CC="+llvm_home+ "/" + LLVM_BIN_selfbuild + "/bin/clang" + print_debug("Now we have compiler for selfbuild: " + selfbuild_compiler + "\n", from_validation, alloy_build) + os.chdir(LLVM_BUILD) + if debug == False: + try_do_LLVM("configure release version ", + "../" + LLVM_SRC + "/configure --prefix=" + llvm_home + "/" + + LLVM_BIN + " --enable-optimized" + selfbuild_compiler, + from_validation) + else: + try_do_LLVM("configure debug version ", + "../" + LLVM_SRC + "/configure --prefix=" + llvm_home + "/" + LLVM_BIN + + " --enable-debug-runtime --enable-debug-symbols --enable-keep-symbols" + selfbuild_compiler, + from_validation) + # building llvm + try_do_LLVM("build LLVM ", MAKE + " -j32", from_validation) + try_do_LLVM("install LLVM ", MAKE + " install", from_validation) + os.chdir(current_path) + +def check_targets(): + answer = [] + answer_sde = [] + SSE2 = False; + SSE4 = False; + AVX = False; + AVX11 = False; + AVX2 = False; + cpu = open("/proc/cpuinfo") + f_lines = cpu.readlines() + cpu.close() + # check what native targets do we have + for i in range(0,len(f_lines)): + if SSE2 == False and "sse2" in f_lines[i]: + SSE2 = True; + answer = answer + ["sse2-i32x4", "sse2-i32x8"] + if SSE4 == False and "sse4_1" in f_lines[i]: + SSE4 = True; + answer = answer + ["sse4-i32x4", "sse4-i32x8", "sse4-i16x8", "sse4-i8x16"] + if AVX == False and "avx" in f_lines[i]: + AVX = True; + answer = answer + ["avx1-i32x8", "avx1-i32x16"] + if AVX11 == False and "rdrand" in f_lines[i]: + AVX11 = True; + answer = answer + ["avx1.1-i32x8", "avx1.1-i32x16"] + if AVX2 == False and "avx2" in f_lines[i]: + AVX2 = True; + answer = answer + ["avx2-i32x8", "avx2-i32x16"] + answer = answer + ["generic-4", "generic-16", "generic-8", "generic-1", "generic-32", "generic-64"] + # now check what targets we have with the help of SDE + sde_exists = "" + PATH_dir = string.split(os.getenv("PATH"), os.pathsep) + for counter in PATH_dir: + if os.path.exists(counter + os.sep + "sde") and sde_exists == "": + sde_exists = counter + os.sep + "sde" + if os.environ.get("SDE_HOME") != None: + if os.path.exists(os.environ.get("SDE_HOME") + os.sep + "sde"): + sde_exists = os.environ.get("SDE_HOME") + os.sep + "sde" + if sde_exists == "": + error("you haven't got sde neither in SDE_HOME nor in your PATH.\n" + + "To test all platforms please set SDE_HOME to path containing SDE.\n" + + "Please refer to http://www.intel.com/software/sde for SDE download information.", 2) + return [answer, answer_sde] + # here we have SDE + os.system(sde_exists + " -help > " + temp_alloy_file) + cpu = open(temp_alloy_file) + f_lines = cpu.readlines() + cpu.close() + for i in range(0,len(f_lines)): + if SSE4 == False and "wsm" in f_lines[i]: + answer_sde = answer_sde + [["-wsm", "sse4-i32x4"], ["-wsm", "sse4-i32x8"], ["-wsm", "sse4-i16x8"], ["-wsm", "sse4-i8x16"]] + if AVX == False and "snb" in f_lines[i]: + answer_sde = answer_sde + [["-snb", "avx1-i32x8"], ["-snb", "avx1-i32x16"]] + if AVX11 == False and "ivb" in f_lines[i]: + answer_sde = answer_sde + [["-ivb", "avx1.1-i32x8"], ["ivb", "avx1.1-i32x16"]] + if AVX2 == False and "hsw" in f_lines[i]: + answer_sde = answer_sde + [["-hsw", "avx2-i32x8"], ["-hsw", "avx2-i32x16"]] + return [answer, answer_sde] + +def build_ispc(version_LLVM): + current_path = os.getcwd() + os.chdir(os.environ["ISPC_HOME"]) + p_temp = os.getenv("PATH") + os.environ["PATH"] = os.environ["LLVM_HOME"] + "/bin-" + version_LLVM + "/bin:" + os.environ["PATH"] + os.system("make clean >> " + alloy_build) + try_do_LLVM("build ISPC with LLVM version " + version_LLVM + " ", "make -j32", True) + os.environ["PATH"] = p_temp + os.chdir(current_path) + +def execute_stability(stability, R, print_version): + stability1 = copy.deepcopy(stability) + temp = run_tests.run_tests(stability1, [], print_version) + for j in range(0,4): + R[j][0] = R[j][0] + temp[j] + for i in range(0,len(temp[j])): + R[j][1].append(temp[4]) + number_of_fails = temp[5] + number_of_new_fails = len(temp[0]) + len(temp[1]) + if number_of_fails == 0: + str_fails = ". No fails" + else: + str_fails = ". Fails: " + str(number_of_fails) + if number_of_new_fails == 0: + str_new_fails = ", No new fails.\n" + else: + str_new_fails = ", New fails: " + str(number_of_new_fails) + ".\n" + print_debug(temp[4][1:-3] + str_fails + str_new_fails, False, stability_log) + +def run_special_tests(): + i = 5 + +def validation_run(only, only_targets, reference_branch, notify, update): + current_path = os.getcwd() + os.chdir(os.environ["ISPC_HOME"]) + os.environ["PATH"] = os.environ["ISPC_HOME"] + ":" + os.environ["PATH"] + if options.notify != "": + if os.environ.get("SMTP_ISPC") == None: + error("you have no SMTP_ISPC in your environment for option notify", 1) + common.remove_if_exists(os.environ["ISPC_HOME"] + os.sep + "all_answer.txt") + smtp_server = os.environ["SMTP_ISPC"] + msg = MIMEMultipart() + msg['Subject'] = 'ISPC test system results' + msg['From'] = 'ISPC_test_system' + msg['To'] = options.notify + print_debug("Command: " + ' '.join(sys.argv) + "\n", False, "") + print_debug("Folder: " + os.environ["ISPC_HOME"] + "\n", False, "") + date = datetime.datetime.now() + print_debug("Date: " + date.strftime('%H:%M %d/%m/%Y') + "\n", False, "") + class options_for_drivers: + pass +# *** *** *** +# Stability validation run +# *** *** *** + if ((("stability" in only) == True) or ("performance" in only) == False): + print_debug("\n\nStability validation run\n\n", False, "") + stability = options_for_drivers() +# stability constant options + stability.random = False + stability.ispc_flags = "" + stability.compiler_exe = None + stability.num_jobs = 1024 + stability.verbose = False + stability.time = False + stability.non_interactive = True + stability.update = update + stability.include_file = None + stability.silent = True + stability.in_file = "." + os.sep + f_date + os.sep + "run_tests_log.log" + stability.verify = False +# stability varying options + stability.target = "" + stability.arch = "" + stability.no_opt = False + stability.wrapexe = "" +# prepare parameters of run + common.check_tools(1) + [targets_t, sde_targets_t] = check_targets() + rebuild = True + opts = [] + archs = [] + LLVM = [] + targets = [] + sde_targets = [] +# parsing option only, update parameters of run + if "-O2" in only: + opts.append(False) + if "-O0" in only: + opts.append(True) + if "x86" in only and not ("x86-64" in only): + archs.append("x86") + if "x86-64" in only: + archs.append("x86-64") + if "native" in only: + sde_targets_t = [] + for i in ["3.1", "3.2", "3.3", "head"]: + if i in only: + LLVM.append(i) + if "current" in only: + LLVM = [" "] + rebuild = False + if only_targets != "": + only_targets_t = only_targets.split(" ") + for i in only_targets_t: + err = True + for j in range(0,len(targets_t)): + if i in targets_t[j]: + targets.append(targets_t[j]) + err = False + for j in range(0,len(sde_targets_t)): + if i in sde_targets_t[j][1]: + sde_targets.append(sde_targets_t[j]) + err = False + if err == True: + error("You haven't sde for target " + i, 1) + else: + targets = targets_t[:-4] + sde_targets = sde_targets_t + if "build" in only: + targets = [] + sde_targets = [] + only = only + " stability " +# finish parameters of run, prepare LLVM + if len(opts) == 0: + opts = [False] + if len(archs) == 0: + archs = ["x86", "x86-64"] + if len(LLVM) == 0: + LLVM = ["3.1", "3.2", "3.3", "head"] + gen_archs = ["x86-64"] + need_LLVM = check_LLVM(LLVM) + for i in range(0,len(need_LLVM)): + build_LLVM(need_LLVM[i], "", "", "", False, False, True, False) +# begin validation run for stabitily + common.remove_if_exists(stability.in_file) + R = [[[],[]],[[],[]],[[],[]],[[],[]]] + print_debug("\n_________________________STABILITY REPORT_________________________\n", False, stability_log) + for i in range(0,len(LLVM)): + print_version = 2 + if rebuild: + build_ispc(LLVM[i]) + for j in range(0,len(targets)): + stability.target = targets[j] + stability.wrapexe = "" + if "generic" in targets[j]: + arch = gen_archs + else: + arch = archs + for i1 in range(0,len(arch)): + for i2 in range(0,len(opts)): + stability.arch = arch[i1] + stability.no_opt = opts[i2] + execute_stability(stability, R, print_version) + print_version = 0 + for j in range(0,len(sde_targets)): + stability.target = sde_targets[j][1] + stability.wrapexe = os.environ["SDE_HOME"] + "/sde " + sde_targets[j][0] + " -- " + for i1 in range(0,len(archs)): + for i2 in range(0,len(opts)): + stability.arch = archs[i1] + stability.no_opt = opts[i2] + execute_stability(stability, R, print_version) + print_version = 0 +# run special tests like embree +# + run_special_tests() + ttt = ["NEW RUNFAILS: ", "NEW COMPFAILS: ", "NEW PASSES RUNFAILS: ", "NEW PASSES COMPFAILS: "] + for j in range(0,4): + if len(R[j][0]) == 0: + print_debug("NO " + ttt[j][:-2] + "\n", False, stability_log) + else: + print_debug(ttt[j] + str(len(R[j][0])) + "\n", False, stability_log) + temp5 = [[],[]] + for i in range(0,len(R[j][0])): + er = True + for k in range(0,len(temp5[0])): + if R[j][0][i] == temp5[0][k]: + temp5[1][k].append(R[j][1][i]) + er = False + if er == True: + temp5[0].append(R[j][0][i]) + temp5[1].append([R[j][1][i]]) + for i in range(0,len(temp5[0])): + print_debug("\t" + temp5[0][i] + "\n", True, stability_log) + for k in range(0,len(temp5[1][i])): + print_debug("\t\t\t" + temp5[1][i][k], True, stability_log) + print_debug("__________________Watch stability.log for details_________________\n", False, stability_log) + if options.notify != "": + attach_mail_file(msg, stability.in_file, "run_tests_log.log") + attach_mail_file(msg, stability_log, "stability.log") + +# *** *** *** +# Performance validation run +# *** *** *** + if ((("performance" in only) == True) or ("stability" in only) == False): + print_debug("\n\nPerformance validation run\n\n", False, "") + performance = options_for_drivers() +# performance constant options + performance.number = 5 + performance.config = "./perf.ini" + performance.path = "./" + performance.silent = True + performance.output = "" + performance.compiler = "" + performance.ref = "ispc_ref" + performance.in_file = "." + os.sep + f_date + os.sep + "performance.log" +# prepare LLVM 3.3 as newest LLVM + need_LLVM = check_LLVM(["3.3"]) + if len(need_LLVM) != 0: + build_LLVM(need_LLVM[i], "", "", "", False, False, True, False) +# prepare reference point. build both test and reference compilers + os.system("git branch > " + temp_alloy_file) + br = open(temp_alloy_file) + temp4 = br.readlines() + br.close() + for line in temp4: + if "*" in line: + current_branch = line[2:-1] + stashing = True + sys.stdout.write("Please, don't interrupt script here! You can have not sync git status after interruption!\n") + if "No local changes" in detect_version("git stash"): + stashing = False + #try_do_LLVM("stash current branch ", "git stash", True) + try_do_LLVM("checkout reference branch " + reference_branch + " ", "git checkout " + reference_branch, True) + sys.stdout.write(".\n") + build_ispc("3.3") + sys.stdout.write(".\n") + os.rename("ispc", "ispc_ref") + try_do_LLVM("checkout test branch " + current_branch + " ", "git checkout " + current_branch, True) + if stashing: + try_do_LLVM("return current branch ", "git stash pop", True) + sys.stdout.write("You can interrupt script now.\n") + build_ispc("3.3") +# begin validation run for performance. output is inserted into perf() + perf.perf(performance, []) + if options.notify != "": + attach_mail_file(msg, performance.in_file, "performance.log") + attach_mail_file(msg, "." + os.sep + "logs" + os.sep + "perf_build.log", "perf_build.log") + + print_debug("Logs are in alloy_results_[date]", False, "") + +# sending e-mail with results + if options.notify != "": + fp = open(os.environ["ISPC_HOME"] + os.sep + "all_answer.txt", 'rb') + f_lines = fp.readlines() + fp.close() + line = "" + for i in range(0,len(f_lines)): + line = line + f_lines[i][:-1] + line = line + ' \n' + text = MIMEText(line, "", "KOI-8") + msg.attach(text) + attach_mail_file(msg, alloy_build, "alloy_build.log") + s = smtplib.SMTP(smtp_server) + s.sendmail('ISPC_test_system', options.notify, msg.as_string()) + s.quit() +# exit of validation routine + common.remove_if_exists(temp_alloy_file) + os.chdir(current_path) + +def Main(): + if (platform.system() == 'Windows' or 'CYGWIN_NT' in platform.system()) == True: + error("Windows isn't supported now", 1) + if (options.build_llvm == False and + options.validation_run == False and + options.llvm_home == "" and + options.ispc_home == "" and + options.sde_home == ""): + parser.print_help() + exit(0) + global f_date + f_date = "logs" + common.remove_if_exists(f_date) + os.makedirs(f_date) + global temp_alloy_file + temp_alloy_file = os.getcwd() + os.sep + f_date + os.sep + "temp_detect_version" + global alloy_build + alloy_build = os.getcwd() + os.sep + f_date + os.sep + "alloy_build.log" + common.remove_if_exists(alloy_build) + global stability_log + stability_log = os.getcwd() + os.sep + f_date + os.sep + "stability.log" + common.remove_if_exists(stability_log) + setting_paths(options.llvm_home, options.ispc_home, options.sde_home) + if os.environ.get("LLVM_HOME") == None: + error("you have no LLVM_HOME", 1) + if os.environ.get("ISPC_HOME") == None: + error("you have no ISPC_HOME", 1) + if options.build_llvm: + build_LLVM(options.version, options.revision, options.folder, options.tarball, + options.debug, options.selfbuild, False, options.force) + if options.validation_run: + validation_run(options.only, options.only_targets, options.branch, options.notify, options.update) + os.rename(f_date, "alloy_results_" + datetime.datetime.now().strftime('%H_%M_%d_%m_%Y')) + +###Main### +from optparse import OptionParser +import sys +import os +import operator +import time +import glob +import string +import platform +import smtplib +import datetime +import copy +from email.MIMEMultipart import MIMEMultipart +from email.MIMEBase import MIMEBase +from email.mime.text import MIMEText +from email.Encoders import encode_base64 +# our drivers +import run_tests +import perf +import common +error = common.error +detect_version = common.detect_version +print_debug = common.print_debug +# parsing options +parser = OptionParser() +# options for activity "build LLVM" +parser.add_option('-b', '--build-llvm', dest='build_llvm', + help='ask to build LLVM', default=False, action="store_true") +parser.add_option('--version', dest='version', + help='version of llvm to build', default="head") +parser.add_option('--revision', dest='revision', + help='revision of llvm to build', default="") +parser.add_option('--debug', dest='debug', + help='debug build of LLVM?', default=False, action="store_true") +parser.add_option('--folder', dest='folder', + help='folder to build LLVM in', default="") +parser.add_option('--tarball', dest='tarball', + help='"llvm_tarball clang_tarball"', default="") +parser.add_option('--selfbuild', dest='selfbuild', + help='make selfbuild of LLVM and clang', default=False, action="store_true") +parser.add_option('--force', dest='force', + help='rebuild LLVM', default=False, action='store_true') +# options for activity "setup PATHS" +parser.add_option('--llvm_home', dest='llvm_home',help='path to LLVM',default="") +parser.add_option('--ispc_home', dest='ispc_home',help='path to ISPC',default="") +parser.add_option('--sde_home', dest='sde_home',help='path to SDE',default="") +# options for activity "validation run" +parser.add_option('-r', '--run', dest='validation_run', + help='ask for validation run', default=False, action="store_true") +parser.add_option('--compare-with', dest='branch', + help='set performance reference point', default="master") +parser.add_option('--only-targets', dest='only_targets', + help='set list of targets to test. Possible values - all subnames of targets.\n' + + 'Example: --only-targets="avx2-i32x8 sse4 i32x16 sse2"', default="") +parser.add_option('--notify', dest='notify', + help='sent results to email', default="") +parser.add_option('--only', dest='only', + help='set types of tests. Possible values:\n' + + '-O0, -O2, x86, x86-64, stability (test only stability), performance (test only performance)\n' + + 'build (only build with different LLVM), 3.1, 3.2, 3.3, head, native (do not use SDE), current (do not rebuild ISPC).\n' + + 'Example: --only="3.2 -O0 stability 3.3"', default="") +parser.add_option('--update-errors', dest='update', + help='rewrite fail_db.txt file according to received results (F or FP)', default="") +(options, args) = parser.parse_args() +Main() diff --git a/check_env.py b/check_env.py new file mode 100755 index 00000000..98deb235 --- /dev/null +++ b/check_env.py @@ -0,0 +1,102 @@ +#!/usr/bin/python +# +# Copyright (c) 2013, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# // Author: Filippov Ilia + +import common +import sys +import os +import string +print_debug = common.print_debug +error = common.error +detect_version = common.detect_version + +exists = [False, False, False, False, False, False, False, False] +names = ["m4", "bison", "flex", "sde", "ispc", "clang", "gcc", "icc"] + +PATH_dir = string.split(os.getenv("PATH"), os.pathsep) +for counter in PATH_dir: + for i in range(0,8): + if os.path.exists(counter + os.sep + names[i]): + exists[i] = True + +print_debug("=== in PATH: ===\n", False, "") +print_debug("Tools:\n", False, "") +for i in range(0,3): + if exists[i]: + print_debug(detect_version(names[i] + " --version"), False, "") + else: + error("you don't have " + names[i], 0) +if exists[0] and exists[1] and exists[2]: + if common.check_tools(2): + print_debug("versions are ok\n", False, "") +print_debug("\nSDE:\n", False, "") +if exists[3]: + print_debug(detect_version(names[3] + " --version"), False, "") +else: + error("you don't have " + names[3], 2) +print_debug("\nISPC:\n", False, "") +if exists[4]: + print_debug(detect_version(names[4] + " --version"), False, "") +else: + error("you don't have " + names[4], 2) +print_debug("\nC/C++ compilers:\n", False, "") +for i in range(5,8): + if exists[i]: + print_debug(detect_version(names[i] + " --version"), False, "") + else: + error("you don't have " + names[i], 2) + +print_debug("\n=== in ISPC specific environment variables: ===\n", False, "") +if os.environ.get("LLVM_HOME") == None: + error("you have no LLVM_HOME", 2) +else: + print_debug("Your LLVM_HOME:" + os.environ.get("LLVM_HOME") + "\n", False, "") +if os.environ.get("ISPC_HOME") == None: + error("you have no ISPC_HOME", 2) +else: + print_debug("Your ISPC_HOME:" + os.environ.get("ISPC_HOME") + "\n", False, "") + if os.path.exists(os.environ.get("ISPC_HOME") + os.sep + "ispc"): + print_debug("You have ISPC in your ISPC_HOME: " + + detect_version(os.environ.get("ISPC_HOME") + os.sep + "ispc" + " --version"), False, "") + else: + error("you don't have ISPC in your ISPC_HOME", 2) +if os.environ.get("SDE_HOME") == None: + error("You have no SDE_HOME", 2) +else: + print_debug("Your SDE_HOME:" + os.environ.get("SDE_HOME") + "\n", False, "") + if os.path.exists(os.environ.get("SDE_HOME") + os.sep + "sde"): + print_debug("You have sde in your SDE_HOME: " + + detect_version(os.environ.get("SDE_HOME") + os.sep + "sde" + " --version"), False, "") + else: + error("you don't have any SDE in your ISPC_HOME", 2) diff --git a/common.py b/common.py new file mode 100644 index 00000000..dd8fb388 --- /dev/null +++ b/common.py @@ -0,0 +1,120 @@ +#!/usr/bin/python +# +# Copyright (c) 2013, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# // Author: Filippov Ilia +import sys +import os +import shutil + +def write_to_file(filename, line): + f = open(filename, 'a') + f.writelines(line) + f.close() + +#remove file if it exists +def remove_if_exists(filename): + if os.path.exists(filename): + if os.path.isdir(filename): + shutil.rmtree(filename) + else: + os.remove(filename) + +# detect version which is printed after command +def detect_version(command): + os.system(command + " > " + "temp_detect_version") + version = open("temp_detect_version") + answer = version.readline() + version.close() + remove_if_exists("temp_detect_version") + return answer + +# print versions of compilers +def print_version(ispc_test, ispc_ref, ref_compiler, s, perf_log, is_windows): + print_debug("\nUsing test compiler: " + detect_version(ispc_test + " --version"), s, perf_log) + if ispc_ref != "": + print_debug("Using ref compiler: " + detect_version(ispc_ref + " --version"), s, perf_log) + if is_windows == False: + temp1 = detect_version(ref_compiler + " --version") + else: + os.system(ref_compiler + " 2>&1" + " 2> temp_detect_version > temp_detect_version1" ) + version = open("temp_detect_version") + temp1 = version.readline() + version.close() + remove_if_exists("temp_detect_version") + remove_if_exists("temp_detect_version1") + print_debug("Using C/C++ compiler: " + temp1 + "\n", s, perf_log) + +# print everything from scripts instead errors +def print_debug(line, silent, filename): + if silent == False: + sys.stdout.write(line) + sys.stdout.flush() + if os.environ.get("ISPC_HOME") != None: + write_to_file(os.environ["ISPC_HOME"] + os.sep + "all_answer.txt", line) + if filename != "": + write_to_file(filename, line) + +# print errors from scripts +# type 1 for error in environment +# type 2 for warning +# type 3 for error of compiler or test which isn't the goal of script +def error(line, error_type): + line = line + "\n" + if error_type == 1: + sys.stderr.write("Fatal error: " + line) + sys.exit(1) + if error_type == 2: + sys.stderr.write("Warning: " + line) + if error_type == 0: + print_debug("FIND ERROR: " + line, False, "") + +def check_tools(m): + input_tools=[[[1,4],"m4 --version", "bad m4 version"], + [[2,4],"bison --version", "bad bison version"], + [[2,5], "flex --version", "bad flex version"]] + + for t in range(0,len(input_tools)): + t1 = ((detect_version(input_tools[t][1]))[:-1].split(" ")) + for i in range(0,len(t1)): + t11 = t1[i].split(".") + f = True + for j in range(0,len(t11)): + if not t11[j].isdigit(): + f = False + if f == True: + for j in range(0,len(t11)): + if j < len(input_tools[t][0]): + if int(t11[j])> "+build_log) - return os.system("make CXX="+ref_compiler+" CC="+refc_compiler+" >> "+build_log+" 2>> "+build_log) - else: - os.system("msbuild /t:clean >> " + build_log) - return os.system("msbuild /V:m /p:Platform=x64 /p:Configuration=Release /p:TargetDir=.\ /t:rebuild >> " + build_log) - -def execute_test(command): - global perf_temp - r = 0 - if os.path.exists(perf_temp): - os.remove(perf_temp) - for k in range(int(options.number)): - r = r + os.system(command) - return r - -#gathers all tests results and made an item test from answer structure -def run_test(command, c1, c2, test, b_serial): - global perf_temp - if build_test() != 0: - sys.stdout.write("ERROR: Compilation fails\n") - return - if execute_test(command) != 0: - sys.stdout.write("ERROR: Execution fails\n") - return - tasks = [] #list of results with tasks, it will be test[2] - ispc = [] #list of results without tasks, it will be test[1] - absolute_tasks = [] #list of absolute results with tasks, it will be test[4] - absolute_ispc = [] #list of absolute results without tasks, ut will be test[3] - serial = [] #list serial times, it will be test[5] - j = 1 - for line in open(perf_temp): # we take test output - if "speedup" in line: # we are interested only in lines with speedup - if j == c1: # we are interested only in lines with c1 numbers - line = line.expandtabs(0) - line = line.replace("("," ") - line = line.split(",") - for i in range(len(line)): - subline = line[i].split(" ") - number = float(subline[1][:-1]) - if "speedup from ISPC + tasks" in line[i]: - tasks.append(number) - else: - ispc.append(number) - c1 = c1 + c2 - j+=1 - if "million cycles" in line: - if j == c1: - line = line.replace("]","[") - line = line.split("[") - number = float(line[3]) - if "tasks" in line[1]: - absolute_tasks.append(number) - else: - if "ispc" in line[1]: - absolute_ispc.append(number) - if "serial" in line[1]: - serial.append(number) - - if len(ispc) != 0: - if len(tasks) != 0: - print_debug("ISPC speedup / ISPC + tasks speedup / ISPC time / ISPC + tasks time / serial time\n") - for i in range(0,len(serial)): - print_debug("%10s /\t%10s\t /%9s / %10s\t /%10s\n" % - (ispc[i], tasks[i], absolute_ispc[i], absolute_tasks[i], serial[i])) - else: - print_debug("ISPC speedup / ISPC time / serial time\n") - for i in range(0,len(serial)): - print_debug("%10s /%9s /%10s\n" % (ispc[i], absolute_ispc[i], serial[i])) - else: - if len(tasks) != 0: - print_debug("ISPC + tasks speedup / ISPC + tasks time / serial time\n") - for i in range(0,len(serial)): - print_debug("%10s\t / %10s\t /%10s\n" % (tasks[i], absolute_tasks[i], serial[i])) - - test[1] = test[1] + ispc - test[2] = test[2] + tasks - test[3] = test[3] + absolute_ispc - test[4] = test[4] + absolute_tasks - if b_serial == True: - #if we concatenate outputs we should use only the first serial answer. - test[5] = test[5] + serial - -def cpu_get(): - p = open("/proc/stat", 'r') - cpu = p.readline() - p.close() - cpu = cpu.split(" ") - cpu_usage = (int(cpu[2]) + int(cpu[3]) + int(cpu[4])) - cpu_all = cpu_usage + int(cpu[5]) - return [cpu_usage, cpu_all] - -#returns cpu_usage -def cpu_check(): - if is_windows == False: - if is_mac == False: - cpu1 = cpu_get() - time.sleep(1) - cpu2 = cpu_get() - cpu_percent = (float(cpu1[0] - cpu2[0])/float(cpu1[1] - cpu2[1]))*100 - else: - os.system("sysctl -n vm.loadavg > cpu_temp") - c = open("cpu_temp", 'r') - c_line = c.readline() - c.close - os.remove("cpu_temp") - R = c_line.split(' ') - cpu_percent = float(R[1]) * 3 - else: - os.system("wmic cpu get loadpercentage /value > cpu_temp") - c = open("cpu_temp", 'r') - c_lines = c.readlines() - c.close() - os.remove("cpu_temp") - t = "0" - for i in c_lines[2]: - if i.isdigit(): - t = t + i - cpu_percent = int(t) - return cpu_percent - -#returns geomean of list -def geomean(par): - temp = 1 - l = len(par) - for i in range(l): - temp = temp * par[i] - temp = temp ** (1.0/l) - return round(temp, 2) - -#takes an answer struct and print it. -#answer struct: list answer contains lists test -#test[0] - name of test -#test[1] - list of results without tasks -#test[2] - list of results with tasks -#test[3] - list of absolute results without tasks -#test[4] - list of absolute results with tasks -#test[5] - list of absolute time without ISPC (serial) -#test[1..4] may be empty -def print_answer(answer): - filelist = [] - print_debug("--------------------------------------------------------------------------\n") - print_debug("test name:\t ISPC speedup: ISPC + tasks speedup: | " + - "ISPC time: ISPC + tasks time: serial:\n") - filelist.append("test name,ISPC speedup,diff," + - "ISPC + tasks speedup,diff,ISPC time,diff,ISPC + tasks time,diff,serial,diff\n") - max_t = [0,0,0,0,0] - diff_t = [0,0,0,0,0] - geomean_t = [0,0,0,0,0] - list_of_max = [[],[],[],[],[]] - for i in range(len(answer)): - for t in range(1,6): - if len(answer[i][t]) == 0: - max_t[t-1] = "n/a" - diff_t[t-1] = "n/a" - else: - if t < 3: - mm = max(answer[i][t]) - else: - mm = min(answer[i][t]) - max_t[t-1] = '%.2f' % mm - list_of_max[t-1].append(mm) - diff_t[t-1] = '%.2f' % (max(answer[i][t]) - min(answer[i][t])) - print_debug("%s:\n" % answer[i][0]) - print_debug("\t\tmax:\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" % - (max_t[0], max_t[1], max_t[2], max_t[3], max_t[4])) - print_debug("\t\tdiff:\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" % - (diff_t[0], diff_t[1], diff_t[2], diff_t[3], diff_t[4])) - for t in range(0,5): - if max_t[t] == "n/a": - max_t[t] = "" - if diff_t[t] == "n/a": - diff_t[t] = "" - filelist.append(answer[i][0] + "," + - max_t[0] + "," + diff_t[0] + "," + max_t[1] + "," + diff_t[1] + "," + - max_t[2] + "," + diff_t[2] + "," + max_t[3] + "," + diff_t[3] + "," + - max_t[4] + "," + diff_t[4] + "\n") - for i in range(0,5): - geomean_t[i] = geomean(list_of_max[i]) - print_debug("---------------------------------------------------------------------------------\n") - print_debug("Geomean:\t\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" % - (geomean_t[0], geomean_t[1], geomean_t[2], geomean_t[3], geomean_t[4])) - filelist.append("Geomean," + str(geomean_t[0]) + ",," + str(geomean_t[1]) - + ",," + str(geomean_t[2]) + ",," + str(geomean_t[3]) + ",," + str(geomean_t[4]) + "\n") - print_file(filelist) - - -###Main### -# parsing options -parser = OptionParser() -parser.add_option('-n', '--number', dest='number', - help='number of repeats', default="3") -parser.add_option('-c', '--config', dest='config', - help='config file of tests', default="./perf.ini") -parser.add_option('-p', '--path', dest='path', - help='path to examples directory', default="./") -parser.add_option('-s', '--silent', dest='silent', - help='silent mode, only table output', default=False, action="store_true") -parser.add_option('-o', '--output', dest='output', - help='output file for script reading', default="") -parser.add_option('--compiler', dest='compiler', - help='reference compiler', default="") -(options, args) = parser.parse_args() - -global is_windows -is_windows = (platform.system() == 'Windows' or - 'CYGWIN_NT' in platform.system()) -global is_mac -is_mac = (platform.system() == 'Darwin') - -# save corrent path -pwd = os.getcwd() -pwd = pwd + os.sep -if is_windows: - pwd = "..\\" - -# check if cpu usage is low now -cpu_percent = cpu_check() -if cpu_percent > 20: - sys.stdout.write("Warning: CPU Usage is very high.\n") - sys.stdout.write("Close other applications.\n") - -# check that required compilers exist -PATH_dir = string.split(os.getenv("PATH"), os.pathsep) -compiler_exists = False -ref_compiler_exists = False -if is_windows == False: - compiler = "ispc" - ref_compiler = "g++" - refc_compiler = "gcc" - if options.compiler != "": - if options.compiler == "clang" or options.compiler == "clang++": - ref_compiler = "clang++" - refc_compiler = "clang" - if options.compiler == "icc" or options.compiler == "icpc": - ref_compiler = "icpc" - refc_compiler = "icc" -else: - compiler = "ispc.exe" - ref_compiler = "cl.exe" -for counter in PATH_dir: - if os.path.exists(counter + os.sep + compiler): - compiler_exists = True - if os.path.exists(counter + os.sep + ref_compiler): - ref_compiler_exists = True -if not compiler_exists: - sys.stderr.write("Fatal error: ISPC compiler not found.\n") - sys.stderr.write("Added path to ispc compiler to your PATH variable.\n") - sys.exit() -if not ref_compiler_exists: - sys.stderr.write("Fatal error: reference compiler %s not found.\n" % ref_compiler) - sys.stderr.write("Added path to %s compiler to your PATH variable.\n" % ref_compiler) - sys.exit() - -# checks that config file exists -path_config = os.path.normpath(options.config) -if os.path.exists(path_config) == False: - sys.stderr.write("Fatal error: config file not found: %s.\n" % options.config) - sys.stderr.write("Set path to your config file in --config.\n") - sys.exit() - -# read lines from config file except comments -f = open(path_config, 'r') -f_lines = f.readlines() -f.close() -lines =[] -for i in range(len(f_lines)): - if f_lines[i][0] != "%": - lines.append(f_lines[i]) -length = len(lines) - -# prepare build.log and perf_temp files -global build_log -build_log = pwd + "build.log" -if is_windows == False: - if os.path.exists(build_log): - os.remove(build_log) -else: - if os.path.exists("build.log"): - os.remove("build.log") -global perf_temp -perf_temp = pwd + "perf_temp" - -i = 0 -answer = [] -print_debug("Okey go go go!\n\n") -os.system(compiler + " --version >" + build_log) -version = open(build_log) -print_debug("Using test compiler: " + version.readline()) -version.close() - -if is_windows == False: - os.system(ref_compiler + " --version >" + build_log) -else: - os.system(ref_compiler + " 2>" + build_log + " 1>&2") - -version = open(build_log) -print_debug("Using reference compiler: " + version.readline()) -version.close() - - -# loop for all tests -while i < length-2: - # we read name of test - print_debug("%s" % lines[i]) - test = [lines[i][:-1],[],[],[],[],[]] - # read location of test - folder = lines[i+1] - folder = folder[:-1] - folder = os.path.normpath(options.path + os.sep + folder) - # check that test exists - if os.path.exists(folder) == False: - sys.stdout.write("Fatal error: Can't find test %s. Your path is: \"%s\".\n" % (lines[i][:-1], options.path)) - sys.stdout.write("Change current location to /examples or set path to /examples in --path.\n") - exit(0) - os.chdir(folder) - # read parameters of test - command = lines[i+2] - command = command[:-1] - if is_windows == False: - command = "./"+command + " >> " + perf_temp - else: - command = "x64\\Release\\"+command + " >> " + perf_temp - # parsing config parameters - next_line = lines[i+3] - if next_line[0] == "!": # we should take only one part of test output - R = next_line.split(' ') - c1 = int(R[1]) #c1 is a number of string which we want to use in test output - c2 = int(R[2]) #c2 is total number of strings in test output - i = i+1 - else: - c1 = 1 - c2 = 1 - next_line = lines[i+3] - if next_line[0] == "^": #we should concatenate result of this test with previous one - run_test(command, c1, c2, answer[len(answer)-1], False) - i = i+1 - else: #we run this test and append it's result to answer structure - run_test(command, c1, c2, test, True) - answer.append(test) - # preparing next loop iteration - os.chdir(pwd) - i+=4 - -# delete temp file -if os.path.exists(perf_temp): - os.remove(perf_temp) -#print collected answer -print_answer(answer) diff --git a/fail_db.txt b/fail_db.txt new file mode 100644 index 00000000..7adc3e41 --- /dev/null +++ b/fail_db.txt @@ -0,0 +1 @@ +% List of known fails diff --git a/llvm_patches/r183327-AVX2-GATHER.patch b/llvm_patches/3_3_r183327-AVX2-GATHER.patch similarity index 100% rename from llvm_patches/r183327-AVX2-GATHER.patch rename to llvm_patches/3_3_r183327-AVX2-GATHER.patch diff --git a/llvm_patches/r184575-x86-shift.patch b/llvm_patches/3_3_r184575-x86-shift.patch similarity index 100% rename from llvm_patches/r184575-x86-shift.patch rename to llvm_patches/3_3_r184575-x86-shift.patch diff --git a/examples/perf.ini b/perf.ini similarity index 84% rename from examples/perf.ini rename to perf.ini index d2a5c73e..d8c7fe71 100755 --- a/examples/perf.ini +++ b/perf.ini @@ -10,44 +10,48 @@ %**************************************************************************************************** AOBench aobench -ao 10 512 512 +10 512 512 #*** Deferred Shading deferred -deferred_shading data/pp1280x720.bin +data/pp1280x720.bin #*** Mandelbrot Set mandelbrot -mandelbrot + #*** Mandelbrot Set mandelbrot_tasks -mandelbrot_tasks + ^ #*** Perlin Noise Function noise -noise + #*** Binomial Options options -options + ! 1 2 #*** Black-Scholes Options options -options + ! 2 2 #*** Ray Tracer rt -rt sponza +sponza #*** 3D Stencil stencil -stencil + #*** Volume Rendering volume_rendering -volume camera.dat density_highres.vol +camera.dat density_highres.vol #*** +%Sort +%sort +% +%#*** diff --git a/perf.py b/perf.py new file mode 100755 index 00000000..d1d7654b --- /dev/null +++ b/perf.py @@ -0,0 +1,489 @@ +#!/usr/bin/python +# +# Copyright (c) 2013, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# // Author: Filippov Ilia + +def print_file(line): + if options.output != "": + output = open(options.output, 'w') + output.writelines(line) + output.close() + +def build_test(commands): + os.system(commands[4]) + test = os.system(commands[1]) + if options.ref: + ref = os.system(commands[3]) + return (options.ref and ref) or test + +def execute_test(commands): + r = 0 + common.remove_if_exists(perf_temp+"_test") + common.remove_if_exists(perf_temp+"_ref") + for k in range(int(options.number)): + r = r + os.system(commands[0]) + if options.ref: + r = r + os.system(commands[2]) + return r + +#gathers all tests results and made an item test from answer structure +def run_test(commands, c1, c2, test, test_ref, b_serial): + if build_test(commands) != 0: + error("Compilation fails of test %s\n" % test[0], 0) + return + if execute_test(commands) != 0: + error("Execution fails of test %s\n" % test[0], 0) + return + print_debug("TEST COMPILER:\n", s, perf_log) + analyse_test(c1, c2, test, b_serial, perf_temp+"_test") + if options.ref: + print_debug("REFERENCE COMPILER:\n", s, perf_log) + analyse_test(c1, c2, test_ref, b_serial, perf_temp+"_ref") + + +def analyse_test(c1, c2, test, b_serial, perf_temp_n): + tasks = [] #list of results with tasks, it will be test[2] + ispc = [] #list of results without tasks, it will be test[1] + absolute_tasks = [] #list of absolute results with tasks, it will be test[4] + absolute_ispc = [] #list of absolute results without tasks, ut will be test[3] + serial = [] #list serial times, it will be test[5] + j = 1 + for line in open(perf_temp_n): # we take test output + if "speedup" in line: # we are interested only in lines with speedup + if j == c1: # we are interested only in lines with c1 numbers + line = line.expandtabs(0) + line = line.replace("("," ") + line = line.split(",") + for i in range(len(line)): + subline = line[i].split(" ") + number = float(subline[1][:-1]) + if "speedup from ISPC + tasks" in line[i]: + tasks.append(number) + else: + ispc.append(number) + c1 = c1 + c2 + j+=1 + if "million cycles" in line: + if j == c1: + line = line.replace("]","[") + line = line.split("[") + number = float(line[3]) + if "tasks" in line[1]: + absolute_tasks.append(number) + else: + if "ispc" in line[1]: + absolute_ispc.append(number) + if "serial" in line[1]: + serial.append(number) + + if len(ispc) != 0: + if len(tasks) != 0: + print_debug("ISPC speedup / ISPC + tasks speedup / ISPC time / ISPC + tasks time / serial time\n", s, perf_log) + for i in range(0,len(serial)): + print_debug("%10s /\t%10s\t /%9s / %10s\t /%10s\n" % + (ispc[i], tasks[i], absolute_ispc[i], absolute_tasks[i], serial[i]), s, perf_log) + else: + print_debug("ISPC speedup / ISPC time / serial time\n", s, perf_log) + for i in range(0,len(serial)): + print_debug("%10s /%9s /%10s\n" % (ispc[i], absolute_ispc[i], serial[i]), s, perf_log) + else: + if len(tasks) != 0: + print_debug("ISPC + tasks speedup / ISPC + tasks time / serial time\n", s, perf_log) + for i in range(0,len(serial)): + print_debug("%10s\t / %10s\t /%10s\n" % (tasks[i], absolute_tasks[i], serial[i]), s, perf_log) + + test[1] = test[1] + ispc + test[2] = test[2] + tasks + test[3] = test[3] + absolute_ispc + test[4] = test[4] + absolute_tasks + if b_serial == True: + #if we concatenate outputs we should use only the first serial answer. + test[5] = test[5] + serial + +def cpu_get(): + p = open("/proc/stat", 'r') + cpu = p.readline() + p.close() + cpu = cpu.split(" ") + cpu_usage = (int(cpu[2]) + int(cpu[3]) + int(cpu[4])) + cpu_all = cpu_usage + int(cpu[5]) + return [cpu_usage, cpu_all] + +#returns cpu_usage +def cpu_check(): + if is_windows == False: + if is_mac == False: + cpu1 = cpu_get() + time.sleep(1) + cpu2 = cpu_get() + cpu_percent = (float(cpu1[0] - cpu2[0])/float(cpu1[1] - cpu2[1]))*100 + else: + os.system("sysctl -n vm.loadavg > cpu_temp") + c = open("cpu_temp", 'r') + c_line = c.readline() + c.close + os.remove("cpu_temp") + R = c_line.split(' ') + cpu_percent = float(R[1]) * 3 + else: + os.system("wmic cpu get loadpercentage /value > cpu_temp") + c = open("cpu_temp", 'r') + c_lines = c.readlines() + c.close() + os.remove("cpu_temp") + t = "0" + for i in c_lines[2]: + if i.isdigit(): + t = t + i + cpu_percent = int(t) + return cpu_percent + +#returns geomean of list +def geomean(par): + temp = 1 + l = len(par) + for i in range(l): + temp = temp * par[i] + temp = temp ** (1.0/l) + return round(temp, 2) + +#takes an answer struct and print it. +#answer struct: list answer contains lists test +#test[0] - name of test +#test[1] - list of results without tasks +#test[2] - list of results with tasks +#test[3] - list of absolute results without tasks +#test[4] - list of absolute results with tasks +#test[5] - list of absolute time without ISPC (serial) +#test[1..4] may be empty +def print_answer(answer): + filelist = [] + print_debug("--------------------------------------------------------------------------\n", s, perf_log) + print_debug("test name:\t ISPC speedup: ISPC + tasks speedup: | " + + "ISPC time: ISPC + tasks time: serial:\n", s, perf_log) + filelist.append("test name,ISPC speedup,diff," + + "ISPC + tasks speedup,diff,ISPC time,diff,ISPC + tasks time,diff,serial,diff\n") + max_t = [0,0,0,0,0] + diff_t = [0,0,0,0,0] + geomean_t = [0,0,0,0,0] + list_of_max = [[],[],[],[],[]] + list_of_compare = [[],[],[],[],[],[]] + for i in range(len(answer)): + list_of_compare[0].append(answer[i][0]) + for t in range(1,6): + if len(answer[i][t]) == 0: + max_t[t-1] = "n/a" + diff_t[t-1] = "n/a" + list_of_compare[t].append(0); + else: + if t < 3: + mm = max(answer[i][t]) + else: + mm = min(answer[i][t]) + list_of_compare[t].append(mm) + max_t[t-1] = '%.2f' % mm + list_of_max[t-1].append(mm) + diff_t[t-1] = '%.2f' % (max(answer[i][t]) - min(answer[i][t])) + print_debug("%s:\n" % answer[i][0], s, perf_log) + print_debug("\t\tmax:\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" % + (max_t[0], max_t[1], max_t[2], max_t[3], max_t[4]), s, perf_log) + print_debug("\t\tdiff:\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" % + (diff_t[0], diff_t[1], diff_t[2], diff_t[3], diff_t[4]), s, perf_log) + for t in range(0,5): + if max_t[t] == "n/a": + max_t[t] = "" + if diff_t[t] == "n/a": + diff_t[t] = "" + filelist.append(answer[i][0] + "," + + max_t[0] + "," + diff_t[0] + "," + max_t[1] + "," + diff_t[1] + "," + + max_t[2] + "," + diff_t[2] + "," + max_t[3] + "," + diff_t[3] + "," + + max_t[4] + "," + diff_t[4] + "\n") + for i in range(0,5): + geomean_t[i] = geomean(list_of_max[i]) + print_debug("---------------------------------------------------------------------------------\n", s, perf_log) + print_debug("Geomean:\t\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" % + (geomean_t[0], geomean_t[1], geomean_t[2], geomean_t[3], geomean_t[4]), s, perf_log) + filelist.append("Geomean," + str(geomean_t[0]) + ",," + str(geomean_t[1]) + + ",," + str(geomean_t[2]) + ",," + str(geomean_t[3]) + ",," + str(geomean_t[4]) + "\n") + print_file(filelist) + return list_of_compare + + +def compare(A, B): + print_debug("\n\n_____________________PERFORMANCE REPORT____________________________\n", False, "") + print_debug("test name: ISPC time: ISPC time ref: %:\n", False, "") + for i in range(0,len(A[0])): + if B[3][i] == 0: + p1 = 0 + else: + p1 = 100 - 100 * A[3][i]/B[3][i] + print_debug("%21s: %10.2f %10.2f %10.2f" % (A[0][i], A[3][i], B[3][i], p1), False, "") + if p1 < -1: + print_debug(" <-", False, "") + if p1 > 1: + print_debug(" <+", False, "") + print_debug("\n", False, "") + print_debug("\n", False, "") + + print_debug("test name: TASKS time: TASKS time ref: %:\n", False, "") + for i in range(0,len(A[0])): + if B[4][i] == 0: + p2 = 0 + else: + p2 = 100 - 100 * A[4][i]/B[4][i] + print_debug("%21s: %10.2f %10.2f %10.2f" % (A[0][i], A[4][i], B[4][i], p2), False, "") + if p2 < -1: + print_debug(" <-", False, "") + if p2 > 1: + print_debug(" <+", False, "") + print_debug("\n", False, "") + if "performance.log" in options.in_file: + print_debug("\n\n_________________Watch performance.log for details________________\n", False, "") + else: + print_debug("\n\n__________________________________________________________________\n", False, "") + + + +def perf(options1, args): + global options + options = options1 + global s + s = options.silent + + # save current OS + global is_windows + is_windows = (platform.system() == 'Windows' or + 'CYGWIN_NT' in platform.system()) + global is_mac + is_mac = (platform.system() == 'Darwin') + + # save current path + pwd = os.getcwd() + pwd = pwd + os.sep + pwd1 = pwd + if is_windows: + pwd1 = "..\\..\\" + + # check if cpu usage is low now + cpu_percent = cpu_check() + if cpu_percent > 20: + error("CPU Usage is very high.\nClose other applications.\n", 2) + + global ispc_test + global ispc_ref + global ref_compiler + global refc_compiler + # check that required compilers exist + PATH_dir = string.split(os.getenv("PATH"), os.pathsep) + ispc_test_exists = False + ispc_ref_exists = False + ref_compiler_exists = False + if is_windows == False: + ispc_test = "ispc" + ref_compiler = "g++" + refc_compiler = "gcc" + if options.compiler != "": + if options.compiler == "clang" or options.compiler == "clang++": + ref_compiler = "clang++" + refc_compiler = "clang" + if options.compiler == "icc" or options.compiler == "icpc": + ref_compiler = "icpc" + refc_compiler = "icc" + else: + ispc_test = "ispc.exe" + ref_compiler = "cl.exe" + ispc_ref = options.ref + if options.ref != "": + options.ref = True + for counter in PATH_dir: + if os.path.exists(counter + os.sep + ispc_test): + ispc_test_exists = True + if os.path.exists(counter + os.sep + ref_compiler): + ref_compiler_exists = True + if os.path.exists(counter + os.sep + ispc_ref): + ispc_ref_exists = True + if not ispc_test_exists: + error("ISPC compiler not found.\nAdded path to ispc compiler to your PATH variable.\n", 1) + if not ref_compiler_exists: + error("C/C++ compiler %s not found.\nAdded path to %s compiler to your PATH variable.\n" % (ref_compiler, ref_compiler), 1) + if options.ref: + if not ispc_ref_exists: + error("ISPC reference compiler not found.\nAdded path to ispc reference compiler to your PATH variable.\n", 1) + + # checks that config file exists + path_config = os.path.normpath(options.config) + if os.path.exists(path_config) == False: + error("config file not found: %s.\nSet path to your config file in --config.\n" % options.config, 1) + sys.exit() + + # read lines from config file except comments + f = open(path_config, 'r') + f_lines = f.readlines() + f.close() + lines =[] + for i in range(len(f_lines)): + if f_lines[i][0] != "%": + lines.append(f_lines[i]) + length = len(lines) + + # prepare build.log, perf_temp and perf.log files + global perf_log + if options.in_file: + perf_log = pwd + options.in_file + common.remove_if_exists(perf_log) + else: + perf_log = "" + global build_log + build_log = pwd + os.sep + "logs" + os.sep + "perf_build.log" + common.remove_if_exists(build_log) + if os.path.exists(pwd + os.sep + "logs") == False: + os.makedirs(pwd + os.sep + "logs") + + global perf_temp + perf_temp = pwd + "perf_temp" + # end of preparations + + print_debug("Okey go go go!\n\n", s, perf_log) + + #print compilers versions + common.print_version(ispc_test, ispc_ref, ref_compiler, False, perf_log, is_windows) + + # begin + i = 0 + answer = [] + answer_ref = [] + + # loop for all tests + while i < length-2: + # we read name of test + print_debug("%s" % lines[i], s, perf_log) + test = [lines[i][:-1],[],[],[],[],[]] + test_ref = [lines[i][:-1],[],[],[],[],[]] + # read location of test + folder = lines[i+1] + folder = folder[:-1] + folder = os.path.normpath(options.path + os.sep + "examples" + os.sep + folder) + # check that test exists + if os.path.exists(folder) == False: + error("Can't find test %s. Your path is: \"%s\".\nChange current location to ISPC_HOME or set path to ISPC_HOME in --path.\n" % + (lines[i][:-1], options.path), 1) + os.chdir(folder) + # read parameters of test + command = lines[i+2] + command = command[:-1] + if is_windows == False: + ex_command_ref = "./ref " + command + " >> " + perf_temp + "_ref" + ex_command = "./test " + command + " >> " + perf_temp + "_test" + bu_command_ref = "make CXX="+ref_compiler+" CC="+refc_compiler+ " EXAMPLE=ref ISPC="+ispc_ref+" >> "+build_log+" 2>> "+build_log + bu_command = "make CXX="+ref_compiler+" CC="+refc_compiler+ " EXAMPLE=test ISPC="+ispc_test+" >> "+build_log+" 2>> "+build_log + re_command = "make clean >> "+build_log + else: + ex_command_ref = "x64\\Release\\ref.exe " + command + " >> " + perf_temp + "_ref" + ex_command = "x64\\Release\\test.exe " + command + " >> " + perf_temp + "_test" + bu_command_ref = "msbuild /V:m /p:Platform=x64 /p:Configuration=Release /p:TargetDir=.\ /p:TargetName=ref /t:rebuild >> " + build_log + bu_command = "msbuild /V:m /p:Platform=x64 /p:Configuration=Release /p:TargetDir=.\ /p:TargetName=test /t:rebuild >> " + build_log + re_command = "msbuild /t:clean >> " + build_log + commands = [ex_command, bu_command, ex_command_ref, bu_command_ref, re_command] + # parsing config parameters + next_line = lines[i+3] + if next_line[0] == "!": # we should take only one part of test output + R = next_line.split(' ') + c1 = int(R[1]) #c1 is a number of string which we want to use in test output + c2 = int(R[2]) #c2 is total number of strings in test output + i = i+1 + else: + c1 = 1 + c2 = 1 + next_line = lines[i+3] + if next_line[0] == "^": #we should concatenate result of this test with previous one + run_test(commands, c1, c2, answer[len(answer)-1], answer_ref[len(answer)-1], False) + i = i+1 + else: #we run this test and append it's result to answer structure + run_test(commands, c1, c2, test, test_ref, True) + answer.append(test) + answer_ref.append(test_ref) + + # preparing next loop iteration + os.chdir(pwd1) + i+=4 + + # delete temp file + common.remove_if_exists(perf_temp+"_test") + common.remove_if_exists(perf_temp+"_ref") + + #print collected answer + print_debug("\n\nTEST COMPILER:\n", s, perf_log) + A = print_answer(answer) + if options.ref != "": + print_debug("\n\nREFERENCE COMPILER:\n", s, perf_log) + B = print_answer(answer_ref) + # print perf report + compare(A,B) + + + +###Main### +from optparse import OptionParser +import sys +import os +import operator +import time +import glob +import string +import platform +# our functions +import common +print_debug = common.print_debug +error = common.error + +if __name__ == "__main__": + # parsing options + parser = OptionParser() + parser.add_option('-n', '--number', dest='number', + help='number of repeats', default="3") + parser.add_option('-c', '--config', dest='config', + help='config file of tests', default="./perf.ini") + parser.add_option('-p', '--path', dest='path', + help='path to test_system directory', default=".") + parser.add_option('-s', '--silent', dest='silent', + help='silent mode, only table output', default=False, action="store_true") + parser.add_option('-o', '--output', dest='output', + help='output file for script reading', default="") + parser.add_option('--compiler', dest='compiler', + help='C/C++ compiler', default="") + parser.add_option('-r', '--ref', dest='ref', + help='set reference compiler for compare', default="") + parser.add_option('-f', '--file', dest='in_file', + help='file to save perf output', default="") + (options, args) = parser.parse_args() + perf(options, args) diff --git a/run_tests.py b/run_tests.py index 9729930f..2471b6cb 100755 --- a/run_tests.py +++ b/run_tests.py @@ -1,165 +1,37 @@ #!/usr/bin/python +# +# Copyright (c) 2013, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # test-running driver for ispc - -from optparse import OptionParser -import multiprocessing -from ctypes import c_int -import os -import sys -import glob -import re -import signal -import random -import string -import subprocess -import shlex -import platform -import tempfile -import os.path -import time - -# disable fancy error/warning printing with ANSI colors, so grepping for error -# messages doesn't get confused -os.environ["TERM"] = "dumb" - -# This script is affected by http://bugs.python.org/issue5261 on OSX 10.5 Leopard -# git history has a workaround for that issue. - -is_windows = (platform.system() == 'Windows' or - 'CYGWIN_NT' in platform.system()) - -parser = OptionParser() -parser.add_option("-r", "--random-shuffle", dest="random", help="Randomly order tests", - default=False, action="store_true") -parser.add_option("-g", "--generics-include", dest="include_file", help="Filename for header implementing functions for generics", - default=None) -parser.add_option("-f", "--ispc-flags", dest="ispc_flags", help="Additional flags for ispc (-g, -O1, ...)", - default="") -parser.add_option('-t', '--target', dest='target', - help='Set compilation target (sse2-i32x4, sse2-i32x8, sse4-i32x4, sse4-i32x8, sse4-i16x8, sse4-i8x16, avx1-i32x8, avx1-i32x16, avx1.1-i32x8, avx1.1-i32x16, avx2-i32x8, avx2-i32x16, generic-x1, generic-x4, generic-x8, generic-x16, generic-x32, generic-x64)', - default="sse4") -parser.add_option('-a', '--arch', dest='arch', - help='Set architecture (arm, x86, x86-64)', - default="x86-64") -parser.add_option("-c", "--compiler", dest="compiler_exe", help="Compiler binary to use to run tests", - default=None) -parser.add_option('-o', '--no-opt', dest='no_opt', help='Disable optimization', - default=False, action="store_true") -parser.add_option('-j', '--jobs', dest='num_jobs', help='Maximum number of jobs to run in parallel', - default="1024", type="int") -parser.add_option('-v', '--verbose', dest='verbose', help='Enable verbose output', - default=False, action="store_true") -parser.add_option('--wrap-exe', dest='wrapexe', - help='Executable to wrap test runs with (e.g. "valgrind")', - default="") -parser.add_option('--time', dest='time', help='Enable time output', - default=False, action="store_true") -parser.add_option('--non-interactive', dest='non_interactive', help='Disable interactive status updates', - default=False, action="store_true") - -(options, args) = parser.parse_args() - -if options.target == 'neon': - options.arch = 'arm' - -# use relative path to not depend on host directory, which may possibly -# have white spaces and unicode characters. -if not is_windows: - ispc_exe = "./ispc" -else: - ispc_exe = ".\\Release\\ispc.exe" - -# checks the required ispc compiler otherwise prints an error message -if not os.path.exists(ispc_exe): - sys.stderr.write("Fatal error: missing ispc compiler: %s\n" % ispc_exe) - sys.exit() - -ispc_exe += " " + options.ispc_flags - -if __name__ == '__main__': - sys.stdout.write("ispc compiler: %s\n" % ispc_exe) - -is_generic_target = (options.target.find("generic-") != -1 and - options.target != "generic-1") -if is_generic_target and options.include_file == None: - if options.target == "generic-4": - sys.stderr.write("No generics #include specified; using examples/intrinsics/sse4.h\n") - options.include_file = "examples/intrinsics/sse4.h" - elif options.target == "generic-8": - sys.stderr.write("No generics #include specified and no default available for \"generic-8\" target.\n") - sys.exit(1) - elif options.target == "generic-16": - sys.stderr.write("No generics #include specified; using examples/intrinsics/generic-16.h\n") - options.include_file = "examples/intrinsics/generic-16.h" - elif options.target == "generic-32": - sys.stderr.write("No generics #include specified; using examples/intrinsics/generic-32.h\n") - options.include_file = "examples/intrinsics/generic-32.h" - elif options.target == "generic-64": - sys.stderr.write("No generics #include specified; using examples/intrinsics/generic-64.h\n") - options.include_file = "examples/intrinsics/generic-64.h" - -if options.compiler_exe == None: - if is_windows: - options.compiler_exe = "cl.exe" - else: - options.compiler_exe = "g++" - -# checks the required compiler otherwise prints an error message -PATH_dir = string.split(os.getenv("PATH"), os.pathsep) -compiler_exists = False - -for counter in PATH_dir: - if os.path.exists(counter + os.sep + options.compiler_exe): - compiler_exists = True - break - -if not compiler_exists: - sys.stderr.write("Fatal error: missing the required compiler: %s \n" % - options.compiler_exe) - sys.exit() - -ispc_root = "." - -# if no specific test files are specified, run all of the tests in tests/, -# failing_tests/, and tests_errors/ -if len(args) == 0: - files = glob.glob(ispc_root + os.sep + "tests" + os.sep + "*ispc") + \ - glob.glob(ispc_root + os.sep + "failing_tests" + os.sep + "*ispc") + \ - glob.glob(ispc_root + os.sep + "tests_errors" + os.sep + "*ispc") -else: - if is_windows: - argfiles = [ ] - for f in args: - # we have to glob ourselves if this is being run under a DOS - # shell, as it passes wildcard as is. - argfiles += glob.glob(f) - else: - argfiles = args - - files = [ ] - for f in argfiles: - if os.path.splitext(string.lower(f))[1] != ".ispc": - sys.stdout.write("Ignoring file %s, which doesn't have an .ispc extension.\n" % f) - else: - files += [ f ] - -# max_test_length is used to issue exact number of whitespace characters when -# updating status. Otherwise update causes new lines standard 80 char terminal -# on both Linux and Windows. -max_test_length = 0 -for f in files: - max_test_length = max(max_test_length, len(f)) - -# randomly shuffle the tests if asked to do so -if (options.random): - random.seed() - random.shuffle(files) - -# counter -total_tests = 0 - - # utility routine to print an update on the number of tests that have been # finished. Should be called with the lock held.. def update_progress(fn, total_tests_arg, counter, max_test_length_arg): @@ -176,7 +48,7 @@ def update_progress(fn, total_tests_arg, counter, max_test_length_arg): def run_command(cmd): if options.verbose: - sys.stdout.write("Running: %s\n" % cmd) + print_debug("Running: %s\n" % cmd, s, run_tests_log) # Here's a bit tricky part. To pass a command for execution we should # break down the line in to arguments. shlex class is designed exactly @@ -204,9 +76,9 @@ def run_cmds(compile_cmds, run_cmd, filename, expect_failure): (return_code, output) = run_command(cmd) compile_failed = (return_code != 0) if compile_failed: - sys.stdout.write("Compilation of test %s failed \n" % filename) + print_debug("Compilation of test %s failed \n" % filename, s, run_tests_log) if output != "": - sys.stdout.write("%s" % output.encode("utf-8")) + print_debug("%s" % output.encode("utf-8"), s, run_tests_log) return (1, 0) (return_code, output) = run_command(run_cmd) @@ -215,11 +87,11 @@ def run_cmds(compile_cmds, run_cmd, filename, expect_failure): surprise = ((expect_failure and not run_failed) or (not expect_failure and run_failed)) if surprise == True: - sys.stderr.write("Test %s %s (return code %d) \n" % \ + print_debug("Test %s %s (return code %d) \n" % \ (filename, "unexpectedly passed" if expect_failure else "failed", - return_code)) + return_code), s, run_tests_log) if output != "": - sys.stdout.write("%s\n" % output.encode("utf-8")) + print_debug("%s\n" % output.encode("utf-8"), s, run_tests_log) if surprise == True: return (0, 1) else: @@ -298,11 +170,11 @@ def run_test(testname): file.close() if re.search(firstline, output) == None: - sys.stderr.write("Didn't see expected error message %s from test %s.\nActual output:\n%s\n" % \ - (firstline, testname, output)) + print_debug("Didn't see expected error message %s from test %s.\nActual output:\n%s\n" % \ + (firstline, testname, output), s, run_tests_log) return (1, 0) elif got_error == False: - sys.stderr.write("Unexpectedly no errors issued from test %s\n" % testname) + print_debug("Unexpectedly no errors issued from test %s\n" % testname, s, run_tests_log) return (1, 0) else: return (0, 0) @@ -328,8 +200,7 @@ def run_test(testname): break file.close() if match == -1: - sys.stderr.write("Fatal error: unable to find function signature " + \ - "in test %s\n" % testname) + error("unable to find function signature in test %s\n" % testname, 0) return (1, 0) else: global is_generic_target @@ -404,7 +275,21 @@ def run_test(testname): # pull tests to run from the given queue and run them. Multiple copies of # this function will be running in parallel across all of the CPU cores of # the system. -def run_tasks_from_queue(queue, queue_ret, queue_skip, total_tests_arg, max_test_length_arg, counter, mutex): +def run_tasks_from_queue(queue, queue_ret, queue_skip, total_tests_arg, max_test_length_arg, counter, mutex, glob_var): + # This is needed on windows because windows doen't copy globals from parent process whili multiprocessing + global is_windows + is_windows = glob_var[0] + global options + options = glob_var[1] + global s + s = glob_var[2] + global ispc_exe + ispc_exe = glob_var[3] + global is_generic_target + is_generic_target = glob_var[4] + global run_tests_log + run_tests_log = glob_var[5] + if is_windows: tmpdir = "tmp%d" % os.getpid() os.mkdir(tmpdir) @@ -454,7 +339,256 @@ def sigint(signum, frame): t.terminate() sys.exit(1) -if __name__ == '__main__': + +def file_check(compfails, runfails): + errors = len(compfails) + len(runfails) + new_compfails = [] + new_runfails = [] + new_passes_compfails = [] + new_passes_runfails = [] +# Open file fail_db.txt + f = open(test_states, 'r') + f_lines = f.readlines() + f.close() +# Detect OS + if platform.system() == 'Windows' or 'CYGWIN_NT' in platform.system(): + OS = "Windows" + else: + if platform.system() == 'Darwin': + OS = "Mac" + else: + OS = "Linux" +# Detect opt_set + if options.no_opt == True: + opt = "-O0" + else: + opt = "-O2" +# Detect LLVM version + temp1 = common.detect_version(ispc_exe + " --version") + llvm_version = temp1[-10:-2] +#Detect compiler version + if is_windows == False: + temp1 = common.detect_version(options.compiler_exe + " --version") + temp2 = temp1.split(" ") + compiler_version = temp2[0] + temp2[2][0:4] + else: + compiler_version = "cl" + new_line = " "+options.arch.rjust(6)+" "+options.target.rjust(14)+" "+OS.rjust(7)+" "+llvm_version+" "+compiler_version.rjust(10)+" "+opt+" *\n" + + new_compfails = compfails[:] + new_runfails = runfails[:] + new_f_lines = f_lines[:] + for j in range(0, len(f_lines)): + if (((" "+options.arch+" ") in f_lines[j]) and + ((" "+options.target+" ") in f_lines[j]) and + ((" "+OS+" ") in f_lines[j]) and + ((" "+llvm_version+" ") in f_lines[j]) and + ((" "+compiler_version+" ") in f_lines[j]) and + ((" "+opt+" ") in f_lines[j])): + if (" compfail " in f_lines[j]): + f = 0 + for i in range(0, len(compfails)): + if compfails[i] in f_lines[j]: + new_compfails.remove(compfails[i]) + else: + f = f + 1 + if f == len(compfails): + temp3 = f_lines[j].split(" ") + new_passes_compfails.append(temp3[0]) + if options.update == "FP": + new_f_lines.remove(f_lines[j]) + if (" runfail " in f_lines[j]): + f = 0 + for i in range(0, len(runfails)): + if runfails[i] in f_lines[j]: + new_runfails.remove(runfails[i]) + else: + f = f + 1 + if f == len(runfails): + temp3 = f_lines[j].split(" ") + new_passes_runfails.append(temp3[0]) + if options.update == "FP": + new_f_lines.remove(f_lines[j]) + if len(new_runfails) != 0: + print_debug("NEW RUNFAILS:\n", s, run_tests_log) + for i in range (0,len(new_runfails)): + new_f_lines.append(new_runfails[i] + " runfail " + new_line) + print_debug("\t" + new_runfails[i] + "\n", s, run_tests_log) + if len(new_compfails) != 0: + print_debug("NEW COMPFAILS:\n", s, run_tests_log) + for i in range (0,len(new_compfails)): + new_f_lines.append(new_compfails[i] + " compfail " + new_line) + print_debug("\t" + new_compfails[i] + "\n", s, run_tests_log) + if len(new_passes_runfails) != 0: + print_debug("NEW PASSES after RUNFAILS:\n", s, run_tests_log) + for i in range (0,len(new_passes_runfails)): + print_debug("\t" + new_passes_runfails[i] + "\n", s, run_tests_log) + if len(new_passes_compfails) != 0: + print_debug("NEW PASSES after COMPFAILS:\n", s, run_tests_log) + for i in range (0,len(new_passes_compfails)): + print_debug("\t" + new_passes_compfails[i] + "\n", s, run_tests_log) + + if options.update != "": + output = open(test_states, 'w') + output.writelines(new_f_lines) + output.close() + return [new_runfails, new_compfails, new_passes_runfails, new_passes_compfails, new_line, errors] + +def verify(): + # Open file fail_db.txt + f = open(test_states, 'r') + f_lines = f.readlines() + f.close() + check = [["g++", "clang", "cl"],["-O0", "-O2"],["x86","x86-64"], + ["Linux","Windows","Mac"],["LLVM 3.1","LLVM 3.2","LLVM 3.3","LLVM head"], + ["sse2-i32x4", "sse2-i32x8", "sse4-i32x4", "sse4-i32x8", "sse4-i16x8", + "sse4-i8x16", "avx1-i32x8", "avx1-i32x16", "avx1.1-i32x8", "avx1.1-i32x16", + "avx2-i32x8", "avx2-i32x16", "generic-1", "generic-4", "generic-8", + "generic-16", "generic-32", "generic-64"]] + for i in range (0,len(f_lines)): + if f_lines[i][0] == "%": + continue + for j in range(0,len(check)): + temp = 0 + for t in range(0,len(check[j])): + if " " + check[j][t] + " " in f_lines[i]: + temp = temp + 1 + if temp != 1: + print_debug("error in line " + str(i) + "\n", False, run_tests_log) + break + + +def run_tests(options1, args, print_version): + global options + options = options1 + global s + s = options.silent + + # prepare run_tests_log and test_states files + global run_tests_log + if options.in_file: + run_tests_log = os.getcwd() + os.sep + options.in_file + if print_version == 1: + common.remove_if_exists(run_tests_log) + else: + run_tests_log = "" + global test_states + test_states = "fail_db.txt" + if options.verify: + verify() + return 0 + + # disable fancy error/warning printing with ANSI colors, so grepping for error + # messages doesn't get confused + os.environ["TERM"] = "dumb" + + # This script is affected by http://bugs.python.org/issue5261 on OSX 10.5 Leopard + # git history has a workaround for that issue. + global is_windows + is_windows = (platform.system() == 'Windows' or + 'CYGWIN_NT' in platform.system()) + + if options.target == 'neon': + options.arch = 'arm' + + # use relative path to not depend on host directory, which may possibly + # have white spaces and unicode characters. + global ispc_exe + if not is_windows: + ispc_exe = "./ispc" + else: + ispc_exe = ".\\Release\\ispc.exe" + + # checks the required ispc compiler otherwise prints an error message + if not os.path.exists(ispc_exe): + error("missing ispc compiler: %s\n" % ispc_exe, 1) + ispc_exe += " " + options.ispc_flags + print_debug("ispc compiler: %s\n" % ispc_exe, s, run_tests_log) + + global is_generic_target + is_generic_target = (options.target.find("generic-") != -1 and + options.target != "generic-1" and options.target != "generic-x1") + if is_generic_target and options.include_file == None: + if options.target == "generic-4" or options.target == "generic-x4": + error("No generics #include specified; using examples/intrinsics/sse4.h\n", 2) + options.include_file = "examples/intrinsics/sse4.h" + options.target = "generic-4" + elif options.target == "generic-8" or options.target == "generic-x8": + error("No generics #include specified and no default available for \"generic-8\" target.\n", 1) + options.target = "generic-8" + elif options.target == "generic-16" or options.target == "generic-x16": + error("No generics #include specified; using examples/intrinsics/generic-16.h\n", 2) + options.include_file = "examples/intrinsics/generic-16.h" + options.target = "generic-16" + elif options.target == "generic-32" or options.target == "generic-x32": + error("No generics #include specified; using examples/intrinsics/generic-32.h\n", 2) + options.include_file = "examples/intrinsics/generic-32.h" + options.target = "generic-32" + elif options.target == "generic-64" or options.target == "generic-x64": + error("No generics #include specified; using examples/intrinsics/generic-64.h\n", 2) + options.include_file = "examples/intrinsics/generic-64.h" + options.target = "generic-64" + + if options.compiler_exe == None: + if is_windows: + options.compiler_exe = "cl.exe" + else: + options.compiler_exe = "g++" + + # checks the required compiler otherwise prints an error message + PATH_dir = string.split(os.getenv("PATH"), os.pathsep) + compiler_exists = False + + for counter in PATH_dir: + if os.path.exists(counter + os.sep + options.compiler_exe): + compiler_exists = True + break + + if not compiler_exists: + error("missing the required compiler: %s \n" % options.compiler_exe, 1) + + # print compilers versions + if print_version > 0: + common.print_version(ispc_exe, "", options.compiler_exe, False, run_tests_log, is_windows) + + ispc_root = "." + + # if no specific test files are specified, run all of the tests in tests/, + # failing_tests/, and tests_errors/ + if len(args) == 0: + files = glob.glob(ispc_root + os.sep + "tests" + os.sep + "*ispc") + \ + glob.glob(ispc_root + os.sep + "failing_tests" + os.sep + "*ispc") + \ + glob.glob(ispc_root + os.sep + "tests_errors" + os.sep + "*ispc") + else: + if is_windows: + argfiles = [ ] + for f in args: + # we have to glob ourselves if this is being run under a DOS + # shell, as it passes wildcard as is. + argfiles += glob.glob(f) + else: + argfiles = args + + files = [ ] + for f in argfiles: + if os.path.splitext(string.lower(f))[1] != ".ispc": + error("Ignoring file %s, which doesn't have an .ispc extension.\n" % f, 2) + else: + files += [ f ] + + # max_test_length is used to issue exact number of whitespace characters when + # updating status. Otherwise update causes new lines standard 80 char terminal + # on both Linux and Windows. + max_test_length = 0 + for f in files: + max_test_length = max(max_test_length, len(f)) + + # randomly shuffle the tests if asked to do so + if (options.random): + random.seed() + random.shuffle(files) + + # counter total_tests = len(files) compile_error_files = [ ] @@ -463,7 +597,7 @@ if __name__ == '__main__': nthreads = min(multiprocessing.cpu_count(), options.num_jobs) nthreads = min(nthreads, len(files)) - sys.stdout.write("Running %d jobs in parallel. Running %d tests.\n" % (nthreads, total_tests)) + print_debug("Running %d jobs in parallel. Running %d tests.\n" % (nthreads, total_tests), s, run_tests_log) # put each of the test filenames into a queue q = multiprocessing.Queue() @@ -483,8 +617,10 @@ if __name__ == '__main__': start_time = time.time() # launch jobs to run tests + glob_var = [is_windows, options, s, ispc_exe, is_generic_target, run_tests_log] for x in range(nthreads): - t = multiprocessing.Process(target=run_tasks_from_queue, args=(q, qret, qskip, total_tests, max_test_length, finished_tests_counter, finished_tests_counter_lock)) + t = multiprocessing.Process(target=run_tasks_from_queue, args=(q, qret, qskip, total_tests, + max_test_length, finished_tests_counter, finished_tests_counter_lock, glob_var)) task_threads.append(t) t.start() @@ -493,35 +629,97 @@ if __name__ == '__main__': for t in task_threads: t.join() if options.non_interactive == False: - sys.stdout.write("\n") + print_debug("\n", s, run_tests_log) elapsed_time = time.time() - start_time while not qret.empty(): - (c, r, s) = qret.get() + (c, r, skip) = qret.get() compile_error_files += c run_error_files += r - skip_files += s + skip_files += skip if options.non_interactive: - sys.stdout.write(" Done %d / %d\n" % (finished_tests_counter.value, total_tests)) + print_debug(" Done %d / %d\n" % (finished_tests_counter.value, total_tests), s, run_tests_log) if len(skip_files) > 0: skip_files.sort() - sys.stdout.write("%d / %d tests SKIPPED:\n" % (len(skip_files), total_tests)) + print_debug("%d / %d tests SKIPPED:\n" % (len(skip_files), total_tests), s, run_tests_log) for f in skip_files: - sys.stdout.write("\t%s\n" % f) + print_debug("\t%s\n" % f, s, run_tests_log) if len(compile_error_files) > 0: compile_error_files.sort() - sys.stdout.write("%d / %d tests FAILED compilation:\n" % (len(compile_error_files), total_tests)) + print_debug("%d / %d tests FAILED compilation:\n" % (len(compile_error_files), total_tests), s, run_tests_log) for f in compile_error_files: - sys.stdout.write("\t%s\n" % f) + print_debug("\t%s\n" % f, s, run_tests_log) if len(run_error_files) > 0: run_error_files.sort() - sys.stdout.write("%d / %d tests FAILED execution:\n" % (len(run_error_files), total_tests)) + print_debug("%d / %d tests FAILED execution:\n" % (len(run_error_files), total_tests), s, run_tests_log) for f in run_error_files: - sys.stdout.write("\t%s\n" % f) + print_debug("\t%s\n" % f, s, run_tests_log) + + R = file_check(compile_error_files, run_error_files) if options.time: - sys.stdout.write("Elapsed time: %d s\n" % elapsed_time) + print_debug("Elapsed time: %d s\n" % elapsed_time, s, run_tests_log) - sys.exit(len(compile_error_files) + len(run_error_files)) + return R + + +from optparse import OptionParser +import multiprocessing +from ctypes import c_int +import os +import sys +import glob +import re +import signal +import random +import string +import subprocess +import shlex +import platform +import tempfile +import os.path +import time +# our functions +import common +print_debug = common.print_debug +error = common.error + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-r", "--random-shuffle", dest="random", help="Randomly order tests", + default=False, action="store_true") + parser.add_option("-g", "--generics-include", dest="include_file", help="Filename for header implementing functions for generics", + default=None) + parser.add_option("-f", "--ispc-flags", dest="ispc_flags", help="Additional flags for ispc (-g, -O1, ...)", + default="") + parser.add_option('-t', '--target', dest='target', + help='Set compilation target (sse2-i32x4, sse2-i32x8, sse4-i32x4, sse4-i32x8, sse4-i16x8, sse4-i8x16, avx1-i32x8, avx1-i32x16, avx1.1-i32x8, avx1.1-i32x16, avx2-i32x8, avx2-i32x16, generic-x1, generic-x4, generic-x8, generic-x16, generic-x32, generic-x64)', + default="sse4") + parser.add_option('-a', '--arch', dest='arch', + help='Set architecture (arm, x86, x86-64)', + default="x86-64") + parser.add_option("-c", "--compiler", dest="compiler_exe", help="C/C++ compiler binary to use to run tests", + default=None) + parser.add_option('-o', '--no-opt', dest='no_opt', help='Disable optimization', + default=False, action="store_true") + parser.add_option('-j', '--jobs', dest='num_jobs', help='Maximum number of jobs to run in parallel', + default="1024", type="int") + parser.add_option('-v', '--verbose', dest='verbose', help='Enable verbose output', + default=False, action="store_true") + parser.add_option('--wrap-exe', dest='wrapexe', + help='Executable to wrap test runs with (e.g. "valgrind")', + default="") + parser.add_option('--time', dest='time', help='Enable time output', + default=False, action="store_true") + parser.add_option('--non-interactive', dest='non_interactive', help='Disable interactive status updates', + default=False, action="store_true") + parser.add_option('-u', "--update", dest='update', help='Update file with fails (F of FP)', default="") + parser.add_option('-s', "--silent", dest='silent', help='enable silent mode without any output', default=False, + action = "store_true") + parser.add_option("--file", dest='in_file', help='file to save run_tests output', default="") + parser.add_option("--verify", dest='verify', help='verify the file fail_db.txt', default=False, action="store_true") + (options, args) = parser.parse_args() + L = run_tests(options, args, 1) + exit(0)