aaron/ispc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

test system

Browse Source
This commit is contained in:
Ilia Filippov
2013-09-17 17:30:34 +04:00
parent 191d9dede5
commit 00cd90c6b0
11 changed files with 1711 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

600
alloy.py Executable file
View File

@@ -0,0 +1,600 @@
#!/usr/bin/python
#
# Copyright (c) 2013, Intel Corporation
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# // Author: Filippov Ilia
def attach_mail_file(msg, filename, name):
if os.path.exists(filename):
fp = open(filename, "rb")
to_attach = MIMEBase("application", "octet-stream")
to_attach.set_payload(fp.read())
encode_base64(to_attach)
to_attach.add_header("Content-Disposition", "attachment", filename=name)
fp.close()
msg.attach(to_attach)
def setting_paths(llvm, ispc, sde):
if llvm != "":
os.environ["LLVM_HOME"]=llvm
if ispc != "":
os.environ["ISPC_HOME"]=ispc
if sde != "":
os.environ["SDE_HOME"]=sde
def check_LLVM(which_LLVM):
answer = []
if which_LLVM[0] == " ":
return answer
p = os.environ["LLVM_HOME"]
for i in range(0,len(which_LLVM)):
if not os.path.exists(p + os.sep + "bin-" + which_LLVM[i] + os.sep + "bin"):
answer.append(which_LLVM[i])
return answer
def try_do_LLVM(text, command, from_validation):
if from_validation == True:
text = text + "\n"
print_debug("Trying to " + text, from_validation, alloy_build)
if os.system(command + " >> " + alloy_build + " 2>> " + alloy_build) != 0:
print_debug("ERROR.\n", from_validation, alloy_build)
error("can't " + text, 1)
print_debug("DONE.\n", from_validation, alloy_build)
def build_LLVM(version_LLVM, revision, folder, tarball, debug, selfbuild, from_validation, force):
print_debug("Building LLVM. Version: " + version_LLVM + ". ", from_validation, alloy_build)
if revision != "":
print_debug("Revision: " + revision + ".\n", from_validation, alloy_build)
else:
print_debug("\n", from_validation, alloy_build)
# Here we understand what and where do we want to build
current_path = os.getcwd()
llvm_home = os.environ["LLVM_HOME"]
os.chdir(llvm_home)
FOLDER_NAME=version_LLVM
if version_LLVM == "head":
SVN_PATH="trunk"
if version_LLVM == "3.3":
SVN_PATH="tags/RELEASE_33/final"
version_LLVM = "3_3"
if version_LLVM == "3.2":
SVN_PATH="tags/RELEASE_32/final"
version_LLVM = "3_2"
if version_LLVM == "3.1":
SVN_PATH="tags/RELEASE_31/final"
version_LLVM = "3_1"
if revision != "":
FOLDER_NAME = FOLDER_NAME + "_" + revision
revision = "-" + revision
if folder == "":
folder = FOLDER_NAME
LLVM_SRC="llvm-" + folder
LLVM_BUILD="build-" + folder
LLVM_BIN="bin-" + folder
if os.path.exists(LLVM_BIN) and not force:
print_debug("You have folder " + LLVM_BIN + ". If you want to rebuild use --force\n", False, "")
exit(0)
LLVM_BUILD_selfbuild = LLVM_BUILD + "_temp"
LLVM_BIN_selfbuild = LLVM_BIN + "_temp"
common.remove_if_exists(LLVM_SRC)
common.remove_if_exists(LLVM_BUILD)
common.remove_if_exists(LLVM_BIN)
if selfbuild:
common.remove_if_exists(LLVM_BUILD_selfbuild)
common.remove_if_exists(LLVM_BIN_selfbuild)
MAKE = "gmake"
print_debug("Using folders: " + LLVM_SRC + " " + LLVM_BUILD + " " + LLVM_BIN + " in " +
llvm_home + "\n", from_validation, alloy_build)
# load llvm
if tarball == "":
try_do_LLVM("load LLVM from http://llvm.org/svn/llvm-project/llvm/" + SVN_PATH + " ",
"svn co " + revision + " http://llvm.org/svn/llvm-project/llvm/" + SVN_PATH + " " + LLVM_SRC,
from_validation)
os.chdir(LLVM_SRC + "/tools")
try_do_LLVM("load clang from http://llvm.org/svn/llvm-project/cfe/" + SVN_PATH + " ",
"svn co " + revision + " http://llvm.org/svn/llvm-project/cfe/" + SVN_PATH + " clang",
from_validation)
os.chdir("../")
else:
tar = tarball.split(" ")
os.makedirs(LLVM_SRC)
os.chdir(LLVM_SRC)
try_do_LLVM("untar LLVM from " + tar[0] + " ",
"tar -xvzf " + tar[0] + " --strip-components 1", from_validation)
os.chdir("./tools")
os.makedirs("clang")
os.chdir("./clang")
try_do_LLVM("untar clang from " + tar[1] + " ",
"tar -xvzf " + tar[1] + " --strip-components 1", from_validation)
os.chdir("../../")
# paching llvm
patches = glob.glob(os.environ["ISPC_HOME"] + "/llvm_patches/*.*")
for patch in patches:
if version_LLVM in os.path.basename(patch):
try_do_LLVM("patch LLVM with patch" + patch + " ", "patch -p0 < " + patch, from_validation)
os.chdir("../")
# configuring llvm, build first part of selfbuild
os.makedirs(LLVM_BUILD)
os.makedirs(LLVM_BIN)
selfbuild_compiler = ""
if selfbuild:
print_debug("Making selfbuild and use folders " + LLVM_BUILD_selfbuild + " and " +
LLVM_BIN_selfbuild + "\n", from_validation, alloy_build)
os.makedirs(LLVM_BUILD_selfbuild)
os.makedirs(LLVM_BIN_selfbuild)
os.chdir(LLVM_BUILD_selfbuild)
try_do_LLVM("configure release version for selfbuild ",
"../" + LLVM_SRC + "/configure --prefix=" + llvm_home + "/" +
LLVM_BIN_selfbuild + " --enable-optimized",
from_validation)
try_do_LLVM("build release version for selfbuild ",
MAKE + " -j32", from_validation)
try_do_LLVM("install release version for selfbuild ",
MAKE + " install",
from_validation)
os.chdir("../")
selfbuild_compiler = " CC="+llvm_home+ "/" + LLVM_BIN_selfbuild + "/bin/clang"
print_debug("Now we have compiler for selfbuild: " + selfbuild_compiler + "\n", from_validation, alloy_build)
os.chdir(LLVM_BUILD)
if debug == False:
try_do_LLVM("configure release version ",
"../" + LLVM_SRC + "/configure --prefix=" + llvm_home + "/" +
LLVM_BIN + " --enable-optimized" + selfbuild_compiler,
from_validation)
else:
try_do_LLVM("configure debug version ",
"../" + LLVM_SRC + "/configure --prefix=" + llvm_home + "/" + LLVM_BIN +
" --enable-debug-runtime --enable-debug-symbols --enable-keep-symbols" + selfbuild_compiler,
from_validation)
# building llvm
try_do_LLVM("build LLVM ", MAKE + " -j32", from_validation)
try_do_LLVM("install LLVM ", MAKE + " install", from_validation)
os.chdir(current_path)
def check_targets():
answer = []
answer_sde = []
SSE2 = False;
SSE4 = False;
AVX = False;
AVX11 = False;
AVX2 = False;
cpu = open("/proc/cpuinfo")
f_lines = cpu.readlines()
cpu.close()
# check what native targets do we have
for i in range(0,len(f_lines)):
if SSE2 == False and "sse2" in f_lines[i]:
SSE2 = True;
answer = answer + ["sse2-i32x4", "sse2-i32x8"]
if SSE4 == False and "sse4_1" in f_lines[i]:
SSE4 = True;
answer = answer + ["sse4-i32x4", "sse4-i32x8", "sse4-i16x8", "sse4-i8x16"]
if AVX == False and "avx" in f_lines[i]:
AVX = True;
answer = answer + ["avx1-i32x8", "avx1-i32x16"]
if AVX11 == False and "rdrand" in f_lines[i]:
AVX11 = True;
answer = answer + ["avx1.1-i32x8", "avx1.1-i32x16"]
if AVX2 == False and "avx2" in f_lines[i]:
AVX2 = True;
answer = answer + ["avx2-i32x8", "avx2-i32x16"]
answer = answer + ["generic-4", "generic-16", "generic-8", "generic-1", "generic-32", "generic-64"]
# now check what targets we have with the help of SDE
sde_exists = ""
PATH_dir = string.split(os.getenv("PATH"), os.pathsep)
for counter in PATH_dir:
if os.path.exists(counter + os.sep + "sde") and sde_exists == "":
sde_exists = counter + os.sep + "sde"
if os.environ.get("SDE_HOME") != None:
if os.path.exists(os.environ.get("SDE_HOME") + os.sep + "sde"):
sde_exists = os.environ.get("SDE_HOME") + os.sep + "sde"
if sde_exists == "":
error("you haven't got sde neither in SDE_HOME nor in your PATH.\n" +
"To test all platforms please set SDE_HOME to path containing SDE.\n" +
"Please refer to http://www.intel.com/software/sde for SDE download information.", 2)
return [answer, answer_sde]
# here we have SDE
os.system(sde_exists + " -help > " + temp_alloy_file)
cpu = open(temp_alloy_file)
f_lines = cpu.readlines()
cpu.close()
for i in range(0,len(f_lines)):
if SSE4 == False and "wsm" in f_lines[i]:
answer_sde = answer_sde + [["-wsm", "sse4-i32x4"], ["-wsm", "sse4-i32x8"], ["-wsm", "sse4-i16x8"], ["-wsm", "sse4-i8x16"]]
if AVX == False and "snb" in f_lines[i]:
answer_sde = answer_sde + [["-snb", "avx1-i32x8"], ["-snb", "avx1-i32x16"]]
if AVX11 == False and "ivb" in f_lines[i]:
answer_sde = answer_sde + [["-ivb", "avx1.1-i32x8"], ["ivb", "avx1.1-i32x16"]]
if AVX2 == False and "hsw" in f_lines[i]:
answer_sde = answer_sde + [["-hsw", "avx2-i32x8"], ["-hsw", "avx2-i32x16"]]
return [answer, answer_sde]
def build_ispc(version_LLVM):
current_path = os.getcwd()
os.chdir(os.environ["ISPC_HOME"])
p_temp = os.getenv("PATH")
os.environ["PATH"] = os.environ["LLVM_HOME"] + "/bin-" + version_LLVM + "/bin:" + os.environ["PATH"]
os.system("make clean >> " + alloy_build)
try_do_LLVM("build ISPC with LLVM version " + version_LLVM + " ", "make -j32", True)
os.environ["PATH"] = p_temp
os.chdir(current_path)
def execute_stability(stability, R, print_version):
stability1 = copy.deepcopy(stability)
temp = run_tests.run_tests(stability1, [], print_version)
for j in range(0,4):
R[j][0] = R[j][0] + temp[j]
for i in range(0,len(temp[j])):
R[j][1].append(temp[4])
number_of_fails = temp[5]
number_of_new_fails = len(temp[0]) + len(temp[1])
if number_of_fails == 0:
str_fails = ". No fails"
else:
str_fails = ". Fails: " + str(number_of_fails)
if number_of_new_fails == 0:
str_new_fails = ", No new fails.\n"
else:
str_new_fails = ", New fails: " + str(number_of_new_fails) + ".\n"
print_debug(temp[4][1:-3] + str_fails + str_new_fails, False, stability_log)
def run_special_tests():
i = 5
def validation_run(only, only_targets, reference_branch, notify, update):
current_path = os.getcwd()
os.chdir(os.environ["ISPC_HOME"])
os.environ["PATH"] = os.environ["ISPC_HOME"] + ":" + os.environ["PATH"]
if options.notify != "":
if os.environ.get("SMTP_ISPC") == None:
error("you have no SMTP_ISPC in your environment for option notify", 1)
common.remove_if_exists(os.environ["ISPC_HOME"] + os.sep + "all_answer.txt")
smtp_server = os.environ["SMTP_ISPC"]
msg = MIMEMultipart()
msg['Subject'] = 'ISPC test system results'
msg['From'] = 'ISPC_test_system'
msg['To'] = options.notify
print_debug("Command: " + ' '.join(sys.argv) + "\n", False, "")
print_debug("Folder: " + os.environ["ISPC_HOME"] + "\n", False, "")
date = datetime.datetime.now()
print_debug("Date: " + date.strftime('%H:%M %d/%m/%Y') + "\n", False, "")
class options_for_drivers:
pass
# *** *** ***
# Stability validation run
# *** *** ***
if ((("stability" in only) == True) or ("performance" in only) == False):
print_debug("\n\nStability validation run\n\n", False, "")
stability = options_for_drivers()
# stability constant options
stability.random = False
stability.ispc_flags = ""
stability.compiler_exe = None
stability.num_jobs = 1024
stability.verbose = False
stability.time = False
stability.non_interactive = True
stability.update = update
stability.include_file = None
stability.silent = True
stability.in_file = "." + os.sep + f_date + os.sep + "run_tests_log.log"
stability.verify = False
# stability varying options
stability.target = ""
stability.arch = ""
stability.no_opt = False
stability.wrapexe = ""
# prepare parameters of run
common.check_tools(1)
[targets_t, sde_targets_t] = check_targets()
rebuild = True
opts = []
archs = []
LLVM = []
targets = []
sde_targets = []
# parsing option only, update parameters of run
if "-O2" in only:
opts.append(False)
if "-O0" in only:
opts.append(True)
if "x86" in only and not ("x86-64" in only):
archs.append("x86")
if "x86-64" in only:
archs.append("x86-64")
if "native" in only:
sde_targets_t = []
for i in ["3.1", "3.2", "3.3", "head"]:
if i in only:
LLVM.append(i)
if "current" in only:
LLVM = [" "]
rebuild = False
if only_targets != "":
only_targets_t = only_targets.split(" ")
for i in only_targets_t:
err = True
for j in range(0,len(targets_t)):
if i in targets_t[j]:
targets.append(targets_t[j])
err = False
for j in range(0,len(sde_targets_t)):
if i in sde_targets_t[j][1]:
sde_targets.append(sde_targets_t[j])
err = False
if err == True:
error("You haven't sde for target " + i, 1)
else:
targets = targets_t[:-4]
sde_targets = sde_targets_t
if "build" in only:
targets = []
sde_targets = []
only = only + " stability "
# finish parameters of run, prepare LLVM
if len(opts) == 0:
opts = [False]
if len(archs) == 0:
archs = ["x86", "x86-64"]
if len(LLVM) == 0:
LLVM = ["3.1", "3.2", "3.3", "head"]
gen_archs = ["x86-64"]
need_LLVM = check_LLVM(LLVM)
for i in range(0,len(need_LLVM)):
build_LLVM(need_LLVM[i], "", "", "", False, False, True, False)
# begin validation run for stabitily
common.remove_if_exists(stability.in_file)
R = [[[],[]],[[],[]],[[],[]],[[],[]]]
print_debug("\n_________________________STABILITY REPORT_________________________\n", False, stability_log)
for i in range(0,len(LLVM)):
print_version = 2
if rebuild:
build_ispc(LLVM[i])
for j in range(0,len(targets)):
stability.target = targets[j]
stability.wrapexe = ""
if "generic" in targets[j]:
arch = gen_archs
else:
arch = archs
for i1 in range(0,len(arch)):
for i2 in range(0,len(opts)):
stability.arch = arch[i1]
stability.no_opt = opts[i2]
execute_stability(stability, R, print_version)
print_version = 0
for j in range(0,len(sde_targets)):
stability.target = sde_targets[j][1]
stability.wrapexe = os.environ["SDE_HOME"] + "/sde " + sde_targets[j][0] + " -- "
for i1 in range(0,len(archs)):
for i2 in range(0,len(opts)):
stability.arch = archs[i1]
stability.no_opt = opts[i2]
execute_stability(stability, R, print_version)
print_version = 0
# run special tests like embree
#
run_special_tests()
ttt = ["NEW RUNFAILS: ", "NEW COMPFAILS: ", "NEW PASSES RUNFAILS: ", "NEW PASSES COMPFAILS: "]
for j in range(0,4):
if len(R[j][0]) == 0:
print_debug("NO " + ttt[j][:-2] + "\n", False, stability_log)
else:
print_debug(ttt[j] + str(len(R[j][0])) + "\n", False, stability_log)
temp5 = [[],[]]
for i in range(0,len(R[j][0])):
er = True
for k in range(0,len(temp5[0])):
if R[j][0][i] == temp5[0][k]:
temp5[1][k].append(R[j][1][i])
er = False
if er == True:
temp5[0].append(R[j][0][i])
temp5[1].append([R[j][1][i]])
for i in range(0,len(temp5[0])):
print_debug("\t" + temp5[0][i] + "\n", True, stability_log)
for k in range(0,len(temp5[1][i])):
print_debug("\t\t\t" + temp5[1][i][k], True, stability_log)
print_debug("__________________Watch stability.log for details_________________\n", False, stability_log)
if options.notify != "":
attach_mail_file(msg, stability.in_file, "run_tests_log.log")
attach_mail_file(msg, stability_log, "stability.log")
# *** *** ***
# Performance validation run
# *** *** ***
if ((("performance" in only) == True) or ("stability" in only) == False):
print_debug("\n\nPerformance validation run\n\n", False, "")
performance = options_for_drivers()
# performance constant options
performance.number = 5
performance.config = "./perf.ini"
performance.path = "./"
performance.silent = True
performance.output = ""
performance.compiler = ""
performance.ref = "ispc_ref"
performance.in_file = "." + os.sep + f_date + os.sep + "performance.log"
# prepare LLVM 3.3 as newest LLVM
need_LLVM = check_LLVM(["3.3"])
if len(need_LLVM) != 0:
build_LLVM(need_LLVM[i], "", "", "", False, False, True, False)
# prepare reference point. build both test and reference compilers
os.system("git branch > " + temp_alloy_file)
br = open(temp_alloy_file)
temp4 = br.readlines()
br.close()
for line in temp4:
if "*" in line:
current_branch = line[2:-1]
stashing = True
sys.stdout.write("Please, don't interrupt script here! You can have not sync git status after interruption!\n")
if "No local changes" in detect_version("git stash"):
stashing = False
#try_do_LLVM("stash current branch ", "git stash", True)
try_do_LLVM("checkout reference branch " + reference_branch + " ", "git checkout " + reference_branch, True)
sys.stdout.write(".\n")
build_ispc("3.3")
sys.stdout.write(".\n")
os.rename("ispc", "ispc_ref")
try_do_LLVM("checkout test branch " + current_branch + " ", "git checkout " + current_branch, True)
if stashing:
try_do_LLVM("return current branch ", "git stash pop", True)
sys.stdout.write("You can interrupt script now.\n")
build_ispc("3.3")
# begin validation run for performance. output is inserted into perf()
perf.perf(performance, [])
if options.notify != "":
attach_mail_file(msg, performance.in_file, "performance.log")
attach_mail_file(msg, "." + os.sep + "logs" + os.sep + "perf_build.log", "perf_build.log")
print_debug("Logs are in alloy_results_[date]", False, "")
# sending e-mail with results
if options.notify != "":
fp = open(os.environ["ISPC_HOME"] + os.sep + "all_answer.txt", 'rb')
f_lines = fp.readlines()
fp.close()
line = ""
for i in range(0,len(f_lines)):
line = line + f_lines[i][:-1]
line = line + ' \n'
text = MIMEText(line, "", "KOI-8")
msg.attach(text)
attach_mail_file(msg, alloy_build, "alloy_build.log")
s = smtplib.SMTP(smtp_server)
s.sendmail('ISPC_test_system', options.notify, msg.as_string())
s.quit()
# exit of validation routine
common.remove_if_exists(temp_alloy_file)
os.chdir(current_path)
def Main():
if (platform.system() == 'Windows' or 'CYGWIN_NT' in platform.system()) == True:
error("Windows isn't supported now", 1)
if (options.build_llvm == False and
options.validation_run == False and
options.llvm_home == "" and
options.ispc_home == "" and
options.sde_home == ""):
parser.print_help()
exit(0)
global f_date
f_date = "logs"
common.remove_if_exists(f_date)
os.makedirs(f_date)
global temp_alloy_file
temp_alloy_file = os.getcwd() + os.sep + f_date + os.sep + "temp_detect_version"
global alloy_build
alloy_build = os.getcwd() + os.sep + f_date + os.sep + "alloy_build.log"
common.remove_if_exists(alloy_build)
global stability_log
stability_log = os.getcwd() + os.sep + f_date + os.sep + "stability.log"
common.remove_if_exists(stability_log)
setting_paths(options.llvm_home, options.ispc_home, options.sde_home)
if os.environ.get("LLVM_HOME") == None:
error("you have no LLVM_HOME", 1)
if os.environ.get("ISPC_HOME") == None:
error("you have no ISPC_HOME", 1)
if options.build_llvm:
build_LLVM(options.version, options.revision, options.folder, options.tarball,
options.debug, options.selfbuild, False, options.force)
if options.validation_run:
validation_run(options.only, options.only_targets, options.branch, options.notify, options.update)
os.rename(f_date, "alloy_results_" + datetime.datetime.now().strftime('%H_%M_%d_%m_%Y'))
###Main###
from optparse import OptionParser
import sys
import os
import operator
import time
import glob
import string
import platform
import smtplib
import datetime
import copy
from email.MIMEMultipart import MIMEMultipart
from email.MIMEBase import MIMEBase
from email.mime.text import MIMEText
from email.Encoders import encode_base64
# our drivers
import run_tests
import perf
import common
error = common.error
detect_version = common.detect_version
print_debug = common.print_debug
# parsing options
parser = OptionParser()
# options for activity "build LLVM"
parser.add_option('-b', '--build-llvm', dest='build_llvm',
help='ask to build LLVM', default=False, action="store_true")
parser.add_option('--version', dest='version',
help='version of llvm to build', default="head")
parser.add_option('--revision', dest='revision',
help='revision of llvm to build', default="")
parser.add_option('--debug', dest='debug',
help='debug build of LLVM?', default=False, action="store_true")
parser.add_option('--folder', dest='folder',
help='folder to build LLVM in', default="")
parser.add_option('--tarball', dest='tarball',
help='"llvm_tarball clang_tarball"', default="")
parser.add_option('--selfbuild', dest='selfbuild',
help='make selfbuild of LLVM and clang', default=False, action="store_true")
parser.add_option('--force', dest='force',
help='rebuild LLVM', default=False, action='store_true')
# options for activity "setup PATHS"
parser.add_option('--llvm_home', dest='llvm_home',help='path to LLVM',default="")
parser.add_option('--ispc_home', dest='ispc_home',help='path to ISPC',default="")
parser.add_option('--sde_home', dest='sde_home',help='path to SDE',default="")
# options for activity "validation run"
parser.add_option('-r', '--run', dest='validation_run',
help='ask for validation run', default=False, action="store_true")
parser.add_option('--compare-with', dest='branch',
help='set performance reference point', default="master")
parser.add_option('--only-targets', dest='only_targets',
help='set list of targets to test. Possible values - all subnames of targets.\n' +
'Example: --only-targets="avx2-i32x8 sse4 i32x16 sse2"', default="")
parser.add_option('--notify', dest='notify',
help='sent results to email', default="")
parser.add_option('--only', dest='only',
help='set types of tests. Possible values:\n' +
'-O0, -O2, x86, x86-64, stability (test only stability), performance (test only performance)\n' +
'build (only build with different LLVM), 3.1, 3.2, 3.3, head, native (do not use SDE), current (do not rebuild ISPC).\n' +
'Example: --only="3.2 -O0 stability 3.3"', default="")
parser.add_option('--update-errors', dest='update',
help='rewrite fail_db.txt file according to received results (F or FP)', default="")
(options, args) = parser.parse_args()
Main()

102
check_env.py Executable file
View File

@@ -0,0 +1,102 @@
#!/usr/bin/python
#
# Copyright (c) 2013, Intel Corporation
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# // Author: Filippov Ilia
import common
import sys
import os
import string
print_debug = common.print_debug
error = common.error
detect_version = common.detect_version
exists = [False, False, False, False, False, False, False, False]
names = ["m4", "bison", "flex", "sde", "ispc", "clang", "gcc", "icc"]
PATH_dir = string.split(os.getenv("PATH"), os.pathsep)
for counter in PATH_dir:
for i in range(0,8):
if os.path.exists(counter + os.sep + names[i]):
exists[i] = True
print_debug("=== in PATH: ===\n", False, "")
print_debug("Tools:\n", False, "")
for i in range(0,3):
if exists[i]:
print_debug(detect_version(names[i] + " --version"), False, "")
else:
error("you don't have " + names[i], 0)
if exists[0] and exists[1] and exists[2]:
if common.check_tools(2):
print_debug("versions are ok\n", False, "")
print_debug("\nSDE:\n", False, "")
if exists[3]:
print_debug(detect_version(names[3] + " --version"), False, "")
else:
error("you don't have " + names[3], 2)
print_debug("\nISPC:\n", False, "")
if exists[4]:
print_debug(detect_version(names[4] + " --version"), False, "")
else:
error("you don't have " + names[4], 2)
print_debug("\nC/C++ compilers:\n", False, "")
for i in range(5,8):
if exists[i]:
print_debug(detect_version(names[i] + " --version"), False, "")
else:
error("you don't have " + names[i], 2)
print_debug("\n=== in ISPC specific environment variables: ===\n", False, "")
if os.environ.get("LLVM_HOME") == None:
error("you have no LLVM_HOME", 2)
else:
print_debug("Your LLVM_HOME:" + os.environ.get("LLVM_HOME") + "\n", False, "")
if os.environ.get("ISPC_HOME") == None:
error("you have no ISPC_HOME", 2)
else:
print_debug("Your ISPC_HOME:" + os.environ.get("ISPC_HOME") + "\n", False, "")
if os.path.exists(os.environ.get("ISPC_HOME") + os.sep + "ispc"):
print_debug("You have ISPC in your ISPC_HOME: " +
detect_version(os.environ.get("ISPC_HOME") + os.sep + "ispc" + " --version"), False, "")
else:
error("you don't have ISPC in your ISPC_HOME", 2)
if os.environ.get("SDE_HOME") == None:
error("You have no SDE_HOME", 2)
else:
print_debug("Your SDE_HOME:" + os.environ.get("SDE_HOME") + "\n", False, "")
if os.path.exists(os.environ.get("SDE_HOME") + os.sep + "sde"):
print_debug("You have sde in your SDE_HOME: " +
detect_version(os.environ.get("SDE_HOME") + os.sep + "sde" + " --version"), False, "")
else:
error("you don't have any SDE in your ISPC_HOME", 2)

120
common.py Normal file
View File

@@ -0,0 +1,120 @@
#!/usr/bin/python
#
# Copyright (c) 2013, Intel Corporation
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# // Author: Filippov Ilia
import sys
import os
import shutil
def write_to_file(filename, line):
f = open(filename, 'a')
f.writelines(line)
f.close()
#remove file if it exists
def remove_if_exists(filename):
if os.path.exists(filename):
if os.path.isdir(filename):
shutil.rmtree(filename)
else:
os.remove(filename)
# detect version which is printed after command
def detect_version(command):
os.system(command + " > " + "temp_detect_version")
version = open("temp_detect_version")
answer = version.readline()
version.close()
remove_if_exists("temp_detect_version")
return answer
# print versions of compilers
def print_version(ispc_test, ispc_ref, ref_compiler, s, perf_log, is_windows):
print_debug("\nUsing test compiler: " + detect_version(ispc_test + " --version"), s, perf_log)
if ispc_ref != "":
print_debug("Using ref compiler: " + detect_version(ispc_ref + " --version"), s, perf_log)
if is_windows == False:
temp1 = detect_version(ref_compiler + " --version")
else:
os.system(ref_compiler + " 2>&1" + " 2> temp_detect_version > temp_detect_version1" )
version = open("temp_detect_version")
temp1 = version.readline()
version.close()
remove_if_exists("temp_detect_version")
remove_if_exists("temp_detect_version1")
print_debug("Using C/C++ compiler: " + temp1 + "\n", s, perf_log)
# print everything from scripts instead errors
def print_debug(line, silent, filename):
if silent == False:
sys.stdout.write(line)
sys.stdout.flush()
if os.environ.get("ISPC_HOME") != None:
write_to_file(os.environ["ISPC_HOME"] + os.sep + "all_answer.txt", line)
if filename != "":
write_to_file(filename, line)
# print errors from scripts
# type 1 for error in environment
# type 2 for warning
# type 3 for error of compiler or test which isn't the goal of script
def error(line, error_type):
line = line + "\n"
if error_type == 1:
sys.stderr.write("Fatal error: " + line)
sys.exit(1)
if error_type == 2:
sys.stderr.write("Warning: " + line)
if error_type == 0:
print_debug("FIND ERROR: " + line, False, "")
def check_tools(m):
input_tools=[[[1,4],"m4 --version", "bad m4 version"],
[[2,4],"bison --version", "bad bison version"],
[[2,5], "flex --version", "bad flex version"]]
for t in range(0,len(input_tools)):
t1 = ((detect_version(input_tools[t][1]))[:-1].split(" "))
for i in range(0,len(t1)):
t11 = t1[i].split(".")
f = True
for j in range(0,len(t11)):
if not t11[j].isdigit():
f = False
if f == True:
for j in range(0,len(t11)):
if j < len(input_tools[t][0]):
if int(t11[j])<input_tools[t][0][j]:
error(input_tools[t][2], m)
return 0
return 1

2
examples/noise/Makefile
View File

@@ -1,6 +1,6 @@
EXAMPLE=noise
CPP_SRC=$(EXAMPLE).cpp $(EXAMPLE)_serial.cpp
CPP_SRC=noise.cpp noise_serial.cpp
ISPC_SRC=noise.ispc
ISPC_IA_TARGETS=sse2,sse4,avx-x2
ISPC_ARM_TARGETS=neon

374
examples/perf.py
View File

@@ -1,374 +0,0 @@
#!/usr/bin/python
# // Author: Filippov Ilia
from optparse import OptionParser
import sys
import os
import operator
import time
import glob
import string
import platform
def print_debug(line):
if options.silent == False:
sys.stdout.write(line)
def print_file(line):
if options.output != "":
output = open(options.output, 'w')
output.writelines(line)
output.close()
def build_test():
global build_log
global is_windows
if is_windows == False:
os.system("make clean >> "+build_log)
return os.system("make CXX="+ref_compiler+" CC="+refc_compiler+" >> "+build_log+" 2>> "+build_log)
else:
os.system("msbuild /t:clean >> " + build_log)
return os.system("msbuild /V:m /p:Platform=x64 /p:Configuration=Release /p:TargetDir=.\ /t:rebuild >> " + build_log)
def execute_test(command):
global perf_temp
r = 0
if os.path.exists(perf_temp):
os.remove(perf_temp)
for k in range(int(options.number)):
r = r + os.system(command)
return r
#gathers all tests results and made an item test from answer structure
def run_test(command, c1, c2, test, b_serial):
global perf_temp
if build_test() != 0:
sys.stdout.write("ERROR: Compilation fails\n")
return
if execute_test(command) != 0:
sys.stdout.write("ERROR: Execution fails\n")
return
tasks = [] #list of results with tasks, it will be test[2]
ispc = [] #list of results without tasks, it will be test[1]
absolute_tasks = [] #list of absolute results with tasks, it will be test[4]
absolute_ispc = [] #list of absolute results without tasks, ut will be test[3]
serial = [] #list serial times, it will be test[5]
j = 1
for line in open(perf_temp): # we take test output
if "speedup" in line: # we are interested only in lines with speedup
if j == c1: # we are interested only in lines with c1 numbers
line = line.expandtabs(0)
line = line.replace("("," ")
line = line.split(",")
for i in range(len(line)):
subline = line[i].split(" ")
number = float(subline[1][:-1])
if "speedup from ISPC + tasks" in line[i]:
tasks.append(number)
else:
ispc.append(number)
c1 = c1 + c2
j+=1
if "million cycles" in line:
if j == c1:
line = line.replace("]","[")
line = line.split("[")
number = float(line[3])
if "tasks" in line[1]:
absolute_tasks.append(number)
else:
if "ispc" in line[1]:
absolute_ispc.append(number)
if "serial" in line[1]:
serial.append(number)
if len(ispc) != 0:
if len(tasks) != 0:
print_debug("ISPC speedup / ISPC + tasks speedup / ISPC time / ISPC + tasks time / serial time\n")
for i in range(0,len(serial)):
print_debug("%10s /\t%10s\t /%9s / %10s\t /%10s\n" %
(ispc[i], tasks[i], absolute_ispc[i], absolute_tasks[i], serial[i]))
else:
print_debug("ISPC speedup / ISPC time / serial time\n")
for i in range(0,len(serial)):
print_debug("%10s /%9s /%10s\n" % (ispc[i], absolute_ispc[i], serial[i]))
else:
if len(tasks) != 0:
print_debug("ISPC + tasks speedup / ISPC + tasks time / serial time\n")
for i in range(0,len(serial)):
print_debug("%10s\t / %10s\t /%10s\n" % (tasks[i], absolute_tasks[i], serial[i]))
test[1] = test[1] + ispc
test[2] = test[2] + tasks
test[3] = test[3] + absolute_ispc
test[4] = test[4] + absolute_tasks
if b_serial == True:
#if we concatenate outputs we should use only the first serial answer.
test[5] = test[5] + serial
def cpu_get():
p = open("/proc/stat", 'r')
cpu = p.readline()
p.close()
cpu = cpu.split(" ")
cpu_usage = (int(cpu[2]) + int(cpu[3]) + int(cpu[4]))
cpu_all = cpu_usage + int(cpu[5])
return [cpu_usage, cpu_all]
#returns cpu_usage
def cpu_check():
if is_windows == False:
if is_mac == False:
cpu1 = cpu_get()
time.sleep(1)
cpu2 = cpu_get()
cpu_percent = (float(cpu1[0] - cpu2[0])/float(cpu1[1] - cpu2[1]))*100
else:
os.system("sysctl -n vm.loadavg > cpu_temp")
c = open("cpu_temp", 'r')
c_line = c.readline()
c.close
os.remove("cpu_temp")
R = c_line.split(' ')
cpu_percent = float(R[1]) * 3
else:
os.system("wmic cpu get loadpercentage /value > cpu_temp")
c = open("cpu_temp", 'r')
c_lines = c.readlines()
c.close()
os.remove("cpu_temp")
t = "0"
for i in c_lines[2]:
if i.isdigit():
t = t + i
cpu_percent = int(t)
return cpu_percent
#returns geomean of list
def geomean(par):
temp = 1
l = len(par)
for i in range(l):
temp = temp * par[i]
temp = temp ** (1.0/l)
return round(temp, 2)
#takes an answer struct and print it.
#answer struct: list answer contains lists test
#test[0] - name of test
#test[1] - list of results without tasks
#test[2] - list of results with tasks
#test[3] - list of absolute results without tasks
#test[4] - list of absolute results with tasks
#test[5] - list of absolute time without ISPC (serial)
#test[1..4] may be empty
def print_answer(answer):
filelist = []
print_debug("--------------------------------------------------------------------------\n")
print_debug("test name:\t ISPC speedup: ISPC + tasks speedup: | " +
"ISPC time: ISPC + tasks time: serial:\n")
filelist.append("test name,ISPC speedup,diff," +
"ISPC + tasks speedup,diff,ISPC time,diff,ISPC + tasks time,diff,serial,diff\n")
max_t = [0,0,0,0,0]
diff_t = [0,0,0,0,0]
geomean_t = [0,0,0,0,0]
list_of_max = [[],[],[],[],[]]
for i in range(len(answer)):
for t in range(1,6):
if len(answer[i][t]) == 0:
max_t[t-1] = "n/a"
diff_t[t-1] = "n/a"
else:
if t < 3:
mm = max(answer[i][t])
else:
mm = min(answer[i][t])
max_t[t-1] = '%.2f' % mm
list_of_max[t-1].append(mm)
diff_t[t-1] = '%.2f' % (max(answer[i][t]) - min(answer[i][t]))
print_debug("%s:\n" % answer[i][0])
print_debug("\t\tmax:\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" %
(max_t[0], max_t[1], max_t[2], max_t[3], max_t[4]))
print_debug("\t\tdiff:\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" %
(diff_t[0], diff_t[1], diff_t[2], diff_t[3], diff_t[4]))
for t in range(0,5):
if max_t[t] == "n/a":
max_t[t] = ""
if diff_t[t] == "n/a":
diff_t[t] = ""
filelist.append(answer[i][0] + "," +
max_t[0] + "," + diff_t[0] + "," + max_t[1] + "," + diff_t[1] + "," +
max_t[2] + "," + diff_t[2] + "," + max_t[3] + "," + diff_t[3] + "," +
max_t[4] + "," + diff_t[4] + "\n")
for i in range(0,5):
geomean_t[i] = geomean(list_of_max[i])
print_debug("---------------------------------------------------------------------------------\n")
print_debug("Geomean:\t\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" %
(geomean_t[0], geomean_t[1], geomean_t[2], geomean_t[3], geomean_t[4]))
filelist.append("Geomean," + str(geomean_t[0]) + ",," + str(geomean_t[1])
+ ",," + str(geomean_t[2]) + ",," + str(geomean_t[3]) + ",," + str(geomean_t[4]) + "\n")
print_file(filelist)
###Main###
# parsing options
parser = OptionParser()
parser.add_option('-n', '--number', dest='number',
help='number of repeats', default="3")
parser.add_option('-c', '--config', dest='config',
help='config file of tests', default="./perf.ini")
parser.add_option('-p', '--path', dest='path',
help='path to examples directory', default="./")
parser.add_option('-s', '--silent', dest='silent',
help='silent mode, only table output', default=False, action="store_true")
parser.add_option('-o', '--output', dest='output',
help='output file for script reading', default="")
parser.add_option('--compiler', dest='compiler',
help='reference compiler', default="")
(options, args) = parser.parse_args()
global is_windows
is_windows = (platform.system() == 'Windows' or
'CYGWIN_NT' in platform.system())
global is_mac
is_mac = (platform.system() == 'Darwin')
# save corrent path
pwd = os.getcwd()
pwd = pwd + os.sep
if is_windows:
pwd = "..\\"
# check if cpu usage is low now
cpu_percent = cpu_check()
if cpu_percent > 20:
sys.stdout.write("Warning: CPU Usage is very high.\n")
sys.stdout.write("Close other applications.\n")
# check that required compilers exist
PATH_dir = string.split(os.getenv("PATH"), os.pathsep)
compiler_exists = False
ref_compiler_exists = False
if is_windows == False:
compiler = "ispc"
ref_compiler = "g++"
refc_compiler = "gcc"
if options.compiler != "":
if options.compiler == "clang" or options.compiler == "clang++":
ref_compiler = "clang++"
refc_compiler = "clang"
if options.compiler == "icc" or options.compiler == "icpc":
ref_compiler = "icpc"
refc_compiler = "icc"
else:
compiler = "ispc.exe"
ref_compiler = "cl.exe"
for counter in PATH_dir:
if os.path.exists(counter + os.sep + compiler):
compiler_exists = True
if os.path.exists(counter + os.sep + ref_compiler):
ref_compiler_exists = True
if not compiler_exists:
sys.stderr.write("Fatal error: ISPC compiler not found.\n")
sys.stderr.write("Added path to ispc compiler to your PATH variable.\n")
sys.exit()
if not ref_compiler_exists:
sys.stderr.write("Fatal error: reference compiler %s not found.\n" % ref_compiler)
sys.stderr.write("Added path to %s compiler to your PATH variable.\n" % ref_compiler)
sys.exit()
# checks that config file exists
path_config = os.path.normpath(options.config)
if os.path.exists(path_config) == False:
sys.stderr.write("Fatal error: config file not found: %s.\n" % options.config)
sys.stderr.write("Set path to your config file in --config.\n")
sys.exit()
# read lines from config file except comments
f = open(path_config, 'r')
f_lines = f.readlines()
f.close()
lines =[]
for i in range(len(f_lines)):
if f_lines[i][0] != "%":
lines.append(f_lines[i])
length = len(lines)
# prepare build.log and perf_temp files
global build_log
build_log = pwd + "build.log"
if is_windows == False:
if os.path.exists(build_log):
os.remove(build_log)
else:
if os.path.exists("build.log"):
os.remove("build.log")
global perf_temp
perf_temp = pwd + "perf_temp"
i = 0
answer = []
print_debug("Okey go go go!\n\n")
os.system(compiler + " --version >" + build_log)
version = open(build_log)
print_debug("Using test compiler: " + version.readline())
version.close()
if is_windows == False:
os.system(ref_compiler + " --version >" + build_log)
else:
os.system(ref_compiler + " 2>" + build_log + " 1>&2")
version = open(build_log)
print_debug("Using reference compiler: " + version.readline())
version.close()
# loop for all tests
while i < length-2:
# we read name of test
print_debug("%s" % lines[i])
test = [lines[i][:-1],[],[],[],[],[]]
# read location of test
folder = lines[i+1]
folder = folder[:-1]
folder = os.path.normpath(options.path + os.sep + folder)
# check that test exists
if os.path.exists(folder) == False:
sys.stdout.write("Fatal error: Can't find test %s. Your path is: \"%s\".\n" % (lines[i][:-1], options.path))
sys.stdout.write("Change current location to /examples or set path to /examples in --path.\n")
exit(0)
os.chdir(folder)
# read parameters of test
command = lines[i+2]
command = command[:-1]
if is_windows == False:
command = "./"+command + " >> " + perf_temp
else:
command = "x64\\Release\\"+command + " >> " + perf_temp
# parsing config parameters
next_line = lines[i+3]
if next_line[0] == "!": # we should take only one part of test output
R = next_line.split(' ')
c1 = int(R[1]) #c1 is a number of string which we want to use in test output
c2 = int(R[2]) #c2 is total number of strings in test output
i = i+1
else:
c1 = 1
c2 = 1
next_line = lines[i+3]
if next_line[0] == "^": #we should concatenate result of this test with previous one
run_test(command, c1, c2, answer[len(answer)-1], False)
i = i+1
else: #we run this test and append it's result to answer structure
run_test(command, c1, c2, test, True)
answer.append(test)
# preparing next loop iteration
os.chdir(pwd)
i+=4
# delete temp file
if os.path.exists(perf_temp):
os.remove(perf_temp)
#print collected answer
print_answer(answer)

1
fail_db.txt Normal file
View File

@@ -0,0 +1 @@
% List of known fails

0
llvm_patches/r183327-AVX2-GATHER.patch → llvm_patches/3_3_r183327-AVX2-GATHER.patch
View File

0
llvm_patches/r184575-x86-shift.patch → llvm_patches/3_3_r184575-x86-shift.patch
View File

24
examples/perf.ini → perf.ini
View File

@@ -10,44 +10,48 @@
%****************************************************************************************************
AOBench
aobench
ao 10 512 512
10 512 512
#***
Deferred Shading
deferred
deferred_shading data/pp1280x720.bin
data/pp1280x720.bin
#***
Mandelbrot Set
mandelbrot
mandelbrot
#***
Mandelbrot Set
mandelbrot_tasks
mandelbrot_tasks
^
#***
Perlin Noise Function
noise
noise
#***
Binomial Options
options
options
! 1 2
#***
Black-Scholes Options
options
options
! 2 2
#***
Ray Tracer
rt
rt sponza
sponza
#***
3D Stencil
stencil
stencil
#***
Volume Rendering
volume_rendering
volume camera.dat density_highres.vol
camera.dat density_highres.vol
#***
%Sort
%sort
%
%#***

489
perf.py Executable file
View File

@@ -0,0 +1,489 @@
#!/usr/bin/python
#
# Copyright (c) 2013, Intel Corporation
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# // Author: Filippov Ilia
def print_file(line):
if options.output != "":
output = open(options.output, 'w')
output.writelines(line)
output.close()
def build_test(commands):
os.system(commands[4])
test = os.system(commands[1])
if options.ref:
ref = os.system(commands[3])
return (options.ref and ref) or test
def execute_test(commands):
r = 0
common.remove_if_exists(perf_temp+"_test")
common.remove_if_exists(perf_temp+"_ref")
for k in range(int(options.number)):
r = r + os.system(commands[0])
if options.ref:
r = r + os.system(commands[2])
return r
#gathers all tests results and made an item test from answer structure
def run_test(commands, c1, c2, test, test_ref, b_serial):
if build_test(commands) != 0:
error("Compilation fails of test %s\n" % test[0], 0)
return
if execute_test(commands) != 0:
error("Execution fails of test %s\n" % test[0], 0)
return
print_debug("TEST COMPILER:\n", s, perf_log)
analyse_test(c1, c2, test, b_serial, perf_temp+"_test")
if options.ref:
print_debug("REFERENCE COMPILER:\n", s, perf_log)
analyse_test(c1, c2, test_ref, b_serial, perf_temp+"_ref")
def analyse_test(c1, c2, test, b_serial, perf_temp_n):
tasks = [] #list of results with tasks, it will be test[2]
ispc = [] #list of results without tasks, it will be test[1]
absolute_tasks = [] #list of absolute results with tasks, it will be test[4]
absolute_ispc = [] #list of absolute results without tasks, ut will be test[3]
serial = [] #list serial times, it will be test[5]
j = 1
for line in open(perf_temp_n): # we take test output
if "speedup" in line: # we are interested only in lines with speedup
if j == c1: # we are interested only in lines with c1 numbers
line = line.expandtabs(0)
line = line.replace("("," ")
line = line.split(",")
for i in range(len(line)):
subline = line[i].split(" ")
number = float(subline[1][:-1])
if "speedup from ISPC + tasks" in line[i]:
tasks.append(number)
else:
ispc.append(number)
c1 = c1 + c2
j+=1
if "million cycles" in line:
if j == c1:
line = line.replace("]","[")
line = line.split("[")
number = float(line[3])
if "tasks" in line[1]:
absolute_tasks.append(number)
else:
if "ispc" in line[1]:
absolute_ispc.append(number)
if "serial" in line[1]:
serial.append(number)
if len(ispc) != 0:
if len(tasks) != 0:
print_debug("ISPC speedup / ISPC + tasks speedup / ISPC time / ISPC + tasks time / serial time\n", s, perf_log)
for i in range(0,len(serial)):
print_debug("%10s /\t%10s\t /%9s / %10s\t /%10s\n" %
(ispc[i], tasks[i], absolute_ispc[i], absolute_tasks[i], serial[i]), s, perf_log)
else:
print_debug("ISPC speedup / ISPC time / serial time\n", s, perf_log)
for i in range(0,len(serial)):
print_debug("%10s /%9s /%10s\n" % (ispc[i], absolute_ispc[i], serial[i]), s, perf_log)
else:
if len(tasks) != 0:
print_debug("ISPC + tasks speedup / ISPC + tasks time / serial time\n", s, perf_log)
for i in range(0,len(serial)):
print_debug("%10s\t / %10s\t /%10s\n" % (tasks[i], absolute_tasks[i], serial[i]), s, perf_log)
test[1] = test[1] + ispc
test[2] = test[2] + tasks
test[3] = test[3] + absolute_ispc
test[4] = test[4] + absolute_tasks
if b_serial == True:
#if we concatenate outputs we should use only the first serial answer.
test[5] = test[5] + serial
def cpu_get():
p = open("/proc/stat", 'r')
cpu = p.readline()
p.close()
cpu = cpu.split(" ")
cpu_usage = (int(cpu[2]) + int(cpu[3]) + int(cpu[4]))
cpu_all = cpu_usage + int(cpu[5])
return [cpu_usage, cpu_all]
#returns cpu_usage
def cpu_check():
if is_windows == False:
if is_mac == False:
cpu1 = cpu_get()
time.sleep(1)
cpu2 = cpu_get()
cpu_percent = (float(cpu1[0] - cpu2[0])/float(cpu1[1] - cpu2[1]))*100
else:
os.system("sysctl -n vm.loadavg > cpu_temp")
c = open("cpu_temp", 'r')
c_line = c.readline()
c.close
os.remove("cpu_temp")
R = c_line.split(' ')
cpu_percent = float(R[1]) * 3
else:
os.system("wmic cpu get loadpercentage /value > cpu_temp")
c = open("cpu_temp", 'r')
c_lines = c.readlines()
c.close()
os.remove("cpu_temp")
t = "0"
for i in c_lines[2]:
if i.isdigit():
t = t + i
cpu_percent = int(t)
return cpu_percent
#returns geomean of list
def geomean(par):
temp = 1
l = len(par)
for i in range(l):
temp = temp * par[i]
temp = temp ** (1.0/l)
return round(temp, 2)
#takes an answer struct and print it.
#answer struct: list answer contains lists test
#test[0] - name of test
#test[1] - list of results without tasks
#test[2] - list of results with tasks
#test[3] - list of absolute results without tasks
#test[4] - list of absolute results with tasks
#test[5] - list of absolute time without ISPC (serial)
#test[1..4] may be empty
def print_answer(answer):
filelist = []
print_debug("--------------------------------------------------------------------------\n", s, perf_log)
print_debug("test name:\t ISPC speedup: ISPC + tasks speedup: | " +
"ISPC time: ISPC + tasks time: serial:\n", s, perf_log)
filelist.append("test name,ISPC speedup,diff," +
"ISPC + tasks speedup,diff,ISPC time,diff,ISPC + tasks time,diff,serial,diff\n")
max_t = [0,0,0,0,0]
diff_t = [0,0,0,0,0]
geomean_t = [0,0,0,0,0]
list_of_max = [[],[],[],[],[]]
list_of_compare = [[],[],[],[],[],[]]
for i in range(len(answer)):
list_of_compare[0].append(answer[i][0])
for t in range(1,6):
if len(answer[i][t]) == 0:
max_t[t-1] = "n/a"
diff_t[t-1] = "n/a"
list_of_compare[t].append(0);
else:
if t < 3:
mm = max(answer[i][t])
else:
mm = min(answer[i][t])
list_of_compare[t].append(mm)
max_t[t-1] = '%.2f' % mm
list_of_max[t-1].append(mm)
diff_t[t-1] = '%.2f' % (max(answer[i][t]) - min(answer[i][t]))
print_debug("%s:\n" % answer[i][0], s, perf_log)
print_debug("\t\tmax:\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" %
(max_t[0], max_t[1], max_t[2], max_t[3], max_t[4]), s, perf_log)
print_debug("\t\tdiff:\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" %
(diff_t[0], diff_t[1], diff_t[2], diff_t[3], diff_t[4]), s, perf_log)
for t in range(0,5):
if max_t[t] == "n/a":
max_t[t] = ""
if diff_t[t] == "n/a":
diff_t[t] = ""
filelist.append(answer[i][0] + "," +
max_t[0] + "," + diff_t[0] + "," + max_t[1] + "," + diff_t[1] + "," +
max_t[2] + "," + diff_t[2] + "," + max_t[3] + "," + diff_t[3] + "," +
max_t[4] + "," + diff_t[4] + "\n")
for i in range(0,5):
geomean_t[i] = geomean(list_of_max[i])
print_debug("---------------------------------------------------------------------------------\n", s, perf_log)
print_debug("Geomean:\t\t%5s\t\t%10s\t|%10s\t%10s\t%10s\n" %
(geomean_t[0], geomean_t[1], geomean_t[2], geomean_t[3], geomean_t[4]), s, perf_log)
filelist.append("Geomean," + str(geomean_t[0]) + ",," + str(geomean_t[1])
+ ",," + str(geomean_t[2]) + ",," + str(geomean_t[3]) + ",," + str(geomean_t[4]) + "\n")
print_file(filelist)
return list_of_compare
def compare(A, B):
print_debug("\n\n_____________________PERFORMANCE REPORT____________________________\n", False, "")
print_debug("test name: ISPC time: ISPC time ref: %:\n", False, "")
for i in range(0,len(A[0])):
if B[3][i] == 0:
p1 = 0
else:
p1 = 100 - 100 * A[3][i]/B[3][i]
print_debug("%21s: %10.2f %10.2f %10.2f" % (A[0][i], A[3][i], B[3][i], p1), False, "")
if p1 < -1:
print_debug(" <-", False, "")
if p1 > 1:
print_debug(" <+", False, "")
print_debug("\n", False, "")
print_debug("\n", False, "")
print_debug("test name: TASKS time: TASKS time ref: %:\n", False, "")
for i in range(0,len(A[0])):
if B[4][i] == 0:
p2 = 0
else:
p2 = 100 - 100 * A[4][i]/B[4][i]
print_debug("%21s: %10.2f %10.2f %10.2f" % (A[0][i], A[4][i], B[4][i], p2), False, "")
if p2 < -1:
print_debug(" <-", False, "")
if p2 > 1:
print_debug(" <+", False, "")
print_debug("\n", False, "")
if "performance.log" in options.in_file:
print_debug("\n\n_________________Watch performance.log for details________________\n", False, "")
else:
print_debug("\n\n__________________________________________________________________\n", False, "")
def perf(options1, args):
global options
options = options1
global s
s = options.silent
# save current OS
global is_windows
is_windows = (platform.system() == 'Windows' or
'CYGWIN_NT' in platform.system())
global is_mac
is_mac = (platform.system() == 'Darwin')
# save current path
pwd = os.getcwd()
pwd = pwd + os.sep
pwd1 = pwd
if is_windows:
pwd1 = "..\\..\\"
# check if cpu usage is low now
cpu_percent = cpu_check()
if cpu_percent > 20:
error("CPU Usage is very high.\nClose other applications.\n", 2)
global ispc_test
global ispc_ref
global ref_compiler
global refc_compiler
# check that required compilers exist
PATH_dir = string.split(os.getenv("PATH"), os.pathsep)
ispc_test_exists = False
ispc_ref_exists = False
ref_compiler_exists = False
if is_windows == False:
ispc_test = "ispc"
ref_compiler = "g++"
refc_compiler = "gcc"
if options.compiler != "":
if options.compiler == "clang" or options.compiler == "clang++":
ref_compiler = "clang++"
refc_compiler = "clang"
if options.compiler == "icc" or options.compiler == "icpc":
ref_compiler = "icpc"
refc_compiler = "icc"
else:
ispc_test = "ispc.exe"
ref_compiler = "cl.exe"
ispc_ref = options.ref
if options.ref != "":
options.ref = True
for counter in PATH_dir:
if os.path.exists(counter + os.sep + ispc_test):
ispc_test_exists = True
if os.path.exists(counter + os.sep + ref_compiler):
ref_compiler_exists = True
if os.path.exists(counter + os.sep + ispc_ref):
ispc_ref_exists = True
if not ispc_test_exists:
error("ISPC compiler not found.\nAdded path to ispc compiler to your PATH variable.\n", 1)
if not ref_compiler_exists:
error("C/C++ compiler %s not found.\nAdded path to %s compiler to your PATH variable.\n" % (ref_compiler, ref_compiler), 1)
if options.ref:
if not ispc_ref_exists:
error("ISPC reference compiler not found.\nAdded path to ispc reference compiler to your PATH variable.\n", 1)
# checks that config file exists
path_config = os.path.normpath(options.config)
if os.path.exists(path_config) == False:
error("config file not found: %s.\nSet path to your config file in --config.\n" % options.config, 1)
sys.exit()
# read lines from config file except comments
f = open(path_config, 'r')
f_lines = f.readlines()
f.close()
lines =[]
for i in range(len(f_lines)):
if f_lines[i][0] != "%":
lines.append(f_lines[i])
length = len(lines)
# prepare build.log, perf_temp and perf.log files
global perf_log
if options.in_file:
perf_log = pwd + options.in_file
common.remove_if_exists(perf_log)
else:
perf_log = ""
global build_log
build_log = pwd + os.sep + "logs" + os.sep + "perf_build.log"
common.remove_if_exists(build_log)
if os.path.exists(pwd + os.sep + "logs") == False:
os.makedirs(pwd + os.sep + "logs")
global perf_temp
perf_temp = pwd + "perf_temp"
# end of preparations
print_debug("Okey go go go!\n\n", s, perf_log)
#print compilers versions
common.print_version(ispc_test, ispc_ref, ref_compiler, False, perf_log, is_windows)
# begin
i = 0
answer = []
answer_ref = []
# loop for all tests
while i < length-2:
# we read name of test
print_debug("%s" % lines[i], s, perf_log)
test = [lines[i][:-1],[],[],[],[],[]]
test_ref = [lines[i][:-1],[],[],[],[],[]]
# read location of test
folder = lines[i+1]
folder = folder[:-1]
folder = os.path.normpath(options.path + os.sep + "examples" + os.sep + folder)
# check that test exists
if os.path.exists(folder) == False:
error("Can't find test %s. Your path is: \"%s\".\nChange current location to ISPC_HOME or set path to ISPC_HOME in --path.\n" %
(lines[i][:-1], options.path), 1)
os.chdir(folder)
# read parameters of test
command = lines[i+2]
command = command[:-1]
if is_windows == False:
ex_command_ref = "./ref " + command + " >> " + perf_temp + "_ref"
ex_command = "./test " + command + " >> " + perf_temp + "_test"
bu_command_ref = "make CXX="+ref_compiler+" CC="+refc_compiler+ " EXAMPLE=ref ISPC="+ispc_ref+" >> "+build_log+" 2>> "+build_log
bu_command = "make CXX="+ref_compiler+" CC="+refc_compiler+ " EXAMPLE=test ISPC="+ispc_test+" >> "+build_log+" 2>> "+build_log
re_command = "make clean >> "+build_log
else:
ex_command_ref = "x64\\Release\\ref.exe " + command + " >> " + perf_temp + "_ref"
ex_command = "x64\\Release\\test.exe " + command + " >> " + perf_temp + "_test"
bu_command_ref = "msbuild /V:m /p:Platform=x64 /p:Configuration=Release /p:TargetDir=.\ /p:TargetName=ref /t:rebuild >> " + build_log
bu_command = "msbuild /V:m /p:Platform=x64 /p:Configuration=Release /p:TargetDir=.\ /p:TargetName=test /t:rebuild >> " + build_log
re_command = "msbuild /t:clean >> " + build_log
commands = [ex_command, bu_command, ex_command_ref, bu_command_ref, re_command]
# parsing config parameters
next_line = lines[i+3]
if next_line[0] == "!": # we should take only one part of test output
R = next_line.split(' ')
c1 = int(R[1]) #c1 is a number of string which we want to use in test output
c2 = int(R[2]) #c2 is total number of strings in test output
i = i+1
else:
c1 = 1
c2 = 1
next_line = lines[i+3]
if next_line[0] == "^": #we should concatenate result of this test with previous one
run_test(commands, c1, c2, answer[len(answer)-1], answer_ref[len(answer)-1], False)
i = i+1
else: #we run this test and append it's result to answer structure
run_test(commands, c1, c2, test, test_ref, True)
answer.append(test)
answer_ref.append(test_ref)
# preparing next loop iteration
os.chdir(pwd1)
i+=4
# delete temp file
common.remove_if_exists(perf_temp+"_test")
common.remove_if_exists(perf_temp+"_ref")
#print collected answer
print_debug("\n\nTEST COMPILER:\n", s, perf_log)
A = print_answer(answer)
if options.ref != "":
print_debug("\n\nREFERENCE COMPILER:\n", s, perf_log)
B = print_answer(answer_ref)
# print perf report
compare(A,B)
###Main###
from optparse import OptionParser
import sys
import os
import operator
import time
import glob
import string
import platform
# our functions
import common
print_debug = common.print_debug
error = common.error
if __name__ == "__main__":
# parsing options
parser = OptionParser()
parser.add_option('-n', '--number', dest='number',
help='number of repeats', default="3")
parser.add_option('-c', '--config', dest='config',
help='config file of tests', default="./perf.ini")
parser.add_option('-p', '--path', dest='path',
help='path to test_system directory', default=".")
parser.add_option('-s', '--silent', dest='silent',
help='silent mode, only table output', default=False, action="store_true")
parser.add_option('-o', '--output', dest='output',
help='output file for script reading', default="")
parser.add_option('--compiler', dest='compiler',
help='C/C++ compiler', default="")
parser.add_option('-r', '--ref', dest='ref',
help='set reference compiler for compare', default="")
parser.add_option('-f', '--file', dest='in_file',
help='file to save perf output', default="")
(options, args) = parser.parse_args()
perf(options, args)

570
run_tests.py
View File

@@ -1,165 +1,37 @@
#!/usr/bin/python
#
# Copyright (c) 2013, Intel Corporation
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# test-running driver for ispc
from optparse import OptionParser
import multiprocessing
from ctypes import c_int
import os
import sys
import glob
import re
import signal
import random
import string
import subprocess
import shlex
import platform
import tempfile
import os.path
import time
# disable fancy error/warning printing with ANSI colors, so grepping for error
# messages doesn't get confused
os.environ["TERM"] = "dumb"
# This script is affected by http://bugs.python.org/issue5261 on OSX 10.5 Leopard
# git history has a workaround for that issue.
is_windows = (platform.system() == 'Windows' or
'CYGWIN_NT' in platform.system())
parser = OptionParser()
parser.add_option("-r", "--random-shuffle", dest="random", help="Randomly order tests",
default=False, action="store_true")
parser.add_option("-g", "--generics-include", dest="include_file", help="Filename for header implementing functions for generics",
default=None)
parser.add_option("-f", "--ispc-flags", dest="ispc_flags", help="Additional flags for ispc (-g, -O1, ...)",
default="")
parser.add_option('-t', '--target', dest='target',
help='Set compilation target (sse2-i32x4, sse2-i32x8, sse4-i32x4, sse4-i32x8, sse4-i16x8, sse4-i8x16, avx1-i32x8, avx1-i32x16, avx1.1-i32x8, avx1.1-i32x16, avx2-i32x8, avx2-i32x16, generic-x1, generic-x4, generic-x8, generic-x16, generic-x32, generic-x64)',
default="sse4")
parser.add_option('-a', '--arch', dest='arch',
help='Set architecture (arm, x86, x86-64)',
default="x86-64")
parser.add_option("-c", "--compiler", dest="compiler_exe", help="Compiler binary to use to run tests",
default=None)
parser.add_option('-o', '--no-opt', dest='no_opt', help='Disable optimization',
default=False, action="store_true")
parser.add_option('-j', '--jobs', dest='num_jobs', help='Maximum number of jobs to run in parallel',
default="1024", type="int")
parser.add_option('-v', '--verbose', dest='verbose', help='Enable verbose output',
default=False, action="store_true")
parser.add_option('--wrap-exe', dest='wrapexe',
help='Executable to wrap test runs with (e.g. "valgrind")',
default="")
parser.add_option('--time', dest='time', help='Enable time output',
default=False, action="store_true")
parser.add_option('--non-interactive', dest='non_interactive', help='Disable interactive status updates',
default=False, action="store_true")
(options, args) = parser.parse_args()
if options.target == 'neon':
options.arch = 'arm'
# use relative path to not depend on host directory, which may possibly
# have white spaces and unicode characters.
if not is_windows:
ispc_exe = "./ispc"
else:
ispc_exe = ".\\Release\\ispc.exe"
# checks the required ispc compiler otherwise prints an error message
if not os.path.exists(ispc_exe):
sys.stderr.write("Fatal error: missing ispc compiler: %s\n" % ispc_exe)
sys.exit()
ispc_exe += " " + options.ispc_flags
if __name__ == '__main__':
sys.stdout.write("ispc compiler: %s\n" % ispc_exe)
is_generic_target = (options.target.find("generic-") != -1 and
options.target != "generic-1")
if is_generic_target and options.include_file == None:
if options.target == "generic-4":
sys.stderr.write("No generics #include specified; using examples/intrinsics/sse4.h\n")
options.include_file = "examples/intrinsics/sse4.h"
elif options.target == "generic-8":
sys.stderr.write("No generics #include specified and no default available for \"generic-8\" target.\n")
sys.exit(1)
elif options.target == "generic-16":
sys.stderr.write("No generics #include specified; using examples/intrinsics/generic-16.h\n")
options.include_file = "examples/intrinsics/generic-16.h"
elif options.target == "generic-32":
sys.stderr.write("No generics #include specified; using examples/intrinsics/generic-32.h\n")
options.include_file = "examples/intrinsics/generic-32.h"
elif options.target == "generic-64":
sys.stderr.write("No generics #include specified; using examples/intrinsics/generic-64.h\n")
options.include_file = "examples/intrinsics/generic-64.h"
if options.compiler_exe == None:
if is_windows:
options.compiler_exe = "cl.exe"
else:
options.compiler_exe = "g++"
# checks the required compiler otherwise prints an error message
PATH_dir = string.split(os.getenv("PATH"), os.pathsep)
compiler_exists = False
for counter in PATH_dir:
if os.path.exists(counter + os.sep + options.compiler_exe):
compiler_exists = True
break
if not compiler_exists:
sys.stderr.write("Fatal error: missing the required compiler: %s \n" %
options.compiler_exe)
sys.exit()
ispc_root = "."
# if no specific test files are specified, run all of the tests in tests/,
# failing_tests/, and tests_errors/
if len(args) == 0:
files = glob.glob(ispc_root + os.sep + "tests" + os.sep + "*ispc") + \
glob.glob(ispc_root + os.sep + "failing_tests" + os.sep + "*ispc") + \
glob.glob(ispc_root + os.sep + "tests_errors" + os.sep + "*ispc")
else:
if is_windows:
argfiles = [ ]
for f in args:
# we have to glob ourselves if this is being run under a DOS
# shell, as it passes wildcard as is.
argfiles += glob.glob(f)
else:
argfiles = args
files = [ ]
for f in argfiles:
if os.path.splitext(string.lower(f))[1] != ".ispc":
sys.stdout.write("Ignoring file %s, which doesn't have an .ispc extension.\n" % f)
else:
files += [ f ]
# max_test_length is used to issue exact number of whitespace characters when
# updating status. Otherwise update causes new lines standard 80 char terminal
# on both Linux and Windows.
max_test_length = 0
for f in files:
max_test_length = max(max_test_length, len(f))
# randomly shuffle the tests if asked to do so
if (options.random):
random.seed()
random.shuffle(files)
# counter
total_tests = 0
# utility routine to print an update on the number of tests that have been
# finished. Should be called with the lock held..
def update_progress(fn, total_tests_arg, counter, max_test_length_arg):
@@ -176,7 +48,7 @@ def update_progress(fn, total_tests_arg, counter, max_test_length_arg):
def run_command(cmd):
if options.verbose:
sys.stdout.write("Running: %s\n" % cmd)
print_debug("Running: %s\n" % cmd, s, run_tests_log)
# Here's a bit tricky part. To pass a command for execution we should
# break down the line in to arguments. shlex class is designed exactly
@@ -204,9 +76,9 @@ def run_cmds(compile_cmds, run_cmd, filename, expect_failure):
(return_code, output) = run_command(cmd)
compile_failed = (return_code != 0)
if compile_failed:
sys.stdout.write("Compilation of test %s failed \n" % filename)
print_debug("Compilation of test %s failed \n" % filename, s, run_tests_log)
if output != "":
sys.stdout.write("%s" % output.encode("utf-8"))
print_debug("%s" % output.encode("utf-8"), s, run_tests_log)
return (1, 0)
(return_code, output) = run_command(run_cmd)
@@ -215,11 +87,11 @@ def run_cmds(compile_cmds, run_cmd, filename, expect_failure):
surprise = ((expect_failure and not run_failed) or
(not expect_failure and run_failed))
if surprise == True:
sys.stderr.write("Test %s %s (return code %d) \n" % \
print_debug("Test %s %s (return code %d) \n" % \
(filename, "unexpectedly passed" if expect_failure else "failed",
return_code))
return_code), s, run_tests_log)
if output != "":
sys.stdout.write("%s\n" % output.encode("utf-8"))
print_debug("%s\n" % output.encode("utf-8"), s, run_tests_log)
if surprise == True:
return (0, 1)
else:
@@ -298,11 +170,11 @@ def run_test(testname):
file.close()
if re.search(firstline, output) == None:
sys.stderr.write("Didn't see expected error message %s from test %s.\nActual output:\n%s\n" % \
(firstline, testname, output))
print_debug("Didn't see expected error message %s from test %s.\nActual output:\n%s\n" % \
(firstline, testname, output), s, run_tests_log)
return (1, 0)
elif got_error == False:
sys.stderr.write("Unexpectedly no errors issued from test %s\n" % testname)
print_debug("Unexpectedly no errors issued from test %s\n" % testname, s, run_tests_log)
return (1, 0)
else:
return (0, 0)
@@ -328,8 +200,7 @@ def run_test(testname):
break
file.close()
if match == -1:
sys.stderr.write("Fatal error: unable to find function signature " + \
"in test %s\n" % testname)
error("unable to find function signature in test %s\n" % testname, 0)
return (1, 0)
else:
global is_generic_target
@@ -404,7 +275,21 @@ def run_test(testname):
# pull tests to run from the given queue and run them. Multiple copies of
# this function will be running in parallel across all of the CPU cores of
# the system.
def run_tasks_from_queue(queue, queue_ret, queue_skip, total_tests_arg, max_test_length_arg, counter, mutex):
def run_tasks_from_queue(queue, queue_ret, queue_skip, total_tests_arg, max_test_length_arg, counter, mutex, glob_var):
# This is needed on windows because windows doen't copy globals from parent process whili multiprocessing
global is_windows
is_windows = glob_var[0]
global options
options = glob_var[1]
global s
s = glob_var[2]
global ispc_exe
ispc_exe = glob_var[3]
global is_generic_target
is_generic_target = glob_var[4]
global run_tests_log
run_tests_log = glob_var[5]
if is_windows:
tmpdir = "tmp%d" % os.getpid()
os.mkdir(tmpdir)
@@ -454,7 +339,256 @@ def sigint(signum, frame):
t.terminate()
sys.exit(1)
if __name__ == '__main__':
def file_check(compfails, runfails):
errors = len(compfails) + len(runfails)
new_compfails = []
new_runfails = []
new_passes_compfails = []
new_passes_runfails = []
# Open file fail_db.txt
f = open(test_states, 'r')
f_lines = f.readlines()
f.close()
# Detect OS
if platform.system() == 'Windows' or 'CYGWIN_NT' in platform.system():
OS = "Windows"
else:
if platform.system() == 'Darwin':
OS = "Mac"
else:
OS = "Linux"
# Detect opt_set
if options.no_opt == True:
opt = "-O0"
else:
opt = "-O2"
# Detect LLVM version
temp1 = common.detect_version(ispc_exe + " --version")
llvm_version = temp1[-10:-2]
#Detect compiler version
if is_windows == False:
temp1 = common.detect_version(options.compiler_exe + " --version")
temp2 = temp1.split(" ")
compiler_version = temp2[0] + temp2[2][0:4]
else:
compiler_version = "cl"
new_line = " "+options.arch.rjust(6)+" "+options.target.rjust(14)+" "+OS.rjust(7)+" "+llvm_version+" "+compiler_version.rjust(10)+" "+opt+" *\n"
new_compfails = compfails[:]
new_runfails = runfails[:]
new_f_lines = f_lines[:]
for j in range(0, len(f_lines)):
if (((" "+options.arch+" ") in f_lines[j]) and
((" "+options.target+" ") in f_lines[j]) and
((" "+OS+" ") in f_lines[j]) and
((" "+llvm_version+" ") in f_lines[j]) and
((" "+compiler_version+" ") in f_lines[j]) and
((" "+opt+" ") in f_lines[j])):
if (" compfail " in f_lines[j]):
f = 0
for i in range(0, len(compfails)):
if compfails[i] in f_lines[j]:
new_compfails.remove(compfails[i])
else:
f = f + 1
if f == len(compfails):
temp3 = f_lines[j].split(" ")
new_passes_compfails.append(temp3[0])
if options.update == "FP":
new_f_lines.remove(f_lines[j])
if (" runfail " in f_lines[j]):
f = 0
for i in range(0, len(runfails)):
if runfails[i] in f_lines[j]:
new_runfails.remove(runfails[i])
else:
f = f + 1
if f == len(runfails):
temp3 = f_lines[j].split(" ")
new_passes_runfails.append(temp3[0])
if options.update == "FP":
new_f_lines.remove(f_lines[j])
if len(new_runfails) != 0:
print_debug("NEW RUNFAILS:\n", s, run_tests_log)
for i in range (0,len(new_runfails)):
new_f_lines.append(new_runfails[i] + " runfail " + new_line)
print_debug("\t" + new_runfails[i] + "\n", s, run_tests_log)
if len(new_compfails) != 0:
print_debug("NEW COMPFAILS:\n", s, run_tests_log)
for i in range (0,len(new_compfails)):
new_f_lines.append(new_compfails[i] + " compfail " + new_line)
print_debug("\t" + new_compfails[i] + "\n", s, run_tests_log)
if len(new_passes_runfails) != 0:
print_debug("NEW PASSES after RUNFAILS:\n", s, run_tests_log)
for i in range (0,len(new_passes_runfails)):
print_debug("\t" + new_passes_runfails[i] + "\n", s, run_tests_log)
if len(new_passes_compfails) != 0:
print_debug("NEW PASSES after COMPFAILS:\n", s, run_tests_log)
for i in range (0,len(new_passes_compfails)):
print_debug("\t" + new_passes_compfails[i] + "\n", s, run_tests_log)
if options.update != "":
output = open(test_states, 'w')
output.writelines(new_f_lines)
output.close()
return [new_runfails, new_compfails, new_passes_runfails, new_passes_compfails, new_line, errors]
def verify():
# Open file fail_db.txt
f = open(test_states, 'r')
f_lines = f.readlines()
f.close()
check = [["g++", "clang", "cl"],["-O0", "-O2"],["x86","x86-64"],
["Linux","Windows","Mac"],["LLVM 3.1","LLVM 3.2","LLVM 3.3","LLVM head"],
["sse2-i32x4", "sse2-i32x8", "sse4-i32x4", "sse4-i32x8", "sse4-i16x8",
"sse4-i8x16", "avx1-i32x8", "avx1-i32x16", "avx1.1-i32x8", "avx1.1-i32x16",
"avx2-i32x8", "avx2-i32x16", "generic-1", "generic-4", "generic-8",
"generic-16", "generic-32", "generic-64"]]
for i in range (0,len(f_lines)):
if f_lines[i][0] == "%":
continue
for j in range(0,len(check)):
temp = 0
for t in range(0,len(check[j])):
if " " + check[j][t] + " " in f_lines[i]:
temp = temp + 1
if temp != 1:
print_debug("error in line " + str(i) + "\n", False, run_tests_log)
break
def run_tests(options1, args, print_version):
global options
options = options1
global s
s = options.silent
# prepare run_tests_log and test_states files
global run_tests_log
if options.in_file:
run_tests_log = os.getcwd() + os.sep + options.in_file
if print_version == 1:
common.remove_if_exists(run_tests_log)
else:
run_tests_log = ""
global test_states
test_states = "fail_db.txt"
if options.verify:
verify()
return 0
# disable fancy error/warning printing with ANSI colors, so grepping for error
# messages doesn't get confused
os.environ["TERM"] = "dumb"
# This script is affected by http://bugs.python.org/issue5261 on OSX 10.5 Leopard
# git history has a workaround for that issue.
global is_windows
is_windows = (platform.system() == 'Windows' or
'CYGWIN_NT' in platform.system())
if options.target == 'neon':
options.arch = 'arm'
# use relative path to not depend on host directory, which may possibly
# have white spaces and unicode characters.
global ispc_exe
if not is_windows:
ispc_exe = "./ispc"
else:
ispc_exe = ".\\Release\\ispc.exe"
# checks the required ispc compiler otherwise prints an error message
if not os.path.exists(ispc_exe):
error("missing ispc compiler: %s\n" % ispc_exe, 1)
ispc_exe += " " + options.ispc_flags
print_debug("ispc compiler: %s\n" % ispc_exe, s, run_tests_log)
global is_generic_target
is_generic_target = (options.target.find("generic-") != -1 and
options.target != "generic-1" and options.target != "generic-x1")
if is_generic_target and options.include_file == None:
if options.target == "generic-4" or options.target == "generic-x4":
error("No generics #include specified; using examples/intrinsics/sse4.h\n", 2)
options.include_file = "examples/intrinsics/sse4.h"
options.target = "generic-4"
elif options.target == "generic-8" or options.target == "generic-x8":
error("No generics #include specified and no default available for \"generic-8\" target.\n", 1)
options.target = "generic-8"
elif options.target == "generic-16" or options.target == "generic-x16":
error("No generics #include specified; using examples/intrinsics/generic-16.h\n", 2)
options.include_file = "examples/intrinsics/generic-16.h"
options.target = "generic-16"
elif options.target == "generic-32" or options.target == "generic-x32":
error("No generics #include specified; using examples/intrinsics/generic-32.h\n", 2)
options.include_file = "examples/intrinsics/generic-32.h"
options.target = "generic-32"
elif options.target == "generic-64" or options.target == "generic-x64":
error("No generics #include specified; using examples/intrinsics/generic-64.h\n", 2)
options.include_file = "examples/intrinsics/generic-64.h"
options.target = "generic-64"
if options.compiler_exe == None:
if is_windows:
options.compiler_exe = "cl.exe"
else:
options.compiler_exe = "g++"
# checks the required compiler otherwise prints an error message
PATH_dir = string.split(os.getenv("PATH"), os.pathsep)
compiler_exists = False
for counter in PATH_dir:
if os.path.exists(counter + os.sep + options.compiler_exe):
compiler_exists = True
break
if not compiler_exists:
error("missing the required compiler: %s \n" % options.compiler_exe, 1)
# print compilers versions
if print_version > 0:
common.print_version(ispc_exe, "", options.compiler_exe, False, run_tests_log, is_windows)
ispc_root = "."
# if no specific test files are specified, run all of the tests in tests/,
# failing_tests/, and tests_errors/
if len(args) == 0:
files = glob.glob(ispc_root + os.sep + "tests" + os.sep + "*ispc") + \
glob.glob(ispc_root + os.sep + "failing_tests" + os.sep + "*ispc") + \
glob.glob(ispc_root + os.sep + "tests_errors" + os.sep + "*ispc")
else:
if is_windows:
argfiles = [ ]
for f in args:
# we have to glob ourselves if this is being run under a DOS
# shell, as it passes wildcard as is.
argfiles += glob.glob(f)
else:
argfiles = args
files = [ ]
for f in argfiles:
if os.path.splitext(string.lower(f))[1] != ".ispc":
error("Ignoring file %s, which doesn't have an .ispc extension.\n" % f, 2)
else:
files += [ f ]
# max_test_length is used to issue exact number of whitespace characters when
# updating status. Otherwise update causes new lines standard 80 char terminal
# on both Linux and Windows.
max_test_length = 0
for f in files:
max_test_length = max(max_test_length, len(f))
# randomly shuffle the tests if asked to do so
if (options.random):
random.seed()
random.shuffle(files)
# counter
total_tests = len(files)
compile_error_files = [ ]
@@ -463,7 +597,7 @@ if __name__ == '__main__':
nthreads = min(multiprocessing.cpu_count(), options.num_jobs)
nthreads = min(nthreads, len(files))
sys.stdout.write("Running %d jobs in parallel. Running %d tests.\n" % (nthreads, total_tests))
print_debug("Running %d jobs in parallel. Running %d tests.\n" % (nthreads, total_tests), s, run_tests_log)
# put each of the test filenames into a queue
q = multiprocessing.Queue()
@@ -483,8 +617,10 @@ if __name__ == '__main__':
start_time = time.time()
# launch jobs to run tests
glob_var = [is_windows, options, s, ispc_exe, is_generic_target, run_tests_log]
for x in range(nthreads):
t = multiprocessing.Process(target=run_tasks_from_queue, args=(q, qret, qskip, total_tests, max_test_length, finished_tests_counter, finished_tests_counter_lock))
t = multiprocessing.Process(target=run_tasks_from_queue, args=(q, qret, qskip, total_tests,
max_test_length, finished_tests_counter, finished_tests_counter_lock, glob_var))
task_threads.append(t)
t.start()
@@ -493,35 +629,97 @@ if __name__ == '__main__':
for t in task_threads:
t.join()
if options.non_interactive == False:
sys.stdout.write("\n")
print_debug("\n", s, run_tests_log)
elapsed_time = time.time() - start_time
while not qret.empty():
(c, r, s) = qret.get()
(c, r, skip) = qret.get()
compile_error_files += c
run_error_files += r
skip_files += s
skip_files += skip
if options.non_interactive:
sys.stdout.write(" Done %d / %d\n" % (finished_tests_counter.value, total_tests))
print_debug(" Done %d / %d\n" % (finished_tests_counter.value, total_tests), s, run_tests_log)
if len(skip_files) > 0:
skip_files.sort()
sys.stdout.write("%d / %d tests SKIPPED:\n" % (len(skip_files), total_tests))
print_debug("%d / %d tests SKIPPED:\n" % (len(skip_files), total_tests), s, run_tests_log)
for f in skip_files:
sys.stdout.write("\t%s\n" % f)
print_debug("\t%s\n" % f, s, run_tests_log)
if len(compile_error_files) > 0:
compile_error_files.sort()
sys.stdout.write("%d / %d tests FAILED compilation:\n" % (len(compile_error_files), total_tests))
print_debug("%d / %d tests FAILED compilation:\n" % (len(compile_error_files), total_tests), s, run_tests_log)
for f in compile_error_files:
sys.stdout.write("\t%s\n" % f)
print_debug("\t%s\n" % f, s, run_tests_log)
if len(run_error_files) > 0:
run_error_files.sort()
sys.stdout.write("%d / %d tests FAILED execution:\n" % (len(run_error_files), total_tests))
print_debug("%d / %d tests FAILED execution:\n" % (len(run_error_files), total_tests), s, run_tests_log)
for f in run_error_files:
sys.stdout.write("\t%s\n" % f)
print_debug("\t%s\n" % f, s, run_tests_log)
R = file_check(compile_error_files, run_error_files)
if options.time:
sys.stdout.write("Elapsed time: %d s\n" % elapsed_time)
print_debug("Elapsed time: %d s\n" % elapsed_time, s, run_tests_log)
sys.exit(len(compile_error_files) + len(run_error_files))
return R
from optparse import OptionParser
import multiprocessing
from ctypes import c_int
import os
import sys
import glob
import re
import signal
import random
import string
import subprocess
import shlex
import platform
import tempfile
import os.path
import time
# our functions
import common
print_debug = common.print_debug
error = common.error
if __name__ == "__main__":
parser = OptionParser()
parser.add_option("-r", "--random-shuffle", dest="random", help="Randomly order tests",
default=False, action="store_true")
parser.add_option("-g", "--generics-include", dest="include_file", help="Filename for header implementing functions for generics",
default=None)
parser.add_option("-f", "--ispc-flags", dest="ispc_flags", help="Additional flags for ispc (-g, -O1, ...)",
default="")
parser.add_option('-t', '--target', dest='target',
help='Set compilation target (sse2-i32x4, sse2-i32x8, sse4-i32x4, sse4-i32x8, sse4-i16x8, sse4-i8x16, avx1-i32x8, avx1-i32x16, avx1.1-i32x8, avx1.1-i32x16, avx2-i32x8, avx2-i32x16, generic-x1, generic-x4, generic-x8, generic-x16, generic-x32, generic-x64)',
default="sse4")
parser.add_option('-a', '--arch', dest='arch',
help='Set architecture (arm, x86, x86-64)',
default="x86-64")
parser.add_option("-c", "--compiler", dest="compiler_exe", help="C/C++ compiler binary to use to run tests",
default=None)
parser.add_option('-o', '--no-opt', dest='no_opt', help='Disable optimization',
default=False, action="store_true")
parser.add_option('-j', '--jobs', dest='num_jobs', help='Maximum number of jobs to run in parallel',
default="1024", type="int")
parser.add_option('-v', '--verbose', dest='verbose', help='Enable verbose output',
default=False, action="store_true")
parser.add_option('--wrap-exe', dest='wrapexe',
help='Executable to wrap test runs with (e.g. "valgrind")',
default="")
parser.add_option('--time', dest='time', help='Enable time output',
default=False, action="store_true")
parser.add_option('--non-interactive', dest='non_interactive', help='Disable interactive status updates',
default=False, action="store_true")
parser.add_option('-u', "--update", dest='update', help='Update file with fails (F of FP)', default="")
parser.add_option('-s', "--silent", dest='silent', help='enable silent mode without any output', default=False,
action = "store_true")
parser.add_option("--file", dest='in_file', help='file to save run_tests output', default="")
parser.add_option("--verify", dest='verify', help='verify the file fail_db.txt', default=False, action="store_true")
(options, args) = parser.parse_args()
L = run_tests(options, args, 1)
exit(0)