Merge pull request #630 from ifilippov/testing

new changes in test system

alloy.py

@@ -294,9 +294,7 @@ def execute_stability(stability, R, print_version):
 def run_special_tests():
    i = 5 
 
-def validation_run(only, only_targets, reference_branch, number, notify, update, make):
+def validation_run(only, only_targets, reference_branch, number, notify, update, speed_number, make, perf_llvm):
-    if os.environ["ISPC_HOME"] != os.getcwd():
-        error("you ISPC_HOME and your current pass are different!\n", 2)
     os.chdir(os.environ["ISPC_HOME"])
     os.environ["PATH"] = os.environ["ISPC_HOME"] + ":" + os.environ["PATH"]
     if options.notify != "":
@@ -322,7 +320,7 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
         stability.random = False
         stability.ispc_flags = ""
         stability.compiler_exe = None
-        stability.num_jobs = 1024
+        stability.num_jobs = speed_number
         stability.verbose = False
         stability.time = False
         stability.non_interactive = True
@@ -476,7 +474,8 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
 # prepare LLVM 3.3 as newest LLVM
         need_LLVM = check_LLVM(["3.3"])
         if len(need_LLVM) != 0:
-            build_LLVM(need_LLVM[i], "", "", "", False, False, False, True, False, make)
+            build_LLVM(need_LLVM[0], "", "", "", False, False, False, True, False, make)
+        if perf_llvm == False:
             # prepare reference point. build both test and reference compilers
             try_do_LLVM("apply git", "git branch", True)
             temp4 = take_lines("git branch", "all")
@@ -498,6 +497,13 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
                 try_do_LLVM("return current branch ", "git stash pop", True)
             sys.stdout.write("You can interrupt script now.\n")
             build_ispc("3.3", make)
+        else:
+            # build compiler with two different LLVM versions
+            if len(check_LLVM([reference_branch])) != 0:
+                error("you haven't got llvm called " + reference_branch, 1)
+            build_ispc("3.3", make)
+            os.rename("ispc", "ispc_ref")
+            build_ispc(reference_branch, make)
 # begin validation run for performance. output is inserted into perf()
         perf.perf(performance, [])
         if options.notify != "":
@@ -560,16 +566,22 @@ def Main():
     stability_log = os.getcwd() + os.sep + f_date + os.sep + "stability.log"
     current_path = os.getcwd()
     make = "make -j" + options.speed
+    if os.environ["ISPC_HOME"] != os.getcwd():
+        error("you ISPC_HOME and your current path are different!\n", 2)
+    if options.perf_llvm == True:
+        if options.branch == "master":
+            options.branch = "trunk"
     try:
         if options.build_llvm:
             build_LLVM(options.version, options.revision, options.folder, options.tarball,
                     options.debug, options.selfbuild, options.extra, False, options.force, make)
         if options.validation_run:
             validation_run(options.only, options.only_targets, options.branch,
-                    options.number_for_performance, options.notify, options.update, make)
+                    options.number_for_performance, options.notify, options.update, int(options.speed),
+                    make, options.perf_llvm)
     finally:
         os.chdir(current_path)
-        date_name = "alloy_results_" + datetime.datetime.now().strftime('%d_%m_%Y_%H_%M_%S')
+        date_name = "alloy_results_" + datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
         if os.path.exists(date_name):
             error("It's forbidden to run alloy two times in a second, logs are in ./logs", 1)
         os.rename(f_date, date_name)
@@ -661,6 +673,8 @@ run_group.add_option('--only', dest='only',
         '-O0, -O2, x86, x86-64, stability (test only stability), performance (test only performance)\n' +
         'build (only build with different LLVM), 3.1, 3.2, 3.3, trunk, native (do not use SDE), current (do not rebuild ISPC).',
         default="")
+run_group.add_option('--perf_LLVM', dest='perf_llvm',
+    help='compare LLVM 3.3 with "--compare-with", default trunk', default=False, action='store_true')
 parser.add_option_group(run_group)
 # options for activity "setup PATHS"
 setup_group = OptionGroup(parser, "Options for setup",

run_tests.py

@@ -378,6 +378,11 @@ def file_check(compfails, runfails):
         compiler_version = options.compiler_exe + temp3.group()
     else:
         compiler_version = "cl"
+    possible_compilers = ["g++4.4", "g++4.7", "clang++3.3", "cl"]
+    if not compiler_version in possible_compilers:
+        error("\n**********\nWe don't have history of fails for compiler " +
+                compiler_version +
+                "\nAll fails will be new!!!\n**********", 2)
     new_line = " "+options.arch.rjust(6)+" "+options.target.rjust(14)+" "+OS.rjust(7)+" "+llvm_version+" "+compiler_version.rjust(10)+" "+opt+" *\n"
 
     new_compfails = compfails[:]
@@ -675,7 +680,11 @@ def run_tests(options1, args, print_version):
     if len(compile_error_files) == 0 and len(run_error_files) == 0:
         print_debug("No fails\n", s, run_tests_log)
 
+    if len(args) == 0:
         R = file_check(compile_error_files, run_error_files)
+    else:
+        error("don't check new fails for incomplete suite of tests", 2)
+        R = 0
 
     if options.time:
         print_debug("Elapsed time: %d s\n" % elapsed_time, s, run_tests_log)