From 09ec2db6e9f1570e22aec99674daf50626729361 Mon Sep 17 00:00:00 2001
From: Anton Mitrokhin <anton.mitrokhin@phystech.edu>
Date: Sun, 20 Jul 2014 13:21:05 +0400
Subject: [PATCH 1/7] add advanced test completion logging

---
 alloy.py           |   5 +-
 common.py          |  33 ++--
 regression.py      | 376 +++++++++++++++++++++++++++++++++++++++++++++
 regression_read.py |  70 +++++++++
 run_tests.py       |  45 ++++--
 5 files changed, 509 insertions(+), 20 deletions(-)
 create mode 100644 regression.py
 create mode 100755 regression_read.py

diff --git a/alloy.py b/alloy.py
index 8bd26d6c..f5f45292 100755
--- a/alloy.py
+++ b/alloy.py
@@ -631,7 +631,8 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
         if options.notify != "":
             attach_mail_file(msg, performance.in_file, "performance.log")
             attach_mail_file(msg, "." + os.sep + "logs" + os.sep + "perf_build.log", "perf_build.log")
-
+# dumping gathered info to the file
+    commom.ex_state.dump(alloy_folder + "test_table.dump", commom.ex_state.tt)
 # sending e-mail with results
     if options.notify != "":
         fp = open(os.environ["ISPC_HOME"] + os.sep + "notify_log.log", 'rb')
@@ -702,6 +703,8 @@ def Main():
     f_date = "logs"
     common.remove_if_exists(f_date)
     os.makedirs(f_date)
+    global alloy_folder
+    alloy_folder = os.getcwd() + os.sep + f_date + os.sep
     global alloy_build
     alloy_build = os.getcwd() + os.sep + f_date + os.sep + "alloy_build.log"
     global stability_log
diff --git a/common.py b/common.py
index 9fe611f0..0fda9b45 100755
--- a/common.py
+++ b/common.py
@@ -36,23 +36,36 @@ import sys
 import os
 import errno
 import shutil
-class ExecutionStatGatherer:
-    def __init__(self):
-        optimizations = ['O0', 'O2']
-        architectures = ['x86', 'x86-64']
-        all_est_targets  = ['sse2-i32x4', 'sse2-i32x8', 'sse4-i32x4', 'sse4-i32x8', 'sse4-i16x8',
-                            'sse4-i8x16', 'avx1-i32x8', 'avx1-i32x16', 'avx1.1-i32x8', 
-                            'avx1.1-i32x16', 'avx2-i32x8', 'avx2-i32x16', 'generic-x1', 
-                            'generic-x4', 'generic-x8', 'generic-x16', 'generic-x32', 
-                            'generic-x64', 'knc']
+from regression import *
 
 
+# this class instance is intended to gather and store all information
+# regarding the testing process. Defined in 'regression.py'
+ex_state = ExecutionStateGatherer()
+
+# load/save almost every object to a file (good for bug reproducing)
+def dump(fname, obj):
+    import pickle
+    with open(fname, 'w') as fp:
+        pickle.dump(obj, fp)  
+
+def undump(fname):
+    import pickle
+    with open(fname, 'r') as fp:
+        obj = pickle.load(fp) 
+    return obj
+
+# retrieve the host name
+def get_host_name():
+    import socket
+    return socket.gethostname()
+
 def write_to_file(filename, line):
     f = open(filename, 'a')
     f.writelines(line)
     f.close()
 
-#remove file if it exists
+# remove file if it exists
 def remove_if_exists(filename):
     if os.path.exists(filename):
         if os.path.isdir(filename):
diff --git a/regression.py b/regression.py
new file mode 100644
index 00000000..6ce0c532
--- /dev/null
+++ b/regression.py
@@ -0,0 +1,376 @@
+#!/usr/bin/python
+#
+#  Copyright (c) 2014, Intel Corporation 
+#  All rights reserved.
+# 
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are
+#  met:
+# 
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+# 
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+# 
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived from
+#      this software without specific prior written permission.
+# 
+# 
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+#   IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+#   TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+#   PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+#   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+#   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+#   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# // Authors: Anton Mitrokhin, Vsevolod Livinskiy
+
+class TestResult(object):
+    """
+    this class stores basicly two integers which stand for the result
+    of the test: (runfail{0/1}, compfail{0/1}). other values are 
+    deemed invalid. the __cmp__ function of this class is used to 
+    define what test regression actually is.
+    """
+    def __init__(self, runfailed, compfailed):
+        self.runfailed, self.compfailed = (runfailed, compfailed)
+
+    def __cmp__(self, other):
+        if isinstance(other, TestResult):
+            if self.runfailed == other.runfailed   and \
+               self.compfailed == other.compfailed:
+                return 0
+            elif self.compfailed > other.compfailed:
+                return 1
+            elif self.runfailed > other.runfailed and \
+                 self.compfailed == other.compfailed:
+                return 1
+            else:
+                return -1
+
+        raise RuntimeError("Wrong type for comparioson")
+        return NotImplemented
+
+    def __repr__(self):
+        if (self.runfailed < 0 or self.compfailed < 0):
+            return "(Undefined)"
+        return "(r%d c%d)" % (self.runfailed, self.compfailed)
+
+
+class TestCase(object):
+    """
+    the TestCase() is a combination of parameters the tast was run with:
+    the architecture (x86, x86-64 ...), compiler optimization (-O0, -O2 ...)
+    and target (sse, avx ...). we also store the result of the test here.
+    """
+    def __init__(self, arch, opt, target):
+        self.arch, self.opt, self.target = (arch, opt, target)
+        self.result = TestResult(-1, -1)
+
+    def __repr__(self):
+        string = "%s %s %s: " % (self.arch, self.opt, self.target)
+        string = string + repr(self.result) + '\n'
+        return string
+
+    def __hash__(self):
+        return hash(self.arch + self.opt + self.target)
+
+    def __ne__(self, other):
+        if isinstance(other, TestCase):
+            if hash(self.arch + self.opt + self.target) != hash(other):
+                return True
+            return False
+        raise RuntimeError("Wrong type for comparioson")
+        return NotImplemented
+
+    def __eq__(self, other):
+        if isinstance(other, TestCase):
+            return not self.__ne__(other)   
+        raise RuntimeError("Wrong type for comparioson")
+        return NotImplemented
+
+
+class Test(object):
+    """
+    Test() stores all TestCase() objects for a given test file name
+    i.e. all archs/opts/targets/ and corresponding testing results.
+    """
+    def __init__(self, name):
+        self.name = name
+        self.test_cases = []
+
+    def add_result(self, test_case):
+        if test_case in self.test_cases:
+            raise RuntimeError("This test case is already in the list: " + repr(test_case))
+            return
+        self.test_cases.append(test_case)
+
+    def __repr__(self):
+        string = self.name + '\n'
+        string = string.rjust(20)
+        for test_case in self.test_cases:
+            string += repr(test_case).rjust(60) + '\n'
+        return string
+    
+    def __hash__(self):
+        return hash(self.name)
+
+    def __ne__(self, other):
+        if isinstance(other, Test):
+            if hash(self) != hash(other):
+                return True
+            return False
+        return NotImplemented
+
+    def __eq__(self, other):
+        if isinstance(other, Test):
+            return not self.__ne__(other)   
+        return NotImplemented
+
+
+class RegressionInfo(object):
+    """
+    service class which provides some statistics on a given regression.
+    the regression test names and cases are given in a form of Test() objects
+    with empty (-1, -1) results 
+    """
+    def __init__(self, revision_old, revision_new, tests):
+        self.revision_old, self.revision_new = (revision_old, revision_new)
+        self.tests = tests
+        self.archfailes = {}
+        self.optfails = {}
+        self.targetfails = {}
+        self.testfails = {}
+        self.archs = []
+        self.opts = []
+        self.targets = []
+
+        for test in tests:
+            for test_case in test.test_cases:
+                self.inc_dictionary(self.testfails, test.name)
+                self.inc_dictionary(self.archfailes, test_case.arch)
+                self.inc_dictionary(self.optfails, test_case.opt)
+                self.inc_dictionary(self.targetfails, test_case.target)
+        
+        self.archs = self.archfailes.keys()
+        self.opts = self.optfails.keys()
+        self.targets = self.targetfails.keys()
+
+    def inc_dictionary(self, dictionary, key):
+        if key not in dictionary:
+            dictionary[key] = 0
+        dictionary[key] += 1
+
+    def __repr__(self):
+        string = "Regression of LLVM revision %s in comparison to %s\n" % (self.revision_new, self.revision_old)
+        string += repr(self.tests) + '\n'
+        string += str(self.testfails) + '\n'
+        string += str(self.archfailes) + '\n'
+        string += str(self.optfails) + '\n'
+        string += str(self.targetfails) + '\n'
+
+        return string
+
+        
+class TestTable(object):
+    """
+    the table which stores a tuple of Test() objects (one per revision) and has some 
+    convenience methods for dealing with them
+    """
+    def __init__(self):
+        """ This dictionary contains {rev: [test1, test2, ...], ...}, where 'rev' is a string (revision name) and 'test#'
+        is a Test() object instance """
+        self.table = {}
+
+    def add_result(self, revision_name, test_name, arch, opt, target, runfailed, compfailed):
+        revision_name = str(revision_name)
+        if revision_name not in self.table:
+            self.table[revision_name] = []
+        
+        test_case = TestCase(arch, opt, target)
+        test_case.result = TestResult(runfailed, compfailed)
+
+        for test in self.table[revision_name]:
+            if test.name == test_name:
+                test.add_result(test_case)
+                return
+        
+        test = Test(test_name)
+        test.add_result(test_case)
+        self.table[revision_name].append(test)
+
+
+    def test_intersection(self, test1, test2):
+        """ Return test cases common for test1 and test2. If test names are different than there is nothing in common """
+        if test1.name != test2.name:
+            return []
+        return list(set(test1.test_cases) & set(test2.test_cases))
+
+    def test_regression(self, test1, test2):
+        """ Return the tuple of empty (i.e. with undefined results) TestCase() objects 
+            corresponding to regression in test2 comparing to test1 """
+        if test1.name != test2.name:
+            return []
+
+        regressed = []
+        for tc1 in test1.test_cases:
+            for tc2 in test2.test_cases:
+                """ If test cases are equal (same arch, opt and target) but tc2 has more runfails or compfails """
+                if tc1 == tc2 and tc1.result < tc2.result:
+                    regressed.append(TestCase(tc1.arch, tc1.opt, tc1.target))
+        return regressed
+
+    def regression(self, revision_old, revision_new):
+        """ Return a tuple of Test() objects containing TestCase() object which show regression along given revisions """
+        revision_old, revision_new = (str(revision_old), str(revision_new))
+        if revision_new not in self.table:
+            raise RuntimeError("This revision in not in the database: " + str(revision_new) + " (" + str(self.table.keys()) + ")")
+            return
+
+        if revision_old not in self.table:
+            raise RuntimeError("This revision in not in the database: " + str(revision_old) + " (" + str(self.table.keys()) + ")")
+            return
+
+        regressed = []
+        for test_old in self.table[revision_old]:
+            for test_new in self.table[revision_new]:
+                tr = self.test_regression(test_old, test_new)
+                if len(tr) == 0:
+                    continue
+                test = Test(test_new.name)
+                for test_case in tr:
+                    test.add_result(test_case)
+                regressed.append(test)
+        return RegressionInfo(revision_old, revision_new, regressed)
+    
+    def __repr__(self):
+        string = ""
+        for rev in self.table.keys():
+            string += "[" + rev + "]:\n"
+            for test in self.table[rev]:
+                string += repr(test) + '\n'
+        return string
+
+ 
+
+class RevisionInfo(object):
+    """
+    this class is intended to store some relevant information about curent LLVM revision
+    """
+    def __init__(self, hostname, revision):
+        self.hostname, self.revision = hostname, revision
+        self.archs = []
+        self.opts = []
+        self.targets = []
+        self.succeed = 0
+        self.runfailed = 0
+        self.compfailed = 0
+        self.skipped = 0
+        self.testall = 0
+        self.regressions = {}
+    
+    def register_test(self, arch, opt, target, succeed, runfailed, compfailed, skipped):
+        if arch not in self.archs:
+            self.archs.append(arch)
+        if opt not in self.opts:
+            self.opts.append(opt)
+        if target not in self.targets:
+            self.targets.append(target)
+        self.runfailed += runfailed
+        self.compfailed += compfailed
+        self.skipped += skipped
+        self.succeed += succeed
+
+    def add_regression(self, revision, regression_info):
+        """ input is intended to be from 'TestTable.regression(..)', 'regression_info' is a tuple of RegressionInfo() object
+        (regression.py) and 'revision' is tested (not current) LLVM revision name """
+        if revision == self.revision:
+            raise RuntimeError("No regression can be found along the same LLVM revision!")
+      
+        if revision in self.regressions:
+            raise RuntimeError("This revision regression info is already in self.regressions!")
+      
+        self.regressions[revision] = regression_info
+
+    def __repr__(self):
+        string = "%s: LLVM(%s)\n" % (self.hostname, self.revision)
+        string += "archs  : %s\n" % (str(self.archs))
+        string += "opts   : %s\n" % (str(self.opts))
+        string += "targets: %s\n" % (str(self.targets))
+        string += "runfails: %d/%d\n" % (self.runfailed, self.testall)
+        string += "compfails: %d/%d\n" % (self.compfailed, self.testall)
+        string += "skipped: %d/%d\n" % (self.skipped, self.testall)
+        string += "succeed: %d/%d\n" % (self.succeed, self.testall)
+        return string
+
+
+
+
+class ExecutionStateGatherer(object):
+    def __init__(self):
+        self.hostname = self.get_host_name()
+        self.revision = ""
+        self.rinf = []
+        self.tt = TestTable()
+
+    def switch_revision(self, revision):
+        print "Switching revision to " + revision
+        self.revision = revision
+        self.rinf.append(RevisionInfo(self.hostname, self.revision))
+
+    def current_rinf(self):
+        if len(self.rinf) == 0:
+            raise RuntimeError("self.rinf is empty. Apparently you've never invoked switch_revision")
+        return self.rinf[len(self.rinf) - 1]
+
+    def add_to_tt(self, test_name, arch, opt, target, runfailed, compfailed):
+        if len(self.rinf) == 0:
+            raise RuntimeError("self.rinf is empty. Apparently you've never invoked switch_revision")
+        self.tt.add_result(self.revision, test_name, arch, opt, target, runfailed, compfailed)
+
+    def add_to_rinf(self, arch, opt, target, succeed, runfailed, compfailed, skipped):
+        self.current_rinf().register_test(arch, opt, target, succeed, runfailed, compfailed, skipped)
+
+    def add_to_rinf_testall(self, tried_to_test):
+        self.current_rinf().testall += tried_to_test
+
+    def load_from_tt(self, tt):
+        # TODO: fill in self.rinf field!
+        self.tt = tt
+        REVISIONS = tt.table.keys()
+        self.revision = ""
+        if len(REVISIONS) != 0:
+            self.revision = REVISIONS[0]
+        print "ESG: loaded from 'TestTable()' with revisions", REVISIONS
+
+    def dump(self, fname, obj):
+        import pickle
+        with open(fname, 'w') as fp:
+            pickle.dump(obj, fp)  
+
+    def undump(self, fname):
+        import pickle
+        with open(fname, 'r') as fp:
+            obj = pickle.load(fp) 
+        return obj
+
+    def get_host_name(self):
+        import socket
+        return socket.gethostname()
+
+    def __repr__(self):
+        string = "Hostname: %s\n" % (self.hostname)
+        string += "Current LLVM Revision = %s\n\n" % (self.revision)
+        for rev_info in self.rinf:
+            string += repr(rev_info) + '\n'
+        return string
+
+
diff --git a/regression_read.py b/regression_read.py
new file mode 100755
index 00000000..10e69046
--- /dev/null
+++ b/regression_read.py
@@ -0,0 +1,70 @@
+#!/usr/bin/python
+#
+#  Copyright (c) 2014, Intel Corporation 
+#  All rights reserved.
+# 
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are
+#  met:
+# 
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+# 
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+# 
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived from
+#      this software without specific prior written permission.
+# 
+# 
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+#   IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+#   TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+#   PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+#   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+#   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+#   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# // Authors: Anton Mitrokhin
+
+from optparse import OptionParser
+from regression import *
+import pickle
+
+
+
+def read_test_table(filename):
+    with open(filename, 'r') as fp:
+        tt = pickle.load(fp) 
+    return tt
+        
+        
+if __name__ == '__main__':
+    # parsing options
+    class MyParser(OptionParser):
+        def format_epilog(self, formatter):
+            return self.epilog
+
+    examples =  ("Examples:\n" +
+    "Load test_table object\n\tregression_read.py -l\n")
+    
+    parser = MyParser(usage="Usage: regression_read.py -l [options]", epilog=examples)
+    parser.add_option('-l', '--load-tt', dest='load_tt',
+        help='load test_table.dump from file', default=None)
+
+    (options, args) = parser.parse_args()
+
+    tt = read_test_table(options.load_tt)
+    print tt
+    print "\n\n ------------------------\n\n"
+    print "Avaluable LLVM revisions:", tt.table.keys()
+    #print tt.regression(tt.table.keys()[1], tt.table.keys()[0])
+
+
+
diff --git a/run_tests.py b/run_tests.py
index c9d8f2ff..be8f78b6 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -322,12 +322,13 @@ def run_tasks_from_queue(queue, queue_ret, queue_skip, total_tests_arg, max_test
         olddir = ""
     
     compile_error_files = [ ]
+    run_succeed_files = [ ]
     run_error_files = [ ]
     skip_files = [ ]
     while True:
         filename = queue.get()
         if (filename == 'STOP'):
-            queue_ret.put((compile_error_files, run_error_files, skip_files))
+            queue_ret.put((compile_error_files, run_error_files, skip_files, run_succeed_files))
             if is_windows:
                 try:
                     os.remove("test_static.obj")
@@ -350,6 +351,8 @@ def run_tasks_from_queue(queue, queue_ret, queue_skip, total_tests_arg, max_test
                 print_debug("ERROR: run_test function raised an exception: %s\n" % (sys.exc_info()[1]), s, run_tests_log)
                 sys.exit(-1) # This is in case the child has unexpectedly died or some other exception happened
             
+            if compile_error == 0 and run_error == 0:
+                run_succeed_files += [ filename ]
             if compile_error != 0:
                 compile_error_files += [ filename ]
             if run_error != 0:
@@ -503,7 +506,7 @@ def run_tests(options1, args, print_version):
     global s
     s = options.silent
     
-    # prepare run_tests_log and fail_db files
+    # prepare run_tests_log and fail_db file
     global run_tests_log
     if options.in_file:
         run_tests_log = os.getcwd() + os.sep + options.in_file
@@ -590,6 +593,7 @@ def run_tests(options1, args, print_version):
         else:
             options.compiler_exe = "clang++"
  
+
     # checks the required compiler otherwise prints an error message
     PATH_dir = string.split(os.getenv("PATH"), os.pathsep) 
     compiler_exists = False
@@ -669,6 +673,7 @@ def run_tests(options1, args, print_version):
     total_tests = len(files)
 
     compile_error_files = [ ]
+    run_succeed_files = [ ]
     run_error_files = [ ]
     skip_files = [ ]
 
@@ -709,20 +714,42 @@ def run_tests(options1, args, print_version):
     if options.non_interactive == False:
         print_debug("\n", s, run_tests_log)
 
-    
-    for jb in task_threads:
-        if not jb.exitcode == 0:
-            raise OSError(2, 'Some test subprocess has thrown an exception', '')
-
-
     temp_time = (time.time() - start_time)
     elapsed_time = time.strftime('%Hh%Mm%Ssec.', time.gmtime(temp_time))
 
     while not qret.empty():
-        (c, r, skip) = qret.get()
+        (c, r, skip, ss) = qret.get()
         compile_error_files += c
         run_error_files += r
         skip_files += skip
+        run_succeed_files += ss
+
+    # Detect opt_set
+    if options.no_opt == True:
+        opt = "-O0"
+    else:
+        opt = "-O2"
+
+    common.ex_state.add_to_rinf_testall(total_tests)
+    for fname in skip_files:
+        # We do not add skipped tests to test table as we do not know the test result
+        common.ex_state.add_to_rinf(options.arch, opt, options.target, 0, 0, 0, 1)
+
+    for fname in compile_error_files:
+        common.ex_state.add_to_tt(fname, options.arch, opt, options.target, 0, 1)
+        common.ex_state.add_to_rinf(options.arch, opt, options.target, 0, 0, 1, 0)
+
+    for fname in run_error_files:
+        common.ex_state.add_to_tt(fname, options.arch, opt, options.target, 1, 0)
+        common.ex_state.add_to_rinf(options.arch, opt, options.target, 0, 1, 0, 0)
+
+    for fname in run_succeed_files:
+        common.ex_state.add_to_tt(fname, options.arch, opt, options.target, 0, 0)
+        common.ex_state.add_to_rinf(options.arch, opt, options.target, 1, 0, 0, 0)
+
+    for jb in task_threads:
+        if not jb.exitcode == 0:
+            raise OSError(2, 'Some test subprocess has thrown an exception', '')
 
     if options.non_interactive:
         print_debug(" Done %d / %d\n" % (finished_tests_counter.value, total_tests), s, run_tests_log)

From 00d435f726cb7db62ac2901c54466ef1c05adee3 Mon Sep 17 00:00:00 2001
From: Anton Mitrokhin <anton.mitrokhin@phystech.edu>
Date: Sun, 20 Jul 2014 15:19:06 +0400
Subject: [PATCH 2/7] fixed typos, added command line arguments check to
 regresion_read.py

---
 alloy.py           | 11 ++++++++---
 regression_read.py |  9 ++++++---
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/alloy.py b/alloy.py
index 331ec34f..5bb189b9 100755
--- a/alloy.py
+++ b/alloy.py
@@ -330,9 +330,13 @@ def build_ispc(version_LLVM, make):
         if options.debug == True:
             folder +=  "dbg"
        
-        p = subprocess.Popen("svnversion " + folder, shell=True, \
+        p = subprocess.Popen("svn info " + folder, shell=True, \
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (revision_llvm, err) = p.communicate()
+        (info_llvm, err) = p.communicate()
+        info_llvm = re.split('\n', info_llvm)
+        for i in info_llvm:
+            if len(i) > 0 and i.startswith("Last Changed Rev: "):
+                common.ex_state.switch_revision(str(i[len("Last Changed Rev: "):]))
         
         try_do_LLVM("recognize LLVM revision", "svn info " + folder, True)
         try_do_LLVM("build ISPC with LLVM version " + version_LLVM + " ", make_ispc, True)
@@ -641,7 +645,7 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
             attach_mail_file(msg, performance.in_file, "performance.log")
             attach_mail_file(msg, "." + os.sep + "logs" + os.sep + "perf_build.log", "perf_build.log")
 # dumping gathered info to the file
-    commom.ex_state.dump(alloy_folder + "test_table.dump", commom.ex_state.tt)
+    common.ex_state.dump(alloy_folder + "test_table.dump", common.ex_state.tt)
 # sending e-mail with results
     if options.notify != "":
         fp = open(os.environ["ISPC_HOME"] + os.sep + "notify_log.log", 'rb')
@@ -762,6 +766,7 @@ import datetime
 import copy
 import multiprocessing
 import subprocess
+import re
 from email.MIMEMultipart import MIMEMultipart
 from email.MIMEBase import MIMEBase
 from email.mime.text import MIMEText
diff --git a/regression_read.py b/regression_read.py
index 10e69046..9d681172 100755
--- a/regression_read.py
+++ b/regression_read.py
@@ -52,14 +52,17 @@ if __name__ == '__main__':
             return self.epilog
 
     examples =  ("Examples:\n" +
-    "Load test_table object\n\tregression_read.py -l\n")
+    "Load test_table object\n\tregression_read.py -l 'test_table.dump'\n")
     
     parser = MyParser(usage="Usage: regression_read.py -l [options]", epilog=examples)
     parser.add_option('-l', '--load-tt', dest='load_tt',
-        help='load test_table.dump from file', default=None)
+        help='load TestTable() from file', default=None)
 
     (options, args) = parser.parse_args()
-
+    if (options.load_tt == None):
+        parser.print_help()
+        exit(0)
+        
     tt = read_test_table(options.load_tt)
     print tt
     print "\n\n ------------------------\n\n"

From ece37ea4f51a4491d564da63aa8f1bc6ef099c2b Mon Sep 17 00:00:00 2001
From: Anton Mitrokhin <anton.mitrokhin@phystech.edu>
Date: Mon, 21 Jul 2014 16:11:27 +0400
Subject: [PATCH 3/7] regression_read.py improved

---
 regression.py      |  2 +-
 regression_read.py | 82 +++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 74 insertions(+), 10 deletions(-)

diff --git a/regression.py b/regression.py
index 6ce0c532..5718e881 100644
--- a/regression.py
+++ b/regression.py
@@ -77,7 +77,7 @@ class TestCase(object):
 
     def __repr__(self):
         string = "%s %s %s: " % (self.arch, self.opt, self.target)
-        string = string + repr(self.result) + '\n'
+        string = string + repr(self.result)
         return string
 
     def __hash__(self):
diff --git a/regression_read.py b/regression_read.py
index 9d681172..42c92363 100755
--- a/regression_read.py
+++ b/regression_read.py
@@ -35,7 +35,7 @@
 
 from optparse import OptionParser
 from regression import *
-import pickle
+import pickle, re
 
 
 
@@ -44,7 +44,22 @@ def read_test_table(filename):
         tt = pickle.load(fp) 
     return tt
         
-        
+
+def print_with_specific_results(tests, runfailed, compfailed):
+    for test in tests:
+        for test_case in test.test_cases:
+            if test_case.result == TestResult(runfailed, compfailed):
+                print test.name.rjust(40) + repr(test_case).rjust(50)
+
+
+
+def check_rev_in_tt(tt, rev, tt_location):
+    if not rev in tt.table.keys():
+        print "Unable to locate", rev, "in table", tt_location
+        print "Available LLVM revisions:", tt.table.keys()
+        exit(0)
+
+
 if __name__ == '__main__':
     # parsing options
     class MyParser(OptionParser):
@@ -52,22 +67,71 @@ if __name__ == '__main__':
             return self.epilog
 
     examples =  ("Examples:\n" +
-    "Load test_table object\n\tregression_read.py -l 'test_table.dump'\n")
-    
+    "Load test_table object\n\tregression_read.py -l 'test_table.dump'\n" +
+    "Show runfailed, compfailed and succeed tests in rev '213493'\n\t" + 
+    "regression_read.py -l 'test_table.dump' -r '213493' --succeed --runfailed --compfailed\n" +
+    "Show regression between two revisions\n\t" +
+    "regression_read.py -l 'test_table.dump' -R '210929 213493'\n")
+
     parser = MyParser(usage="Usage: regression_read.py -l [options]", epilog=examples)
     parser.add_option('-l', '--load-tt', dest='load_tt',
         help='load TestTable() from file', default=None)
+    parser.add_option('-r', '--revision', dest='revision',
+        help='show only specified revision', default=None)
+    parser.add_option('--runfailed', dest='runfailed',
+        help='show runfailed tests', default=False, action='store_true')
+    parser.add_option('--compfailed', dest='compfailed',
+        help='show compfailed tests', default=False, action='store_true')
+    parser.add_option('--succeed', dest='succeed',
+        help='show succeed tests', default=False, action='store_true')
+    parser.add_option('-R', '--regression', dest='regression',
+        help='show regression between two specified revisions', default="")
 
     (options, args) = parser.parse_args()
     if (options.load_tt == None):
         parser.print_help()
         exit(0)
-        
+    
     tt = read_test_table(options.load_tt)
-    print tt
-    print "\n\n ------------------------\n\n"
-    print "Avaluable LLVM revisions:", tt.table.keys()
-    #print tt.regression(tt.table.keys()[1], tt.table.keys()[0])
+    print "Available LLVM revisions:", tt.table.keys()
+
+    if options.revision != None:
+        check_rev_in_tt(tt, options.revision, options.load_tt)
+        revisions = [options.revision]
+    else:
+        revisions = tt.table.keys()
 
 
+    # print test cases
+    if (options.succeed):
+        print "\n\n Succeed:"
+        for rev in revisions:
+            print "Revision %s" % (rev)
+            print_with_specific_results(tt.table[rev], 0, 0)
+
+    if (options.runfailed):
+        print "\n\n Runfailed:"
+        for rev in revisions:
+            print "Revision %s" % (rev)
+            print_with_specific_results(tt.table[rev], 1, 0)
+
+    if (options.compfailed):
+        print "\n\n Compfailed:"
+        for rev in revisions:
+            print "Revision %s" % (rev)
+            print_with_specific_results(tt.table[rev], 0, 1)
+            
+     
+    # print regression
+    if options.regression != "":
+        regr_revs = re.split('\ ', options.regression)
+        if len(regr_revs) != 2:
+            print "Invalid input:", regr_revs
+            exit(0)
+
+        check_rev_in_tt(tt, regr_revs[0], options.load_tt)
+        check_rev_in_tt(tt, regr_revs[1], options.load_tt)
+
+        print tt.regression(regr_revs[0], regr_revs[1])
+
 

From ad3b66a1781ff20494f07cb5433f5842c4e727d2 Mon Sep 17 00:00:00 2001
From: Anton Mitrokhin <anton.mitrokhin@phystech.edu>
Date: Mon, 21 Jul 2014 16:12:44 +0400
Subject: [PATCH 4/7] renamed regression_read.py -> tt_read.py

---
 regression_read.py => tt_read.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename regression_read.py => tt_read.py (100%)

diff --git a/regression_read.py b/tt_read.py
similarity index 100%
rename from regression_read.py
rename to tt_read.py

From fb82d37f09798690e359fc2ebf74a184b1f12ef2 Mon Sep 17 00:00:00 2001
From: Anton Mitrokhin <anton.mitrokhin@phystech.edu>
Date: Mon, 21 Jul 2014 16:18:40 +0400
Subject: [PATCH 5/7] add host name printing in stability.log and e-mail body

---
 alloy.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/alloy.py b/alloy.py
index 5bb189b9..cfd1dc38 100755
--- a/alloy.py
+++ b/alloy.py
@@ -515,6 +515,7 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
 # begin validation run for stabitily
         common.remove_if_exists(stability.in_file)
         R = [[[],[]],[[],[]],[[],[]],[[],[]]]
+        print_debug("\n" + common.get_host_name() + "\n", False, stability_log)
         print_debug("\n_________________________STABILITY REPORT_________________________\n", False, stability_log)
         for i in range(0,len(LLVM)):
             print_version = 2
@@ -652,6 +653,8 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
         f_lines = fp.readlines()
         fp.close()
         body = ""
+        body += "Hostname: " + common.get_host_name() + "\n\n"
+
         if  not sys.exc_info()[0] == None:
             body = body + "Last exception: " + str(sys.exc_info()) + '\n'
         for i in range(0,len(f_lines)):

From 48e4373d28de78f47ef1529fa9bb210933e1be49 Mon Sep 17 00:00:00 2001
From: Anton Mitrokhin <anton.mitrokhin@phystech.edu>
Date: Mon, 21 Jul 2014 19:36:35 +0400
Subject: [PATCH 6/7] comments on merge request are taken into account

---
 alloy.py                      |   4 +-
 common.py                     | 352 ++++++++++++++++++++++++++++++-
 regression.py                 | 376 ----------------------------------
 tt_read.py => tt_dump_read.py |   0
 4 files changed, 349 insertions(+), 383 deletions(-)
 delete mode 100644 regression.py
 rename tt_read.py => tt_dump_read.py (100%)

diff --git a/alloy.py b/alloy.py
index cfd1dc38..6fcfca96 100755
--- a/alloy.py
+++ b/alloy.py
@@ -722,9 +722,9 @@ def Main():
     global alloy_folder
     alloy_folder = os.getcwd() + os.sep + f_date + os.sep
     global alloy_build
-    alloy_build = os.getcwd() + os.sep + f_date + os.sep + "alloy_build.log"
+    alloy_build = alloy_folder + "alloy_build.log"
     global stability_log
-    stability_log = os.getcwd() + os.sep + f_date + os.sep + "stability.log"
+    stability_log = alloy_folder + "stability.log"
     current_path = os.getcwd()
     make = "make -j" + options.speed
     if os.environ["ISPC_HOME"] != os.getcwd():
diff --git a/common.py b/common.py
index 0fda9b45..e342d8cf 100755
--- a/common.py
+++ b/common.py
@@ -31,17 +31,13 @@
 #   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 #   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# // Author: Filippov Ilia
+# // Author: Filippov Ilia, Anton Mitrokhin, Vsevolod Livinskiy
 import sys
 import os
 import errno
 import shutil
-from regression import *
 
 
-# this class instance is intended to gather and store all information
-# regarding the testing process. Defined in 'regression.py'
-ex_state = ExecutionStateGatherer()
 
 # load/save almost every object to a file (good for bug reproducing)
 def dump(fname, obj):
@@ -157,3 +153,349 @@ def check_tools(m):
                         if int(t11[j])>input_tools[t][0][j]:
                             break
     return ret
+
+
+
+
+
+# regression testing functionality
+class TestResult(object):
+    """
+    this class stores basicly two integers which stand for the result
+    of the test: (runfail{0/1}, compfail{0/1}). other values are 
+    deemed invalid. the __cmp__ function of this class is used to 
+    define what test regression actually is.
+    """
+    def __init__(self, runfailed, compfailed):
+        self.runfailed, self.compfailed = (runfailed, compfailed)
+
+    def __cmp__(self, other):
+        if isinstance(other, TestResult):
+            if self.runfailed == other.runfailed   and \
+               self.compfailed == other.compfailed:
+                return 0
+            elif self.compfailed > other.compfailed:
+                return 1
+            elif self.runfailed > other.runfailed and \
+                 self.compfailed == other.compfailed:
+                return 1
+            else:
+                return -1
+
+        raise RuntimeError("Wrong type for comparioson")
+        return NotImplemented
+
+    def __repr__(self):
+        if (self.runfailed < 0 or self.compfailed < 0):
+            return "(Undefined)"
+        return "(r%d c%d)" % (self.runfailed, self.compfailed)
+
+
+class TestCase(object):
+    """
+    the TestCase() is a combination of parameters the tast was run with:
+    the architecture (x86, x86-64 ...), compiler optimization (-O0, -O2 ...)
+    and target (sse, avx ...). we also store the result of the test here.
+    """
+    def __init__(self, arch, opt, target):
+        self.arch, self.opt, self.target = (arch, opt, target)
+        self.result = TestResult(-1, -1)
+
+    def __repr__(self):
+        string = "%s %s %s: " % (self.arch, self.opt, self.target)
+        string = string + repr(self.result)
+        return string
+
+    def __hash__(self):
+        return hash(self.arch + self.opt + self.target)
+
+    def __ne__(self, other):
+        if isinstance(other, TestCase):
+            if hash(self.arch + self.opt + self.target) != hash(other):
+                return True
+            return False
+        raise RuntimeError("Wrong type for comparioson")
+        return NotImplemented
+
+    def __eq__(self, other):
+        if isinstance(other, TestCase):
+            return not self.__ne__(other)   
+        raise RuntimeError("Wrong type for comparioson")
+        return NotImplemented
+
+
+class Test(object):
+    """
+    Test() stores all TestCase() objects for a given test file name
+    i.e. all archs/opts/targets/ and corresponding testing results.
+    """
+    def __init__(self, name):
+        self.name = name
+        self.test_cases = []
+
+    def add_result(self, test_case):
+        if test_case in self.test_cases:
+            raise RuntimeError("This test case is already in the list: " + repr(test_case))
+            return
+        self.test_cases.append(test_case)
+
+    def __repr__(self):
+        string = self.name + '\n'
+        string = string.rjust(20)
+        for test_case in self.test_cases:
+            string += repr(test_case).rjust(60) + '\n'
+        return string
+    
+    def __hash__(self):
+        return hash(self.name)
+
+    def __ne__(self, other):
+        if isinstance(other, Test):
+            if hash(self) != hash(other):
+                return True
+            return False
+        return NotImplemented
+
+    def __eq__(self, other):
+        if isinstance(other, Test):
+            return not self.__ne__(other)   
+        return NotImplemented
+
+
+class RegressionInfo(object):
+    """
+    service class which provides some statistics on a given regression.
+    the regression test names and cases are given in a form of Test() objects
+    with empty (-1, -1) results 
+    """
+    def __init__(self, revision_old, revision_new, tests):
+        self.revision_old, self.revision_new = (revision_old, revision_new)
+        self.tests = tests
+        self.archfailes = {}
+        self.optfails = {}
+        self.targetfails = {}
+        self.testfails = {}
+        self.archs = []
+        self.opts = []
+        self.targets = []
+
+        for test in tests:
+            for test_case in test.test_cases:
+                self.inc_dictionary(self.testfails, test.name)
+                self.inc_dictionary(self.archfailes, test_case.arch)
+                self.inc_dictionary(self.optfails, test_case.opt)
+                self.inc_dictionary(self.targetfails, test_case.target)
+        
+        self.archs = self.archfailes.keys()
+        self.opts = self.optfails.keys()
+        self.targets = self.targetfails.keys()
+
+    def inc_dictionary(self, dictionary, key):
+        if key not in dictionary:
+            dictionary[key] = 0
+        dictionary[key] += 1
+
+    def __repr__(self):
+        string = "Regression of LLVM revision %s in comparison to %s\n" % (self.revision_new, self.revision_old)
+        string += repr(self.tests) + '\n'
+        string += str(self.testfails) + '\n'
+        string += str(self.archfailes) + '\n'
+        string += str(self.optfails) + '\n'
+        string += str(self.targetfails) + '\n'
+
+        return string
+
+
+class TestTable(object):
+    """
+    the table which stores a tuple of Test() objects (one per revision) and has some 
+    convenience methods for dealing with them
+    """
+    def __init__(self):
+        """ This dictionary contains {rev: [test1, test2, ...], ...}, where 'rev' is a string (revision name) and 'test#'
+        is a Test() object instance """
+        self.table = {}
+
+    def add_result(self, revision_name, test_name, arch, opt, target, runfailed, compfailed):
+        revision_name = str(revision_name)
+        if revision_name not in self.table:
+            self.table[revision_name] = []
+        
+        test_case = TestCase(arch, opt, target)
+        test_case.result = TestResult(runfailed, compfailed)
+
+        for test in self.table[revision_name]:
+            if test.name == test_name:
+                test.add_result(test_case)
+                return
+        
+        test = Test(test_name)
+        test.add_result(test_case)
+        self.table[revision_name].append(test)
+
+    def test_intersection(self, test1, test2):
+        """ Return test cases common for test1 and test2. If test names are different than there is nothing in common """
+        if test1.name != test2.name:
+            return []
+        return list(set(test1.test_cases) & set(test2.test_cases))
+
+    def test_regression(self, test1, test2):
+        """ Return the tuple of empty (i.e. with undefined results) TestCase() objects 
+            corresponding to regression in test2 comparing to test1 """
+        if test1.name != test2.name:
+            return []
+
+        regressed = []
+        for tc1 in test1.test_cases:
+            for tc2 in test2.test_cases:
+                """ If test cases are equal (same arch, opt and target) but tc2 has more runfails or compfails """
+                if tc1 == tc2 and tc1.result < tc2.result:
+                    regressed.append(TestCase(tc1.arch, tc1.opt, tc1.target))
+        return regressed
+ 
+    def regression(self, revision_old, revision_new):
+        """ Return a tuple of Test() objects containing TestCase() object which show regression along given revisions """
+        revision_old, revision_new = (str(revision_old), str(revision_new))
+        if revision_new not in self.table:
+            raise RuntimeError("This revision in not in the database: " + str(revision_new) + " (" + str(self.table.keys()) + ")")
+            return
+
+        if revision_old not in self.table:
+            raise RuntimeError("This revision in not in the database: " + str(revision_old) + " (" + str(self.table.keys()) + ")")
+            return
+
+        regressed = []
+        for test_old in self.table[revision_old]:
+            for test_new in self.table[revision_new]:
+                tr = self.test_regression(test_old, test_new)
+                if len(tr) == 0:
+                    continue
+                test = Test(test_new.name)
+                for test_case in tr:
+                    test.add_result(test_case)
+                regressed.append(test)
+        return RegressionInfo(revision_old, revision_new, regressed)
+    
+    def __repr__(self):
+        string = ""
+        for rev in self.table.keys():
+            string += "[" + rev + "]:\n"
+            for test in self.table[rev]:
+                string += repr(test) + '\n'
+        return string
+
+
+class RevisionInfo(object):
+    """
+    this class is intended to store some relevant information about curent LLVM revision
+    """
+    def __init__(self, hostname, revision):
+        self.hostname, self.revision = hostname, revision
+        self.archs = []
+        self.opts = []
+        self.targets = []
+        self.succeed = 0
+        self.runfailed = 0
+        self.compfailed = 0
+        self.skipped = 0
+        self.testall = 0
+        self.regressions = {}
+    
+    def register_test(self, arch, opt, target, succeed, runfailed, compfailed, skipped):
+        if arch not in self.archs:
+            self.archs.append(arch)
+        if opt not in self.opts:
+            self.opts.append(opt)
+        if target not in self.targets:
+            self.targets.append(target)
+        self.runfailed += runfailed
+        self.compfailed += compfailed
+        self.skipped += skipped
+        self.succeed += succeed
+
+    def add_regression(self, revision, regression_info):
+        """ input is intended to be from 'TestTable.regression(..)', 'regression_info' is a tuple of RegressionInfo() object
+        (regression.py) and 'revision' is tested (not current) LLVM revision name """
+        if revision == self.revision:
+            raise RuntimeError("No regression can be found along the same LLVM revision!")
+      
+        if revision in self.regressions:
+            raise RuntimeError("This revision regression info is already in self.regressions!")
+      
+        self.regressions[revision] = regression_info
+
+    def __repr__(self):
+        string = "%s: LLVM(%s)\n" % (self.hostname, self.revision)
+        string += "archs  : %s\n" % (str(self.archs))
+        string += "opts   : %s\n" % (str(self.opts))
+        string += "targets: %s\n" % (str(self.targets))
+        string += "runfails: %d/%d\n" % (self.runfailed, self.testall)
+        string += "compfails: %d/%d\n" % (self.compfailed, self.testall)
+        string += "skipped: %d/%d\n" % (self.skipped, self.testall)
+        string += "succeed: %d/%d\n" % (self.succeed, self.testall)
+        return string
+
+
+class ExecutionStateGatherer(object):
+    def __init__(self):
+        self.hostname = self.get_host_name()
+        self.revision = ""
+        self.rinf = []
+        self.tt = TestTable()
+
+    def switch_revision(self, revision):
+        print "Switching revision to " + revision
+        self.revision = revision
+        self.rinf.append(RevisionInfo(self.hostname, self.revision))
+
+    def current_rinf(self):
+        if len(self.rinf) == 0:
+            raise RuntimeError("self.rinf is empty. Apparently you've never invoked switch_revision")
+        return self.rinf[len(self.rinf) - 1]
+
+    def add_to_tt(self, test_name, arch, opt, target, runfailed, compfailed):
+        if len(self.rinf) == 0:
+            raise RuntimeError("self.rinf is empty. Apparently you've never invoked switch_revision")
+        self.tt.add_result(self.revision, test_name, arch, opt, target, runfailed, compfailed)
+
+    def add_to_rinf(self, arch, opt, target, succeed, runfailed, compfailed, skipped):
+        self.current_rinf().register_test(arch, opt, target, succeed, runfailed, compfailed, skipped)
+
+    def add_to_rinf_testall(self, tried_to_test):
+        self.current_rinf().testall += tried_to_test
+
+    def load_from_tt(self, tt):
+        # TODO: fill in self.rinf field!
+        self.tt = tt
+        REVISIONS = tt.table.keys()
+        self.revision = ""
+        if len(REVISIONS) != 0:
+            self.revision = REVISIONS[0]
+        print "ESG: loaded from 'TestTable()' with revisions", REVISIONS
+
+    def dump(self, fname, obj):
+        import pickle
+        with open(fname, 'w') as fp:
+            pickle.dump(obj, fp)  
+
+    def undump(self, fname):
+        import pickle
+        with open(fname, 'r') as fp:
+            obj = pickle.load(fp) 
+        return obj
+
+    def get_host_name(self):
+        import socket
+        return socket.gethostname()
+
+    def __repr__(self):
+        string = "Hostname: %s\n" % (self.hostname)
+        string += "Current LLVM Revision = %s\n\n" % (self.revision)
+        for rev_info in self.rinf:
+            string += repr(rev_info) + '\n'
+        return string
+
+
+# this class instance is intended to gather and store all information
+# regarding the testing process.
+ex_state = ExecutionStateGatherer()   
diff --git a/regression.py b/regression.py
deleted file mode 100644
index 5718e881..00000000
--- a/regression.py
+++ /dev/null
@@ -1,376 +0,0 @@
-#!/usr/bin/python
-#
-#  Copyright (c) 2014, Intel Corporation 
-#  All rights reserved.
-# 
-#  Redistribution and use in source and binary forms, with or without
-#  modification, are permitted provided that the following conditions are
-#  met:
-# 
-#    * Redistributions of source code must retain the above copyright
-#      notice, this list of conditions and the following disclaimer.
-# 
-#    * Redistributions in binary form must reproduce the above copyright
-#      notice, this list of conditions and the following disclaimer in the
-#      documentation and/or other materials provided with the distribution.
-# 
-#    * Neither the name of Intel Corporation nor the names of its
-#      contributors may be used to endorse or promote products derived from
-#      this software without specific prior written permission.
-# 
-# 
-#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-#   IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-#   TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-#   PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
-#   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-#   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-#   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# // Authors: Anton Mitrokhin, Vsevolod Livinskiy
-
-class TestResult(object):
-    """
-    this class stores basicly two integers which stand for the result
-    of the test: (runfail{0/1}, compfail{0/1}). other values are 
-    deemed invalid. the __cmp__ function of this class is used to 
-    define what test regression actually is.
-    """
-    def __init__(self, runfailed, compfailed):
-        self.runfailed, self.compfailed = (runfailed, compfailed)
-
-    def __cmp__(self, other):
-        if isinstance(other, TestResult):
-            if self.runfailed == other.runfailed   and \
-               self.compfailed == other.compfailed:
-                return 0
-            elif self.compfailed > other.compfailed:
-                return 1
-            elif self.runfailed > other.runfailed and \
-                 self.compfailed == other.compfailed:
-                return 1
-            else:
-                return -1
-
-        raise RuntimeError("Wrong type for comparioson")
-        return NotImplemented
-
-    def __repr__(self):
-        if (self.runfailed < 0 or self.compfailed < 0):
-            return "(Undefined)"
-        return "(r%d c%d)" % (self.runfailed, self.compfailed)
-
-
-class TestCase(object):
-    """
-    the TestCase() is a combination of parameters the tast was run with:
-    the architecture (x86, x86-64 ...), compiler optimization (-O0, -O2 ...)
-    and target (sse, avx ...). we also store the result of the test here.
-    """
-    def __init__(self, arch, opt, target):
-        self.arch, self.opt, self.target = (arch, opt, target)
-        self.result = TestResult(-1, -1)
-
-    def __repr__(self):
-        string = "%s %s %s: " % (self.arch, self.opt, self.target)
-        string = string + repr(self.result)
-        return string
-
-    def __hash__(self):
-        return hash(self.arch + self.opt + self.target)
-
-    def __ne__(self, other):
-        if isinstance(other, TestCase):
-            if hash(self.arch + self.opt + self.target) != hash(other):
-                return True
-            return False
-        raise RuntimeError("Wrong type for comparioson")
-        return NotImplemented
-
-    def __eq__(self, other):
-        if isinstance(other, TestCase):
-            return not self.__ne__(other)   
-        raise RuntimeError("Wrong type for comparioson")
-        return NotImplemented
-
-
-class Test(object):
-    """
-    Test() stores all TestCase() objects for a given test file name
-    i.e. all archs/opts/targets/ and corresponding testing results.
-    """
-    def __init__(self, name):
-        self.name = name
-        self.test_cases = []
-
-    def add_result(self, test_case):
-        if test_case in self.test_cases:
-            raise RuntimeError("This test case is already in the list: " + repr(test_case))
-            return
-        self.test_cases.append(test_case)
-
-    def __repr__(self):
-        string = self.name + '\n'
-        string = string.rjust(20)
-        for test_case in self.test_cases:
-            string += repr(test_case).rjust(60) + '\n'
-        return string
-    
-    def __hash__(self):
-        return hash(self.name)
-
-    def __ne__(self, other):
-        if isinstance(other, Test):
-            if hash(self) != hash(other):
-                return True
-            return False
-        return NotImplemented
-
-    def __eq__(self, other):
-        if isinstance(other, Test):
-            return not self.__ne__(other)   
-        return NotImplemented
-
-
-class RegressionInfo(object):
-    """
-    service class which provides some statistics on a given regression.
-    the regression test names and cases are given in a form of Test() objects
-    with empty (-1, -1) results 
-    """
-    def __init__(self, revision_old, revision_new, tests):
-        self.revision_old, self.revision_new = (revision_old, revision_new)
-        self.tests = tests
-        self.archfailes = {}
-        self.optfails = {}
-        self.targetfails = {}
-        self.testfails = {}
-        self.archs = []
-        self.opts = []
-        self.targets = []
-
-        for test in tests:
-            for test_case in test.test_cases:
-                self.inc_dictionary(self.testfails, test.name)
-                self.inc_dictionary(self.archfailes, test_case.arch)
-                self.inc_dictionary(self.optfails, test_case.opt)
-                self.inc_dictionary(self.targetfails, test_case.target)
-        
-        self.archs = self.archfailes.keys()
-        self.opts = self.optfails.keys()
-        self.targets = self.targetfails.keys()
-
-    def inc_dictionary(self, dictionary, key):
-        if key not in dictionary:
-            dictionary[key] = 0
-        dictionary[key] += 1
-
-    def __repr__(self):
-        string = "Regression of LLVM revision %s in comparison to %s\n" % (self.revision_new, self.revision_old)
-        string += repr(self.tests) + '\n'
-        string += str(self.testfails) + '\n'
-        string += str(self.archfailes) + '\n'
-        string += str(self.optfails) + '\n'
-        string += str(self.targetfails) + '\n'
-
-        return string
-
-        
-class TestTable(object):
-    """
-    the table which stores a tuple of Test() objects (one per revision) and has some 
-    convenience methods for dealing with them
-    """
-    def __init__(self):
-        """ This dictionary contains {rev: [test1, test2, ...], ...}, where 'rev' is a string (revision name) and 'test#'
-        is a Test() object instance """
-        self.table = {}
-
-    def add_result(self, revision_name, test_name, arch, opt, target, runfailed, compfailed):
-        revision_name = str(revision_name)
-        if revision_name not in self.table:
-            self.table[revision_name] = []
-        
-        test_case = TestCase(arch, opt, target)
-        test_case.result = TestResult(runfailed, compfailed)
-
-        for test in self.table[revision_name]:
-            if test.name == test_name:
-                test.add_result(test_case)
-                return
-        
-        test = Test(test_name)
-        test.add_result(test_case)
-        self.table[revision_name].append(test)
-
-
-    def test_intersection(self, test1, test2):
-        """ Return test cases common for test1 and test2. If test names are different than there is nothing in common """
-        if test1.name != test2.name:
-            return []
-        return list(set(test1.test_cases) & set(test2.test_cases))
-
-    def test_regression(self, test1, test2):
-        """ Return the tuple of empty (i.e. with undefined results) TestCase() objects 
-            corresponding to regression in test2 comparing to test1 """
-        if test1.name != test2.name:
-            return []
-
-        regressed = []
-        for tc1 in test1.test_cases:
-            for tc2 in test2.test_cases:
-                """ If test cases are equal (same arch, opt and target) but tc2 has more runfails or compfails """
-                if tc1 == tc2 and tc1.result < tc2.result:
-                    regressed.append(TestCase(tc1.arch, tc1.opt, tc1.target))
-        return regressed
-
-    def regression(self, revision_old, revision_new):
-        """ Return a tuple of Test() objects containing TestCase() object which show regression along given revisions """
-        revision_old, revision_new = (str(revision_old), str(revision_new))
-        if revision_new not in self.table:
-            raise RuntimeError("This revision in not in the database: " + str(revision_new) + " (" + str(self.table.keys()) + ")")
-            return
-
-        if revision_old not in self.table:
-            raise RuntimeError("This revision in not in the database: " + str(revision_old) + " (" + str(self.table.keys()) + ")")
-            return
-
-        regressed = []
-        for test_old in self.table[revision_old]:
-            for test_new in self.table[revision_new]:
-                tr = self.test_regression(test_old, test_new)
-                if len(tr) == 0:
-                    continue
-                test = Test(test_new.name)
-                for test_case in tr:
-                    test.add_result(test_case)
-                regressed.append(test)
-        return RegressionInfo(revision_old, revision_new, regressed)
-    
-    def __repr__(self):
-        string = ""
-        for rev in self.table.keys():
-            string += "[" + rev + "]:\n"
-            for test in self.table[rev]:
-                string += repr(test) + '\n'
-        return string
-
- 
-
-class RevisionInfo(object):
-    """
-    this class is intended to store some relevant information about curent LLVM revision
-    """
-    def __init__(self, hostname, revision):
-        self.hostname, self.revision = hostname, revision
-        self.archs = []
-        self.opts = []
-        self.targets = []
-        self.succeed = 0
-        self.runfailed = 0
-        self.compfailed = 0
-        self.skipped = 0
-        self.testall = 0
-        self.regressions = {}
-    
-    def register_test(self, arch, opt, target, succeed, runfailed, compfailed, skipped):
-        if arch not in self.archs:
-            self.archs.append(arch)
-        if opt not in self.opts:
-            self.opts.append(opt)
-        if target not in self.targets:
-            self.targets.append(target)
-        self.runfailed += runfailed
-        self.compfailed += compfailed
-        self.skipped += skipped
-        self.succeed += succeed
-
-    def add_regression(self, revision, regression_info):
-        """ input is intended to be from 'TestTable.regression(..)', 'regression_info' is a tuple of RegressionInfo() object
-        (regression.py) and 'revision' is tested (not current) LLVM revision name """
-        if revision == self.revision:
-            raise RuntimeError("No regression can be found along the same LLVM revision!")
-      
-        if revision in self.regressions:
-            raise RuntimeError("This revision regression info is already in self.regressions!")
-      
-        self.regressions[revision] = regression_info
-
-    def __repr__(self):
-        string = "%s: LLVM(%s)\n" % (self.hostname, self.revision)
-        string += "archs  : %s\n" % (str(self.archs))
-        string += "opts   : %s\n" % (str(self.opts))
-        string += "targets: %s\n" % (str(self.targets))
-        string += "runfails: %d/%d\n" % (self.runfailed, self.testall)
-        string += "compfails: %d/%d\n" % (self.compfailed, self.testall)
-        string += "skipped: %d/%d\n" % (self.skipped, self.testall)
-        string += "succeed: %d/%d\n" % (self.succeed, self.testall)
-        return string
-
-
-
-
-class ExecutionStateGatherer(object):
-    def __init__(self):
-        self.hostname = self.get_host_name()
-        self.revision = ""
-        self.rinf = []
-        self.tt = TestTable()
-
-    def switch_revision(self, revision):
-        print "Switching revision to " + revision
-        self.revision = revision
-        self.rinf.append(RevisionInfo(self.hostname, self.revision))
-
-    def current_rinf(self):
-        if len(self.rinf) == 0:
-            raise RuntimeError("self.rinf is empty. Apparently you've never invoked switch_revision")
-        return self.rinf[len(self.rinf) - 1]
-
-    def add_to_tt(self, test_name, arch, opt, target, runfailed, compfailed):
-        if len(self.rinf) == 0:
-            raise RuntimeError("self.rinf is empty. Apparently you've never invoked switch_revision")
-        self.tt.add_result(self.revision, test_name, arch, opt, target, runfailed, compfailed)
-
-    def add_to_rinf(self, arch, opt, target, succeed, runfailed, compfailed, skipped):
-        self.current_rinf().register_test(arch, opt, target, succeed, runfailed, compfailed, skipped)
-
-    def add_to_rinf_testall(self, tried_to_test):
-        self.current_rinf().testall += tried_to_test
-
-    def load_from_tt(self, tt):
-        # TODO: fill in self.rinf field!
-        self.tt = tt
-        REVISIONS = tt.table.keys()
-        self.revision = ""
-        if len(REVISIONS) != 0:
-            self.revision = REVISIONS[0]
-        print "ESG: loaded from 'TestTable()' with revisions", REVISIONS
-
-    def dump(self, fname, obj):
-        import pickle
-        with open(fname, 'w') as fp:
-            pickle.dump(obj, fp)  
-
-    def undump(self, fname):
-        import pickle
-        with open(fname, 'r') as fp:
-            obj = pickle.load(fp) 
-        return obj
-
-    def get_host_name(self):
-        import socket
-        return socket.gethostname()
-
-    def __repr__(self):
-        string = "Hostname: %s\n" % (self.hostname)
-        string += "Current LLVM Revision = %s\n\n" % (self.revision)
-        for rev_info in self.rinf:
-            string += repr(rev_info) + '\n'
-        return string
-
-
diff --git a/tt_read.py b/tt_dump_read.py
similarity index 100%
rename from tt_read.py
rename to tt_dump_read.py

From a051c9bbd8cfbd3485733ee3bfc817d01641af48 Mon Sep 17 00:00:00 2001
From: Anton Mitrokhin <anton.mitrokhin@phystech.edu>
Date: Mon, 21 Jul 2014 19:48:42 +0400
Subject: [PATCH 7/7] tt_dump_read.py import bux fixed

---
 tt_dump_read.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tt_dump_read.py b/tt_dump_read.py
index 42c92363..0263ec32 100755
--- a/tt_dump_read.py
+++ b/tt_dump_read.py
@@ -34,7 +34,7 @@
 # // Authors: Anton Mitrokhin
 
 from optparse import OptionParser
-from regression import *
+from common import *
 import pickle, re