Added some basic test subprocess exception handling and remapped error messages to the e-mail

alloy.py

@@ -473,7 +473,10 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
                     for i2 in range(0,len(opts)):
                         stability.arch = arch[i1]
                         stability.no_opt = opts[i2]
-                        execute_stability(stability, R, print_version)
+                        try:
+                            execute_stability(stability, R, print_version)
+                        except:
+                            print_debug("Exception in execute_stability - maybe some test subprocess terminated before it should have\n", False, stability_log)
                         print_version = 0
             for j in range(0,len(sde_targets)):
                 stability.target = sde_targets[j][1]
@@ -580,6 +583,8 @@ def validation_run(only, only_targets, reference_branch, number, notify, update,
         f_lines = fp.readlines()
         fp.close()
         line = ""
+        if  not sys.exc_info()[0] == None:
+            line = line + "Last exception: " + str(sys.exc_info()) + '\n'
         for i in range(0,len(f_lines)):
             line = line + f_lines[i][:-1]
             line = line + '   \n'
@@ -664,6 +669,7 @@ from optparse import OptionParser
 from optparse import OptionGroup
 import sys
 import os
+import errno
 import operator
 import time
 import glob

run_tests.py

@@ -59,10 +59,15 @@ def run_command(cmd):
     lexer.whitespace_split = True
     lexer.escape = ''
     arg_list = list(lexer)
-
-    sp = subprocess.Popen(arg_list, stdin=None,
+    
+    try:
+        sp = subprocess.Popen(arg_list, stdin=None,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
+    except:
+        print_debug("ERROR: The child (%s) raised an esception: %s\n" % (cmd, sys.exc_info()[1]), s, run_tests_log)
+        raise
+
     out = sp.communicate()
     output = ""
     output += out[0].decode("utf-8")
@@ -325,7 +330,11 @@ def run_tasks_from_queue(queue, queue_ret, queue_skip, total_tests_arg, max_test
             sys.exit(0)
 
         if check_test(filename):
-            (compile_error, run_error) = run_test(filename)
+            try:
+                (compile_error, run_error) = run_test(filename)
+            except:
+                sys.exit(-1) # This is in case the child has unexpectedly died
+            
             if compile_error != 0:
                 compile_error_files += [ filename ]
             if run_error != 0:
@@ -646,8 +655,9 @@ def run_tests(options1, args, print_version):
     task_threads = [0] * nthreads
     for x in range(nthreads):
         task_threads[x] = multiprocessing.Process(target=run_tasks_from_queue, args=(q, qret, qskip, total_tests,
-            max_test_length, finished_tests_counter, finished_tests_counter_lock, glob_var))
+                max_test_length, finished_tests_counter, finished_tests_counter_lock, glob_var))
         task_threads[x].start()
+
     # wait for them to all finish and then return the number that failed
     # (i.e. return 0 if all is ok)
     for t in task_threads:
@@ -655,6 +665,12 @@ def run_tests(options1, args, print_version):
     if options.non_interactive == False:
         print_debug("\n", s, run_tests_log)
 
+    
+    for jb in task_threads:
+        if not jb.exitcode == 0:
+            raise OSError(2, 'Some test subprocess has thrown an exception', '')
+
+
     temp_time = (time.time() - start_time)
     elapsed_time = time.strftime('%Hh%Mm%Ssec.', time.gmtime(temp_time))