#!/usr/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import print_function usage = """%prog: A test for OOM conditions in the shell. %prog finds segfaults and other errors caused by incorrect handling of allocation during OOM (out-of-memory) conditions. """ help = """Check for regressions only. This runs a set of files with a known number of OOM errors (specified by REGRESSION_COUNT), and exits with a non-zero result if more or less errors are found. See js/src/Makefile.in for invocation. """ import hashlib import re import shlex import subprocess import sys import threading import time from optparse import OptionParser ##################################################################### # Utility functions ##################################################################### def run(args, stdin=None): class ThreadWorker(threading.Thread): def __init__(self, pipe): super(ThreadWorker, self).__init__() self.all = "" self.pipe = pipe self.setDaemon(True) def run(self): while True: line = self.pipe.readline() if line == '': break else: self.all += line try: if type(args) == str: args = shlex.split(args) args = [str(a) for a in args] # convert to strs stdin_pipe = subprocess.PIPE if stdin else None proc = subprocess.Popen(args, stdin=stdin_pipe, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if stdin_pipe: proc.stdin.write(stdin) proc.stdin.close() stdout_worker = ThreadWorker(proc.stdout) stderr_worker = ThreadWorker(proc.stderr) stdout_worker.start() stderr_worker.start() proc.wait() stdout_worker.join() stderr_worker.join() except KeyboardInterrupt as e: sys.exit(-1) stdout, stderr = stdout_worker.all, stderr_worker.all result = (stdout, stderr, proc.returncode) return result def get_js_files(): (out, err, exit) = run('find ../jit-test/tests -name "*.js"') if (err, exit) != ("", 0): sys.exit("Wrong directory, run from an objdir") return out.split() ##################################################################### # Blacklisting ##################################################################### def in_blacklist(sig): return sig in blacklist def add_to_blacklist(sig): blacklist[sig] = blacklist.get(sig, 0) blacklist[sig] += 1 # How often is a particular lines important for this. def count_lines(): """Keep track of the amount of times individual lines occur, in order to prioritize the errors which occur most frequently.""" counts = {} for string,count in blacklist.items(): for line in string.split("\n"): counts[line] = counts.get(line, 0) + count lines = [] for k,v in counts.items(): lines.append("{0:6}: {1}".format(v, k)) lines.sort() countlog = file("../OOM_count_log", "w") countlog.write("\n".join(lines)) countlog.flush() countlog.close() ##################################################################### # Output cleaning ##################################################################### def clean_voutput(err): # Skip what we can't reproduce err = re.sub(r"^--\d+-- run: /usr/bin/dsymutil \"shell/js\"$", "", err, flags=re.MULTILINE) err = re.sub(r"^==\d+==", "", err, flags=re.MULTILINE) err = re.sub(r"^\*\*\d+\*\*", "", err, flags=re.MULTILINE) err = re.sub(r"^\s+by 0x[0-9A-Fa-f]+: ", "by: ", err, flags=re.MULTILINE) err = re.sub(r"^\s+at 0x[0-9A-Fa-f]+: ", "at: ", err, flags=re.MULTILINE) err = re.sub(r"(^\s+Address 0x)[0-9A-Fa-f]+( is not stack'd)", r"\1\2", err, flags=re.MULTILINE) err = re.sub(r"(^\s+Invalid write of size )\d+", r"\1x", err, flags=re.MULTILINE) err = re.sub(r"(^\s+Invalid read of size )\d+", r"\1x", err, flags=re.MULTILINE) err = re.sub(r"(^\s+Address 0x)[0-9A-Fa-f]+( is )\d+( bytes inside a block of size )[0-9,]+( free'd)", r"\1\2\3\4", err, flags=re.MULTILINE) # Skip the repeating bit due to the segfault lines = [] for l in err.split('\n'): if l == " Process terminating with default action of signal 11 (SIGSEGV)": break lines.append(l) err = '\n'.join(lines) return err def remove_failed_allocation_backtraces(err): lines = [] add = True for l in err.split('\n'): # Set start and end conditions for including text if l == " The site of the failed allocation is:": add = False elif l[:2] not in ['by: ', 'at:']: add = True if add: lines.append(l) err = '\n'.join(lines) return err def clean_output(err): err = re.sub(r"^js\(\d+,0x[0-9a-f]+\) malloc: \*\*\* error for object 0x[0-9a-f]+: pointer being freed was not allocated\n\*\*\* set a breakppoint in malloc_error_break to debug\n$", "pointer being freed was not allocated", err, flags=re.MULTILINE) return err ##################################################################### # Consts, etc ##################################################################### command_template = 'shell/js' \ + ' -m -j -p' \ + ' -e "const platform=\'darwin\'; const libdir=\'../jit-test/lib/\';"' \ + ' -f ../jit-test/lib/prolog.js' \ + ' -f {0}' # Blacklists are things we don't want to see in our logs again (though we do # want to count them when they happen). Whitelists we do want to see in our # logs again, principally because the information we have isn't enough. blacklist = {} add_to_blacklist(r"('', '', 1)") # 1 means OOM if the shell hasn't launched yet. add_to_blacklist(r"('', 'out of memory\n', 1)") whitelist = set() whitelist.add(r"('', 'out of memory\n', -11)") # -11 means OOM whitelist.add(r"('', 'out of memory\nout of memory\n', -11)") ##################################################################### # Program ##################################################################### # Options parser = OptionParser(usage=usage) parser.add_option("-r", "--regression", action="store", metavar="REGRESSION_COUNT", help=help, type="int", dest="regression", default=None) (OPTIONS, args) = parser.parse_args() if OPTIONS.regression != None: # TODO: This should be expanded as we get a better hang of the OOM problems. # For now, we'll just check that the number of OOMs in one short file does not # increase. files = ["../jit-test/tests/arguments/args-createontrace.js"] else: files = get_js_files() # Use a command-line arg to reduce the set of files if len (args): files = [f for f in files if f.find(args[0]) != -1] if OPTIONS.regression == None: # Don't use a logfile, this is automated for tinderbox. log = file("../OOM_log", "w") num_failures = 0 for f in files: # Run it once to establish boundaries command = (command_template + ' -O').format(f) out, err, exit = run(command) max = re.match(".*OOM max count: (\d+).*", out, flags=re.DOTALL).groups()[0] max = int(max) # OOMs don't recover well for the first 20 allocations or so. # TODO: revisit this. for i in range(20, max): if OPTIONS.regression == None: print("Testing allocation {0}/{1} in {2}".format(i,max,f)) else: sys.stdout.write('.') # something short for tinderbox, no space or \n command = (command_template + ' -A {0}').format(f, i) out, err, exit = run(command) # Success (5 is SM's exit code for controlled errors) if exit == 5 and err.find("out of memory") != -1: continue # Failure else: if OPTIONS.regression != None: # Just count them num_failures += 1 continue ######################################################################### # The regression tests ends above. The rest of this is for running the # script manually. ######################################################################### problem = str((out, err, exit)) if in_blacklist(problem) and problem not in whitelist: add_to_blacklist(problem) continue add_to_blacklist(problem) # Get valgrind output for a good stack trace vcommand = "valgrind --dsymutil=yes -q --log-file=OOM_valgrind_log_file " + command run(vcommand) vout = file("OOM_valgrind_log_file").read() vout = clean_voutput(vout) sans_alloc_sites = remove_failed_allocation_backtraces(vout) # Don't print duplicate information if in_blacklist(sans_alloc_sites): add_to_blacklist(sans_alloc_sites) continue add_to_blacklist(sans_alloc_sites) log.write ("\n") log.write ("\n") log.write ("=========================================================================") log.write ("\n") log.write ("An allocation failure at\n\tallocation {0}/{1} in {2}\n\t" "causes problems (detected using bug 624094)" .format(i, max, f)) log.write ("\n") log.write ("\n") log.write ("Command (from obj directory, using patch from bug 624094):\n " + command) log.write ("\n") log.write ("\n") log.write ("stdout, stderr, exitcode:\n " + problem) log.write ("\n") log.write ("\n") double_free = err.find("pointer being freed was not allocated") != -1 oom_detected = err.find("out of memory") != -1 multiple_oom_detected = err.find("out of memory\nout of memory") != -1 segfault_detected = exit == -11 log.write ("Diagnosis: ") log.write ("\n") if multiple_oom_detected: log.write (" - Multiple OOMs reported") log.write ("\n") if segfault_detected: log.write (" - segfault") log.write ("\n") if not oom_detected: log.write (" - No OOM checking") log.write ("\n") if double_free: log.write (" - Double free") log.write ("\n") log.write ("\n") log.write ("Valgrind info:\n" + vout) log.write ("\n") log.write ("\n") log.flush() if OPTIONS.regression == None: count_lines() print() # Do the actual regression check if OPTIONS.regression != None: expected_num_failures = OPTIONS.regression if num_failures != expected_num_failures: print("TEST-UNEXPECTED-FAIL |", end='') if num_failures > expected_num_failures: print("More out-of-memory errors were found ({0}) than expected ({1}). " "This probably means an allocation site has been added without a " "NULL-check. If this is unavoidable, you can account for it by " "updating Makefile.in.".format(num_failures, expected_num_failures), end='') else: print("Congratulations, you have removed {0} out-of-memory error(s) " "({1} remain)! Please account for it by updating Makefile.in." .format(expected_num_failures - num_failures, num_failures), end='') sys.exit(-1) else: print('TEST-PASS | find_OOM_errors | Found the expected number of OOM ' 'errors ({0})'.format(expected_num_failures))