#!/usr/bin/env python
#
# Rewrite of sub_test by Sung-Eun Choi (sungeun@cray.com)
#  sub_test is used by start_test in the Chapel Testing system
#  August 2009
#
# This script can be overridden with a script by the same name
#  placed in the test directory.
#
# The use and behavior of the various environment variables,
#  settings, and files were copied straight from sub_test.  They
#  were added/modified to sub_test over the years, and their use
#  is inconsistent and a bit of a mess.  I like to think that this
#  is due to the fact that the original sub_test was written in csh,
#  which was probably pretty novel at the time but is quite limited
#  by today's standards.  In addition, I implemented the timeout
#  mechanism directly rather than calling out to the timedexec
#  (perl) script.
#
# For compatibility reasons, I have maintained the behavior of the
#  original sub_test.  Any new features (e.g., internal timeout
#  mechanism) or modified behaviors (e.g., multiple .compopts,
#  multiple .execopts, custom .good files) will not interfere with
#  the expected behavior of tests that do not use the features or
#  behaviors.
#
#
# ENVIRONMENT VARIABLES:
#
# CHPL_HOME: Grabbed from the environment or deduced based on the path to
#    the compiler.
# CHPL_TEST_VGRND_COMP: Use valgrind on the compiler
# CHPL_TEST_VBRND_EXE: Use valgrind on the test program
# CHPL_VALGRIND_OPTS: Options to valgrind
# CHPL_TEST_FUTURES: 2 == test futures only
#                    1 == test futures and non-futures
#                    0 == test non-futures only
# CHPL_TEST_NOTESTS: Test the tests that are marked "notest" (see below)
# LAUNCHCMD: Uses this command to launch the test program
# CHPL_TEST_INTERP: DEPRECATED
# CHPL_TEST_PERF: Run as a performance test (same as -performance flag)
# CHPL_TEST_PERF_LABEL: The performance label, e.g. "perf"
# CHPL_TEST_PERF_DIR: Scratch directory for performance data
# CHPL_TEST_PERF_TRIALS: Default number of trials for perf tests
# CHPL_ONETEST: Name of the one test in this directory to run
# CHPL_TEST_SINGLES: If false, test the entire directory
# CHPL_SYSTEM_PREDIFF: If set, run that script on each test output
# CHPL_COMM: Chapel communication layer
# CHPL_COMPONLY: Only build the test (same as -noexec flag)
# CHPL_NO_STDIN_REDIRECT: do not redirect stdin when running tests
#                         also, skip tests with .stdin files
# CHPL_LAUNCHER_TIMEOUT: if defined, pass an option/options to the executable
#                        for it to enforce timeout instead of using timedexec;
#                        the value of the variable determines the option format.
#
#
# DIRECTORY-WIDE FILES:  These settings are for the entire directory and
#  in many cases can be overridden or augmented with test-specific settings.
#
# NOEXEC: Do not execute tests in this directory
# NOVGRBIN: Do not execute valgrind
# COMPSTDIN: Get stdin from this file (default /dev/null)
# COMPOPTS: Compiler flags
# LASTCOMPOPTS: Compiler flags to be put at the end of the command line
# EXECENV: Environment variables to be applied to the entire directory
# EXECOPTS: Test program flags to be applied to the entire directory
# LASTEXECOPTS: Test program flags to be put at the end of the command line
# NUMLOCALES: Number of locales to use
# CATFILES: List of files whose contents are added to end of test output
# PREDIFF: Script to execute before diff'ing output (arguments: <test
#    executable>, <log>, <compiler executable>)
# PREEXEC: Script to execute before executing test program (arguments: <test
#    executable>, <log>, <compiler executable>)
# PRECOMP: Script to execute before running the compiler (arguments: <test
#    executable>, <log>, <compiler executable>).
# PERFNUMTRIALS: Number of trials to run for performance testing
#
#
# TEST-SPECIFIC FILES:  These setting override or augment the directory-wide
#  settings.  Unless otherwise specified, these files are named
#  <test executable>.suffix (where suffix is one of the following).
#
# .good: "Golden" output file (can have different basenames)
# .compopts: Additional compiler options
# .perfcompopts: Additional compiler options for performance testing
# .lastcompopts: Additional compiler options to be added at the end of the
#    command line
# .execenv: Additional environment variables for the test
# .execopts: Additional test options
# .perfexecenv: Additional environment variables for performance testing
# .perfexecopts: Additional test options for performance testing
# .perfnumtrials: Number of trials to run for performance testing
# .notest: Do not run this test
# .numlocales: Number of locales to use (overrides NUMLOCALES)
# .future: Future test
# .ifuture: Future test
# .noexec: Do not execute this test
# .skipif: Skip this test if certain environment conditions hold true
# .timeout: Test timeout (overrides TIMEOUT)
# .perftimeout: Performance test timeout
# .killtimeout: Kill timeout (overrides KILLTIMEOUT)
# .catfiles: Additional list of files whose contents are added to end of
#    test output
# .precomp: Additional script to execute before compiling the test
# .prediff: Additional script to execute before diff'ing output
# .preexec: Additional script to execute before executing test program
# .perfkeys: Existence indicates a performance test.  Contents specifies
#    performance "keys"
#
# In general, the performance label from CHPL_TEST_PERF_LABEL is used
# instead of "perf" in the above suffixes, and its all-caps version is used
# in the all-caps file names instead "PERF".


import sys, os, subprocess, string, signal
import select, fcntl
import fnmatch, time
import re
import shlex
import datetime


#
# Time out class:  Read from a stream until time out
#  A little ugly but sending SIGALRM (or any other signal) to Python
#   can be unreliable (will not respond if holding certain locks).
#
class ReadTimeoutException(Exception): pass

def SetNonBlock(stream):
    flags = fcntl.fcntl(stream.fileno(), fcntl.F_GETFL)
    flags |= os.O_NONBLOCK
    fcntl.fcntl(stream.fileno(), fcntl.F_SETFL, flags)

def SuckOutputWithTimeout(stream, timeout):
    SetNonBlock(stream)
    buffer = ''
    end_time = time.time() + timeout
    while True:
        now = time.time()
        if end_time <= now:
            # Maybe return partial result instead?
            raise ReadTimeoutException('Teh tiem iz out!');
        ready_set = select.select([stream], [], [], end_time - now)[0]
        if stream in ready_set:
            bytes = stream.read()
            if len(bytes) == 0:
                break           # EOF
            buffer += bytes     # Inefficient way to accumulate bytes.
            # len(ready_set) == 0 is also an indication of timeout. However,
            # if we relied on that, we would require no data ready in order
            # to timeout  which doesn't seem quite right either.
    return buffer

def LauncherTimeoutArgs(timeout):
    if useLauncherTimeout == 'pbs':
        # --walltime=hh:mm:ss
        hours = 0
        minutes = timeout / 60  # might be OK even if over an hour
        seconds = timeout % 60
        option = '--walltime=' + str(hours) + ':' + str(minutes) \
                 + ':' + str(seconds)
        return [option]
    if useLauncherTimeout == 'slurm':
        # slurm timeout is currently set with CHPL_LAUNCHER_WALLTIME
        return []
    else:
        Fatal('LauncherTimeoutArgs encountered an unknown format spec: ' + \
              useLauncherTimeout)


#
# Auxilliary functions
#

# Escape all special characters
def ShellEscape(str):
    return re.sub(r'([\\!@#$%^&*()?\'"|<>[\]{} ])', r'\\\1', str)

# return True if f has .chpl extension
def IsChplSource(f):
    if ((f.count('.') !=0) and (len(f) >= 6) and (len(f)-f.find('.chpl')==5)):
        return True
    else:
        return False

perflabel = '' # declare it for the following functions

# file suffix: 'keys' -> '.perfkeys' etc.
def PerfSfx(s):
    return '.' + perflabel + s

# directory-wide file: 'COMPOPTS' or 'compopts' -> './PERFCOMPOPTS' etc.
def PerfDirFile(s):
    return './' + perflabel.upper() + s.upper()

# test-specific file: (foo,keys) -> foo.perfkeys etc.
def PerfTFile(execname, sfx):
    return execname + '.' + perflabel + sfx

# read file with comments
def ReadFileWithComments(f, ignoreLeadingSpace=True):
    # sys.stdout.write('Opening: %s\n'%(f))
    myfile = open(f, 'r')
    mylines = myfile.readlines()
    myfile.close()
    mylist=list()
    for line in mylines:
        line=line.rstrip()
        # ignore blank lines
        if not line.strip(): continue
        # ignore comments
        if ignoreLeadingSpace:
            if line.lstrip()[0] == '#': continue
        else:
            if line[0] == '#': continue
        mylist.append(line)
    return mylist

# diff 2 files
def DiffFiles(f1, f2):
    sys.stdout.write('[Executing diff %s %s]\n'%(f1, f2))
    p = subprocess.Popen(['diff',f1,f2],
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    myoutput = p.communicate()[0] # grab stdout to avoid potential deadlock
    if p.returncode != 0:
        sys.stdout.write(myoutput)
    return p.returncode

# kill process
def KillProc(p, timeout):
    k = subprocess.Popen(['kill',str(p.pid)])
    k.wait()
    now = time.time()
    end_time = now + timeout # give it a little time
    while end_time > now:
        if p.poll():
            return
        now = time.time()
    # use the big hammer (and don't bother waiting)
    subprocess.Popen(['kill','-9', str(p.pid)])
    return

# clean up after the test has been built
def cleanup(execname):
    if execname is not None:
        if os.path.isfile(execname):
            os.unlink(execname)
        if os.path.isfile(execname+'_real'):
            os.unlink(execname+'_real')
    return

# print (compopts: XX, execopts: XX) for later decoding of failed tests
def printTestVariation(compoptsnum, execoptsnum=0):
    if compoptsnum==0 and execoptsnum==0:
        return;

    sys.stdout.write(' (')
    if compoptsnum != 0:
        sys.stdout.write('compopts: %d'%(compoptsnum))
    if execoptsnum != 0:
        if compoptsnum != 0:
            sys.stdout.write(', ')
        sys.stdout.write('execopts: %d'%(execoptsnum))
    sys.stdout.write(')')
    return

# return true if string is an integer
def IsInteger(str):
    try:
        int(str)
        return True
    except ValueError:
        return False

# read integer value from a file
def ReadIntegerValue(f, localdir):
    to = ReadFileWithComments(f)
    if to:
        for l in to:
            if l[0] == '#':
                continue
            if IsInteger(l):
                return string.atoi(l)
            else:
                break
    Fatal('Invalid integer value in '+f+' ('+localdir+')');

# report an error message and exit
def Fatal(message):
    sys.stdout.write('[Error (sub_test): '+message+']\n')
    sys.exit(-1)

#
# Start of sub_test proper
#

if len(sys.argv)!=2:
    print 'usage: sub_test COMPILER'
    sys.exit(0)

# Find the base installation
compiler=sys.argv[1]
if not os.access(compiler,os.R_OK|os.X_OK):
    Fatal('Cannot execute compiler \''+compiler+'\'')

path_to_compiler=os.path.abspath(os.path.dirname(compiler))
# Assume chpl binary is 2 directory levels down in the base installation
(chpl_base, tmp) = os.path.split(path_to_compiler)
(chpl_base, tmp) = os.path.split(chpl_base)
chpl_base=os.path.normpath(chpl_base)
# sys.stdout.write('CHPL_BASE='+chpl_base+'\n')

# If $CHPL_HOME is not set, use the base installation of the compiler
chpl_home=os.getenv('CHPL_HOME', chpl_base);
chpl_home=os.path.normpath(chpl_home)
# sys.stdout.write('CHPL_HOME='+chpl_home+'\n');

# Find the test util directory -- set this in start_test to permit
# a version of start_test other than the one in CHPL_HOME to be used
utildir=os.getenv('CHPL_TEST_UTIL_DIR');
if utildir is None or not os.path.isdir(utildir):
    Fatal('Cannot find test util directory {0}'.format(utildir))

# Needed for MacOS mount points
utildir = os.path.realpath(utildir)
# sys.stdout.write('utildir='+utildir+'\n');

# Find the test directory
testdir=chpl_home+'/test'
if os.path.isdir(testdir)==0:
    testdir=chpl_home+'/examples'
    if os.path.isdir(testdir)==0:
        Fatal('Cannot find test directory '+chpl_home+'/test or '+testdir)
# Needed for MacOS mount points
testdir = os.path.realpath(testdir)
# sys.stdout.write('testdir='+testdir+'\n');

# Use timedexec
# As much as I hate calling out to another script for the time out stuff,
#  subprocess doesn't quite cut it for this kind of stuff
useTimedExec=True
if useTimedExec:
    timedexec=utildir+'/timedexec'
    if not os.access(timedexec,os.R_OK|os.X_OK):
        Fatal('Cannot execute timedexec script \''+timedexec+'\'')
# sys.stdout.write('timedexec='+timedexec+'\n');

# HW platform
platform=subprocess.Popen([chpl_home+'/util/chplenv/platform', '--target'], stdout=subprocess.PIPE).communicate()[0]
platform = platform.strip()
# sys.stdout.write('platform='+platform+'\n')

# Machine name we are running on
machine=os.uname()[1].split('.', 1)[0]
# sys.stdout.write('machine='+machine+'\n')

# Get the system-wide prediff
systemPrediff = os.getenv('CHPL_SYSTEM_PREDIFF')
if systemPrediff:
  if not os.access(systemPrediff,os.R_OK|os.X_OK):
    Fatal('Cannot execute system-wide prediff \''+systemPrediff+'\'')

# Use the launcher walltime option for timeout
useLauncherTimeout = os.getenv('CHPL_LAUNCHER_TIMEOUT')

# Get the current directory (normalize for MacOS case-sort-of-sensitivity)
localdir = string.replace(os.path.normpath(os.getcwd()), testdir, '.')
# sys.stdout.write('localdir=%s\n'%(localdir))

if localdir.find('./') == 0:
    # strip off the leading './'
    localdir = string.lstrip(localdir, '.')
    localdir = string.lstrip(localdir, '/')
# sys.stdout.write('localdir=%s\n'%(localdir))

# CHPL_COMM
chplcomm=os.getenv('CHPL_COMM','none').strip()
chplcommstr='.comm-'+chplcomm
# sys.stdout.write('chplcomm=%s\n'%(chplcomm))

#
# Test options for all tests in this directory
#

if os.getenv('CHPL_TEST_PERF')!=None:
    perftest=True
    perflabel=os.getenv('CHPL_TEST_PERF_LABEL')
    perfdir=os.getenv('CHPL_TEST_PERF_DIR')
    if perflabel==None or perfdir==None:
        Fatal('$CHPL_TEST_PERF_DIR and $CHPL_TEST_PERF_LABEL must be set for performance testing')
else:
    perftest=False
    perflabel=''

compoptssuffix = PerfSfx('compopts')  # .compopts or .perfcompopts or ...
execenvsuffix  = PerfSfx('execenv')   # .execenv  or .perfexecenv  or ...
execoptssuffix = PerfSfx('execopts')  # .execopts or .perfexecopts or ...
timeoutsuffix  = PerfSfx('timeout')   # .timeout  or .perftimeout  or ...
# sys.stdout.write('perftest=%d perflabel=%s\n'%(perftest,perflabel))

# Get global timeout
if os.access('./TIMEOUT',os.R_OK):
    globalTimeout = ReadIntegerValue('./TIMEOUT', localdir)
elif os.getenv('CHPL_TEST_VGRND_COMP')=='on':
    globalTimeout=1000
else:
    globalTimeout=300
# sys.stdout.write('globalTimeout=%d\n'%(globalTimeout))

#
# Check for global PERFTIMEEXEC option
#
# TODO: Eventually, we could store the time command to use within this
# file itself to permit various other options to be used; on the other
# hand, that could be considered a potential security risk...
#
if os.access(PerfDirFile('TIMEEXEC'),os.R_OK):  # e.g. ./PERFTIMEEXEC
    globalTimeExec=True
else:
    globalTimeExec=False

# Get global timeout for kill
if os.access('./KILLTIMEOUT',os.R_OK):
    globalKillTimeout = ReadIntegerValue('./KILLTIMEOUT', localdir)
else:
    globalKillTimeout=10
# sys.stdout.write('globalKillTimeout=%d\n'%(globalKillTimeout))

if os.access('./NOEXEC',os.R_OK):
    execute=False
else:
    execute=True
# sys.stdout.write('execute=%d\n'%(execute))

if os.access('./NOVGRBIN',os.R_OK):
    vgrbin=False
else:
    vgrbin=True
# sys.stdout.write('vgrbin=%d\n'%(vgrbin))

if os.access('./COMPSTDIN',os.R_OK):
    compstdin='./COMPSTDIN'
else:
    compstdin='/dev/null'
# sys.stdout.write('compstdin=%s\n'%(compstdin))

globalLastcompopts=list();
if os.access('./LASTCOMPOPTS',os.R_OK):
    globalLastcompopts+=subprocess.Popen(['cat', './LASTCOMPOPTS'], stdout=subprocess.PIPE).communicate()[0].strip().split()
# sys.stdout.write('globalLastcompopts=%s\n'%(globalLastcompopts))

globalLastexecopts=list();
if os.access('./LASTEXECOPTS',os.R_OK):
    globalLastexecopts+=subprocess.Popen(['cat', './LASTEXECOPTS'], stdout=subprocess.PIPE).communicate()[0].strip().split()
# sys.stdout.write('globalLastexecopts=%s\n'%(globalLastexecopts))

if os.access(PerfDirFile('NUMLOCALES'),os.R_OK):
    globalNumlocales=ReadIntegerValue(PerfDirFile('NUMLOCALES'), localdir)
    # globalNumlocales.strip(globalNumlocales)
else:
    # start_test sets this, so we'll assume it's right :)
    globalNumlocales=int(os.getenv('NUMLOCALES', '0'))
# sys.stdout.write('globalNumlocales=%s\n'%(globalNumlocales))

if os.access('./CATFILES',os.R_OK):
    globalCatfiles=subprocess.Popen(['cat', './CATFILES'], stdout=subprocess.PIPE).communicate()[0]
    globalCatfiles.strip(globalCatfiles)
else:
    globalCatfiles=None
# sys.stdout.write('globalCatfiles=%s\n'%(globalCatfiles))


#
# valgrind stuff
#
chpl_valgrind_opts=os.getenv('CHPL_VALGRIND_OPTS', '--tool=memcheck')
# sys.stdout.write('chpl_valgrind_opts=%s\n'%(chpl_valgrind_opts))

if os.getenv('CHPL_TEST_VGRND_COMP')=='on':
    valgrindcomp = 'valgrind'
    valgrindcompopts=chpl_valgrind_opts.split()
    valgrindcompopts+=['--gen-suppressions=all']
    valgrindcompopts+=['--suppressions=%s/compiler/etc/valgrind.suppressions'%(chpl_home)]
    valgrindcompopts+=['-q']
else:
    valgrindcomp = None
    valgrindcompopts = None
# sys.stdout.write('valgrindcomp=%s %s\n'%(valgrindcomp, valgrindcompopts))

if (os.getenv('CHPL_TEST_VGRND_EXE')=='on' and vgrbin):
    valgrindbin = 'valgrind'
    valgrindbinopts = chpl_valgrind_opts.split()+['-q']
    if (chplcomm!='none'):
        valgrindbinopts+=['--trace-children=yes']
else:
    valgrindbin = None
    valgrindbinopts = None
# sys.stdout.write('valgrindbin=%s %s\n'%(valgrindbin, valgrindbinopts))


#
# Misc set up
#

testfutures=string.atoi(os.getenv('CHPL_TEST_FUTURES','0'))
# sys.stdout.write('testfutures=%s\n'%(testfutures))

testnotests=os.getenv('CHPL_TEST_NOTESTS')
# sys.stdout.write('testnotests=%s\n'%(testnotests))

launchcmd=os.getenv('LAUNCHCMD')
# sys.stdout.write('launchcmd=%s\n'%(launchcmd))

if os.getenv('CHPL_TEST_INTERP')=='on':
    execute=False
    futureSuffix='.ifuture'
else:
    futureSuffix='.future'
# sys.stdout.write('futureSuffix=%s\n'%(futureSuffix))

printpassesfile = None
if os.getenv('CHPL_TEST_COMP_PERF')!=None:
    compperftest=True
    
    # check for the main compiler performance directory
    if os.getenv('CHPL_TEST_COMP_PERF_DIR')!=None:
        compperfdir=os.getenv('CHPL_TEST_COMP_PERF_DIR')
    else:
        compperfdir=chpl_home+'/test/compperfdat/' 
    
    # The env var CHPL_PRINT_PASSES_FILE will cause the 
    # compiler to save the pass timings to specified file.
    if os.getenv('CHPL_PRINT_PASSES_FILE')!=None:
        printpassesfile=os.getenv('CHPL_PRINT_PASSES_FILE')
    else:
        printpassesfile='timing.txt'
        os.putenv('CHPL_PRINT_PASSES_FILE', 'timing.txt')
      
    # check for the perfkeys file
    if os.getenv('CHPL_TEST_COMP_PERF_KEYS')!=None:
        keyfile=os.getenv('CHPL_TEST_COMP_PERF_KEYS')
    else: 
        keyfile=chpl_home+'/test/performance/compiler/compilerPerformance.perfkeys'
        
    # Check for the directory to store the tempory .dat files that will get 
    # combined into one.    
    if os.getenv('CHPL_TEST_COMP_PERF_TEMP_DAT_DIR')!=None:
        tempDatFilesDir = os.getenv('CHPL_TEST_COMP_PERF_TEMP_DAT_DIR')
    else: 
        tempDatFilesDir = compperfdir + 'tempCompPerfDatFiles/'
        
else: 
    compperftest=False

#
# Global COMPOPTS/PERFCOMPOPTS:
#
#   Prefer PERFCOMPOPTS if doing performance testing; otherwise, use
#   COMPOPTS.  Note that COMPOPTS is used for performance testing
#   currently in the absence of a PERFCOMPOPTS file.  Not sure whether
#   or not this is a good idea, but preserving it for now for backwards
#   compatibility.
#
if (perftest and os.access(PerfDirFile('COMPOPTS'),os.R_OK)): # ./PERFCOMPOPTS
    tgco=ReadFileWithComments(PerfDirFile('COMPOPTS'))
    globalCompopts = shlex.split(tgco[0])
elif os.access('./COMPOPTS',os.R_OK):
    tgco=ReadFileWithComments('./COMPOPTS')
    globalCompopts = shlex.split(tgco[0])
else:
    globalCompopts=list()
envCompopts = os.getenv('COMPOPTS')
if envCompopts != None:
    globalCompopts += shlex.split(envCompopts)
# sys.stdout.write('globalCompopts=%s\n'%(globalCompopts))

#
# Global PERFNUMTRIALS
#
if perftest and os.access(PerfDirFile('NUMTRIALS'), os.R_OK): # ./PERFNUMTRIALS
    globalNumTrials = ReadIntegerValue(PerfDirFile('NUMTRIALS'), localdir)
else:
    globalNumTrials=int(os.getenv('CHPL_TEST_NUM_TRIALS', '1'))

#
# Global EXECENV
#
if os.access('./EXECENV',os.R_OK):
    globalExecenv=ReadFileWithComments('./EXECENV')
else:
    globalExecenv=list()
# sys.stdout.write('globalExecenv=%s\n'%(globalExecenv))

#
# Global EXECOPTS/PERFEXECOPTS
#
#
#   Prefer PERFEXECOPTS if doing performance testing; otherwise, use
#   EXECOPTS.  Note that EXECOPTS is used for performance testing
#   currently in the absence of a PERFEXECOPTS file.  Not sure whether
#   or not this is a good idea, but preserving it for now for backwards
#   compatibility.
#
if (perftest and os.access(PerfDirFile('EXECOPTS'),os.R_OK)): # ./PERFEXECOPTS
    tgeo=ReadFileWithComments(PerfDirFile('EXECOPTS'))
    globalExecopts= shlex.split(tgeo[0])
elif os.access('./EXECOPTS',os.R_OK):
    tgeo=ReadFileWithComments('./EXECOPTS')
    globalExecopts= shlex.split(tgeo[0])
else:
    globalExecopts=list()
envExecopts = os.getenv('EXECOPTS')
# sys.stdout.write('globalExecopts=%s\n'%(globalExecopts))

#
# Global PRECOMP, PREDIFF & PREEXEC
#
if os.access('./PRECOMP', os.R_OK|os.X_OK):
    globalPrecomp='./PRECOMP'
else:
    globalPrecomp=None
#
if os.access('./PREDIFF',os.R_OK|os.X_OK):
    globalPrediff='./PREDIFF'
else:
    globalPrediff=None
# sys.stdout.write('globalPrediff=%s\n'%(globalPrediff))
if os.access('./PREEXEC',os.R_OK|os.X_OK):
    globalPreexec='./PREEXEC'
else:
    globalPreexec=None
#
# Start running tests
#
sys.stdout.write('[Starting subtest - %s]\n'%(time.strftime('%a %b %d %H:%M:%S %Z %Y', time.localtime())))
#sys.stdout.write('[compiler: \'%s\']\n'%(compiler))
if systemPrediff:
    sys.stdout.write('[system-wide prediff: \'%s\']\n'%(systemPrediff))

if os.path.isdir(testdir+'/'+localdir):
    dirlist=os.listdir(testdir+'/'+localdir)
elif os.path.isdir(localdir):
    dirlist=os.listdir(localdir)
else:
    Fatal('Could not find local test directory: \''+localdir+'\'')

onetestsrc = os.getenv('CHPL_ONETEST')
if onetestsrc==None:
    testsrc=filter(IsChplSource, dirlist)
else:
    testsrc=list()
    testsrc.append(onetestsrc)

for testname in testsrc:
    sys.stdout.flush()
    exectimeout = False

    # print testname
    sys.stdout.write('[test: %s/%s]\n'%(localdir,testname))

    execname=testname[:len(testname)-5]
    # print execname

    # Test specific settings
    catfiles = globalCatfiles
    numlocales = globalNumlocales
    lastcompopts = list()
    if globalLastcompopts:
        lastcompopts += globalLastcompopts
    # sys.stdout.write("lastcompopts=%s\n"%(lastcompopts))
    lastexecopts = list()
    if globalLastexecopts:
        lastexecopts += globalLastexecopts
    # sys.stdout.write("lastexecopts=%s\n"%(lastexecopts))

    # Get the list of files starting with 'execname.'
    execname_files = fnmatch.filter(dirlist, execname+'.*')
    # print execname_files, dirlist

    if (perftest and (execname_files.count(PerfTFile(execname,'keys'))==0) and
        (execname_files.count(PerfTFile(execname,'execopts'))==0)):
        sys.stdout.write('[Skipping noperf test: %s/%s]\n'%(localdir,execname))
        continue # on to next test

    timeout = globalTimeout
    killtimeout = globalKillTimeout
    numTrials = globalNumTrials
    if (perftest):
        timeExec = globalTimeExec
    else:
        timeExec = False;
    futuretest=''
    executebin=execute
    testfuturesfile=False
    noexecfile=False
    execoptsfile=False
    precomp=None
    prediff=None
    preexec=None

    if os.getenv('CHPL_NO_STDIN_REDIRECT') == None:
        redirectin = '/dev/null'
    else:
        redirectin = None

    # Deal with these files
    do_not_test=False
    for f in execname_files:
        (root, suffix) = os.path.splitext(f)
        # sys.stdout.write("**** %s ****\n"%(f))

        # 'f' is of the form execname.SOMETHING.suffix,
        # not pertinent at the moment
        if root != execname:
            continue

        # Deal with these later
        if (suffix == '.good' or
            suffix=='.compopts' or suffix=='.perfcompopts' or
            suffix=='.execenv' or suffix=='.perfexecenv' or
            suffix=='.execopts' or suffix=='.perfexecopts'):
            continue # on to next file

        elif (suffix=='.notest' and (os.access(f, os.R_OK) and
                                     testnotests=='0')):
            sys.stdout.write('[Skipping notest test: %s/%s]\n'%(localdir,execname))
            do_not_test=True
            break

        elif (suffix=='.skipif' and (os.access(f, os.R_OK) and
               (os.getenv('CHPL_TEST_SINGLES')=='0'))):
            skiptest=subprocess.Popen([utildir+'/testEnv', './'+f], stdout=subprocess.PIPE).communicate()[0]
            try:
                skipme = int(skiptest)
                if skipme:
                    sys.stdout.write('[Skipping test based on .skipif environment settings: %s/%s]\n'%(localdir,execname))
                    do_not_test=True
            except ValueError:
                sys.stdout.write('[Error processing .skipif file %s/%s]\n'%(localdir,f))
                do_not_test=True
            if do_not_test:
                break

        elif (suffix==timeoutsuffix and os.access(f, os.R_OK)):
            timeout=ReadIntegerValue(f, localdir)
            sys.stdout.write('[Overriding default timeout with %d]\n'%(timeout))
        elif (perftest and suffix==PerfSfx('timeexec') and os.access(f, os.R_OK)): #e.g. .perftimeexec
            timeExec = True

        elif (perftest and suffix==PerfSfx('numtrials') and os.access(f, os.R_OK)): #e.g. .perfnumtrials
            numTrials = ReadIntegerValue(f, localdir)

        elif (suffix=='.killtimeout' and os.access(f, os.R_OK)):
            killtimeout=ReadIntegerValue(f, localdir)

        elif (suffix=='.catfiles' and os.access(f, os.R_OK)):
            execcatfiles=subprocess.Popen(['cat', f], stdout=subprocess.PIPE).communicate()[0].strip()
            if catfiles:
                catfiles+=execcatfiles
            else:
                catfiles=execcatfiles

        elif (suffix=='.lastcompopts' and os.access(f, os.R_OK)):
            lastcompopts+=subprocess.Popen(['cat', f], stdout=subprocess.PIPE).communicate()[0].strip().split()
            # sys.stdout.write("lastcompopts=%s\n"%(lastcompopts))

        elif (suffix=='.lastexecopts' and os.access(f, os.R_OK)):
            lastexecopts+=subprocess.Popen(['cat', f], stdout=subprocess.PIPE).communicate()[0].strip().split()
            # sys.stdout.write("lastexecopts=%s\n"%(lastexecopts))

        elif (suffix==PerfSfx('numlocales') and os.access(f, os.R_OK)):
            numlocales=ReadIntegerValue(f, localdir)

        elif suffix==futureSuffix and os.access(f, os.R_OK):
            futurefile = open('./'+execname+futureSuffix, 'r')
            futuretest='Future ('+futurefile.readline().strip()+') '
            futurefile.close()

        elif (suffix=='.noexec' and os.access(f, os.R_OK)):
            noexecfile=True
            executebin=False

        elif (suffix=='.precomp' and os.access(f, os.R_OK|os.X_OK)):
            precomp=f

        elif (suffix=='.prediff' and os.access(f, os.R_OK|os.X_OK)):
            prediff=f

        elif (suffix=='.preexec' and os.access(f, os.R_OK|os.X_OK)):
            preexec=f

        elif (suffix=='.stdin' and os.access(f, os.R_OK)):
            if redirectin == None:
                sys.stdout.write('[Skipping test with .stdin input since -nostdinredirect is given: %s/%s]\n'%(localdir,execname))
                do_not_test=True
                break
            else:
                redirectin=f

        if suffix==futureSuffix:
            if testfutures==0:
                sys.stdout.write('[Skipping future test: %s/%s]\n'%(localdir,execname))
                do_not_test=True
                break
            testfuturesfile=True

    del execname_files

    # Skip to the next test
    if do_not_test:
        continue # on to next test

    # Skip non-future tests if specified
    if (testfuturesfile==False and testfutures==2):
        sys.stdout.write('[Skipping non-future test: %s/%s]\n'%(localdir,execname))
        continue # on to next test

    # Set numlocales
    if (numlocales == 0) or (chplcomm=='none'):
        numlocexecopts = None
    else:
        numlocexecopts = ' -nl '+str(numlocales)

    # if any performance test has a timeout longer than the default we only
    # want to run it once
    if (timeout != globalTimeout):
      numTrials = 1

    # Get list of test specific compiler options
    if os.access(execname+compoptssuffix, os.R_OK):
        compoptslist = ReadFileWithComments(execname+compoptssuffix, False)
        if not compoptslist:
            # cf. for execoptslist no warning is issued
            sys.stdout.write('[Warning: ignoring an empty compopts file %s]\n'%(execname+compoptssuffix))
            compoptslist = list(' ')
    else:
        compoptslist = list(' ')

    # The test environment is that of this process, augmented as specified
    if os.access(execname+execenvsuffix, os.R_OK):
        execenv = ReadFileWithComments(execname+execenvsuffix)
    else:
        execenv = list()

    if len(globalExecenv)==0 and len(execenv)==0:
        testenv = os.environ
    else:
        testenv = os.environ.copy()
        for tev in globalExecenv:
            testenv[tev.split('=')[0].strip()] = tev.split('=')[1].strip()
        for tev in execenv:
            testenv[tev.split('=')[0].strip()] = tev.split('=')[1].strip()
        del tev

    # Get list of test specific exec options
    if os.access(execname+execoptssuffix, os.R_OK):
        execoptsfile=True
        execoptslist = ReadFileWithComments(execname+execoptssuffix, False)
    else:
        execoptslist = list()

    if (os.getenv('CHPL_TEST_INTERP')=='on' and
        (noexecfile or testfuturesfile or execoptsfile)):
        sys.stdout.write('[Skipping interpretation of: %s/%s]\n'%(localdir,execname))
        continue # on to next test

    clist = list()
    # For all compopts + execopts combos..
    compoptsnum = 0
    for compopts in compoptslist:
        sys.stdout.flush()
        del clist
        # use the remaining portion as a .good file for executing tests
        #  clist will be *added* to execopts if it is empty, or just used
        #  as the default .good file if not empty
        clist = compopts.split('#')
        if len(clist) >= 2:
            compopts = clist.pop(0)
            cstr = ' #' + '#'.join(clist)
            del clist[:]
            clist.append(cstr)
        else:
            del clist[:]

        if compopts == ' ':
            complog=execname+'.comp.out.tmp'
        else:
            compoptsnum += 1
            complog = execname+'.'+str(compoptsnum)+'.comp.out.tmp'

        #
        # Run the precompile script
        #
        if globalPrecomp:
            sys.stdout.write('[Executing ./PRECOMP]\n')
            sys.stdout.flush()
            subprocess.Popen(['./PRECOMP',
                             execname,complog,compiler]).wait()

        if precomp:
            sys.stdout.write('[Executing precomp %s.precomp]\n'%(execname))
            sys.stdout.flush()
            subprocess.Popen(['./'+execname+'.precomp',
                             execname,complog,compiler]).wait()


        #
        # Build the test program
        #
        args=['-o']+[execname]+globalCompopts+shlex.split(compopts)+[testname]
        if lastcompopts:
            args += lastcompopts
        # sys.stdout.write("args=%s\n"%(args))

        if valgrindcomp:
            cmd = valgrindcomp
            args = valgrindcompopts+[compiler]+args
        else:
            cmd = compiler

        #
        # Compile (with timeout)
        #
        sys.stdout.write('[Executing compiler %s'%(cmd))
        if args:
            sys.stdout.write(' %s'%(' '.join(args)))
        sys.stdout.write(' < %s]\n'%(compstdin))
        sys.stdout.flush()
        if useTimedExec:
            wholecmd = cmd+' '+' '.join(map(ShellEscape, args))
            p = subprocess.Popen([timedexec, str(timeout), wholecmd],
                                 stdin=open(compstdin, 'r'),
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
            output = p.communicate()[0]
            status = p.returncode

            if status == 222:
                sys.stdout.write('%s[Error: Timed out compilation for %s/%s'%
                                 (futuretest, localdir, execname))
                printTestVariation(compoptsnum);
                sys.stdout.write(']\n')
                cleanup(execname)
                cleanup(printpassesfile)
                continue # on to next compopts

        else:
            p = subprocess.Popen([cmd]+args, stdin=open(cmpstdin, 'r'),
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
            try:
                output = SuckOutputWithTimeout(p.stdout, timeout)
            except ReadTimeoutException:
                sys.stdout.write('%s[Error: Timed out compilation for %s/%s'%
                                 (futuretest, localdir, execname))
                printTestVariation(compoptsnum);
                sys.stdout.write(']\n')
                KillProc(p, killtimeout)
                cleanup(execname)
                cleanup(printpassesfile)
                continue # on to next compopts

            status = p.returncode

        if (status!=0 or not executebin):
            # Save original output
            origoutput = output;

            # Compare compiler output with expected program output
            if catfiles:
                sys.stdout.write('[Concatenating extra files: %s]\n'%
                                 (execname+'.catfiles'))
                sys.stdout.flush()
                output+=subprocess.Popen(['cat']+catfiles.split(),
                                         stdout=subprocess.PIPE,
                                         stderr=subprocess.STDOUT).communicate()[0]

            # Sadly these scripts require an actual file
            complogfile=file(complog, 'w')
            complogfile.write('%s'%(output))
            complogfile.close()

            if globalPrediff:
                sys.stdout.write('[Executing ./PREDIFF]\n')
                sys.stdout.flush()
                subprocess.Popen(['./PREDIFF',
                                  execname,complog,compiler,
                                  ' '.join(globalCompopts)+' '+compopts,
                                  ' '.join(args)]).wait()

            if prediff:
                sys.stdout.write('[Executing prediff %s.prediff]\n'%(execname))
                sys.stdout.flush()
                subprocess.Popen(['./'+execname+'.prediff',
                                  execname,complog,compiler,
                                  ' '.join(globalCompopts)+' '+compopts,
                                  ' '.join(args)]).wait()

            # Find the default 'golden' output file
            checkfile = ''
            # Use specified golden file if there is one.
            if len(clist) != 0:
                checkfile = clist[0].split('#')[1].strip()
            # Else use machine-specific file
            if not os.path.isfile(checkfile):
                checkfile = execname+'.'+machine+'.good'
            # Else if no-local use no-local file.
            if not os.path.isfile(checkfile):
                nlString = '--no-local'
                if globalCompopts.count(nlString) > 0:
                    checkfile=execname+'.no-local.good'
            # Else use comm-specific file.
            if not os.path.isfile(checkfile):
                checkfile=execname+chplcommstr+'.good'
            # Else use platform-specific file.
            if not os.path.isfile(checkfile):
                checkfile=execname+'.'+platform+'.good'
            # Else use general file.
            if not os.path.isfile(checkfile):
                checkfile=execname+'.good'
            # sys.stdout.write('default checkfile=%s\n'%(checkfile))

            if not os.access(checkfile, os.R_OK):
                sys.stdout.write('[Error cannot locate compiler output comparison file %s/%s]\n'%(localdir, checkfile))
                sys.stdout.write('[Compiler output was as follows:]\n')
                sys.stdout.write(origoutput)
                cleanup(execname)
                cleanup(printpassesfile)
                continue # on to next compopts

            result = DiffFiles(checkfile, complog)
            if result==0:
                os.unlink(complog)
                sys.stdout.write('%s[Success '%(futuretest))
            else:
                sys.stdout.write('%s[Error '%(futuretest))
            sys.stdout.write('matching compiler output for %s/%s'%
                                 (localdir, execname))
            printTestVariation(compoptsnum);
            sys.stdout.write(']\n')

            if (result != 0 and futuretest != ''):
                badfile=execname+'.bad'
                if os.access(badfile, os.R_OK):
                    badresult = DiffFiles(badfile, complog)
                    if badresult==0:
                        os.unlink(complog);
                        sys.stdout.write('[Clean match against .bad file ')
                    else:
                        # bad file doesn't match, which is bad
                        sys.stdout.write('[Error matching .bad file ')
                    sys.stdout.write('for %s/%s'%(localdir, execname))
                    printTestVariation(compoptsnum);
                    sys.stdout.write(']\n');

            cleanup(execname)
            cleanup(printpassesfile)
            continue # on to next compopts
        else:
            compoutput = output # save for diff


        #
        # Compile successful
        #
        sys.stdout.write('[Success compiling %s/%s]\n'%(localdir, execname))

        # Note that compiler performance only times successful compilations. 
        # Tests that are designed to fail before compilation is complete will 
        # not get timed, so the total time compiling might be off slightly.   
        if compperftest:
            # make the compiler performance directories if they don't exist 
            timePasses = True
            if not os.path.isdir(compperfdir) and not os.path.isfile(compperfdir):
                os.makedirs(compperfdir)
            if not os.access(compperfdir, os.R_OK|os.X_OK):
                sys.stdout.write('[Error creating compiler performance test directory %s]\n'%(compperfdir))
                timePasses = False

            if not os.path.isdir(tempDatFilesDir) and not os.path.isfile(tempDatFilesDir):
                os.makedirs(tempDatFilesDir)
            if not os.access(compperfdir, os.R_OK|os.X_OK):
                sys.stdout.write('[Error creating compiler performance temp dat file test directory %s]\n'%(tempDatFilesDir))
                timePasses = False

            # so long as we have to the directories 
            if timePasses: 
                # make the datFileName the full path with / replaced with -- so
                # we can keep the full path but not create a bunch of new
                # directories
                datFileName = localdir.replace('/', '--') + '--' + execname + str(compoptsnum) 

                # computePerfStats for the current test 
                sys.stdout.write('[Executing computePerfStats %s %s %s %s %s]\n'%(datFileName, tempDatFilesDir, keyfile, printpassesfile, str(exectimeout)))
                sys.stdout.flush()
                p = subprocess.Popen([utildir+'/computePerfStats', datFileName, tempDatFilesDir, keyfile, printpassesfile, str(exectimeout)], stdout=subprocess.PIPE)
                sys.stdout.write('%s'%(p.communicate()[0]))
                sys.stdout.flush()

                status = p.returncode

                if status == 0:
                    sys.stdout.write('[Success')
                else:
                    sys.stdout.write('[Error')
                sys.stdout.write(' finding  compiler performance keys for %s/%s'%
                                 (localdir, execname))
                if status!=0:
                    printTestVariation(compoptsnum, execoptsnum);
                sys.stdout.write(']\n')    

            #delete the timing file     
            cleanup(printpassesfile)


        if os.getenv('CHPL_COMPONLY'):
            sys.stdout.write('[Note: Not executing or comparing the output due to -noexec flags]\n')
            cleanup(execname)
            continue # on to next compopts

        dfltexeccheckfile = None
        # Execute the test for all requested execopts
        execoptsnum = 0
        # Handle empty execopts list
        if len(execoptslist)==0:
            # cf. for compoptslist, a warning is issued in this case
            execoptslist.append(' ')
        if len(clist)!=0:
            if len(clist[0].split('#')) > 1:
                dfltexeccheckfile = clist[0].split('#')[1].strip()
        for texecopts in execoptslist:
            sys.stdout.flush()

            if (len(compoptslist)==1) and (len(execoptslist)==1):
                onlyone = True
                execlog=execname+'.exec.out.tmp'
            else:
                onlyone = False
                if texecopts != ' ':
                    execoptsnum += 1
                execlog = execname+'.'+str(compoptsnum)+'-'+str(execoptsnum)+'.exec.out.tmp'

            tlist = texecopts.split('#')
            execopts = tlist[0].strip()

            if numlocexecopts != None:
                execopts += numlocexecopts;
            if len(tlist) > 1:
                # Ignore everything after the first token
                execcheckfile = tlist[1].strip().split()[0]
            else:
                execcheckfile = dfltexeccheckfile
            del tlist

            if globalPreexec:
                sys.stdout.write('[Executing ./PREEXEC]\n')
                sys.stdout.flush()
                subprocess.Popen(['./PREEXEC',
                                  execname,execlog,compiler]).wait()

            if preexec:
                sys.stdout.write('[Executing preexec %s.preexec]\n'%(execname))
                sys.stdout.flush()
                subprocess.Popen(['./'+execname+'.preexec',
                                  execname,execlog,compiler]).wait()

            if not os.access(execname, os.R_OK|os.X_OK):
                sys.stdout.write('%s[Error could not locate executable %s for %s/%s'%
                                 (futuretest, execname, localdir, execname))
                printTestVariation(compoptsnum, execoptsnum)
                sys.stdout.write(']\n')
                break; # on to next compopts

            args=list()
            if timeExec:
                cmd='time'
                args+=['-p']+['./'+execname]
            elif launchcmd:
                cmd=launchcmd
                args+=['./'+execname]
            elif valgrindbin:
                cmd=valgrindbin
                args+=valgrindbinopts+['./'+execname]
            else:
                cmd='./'+execname

            args+=globalExecopts
            args+=shlex.split(execopts)
            if envExecopts != None:
                args+=shlex.split(envExecopts)
            if lastexecopts:
                args += lastexecopts
            # sys.stdout.write("args=%s\n"%(args))

            #
            # Run program (with timeout)
            #
            for count in xrange(numTrials):
                sys.stdout.write('[Executing program %s %s'%(cmd, ' '.join(args)))
                if redirectin:
                    sys.stdout.write(' < %s'%(redirectin))
                sys.stdout.write(']\n')
                sys.stdout.flush()

                if useLauncherTimeout:
                    if redirectin == None:
                        my_stdin = None
                    else:
                        my_stdin=file(redirectin, 'r')
                    p = subprocess.Popen([cmd]+LauncherTimeoutArgs(timeout)+args,
                                        env=testenv,
                                        stdin=my_stdin,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
                    output = p.communicate()[0]
                    status = p.returncode

                    if re.search('PBS: job killed: walltime', output) != None:
                        exectimeout = True
                        sys.stdout.write('%s[Error: Timed out executing program %s/%s'%
                                        (futuretest, localdir, execname))
                        printTestVariation(compoptsnum, execoptsnum);
                        sys.stdout.write(']\n')
                        sys.stdout.write('[Execution output was as follows:]\n')
                        sys.stdout.write(output)
                    
                    # so far as I can tell, an interactive slurm run doesn't tell 
                    # you when walltime has been hit. If batch walltime is hit an 
                    # error message is put in the output file but we don't have 
                    # here. For slurm sub_test just queses up the job and then 
                    # there is a slurm prediff that waits for the job to complete
                    # or timeout so the timeout will be caught by the .good diff 
                    # failures. The error message isn't very pretty though

                elif useTimedExec:
                    wholecmd = cmd+' '+' '.join(map(ShellEscape, args))

                    if redirectin == None:
                        my_stdin = sys.stdin
                    else:
                        my_stdin = file(redirectin, 'r')
                    p = subprocess.Popen([timedexec, str(timeout), wholecmd],
                                        env=testenv,
                                        stdin=my_stdin,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
                    output = p.communicate()[0]
                    status = p.returncode

                    if status == 222:
                        exectimeout = True
                        sys.stdout.write('%s[Error: Timed out executing program %s/%s'%
                                        (futuretest, localdir, execname))
                        printTestVariation(compoptsnum, execoptsnum);
                        sys.stdout.write(']\n')
                        sys.stdout.write('[Execution output was as follows:]\n')
                        sys.stdout.write(output)

                else:
                    if redirectin == None:
                        my_stdin = None
                    else:
                        my_stdin=file(redirectin, 'r')
                    p = subprocess.Popen([cmd]+args,
                                        env=testenv,
                                        stdin=my_stdin,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
                    try:
                        output = SuckOutputWithTimeout(p.stdout, timeout)
                    except ReadTimeoutException:
                        exectimeout = True
                        sys.stdout.write('%s[Error: Timed out executing program %s/%s'%
                                        (futuretest, localdir, execname))
                        printTestVariation(compoptsnum, execoptsnum);
                        sys.stdout.write(']\n')
                        KillProc(p, killtimeout)

                    status = p.returncode

                if catfiles:
                    sys.stdout.write('[Concatenating extra files: %s]\n'%
                                    (execname+'.catfiles'))
                    sys.stdout.flush()
                    output+=subprocess.Popen(['cat']+catfiles.split(),
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.STDOUT).communicate()[0]

                # Sadly the scripts used below require an actual file
                execlogfile=file(execlog, 'w')
                execlogfile.write('%s'%(compoutput))
                execlogfile.write('%s'%(output))
                execlogfile.close()

                if not exectimeout:
                    if systemPrediff:
                        sys.stdout.write('[Executing system-wide prediff]\n')
                        sys.stdout.flush()
                        sys.stdout.write(subprocess.Popen([systemPrediff,
                                                          execname,execlog,compiler,
                                                          ' '.join(globalCompopts)+
                                                          ' '+compopts,
                                                          ' '.join(args)],
                                                          stdout=subprocess.PIPE).
                                        communicate()[0])

                    if globalPrediff:
                        sys.stdout.write('[Executing ./PREDIFF]\n')
                        sys.stdout.flush()
                        sys.stdout.write(subprocess.Popen(['./PREDIFF',
                                                          execname,execlog,compiler,
                                                          ' '.join(globalCompopts)+
                                                          ' '+compopts,
                                                          ' '.join(args)],
                                                          stdout=subprocess.PIPE).
                                        communicate()[0])

                    if prediff:
                        sys.stdout.write('[Executing prediff ./%s]\n'%(prediff))
                        sys.stdout.flush()
                        sys.stdout.write(subprocess.Popen(['./'+prediff,
                                                          execname,execlog,compiler,
                                                          ' '.join(globalCompopts)+
                                                          ' '+compopts,
                                                          ' '.join(args)],
                                                          stdout=subprocess.PIPE).
                                        communicate()[0])

                    if not perftest:
                        if (execcheckfile == None):
                            # Look for the "golden" output
                            if onlyone:
                                ceident=''
                            else:
                                ceident = '.'+str(compoptsnum)+'-'+str(execoptsnum)

                            execcheckfile = execname+'.'+machine+ceident+'.good'
                            # Else if no-local use the no-local .good file.
                            if not os.path.isfile(execcheckfile):
                                nlString = '--no-local'
                                if globalCompopts.count(nlString) > 0:
                                    execcheckfile=execname+'.no-local'+ceident+'.good'
                            # Else use the comm-specific .good file.
                            if not os.path.isfile(execcheckfile):
                                execcheckfile=execname+chplcommstr+ceident+'.good'
                            # Else use the platform-specific .good file.
                            if not os.path.isfile(execcheckfile):
                                execcheckfile=execname+'.'+platform+ceident+'.good'
                            # Else use the execopts-specific .good file.
                            if not os.path.isfile(execcheckfile):
                                execcheckfile=execname+ceident+'.good'
                            # Else use the general .good file.
                            if not os.path.isfile(execcheckfile):
                                execcheckfile=execname+'.good'

                        if not os.access(execcheckfile, os.R_OK):
                            sys.stdout.write('[Error cannot locate program output comparison file %s/%s]\n'%(localdir, execcheckfile))
                            sys.stdout.write('[Execution output was as follows:]\n')
                            sys.stdout.write(subprocess.Popen(['cat', execlog],
                                                              stdout=subprocess.PIPE).
                                            communicate()[0])

                            continue # on to next execopts

                        result = DiffFiles(execcheckfile, execlog)
                        if result==0:
                            os.unlink(execlog)
                            sys.stdout.write('%s[Success '%(futuretest))
                        else:
                            sys.stdout.write('%s[Error '%(futuretest))
                        sys.stdout.write('matching program output for %s/%s'%
                                        (localdir, execname))
                        if result!=0:
                            printTestVariation(compoptsnum, execoptsnum);
                        sys.stdout.write(']\n')

                        if (result != 0 and futuretest != ''):
                            badfile=execname+'.bad'
                            if os.access(badfile, os.R_OK):
                                badresult = DiffFiles(badfile, execlog)
                                if badresult==0:
                                    os.unlink(execlog);
                                    sys.stdout.write('[Clean match against .bad file ')
                                else:
                                    # bad file doesn't match, which is bad
                                    sys.stdout.write('[Error matching .bad file ')
                                sys.stdout.write('for %s/%s'%(localdir, execname))
                                printTestVariation(compoptsnum);
                                sys.stdout.write(']\n');


                if perftest:
                    if not os.path.isdir(perfdir) and not os.path.isfile(perfdir):
                        os.makedirs(perfdir)
                    if not os.access(perfdir, os.R_OK|os.X_OK):
                        sys.stdout.write('[Error creating performance test directory %s]\n'%(perfdir))
                        break # on to next compopts

                    if execcheckfile==None:
                        perfexecname = execname
                        keyfile = PerfTFile(execname,'keys') #e.g. .perfkeys
                    else:
                        perfexecname = re.sub(r'\{0}$'.format(PerfSfx('keys')), '', execcheckfile)
                        if os.path.isfile(execcheckfile):
                            keyfile = execcheckfile
                        else:
                            keyfile = PerfTFile(execname,'keys')

                    perfdate = os.getenv('CHPL_TEST_PERF_DATE')
                    if perfdate == None:
                        perfdate = datetime.date.today().strftime("%m/%d/%y")

                    sys.stdout.write('[Executing %s/computePerfStats %s %s %s %s %s %s]\n'%(utildir, perfexecname, perfdir, keyfile, execlog, str(exectimeout), perfdate))
                    sys.stdout.flush()

                    p = subprocess.Popen([utildir+'/computePerfStats',
                                          perfexecname, perfdir, keyfile, execlog, str(exectimeout), perfdate],
                                         stdout=subprocess.PIPE)
                    sys.stdout.write('%s'%(p.communicate()[0]))
                    sys.stdout.flush()

                    status = p.returncode

                    if status == 0:
                        sys.stdout.write('[Success')
                    else:
                        sys.stdout.write('[Error')
                    sys.stdout.write(' matching performance keys for %s/%s'%
                                    (localdir, execname))
                    if status!=0:
                        printTestVariation(compoptsnum, execoptsnum);
                    sys.stdout.write(']\n')

                    if exectimeout or status != 0:
                        break

        cleanup(execname)

    del execoptslist
    del compoptslist

sys.exit(0)
