Merge pull request #337 from PRUNERS/issue333-bisect-extraneous-runs

Closes Issue #333 bisect extraneous runs

data/Makefile.in

@@ -291,6 +291,20 @@ Makefile: $(FLIT_SCRIPT_DIR)/flitutil.py
 Makefile: $(FLIT_SCRIPT_DIR)/flit_update.py
 	$(FLIT_SCRIPT_DIR)/flit.py update
 
+# Since this makefile has a huge dependency tree, it was causing the parsing to
+# take way too long to generate, especially for small requested tasks.
+# Therefore, we split each executable's dependencies into a recursive call on
+# this Makefile, setting R_TYPE to compile so that it enumerates the inner
+# dependencies of the specified executable (specified in R_TARGET).
+#
+# This function creates the recursive call for a target.
+#
+# Note: The target itself should not be phony, otherwise any targets depending
+# on $1 would always be rebuilt even if $1 does not need to be.  But, we can
+# instead have the $1 target depending on a phony target, then having a no-op
+# recipe so that other targets that depend on $1 will recheck the timestamp of
+# $1 after the phony target is called.
+#
 # @param 1: binary compile target
 # @param 2: compiler binary
 # @param 3: optimization level
@@ -300,8 +314,11 @@ Makefile: $(FLIT_SCRIPT_DIR)/flit_update.py
 # @param 7: directory for object files
 define REC_COMPILE_RULE
 ifneq ($$(R_TARGET),$1)
-.PHONY: $1
-$1: Makefile custom.mk
+$1: $1-phony
+	@true
+
+.PHONY: $1-phony
+$1-phony: Makefile custom.mk
 	+$$(REC_MAKE) R_TYPE=compile \
 	  R_TARGET="$1" \
 	  R_COMPILER="$2" \

scripts/flitcli/flit_bisect.py

@@ -524,10 +524,15 @@ def run_make(makefilename='Makefile', directory='.', verbose=False,
                 subp.check_call(command, stdout=tmpout, stderr=subp.STDOUT)
             else:
                 ps = subp.Popen(command, stdout=subp.PIPE, stderr=subp.STDOUT)
-                subp.check_call(['tee', tmpout.name], stdin=ps.stdout)
+                out = subp.check_output(['tee', tmpout.name], stdin=ps.stdout, universal_newlines=True)
                 ps.communicate()
+
                 if ps.returncode != 0:
                     raise subp.CalledProcessError(ps.returncode, command)
+
+                # write to stdout; can't set stdout in subprocess as non-file streams throw exceptions
+                if sys.stdout != sys.__stdout__:
+                    _ = sys.stdout.write(out)
         except:
             tmpout.flush()
             with open(tmpout.name, 'r') as tmpin:

tests/flit_cli/flit_bisect/tst_bisect_duplicateGTtest.py

@@ -0,0 +1,146 @@
+# -- LICENSE BEGIN --
+#
+# Copyright (c) 2015-2020, Lawrence Livermore National Security, LLC.
+#
+# Produced at the Lawrence Livermore National Laboratory
+#
+# Written by
+#   Michael Bentley ([email protected]),
+#   Geof Sawaya ([email protected]),
+#   and Ian Briggs ([email protected])
+# under the direction of
+#   Ganesh Gopalakrishnan
+#   and Dong H. Ahn.
+#
+# LLNL-CODE-743137
+#
+# All rights reserved.
+#
+# This file is part of FLiT. For details, see
+#   https://pruners.github.io/flit
+# Please also read
+#   https://github.com/PRUNERS/FLiT/blob/master/LICENSE
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+# - Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the disclaimer below.
+#
+# - Redistributions in binary form must reproduce the above
+#   copyright notice, this list of conditions and the disclaimer
+#   (as noted below) in the documentation and/or other materials
+#   provided with the distribution.
+#
+# - Neither the name of the LLNS/LLNL nor the names of its
+#   contributors may be used to endorse or promote products derived
+#   from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL
+# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+# THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Additional BSD Notice
+#
+# 1. This notice is required to be provided under our contract
+#    with the U.S. Department of Energy (DOE). This work was
+#    produced at Lawrence Livermore National Laboratory under
+#    Contract No. DE-AC52-07NA27344 with the DOE.
+#
+# 2. Neither the United States Government nor Lawrence Livermore
+#    National Security, LLC nor any of their employees, makes any
+#    warranty, express or implied, or assumes any liability or
+#    responsibility for the accuracy, completeness, or usefulness of
+#    any information, apparatus, product, or process disclosed, or
+#    represents that its use would not infringe privately-owned
+#    rights.
+#
+# 3. Also, reference herein to any specific commercial products,
+#    process, or services by trade name, trademark, manufacturer or
+#    otherwise does not necessarily constitute or imply its
+#    endorsement, recommendation, or favoring by the United States
+#    Government or Lawrence Livermore National Security, LLC. The
+#    views and opinions of authors expressed herein do not
+#    necessarily state or reflect those of the United States
+#    Government or Lawrence Livermore National Security, LLC, and
+#    shall not be used for advertising or product endorsement
+#    purposes.
+#
+# -- LICENSE END --
+
+'''
+Tests FLiT's capabilities to run bisect and identify the problem files and
+functions
+
+The tests are below using doctest
+
+Let's now make a temporary directory and test that we can successfully compile
+and run FLiT bisect
+
+>>> import glob
+>>> import os
+>>> import shutil
+>>> import subprocess as subp
+>>> from io import StringIO
+>>> import flitutil as util
+>>> from common import BisectTestError, flit_init
+
+>>> class BisectTestError(RuntimeError): pass
+
+>>> with th.tempdir() as temp_dir:
+...     init_out = flit_init(temp_dir)
+...     shutil.rmtree(os.path.join(temp_dir, 'tests'))
+...     _ = shutil.copytree(os.path.join('data', 'tests'),
+...                         os.path.join(temp_dir, 'tests'))
+...     with open(os.path.join(temp_dir, 'custom.mk'), 'a') as mkout:
+...         _ = mkout.write('SOURCE += tests/file4.cxx\\n')
+...     _ = subp.check_call(['make', '-C', os.path.relpath(temp_dir), 'ground-truth.csv'])
+...     outfile = os.path.join(temp_dir, 'out.log')
+...     os.environ['VERBOSE'] = '1'
+...     with StringIO() as ostream:
+...         retval = th.flit.main(['bisect', '-C', os.path.relpath(temp_dir),
+...                                '--verbose',
+...                                '--compiler-type', 'gcc',
+...                                '--precision', 'double',
+...                                'g++ -O2', 'BisectTest'],
+...                                outstream=ostream)
+...         if retval != 0:
+...             raise BisectTestError(
+...                 'Could not bisect (1) (retval={0}):\\n'.format(retval) +
+...                 ostream.getvalue())
+...         bisect_out = ostream.getvalue().splitlines()
+...     with open(os.path.join(temp_dir, 'bisect-01', 'bisect.log')) as fin:
+...         raw_log = fin.readlines()
+...         log_contents = [line for line in raw_log if line.strip() != '']
+
+Verify ground-truth.csv is not generated in bisect run,
+since it was generated prior to bisect call
+>>> gt_lines = [line for line in bisect_out if line.find('--output ground-truth.csv') != -1]
+>>> len(gt_lines)
+0
+
+'''
+
+# Test setup before the docstring is run.
+import sys
+before_path = sys.path[:]
+sys.path.append('../..')
+import test_harness as th
+sys.path = before_path
+
+if __name__ == '__main__':
+    from doctest import testmod
+    failures, tests = testmod()
+    sys.exit(failures)

tests/flit_makefile/tst_incremental_build.py

@@ -138,8 +138,10 @@
 >>> touched_files(before_build)
 ['devrun', 'obj/dev/ALL-FLIT.cpp.o', 'obj/dev/Empty.cpp.o', 'obj/dev/main.cpp.o']
 
+This is erroneously touched by `make --touch` due to a no-op recipe in the makefile.
+Should be removed in Issue 338 fix
 >>> touched_files(after_build)
-[]
+['devrun']
 
 >>> touched_files(after_modify)
 ['devrun', 'obj/dev/main.cpp.o']
@@ -176,8 +178,10 @@
 >>> touched_files(before_build)
 ['gtrun', 'obj/gt/ALL-FLIT.cpp.o', 'obj/gt/Empty.cpp.o', 'obj/gt/main.cpp.o']
 
+This is erroneously touched by `make --touch` due to a no-op recipe in the makefile.
+Should be removed in Issue 338 fix
 >>> touched_files(after_build)
-[]
+['gtrun']
 
 >>> touched_files(after_modify)
 ['gtrun', 'obj/gt/main.cpp.o']