Merge pull request #332 from PRUNERS/issue328-flit-disguise

Issue328 flit disguise

Fixes issue #328, issue #277, and issue #263

.travis.yml

@@ -82,18 +82,19 @@
 
 language: cpp
 os: linux
+#dist: bionic
 addons:
   apt:
     packages: &native_deps
       - python3
       - python3-pip
+      - bash-completion
 
 matrix:
   include:
     # Job 1: OpenMPI
     - env:
       - mpi_type=openmpi
-      - extra_pip=pyelftools
       addons:
         apt:
           packages:
@@ -103,7 +104,6 @@ matrix:
     # Job 2: MPICH
     - env:
       - mpi_type=mpich
-      - extra_pip=
       addons:
         apt:
           packages:
@@ -113,17 +113,16 @@ matrix:
     # Job 3: No MPI
     - env:
       - mpi_type=none
-      - extra_pip=
       addons:
         apt:
           packages:
             - *native_deps
 
 before_install:
   - pip3 install --user setuptools
-  - pip3 install --user toml $extra_pip
+  - pip3 install --user toml
 
-script: make -j4 && make -j4 -C tests && make check
+script: dpkg --list | grep binutil && make -j4 && make -j4 -C tests && make check
 
 #notifications:
 #  email: false

documentation/flit-command-line.md

@@ -162,11 +162,6 @@ flit import --dbfile temporary.sqlite backup/results/*.csv
 
 ## flit bisect
 
-There is an additional optional dependency in order to run `flit bisect`.  That
-is [pyelftools](https://github.com/eliben/pyelftools) as discussed in [FLiT
-Installation](installation.md).  If `pyelftools` is not installed, then
-`bisect` is disabled.
-
 After FLiT identifies compilations that cause some tests to exhibit
 variability, one may want to investigate further and understand where the
 compiler introduced overly aggressive optimizations.
@@ -177,6 +172,9 @@ blamed source files.  You can run `flit bisect` directly giving it a specific
 compilation, precision, and test case, or you can tell it to automatically run
 for all differences in a given SQLite3 database.
 
+FLiT Bisect depends on binutils to extract symbols, filenames, and line numbers
+from compiled object files.
+
 Here is an example of giving a single test case (named `subnormal`) known to
 show variability:
 

documentation/installation.md

@@ -31,9 +31,6 @@ Stuff you may need to get
 * [python3](https://www.python.org)
   * [toml](https://github.com/uiri/toml) module (for
     [TOML](https://github.com/toml-lang/toml) configuration files)
-  * (optional) [pyelftools](https://github.com/eliben/pyelftools) module for
-    parsing ELF files.  This is used for `flit bisect`; all other functionality
-    will work without it.
 * [make](https://www.gnu.org/software/make)
 * A C++11 compatible compiler
   (see section [Compilers](#compilers) for supported versions)
@@ -51,21 +48,21 @@ sudo apt install \
 The python modules can be installed with `apt`
 
 ```bash
-sudo apt install python3-toml python3-pyelftools
+sudo apt install python3-toml
 ```
 
 or with `pip`
 
 ```bash
 sudo apt install python3-pip
-pip3 install --user toml pyelftools
+pip3 install --user toml
 ```
 
 For homebrew on OSX (besides installing [Xcode](https://developer.apple.com/xcode))
 
 ```bash
 brew install make python3 gcc git
-pip3 install toml pyelftools
+pip3 install toml
 ```
 
 If you install python version 3.0 or later, then you will need to have a

scripts/bash-completion/flit

@@ -23,7 +23,8 @@ _flit__sqlite_files()
 _flit_help()
 {
   local cur available_subcommands
-  available_subcommands="-h --help bisect experimental init make update import"
+  available_subcommands="-h --help
+    bisect experimental disguise init make update import"
   cur="${COMP_WORDS[COMP_CWORD]}"
   COMPREPLY=( $(compgen -W "${available_subcommands}" -- ${cur}) )
 }
@@ -93,6 +94,39 @@ _flit_bisect()
   return 0
 }
 
+_flit_disguise()
+{
+  local cur prev opts
+  COMPREPLY=()
+  cur="${COMP_WORDS[COMP_CWORD]}"
+  prev="${COMP_WORDS[COMP_CWORD-1]}"
+  opts="-h --help
+    -g --generate
+    -o --output
+    -m --disguise-map
+    -u --undo
+    -j --jobs"
+  # file field
+
+  case "${prev}" in
+
+    -m|--disguise-map|-o|--output)
+      _filedir  # match with a file
+      return 0
+      ;;
+
+    -j|--jobs)
+      # do no completion -- numbers
+      return 0
+      ;;
+
+  esac
+
+  COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+  _filedir  # positional argument, match on files
+  return 0
+}
+
 _flit_init()
 {
   local cur prev opts
@@ -258,7 +292,7 @@ _flit()
   available_subcommands="
     -h --help
     -v --version
-    experimental help bisect init make update import"
+    experimental help bisect disguise init make update import"
 
   # subcommand completion
   if [ ${COMP_CWORD} -le 1 ]; then
@@ -270,6 +304,7 @@ _flit()
   case "${subcommand}" in
     help)         _flit_help         ;;
     bisect)       _flit_bisect       ;;
+    disguise)     _flit_disguise     ;;
     init)         _flit_init         ;;
     make)         _flit_make         ;;
     update)       _flit_update       ;;

scripts/flitcli/flit_bisect.py

@@ -85,6 +85,7 @@
 files that cause the variability.
 '''
 
+from collections import namedtuple
 from tempfile import NamedTemporaryFile
 import argparse
 import csv
@@ -699,6 +700,8 @@ def is_result_differing(resultfile):
     return float(get_comparison_result(resultfile)) != 0.0
 
 _extract_symbols_memos = {}
+BisectSymbolTuple = namedtuple('BisectSymbolTuple',
+                               'src, symbol, demangled, fname, lineno')
 def extract_symbols(file_or_filelist, objdir):
     '''
     Extracts symbols for the given source file(s).  The corresponding object is
@@ -733,7 +736,16 @@ def extract_symbols(file_or_filelist, objdir):
     if fobj in _extract_symbols_memos:
         return _extract_symbols_memos[fobj]
 
-    _extract_symbols_memos[fobj] = elf.extract_symbols(fobj, fname)
+    func_symbols, remaining_symbols = elf.extract_symbols(fobj)
+    func_symbols = [
+        BisectSymbolTuple(fname, sym.symbol, sym.demangled, sym.fname,
+                          sym.lineno)
+        for sym in func_symbols]
+    remaining_symbols = [
+        BisectSymbolTuple(fname, sym.symbol, sym.demangled, sym.fname,
+                          sym.lineno)
+        for sym in remaining_symbols]
+    _extract_symbols_memos[fobj] = (func_symbols, remaining_symbols)
     return _extract_symbols_memos[fobj]
 
 def memoize_strlist_func(func):
@@ -2215,7 +2227,7 @@ def main(arguments, prog=None):
     '''
 
     if elf is None:
-        print('Error: pyelftools is not installed, bisect disabled',
+        print('Error: binutils is not installed, bisect disabled',
               file=sys.stderr)
         return 1