diff --git a/.travis.yml b/.travis.yml index 03996658..2d7b222b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,18 +82,19 @@ language: cpp os: linux +#dist: bionic addons: apt: packages: &native_deps - python3 - python3-pip + - bash-completion matrix: include: # Job 1: OpenMPI - env: - mpi_type=openmpi - - extra_pip=pyelftools addons: apt: packages: @@ -103,7 +104,6 @@ matrix: # Job 2: MPICH - env: - mpi_type=mpich - - extra_pip= addons: apt: packages: @@ -113,7 +113,6 @@ matrix: # Job 3: No MPI - env: - mpi_type=none - - extra_pip= addons: apt: packages: @@ -121,9 +120,9 @@ matrix: before_install: - pip3 install --user setuptools - - pip3 install --user toml $extra_pip + - pip3 install --user toml -script: make -j4 && make -j4 -C tests && make check +script: dpkg --list | grep binutil && make -j4 && make -j4 -C tests && make check #notifications: # email: false diff --git a/documentation/flit-command-line.md b/documentation/flit-command-line.md index 0ab2a047..dbff00b4 100644 --- a/documentation/flit-command-line.md +++ b/documentation/flit-command-line.md @@ -162,11 +162,6 @@ flit import --dbfile temporary.sqlite backup/results/*.csv ## flit bisect -There is an additional optional dependency in order to run `flit bisect`. That -is [pyelftools](https://github.com/eliben/pyelftools) as discussed in [FLiT -Installation](installation.md). If `pyelftools` is not installed, then -`bisect` is disabled. - After FLiT identifies compilations that cause some tests to exhibit variability, one may want to investigate further and understand where the compiler introduced overly aggressive optimizations. @@ -177,6 +172,9 @@ blamed source files. You can run `flit bisect` directly giving it a specific compilation, precision, and test case, or you can tell it to automatically run for all differences in a given SQLite3 database. +FLiT Bisect depends on binutils to extract symbols, filenames, and line numbers +from compiled object files. + Here is an example of giving a single test case (named `subnormal`) known to show variability: diff --git a/documentation/installation.md b/documentation/installation.md index d26c1724..3ac3df75 100644 --- a/documentation/installation.md +++ b/documentation/installation.md @@ -31,9 +31,6 @@ Stuff you may need to get * [python3](https://www.python.org) * [toml](https://github.com/uiri/toml) module (for [TOML](https://github.com/toml-lang/toml) configuration files) - * (optional) [pyelftools](https://github.com/eliben/pyelftools) module for - parsing ELF files. This is used for `flit bisect`; all other functionality - will work without it. * [make](https://www.gnu.org/software/make) * A C++11 compatible compiler (see section [Compilers](#compilers) for supported versions) @@ -51,21 +48,21 @@ sudo apt install \ The python modules can be installed with `apt` ```bash -sudo apt install python3-toml python3-pyelftools +sudo apt install python3-toml ``` or with `pip` ```bash sudo apt install python3-pip -pip3 install --user toml pyelftools +pip3 install --user toml ``` For homebrew on OSX (besides installing [Xcode](https://developer.apple.com/xcode)) ```bash brew install make python3 gcc git -pip3 install toml pyelftools +pip3 install toml ``` If you install python version 3.0 or later, then you will need to have a diff --git a/scripts/bash-completion/flit b/scripts/bash-completion/flit index b636d4b3..c00fef14 100644 --- a/scripts/bash-completion/flit +++ b/scripts/bash-completion/flit @@ -23,7 +23,8 @@ _flit__sqlite_files() _flit_help() { local cur available_subcommands - available_subcommands="-h --help bisect experimental init make update import" + available_subcommands="-h --help + bisect experimental disguise init make update import" cur="${COMP_WORDS[COMP_CWORD]}" COMPREPLY=( $(compgen -W "${available_subcommands}" -- ${cur}) ) } @@ -93,6 +94,39 @@ _flit_bisect() return 0 } +_flit_disguise() +{ + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-h --help + -g --generate + -o --output + -m --disguise-map + -u --undo + -j --jobs" + # file field + + case "${prev}" in + + -m|--disguise-map|-o|--output) + _filedir # match with a file + return 0 + ;; + + -j|--jobs) + # do no completion -- numbers + return 0 + ;; + + esac + + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + _filedir # positional argument, match on files + return 0 +} + _flit_init() { local cur prev opts @@ -258,7 +292,7 @@ _flit() available_subcommands=" -h --help -v --version - experimental help bisect init make update import" + experimental help bisect disguise init make update import" # subcommand completion if [ ${COMP_CWORD} -le 1 ]; then @@ -270,6 +304,7 @@ _flit() case "${subcommand}" in help) _flit_help ;; bisect) _flit_bisect ;; + disguise) _flit_disguise ;; init) _flit_init ;; make) _flit_make ;; update) _flit_update ;; diff --git a/scripts/flitcli/flit_bisect.py b/scripts/flitcli/flit_bisect.py index 940254e9..7458d8c9 100644 --- a/scripts/flitcli/flit_bisect.py +++ b/scripts/flitcli/flit_bisect.py @@ -85,6 +85,7 @@ files that cause the variability. ''' +from collections import namedtuple from tempfile import NamedTemporaryFile import argparse import csv @@ -699,6 +700,8 @@ def is_result_differing(resultfile): return float(get_comparison_result(resultfile)) != 0.0 _extract_symbols_memos = {} +BisectSymbolTuple = namedtuple('BisectSymbolTuple', + 'src, symbol, demangled, fname, lineno') def extract_symbols(file_or_filelist, objdir): ''' Extracts symbols for the given source file(s). The corresponding object is @@ -733,7 +736,16 @@ def extract_symbols(file_or_filelist, objdir): if fobj in _extract_symbols_memos: return _extract_symbols_memos[fobj] - _extract_symbols_memos[fobj] = elf.extract_symbols(fobj, fname) + func_symbols, remaining_symbols = elf.extract_symbols(fobj) + func_symbols = [ + BisectSymbolTuple(fname, sym.symbol, sym.demangled, sym.fname, + sym.lineno) + for sym in func_symbols] + remaining_symbols = [ + BisectSymbolTuple(fname, sym.symbol, sym.demangled, sym.fname, + sym.lineno) + for sym in remaining_symbols] + _extract_symbols_memos[fobj] = (func_symbols, remaining_symbols) return _extract_symbols_memos[fobj] def memoize_strlist_func(func): @@ -2215,7 +2227,7 @@ def main(arguments, prog=None): ''' if elf is None: - print('Error: pyelftools is not installed, bisect disabled', + print('Error: binutils is not installed, bisect disabled', file=sys.stderr) return 1 diff --git a/scripts/flitcli/flit_disguise.py b/scripts/flitcli/flit_disguise.py new file mode 100644 index 00000000..85bec40c --- /dev/null +++ b/scripts/flitcli/flit_disguise.py @@ -0,0 +1,334 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2020, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +'Implements the disguise subcommand to anonymize project-specific data' + +import flitutil as util +import flit_bisect as bisect + +import argparse +import csv +import glob +import multiprocessing as mp +import os +import re +import subprocess as subp +import sys +try: + import flitelf as elf +except ImportError: + elf = None + +brief_description = 'Anonymizes project-specific data from text files' + +def populate_parser(parser=None): + 'Populate or create an ArgumentParser' + if parser is None: + parser = argparse.ArgumentParser() + parser.description = ''' + This command disguises (a.k.a., anonymizes) text or log files. Fields + that can be disguised are source file names, test names, and function names. + + To accomplish this feat, a mapping csv file will either be provided by + the user (with --disguise-map) or will be autogenerated as + "disguise.csv" (default behavior) from the contents of the Makefile and + the object files for the baseline compilation. The mapping is applied + as a very simple search and replace. + + To undo the disguise, use the --undo flag either allowing the default + "disguise.csv" file to be used or specifying one with --disguise-map. + ''' + parser.add_argument('-g', '--generate', action='store_true', + help=''' + Just generate the disguise map as "disguise.csv" + and then exit. It will be overwritten if it + already exists. Most users will not need to use + this flag. If you use this flag with the + --disguise-map flag, then it will output the map to + the specified disguise map. + ''') + parser.add_argument('-o', '--output', + help=''' + Output the disguised version of the input file to + this specified file. The default behavior is to + output to standard output. + ''') + parser.add_argument('-m', '--disguise-map', default='disguise.csv', + help=''' + Specify a specific CSV file to use as the disguise + map. The CSV file is expected to have a header row + with the column names "value" and "disguise". Both + columns should have unique values, i.e., a + one-to-one mapping. The default is "disguise.csv" + which is autogenerated if it is not there. + ''') + parser.add_argument('-u', '--undo', action='store_true', + help=''' + Undo the disguising. This will use the disguise + map to do search and replace in reverse. For + example, this can be used to de-anonymize the + analysis done by someone with the anonymized + file(s). + ''') + parser.add_argument('-j', '--jobs', default=None, + help=''' + When generating the disguise map, we may need to + compile the gtrun executable using the + autogenerated flit Makefile. This flag specifies + the number of jobs to give to GNU make. The + default behavior is to defer to the MAKEFLAGS + environment variable. If that variable is not set, + then we will use the number of processors. + ''') + parser.add_argument('file', nargs='?', + help=''' + Text file to disguise. Disguising is done with + simple search and replace. The "value" column of + the disguise map CSV file is searched for and + replaced with the "value" field. If --undo is + specified, then it is done in reverse. + + If the file is not specified, then it is read from + standard in. + ''') + return parser + +def generate_disguise_map(outfile='disguise.csv', jobs=None): + 'Generate the disguise map, often called from the Makefile' + + if not jobs and not hasattr(os.environ, 'MAKEFLAGS'): + jobs = mp.cpu_count() + + # make sure gtrun is compiled + make_args = ['make', 'gtrun'] + if jobs: + make_args.append('-j{}'.format(jobs)) + subp.check_call(make_args) + + makevars = util.extract_make_vars() + + # get list of source files + sources = sorted(makevars['SOURCE']) + + # get list of object files + objdir = makevars['GT_OBJ_DIR'][0] + objects = sorted([os.path.basename(source) + '.o' for source in sources]) + + # get list of function symbols and demangled signatures + symbol_objects, _ = elf.extract_symbols([ + os.path.join(objdir, obj) for obj in objects]) + symbols = sorted(sym.symbol for sym in symbol_objects) + demangled = sorted(sym.demangled for sym in symbol_objects) + + # get list of tests + tests = subp.check_output(['./gtrun', '--list-tests']).decode('utf-8').splitlines() + + seen_values = set() + + # write mapping to file + with open(outfile, 'w') as fout: + writer = csv.DictWriter(fout, ['disguise', 'value']) + writer.writeheader() + + def writerows(disguise_base, values): + 'Only write rows that have a unique value' + unique_values = [val for val in values if val not in seen_values] + seen_values.update(unique_values) + writer.writerows(gen_disguise_list(disguise_base, unique_values)) + + writerows('objfile', objects) + writerows('filepath', sources) + writerows('filename', sorted(os.path.basename(x) for x in sources)) + writerows('symbol', symbols) + writerows('demangled', demangled) + writerows('test', sorted(tests)) + + print('Created {}'.format(outfile)) + +def gen_disguise_list(disguise_base, values): + ''' + Generates a list of dictionaries for insertion into a disguise map. + Will add an integer to the disguise base, zero padded based on the number + of values in the given list of values. + + @param disguise_base (str): basename of the disguise value + @param values (list(str)): values to be disguised in this order + + >>> gen_disguise_list('ababab', []) + [] + + >>> expected = [ + ... {'disguise': 'happy-1', 'value': 'me'}, + ... {'disguise': 'happy-2', 'value': 'myself'}, + ... {'disguise': 'happy-3', 'value': 'I'}, + ... ] + >>> expected == gen_disguise_list('happy', ['me', 'myself', 'I']) + True + + >>> expected = [ + ... {'disguise': 'sad-01', 'value': '0'}, + ... {'disguise': 'sad-02', 'value': '1'}, + ... {'disguise': 'sad-03', 'value': '2'}, + ... {'disguise': 'sad-04', 'value': '3'}, + ... {'disguise': 'sad-05', 'value': '4'}, + ... {'disguise': 'sad-06', 'value': '5'}, + ... {'disguise': 'sad-07', 'value': '6'}, + ... {'disguise': 'sad-08', 'value': '7'}, + ... {'disguise': 'sad-09', 'value': '8'}, + ... {'disguise': 'sad-10', 'value': '9'}, + ... {'disguise': 'sad-11', 'value': '10'}, + ... ] + >>> expected == gen_disguise_list('sad', [str(i) for i in range(11)]) + True + ''' + ndigits = len(str(len(values))) + format_str = '{}-{{i:0{}d}}'.format(disguise_base, ndigits) + disguises = [{'disguise': format_str.format(i=i+1), 'value': val} + for i, val in enumerate(values)] + return disguises + +def read_disguise_map(fname): + 'Read and return the forward and reverse dictionary of the disguise map' + forward_map = {} + reverse_map = {} + with open(fname, 'r') as fin: + reader = csv.DictReader(fin) + assert 'disguise' in reader.fieldnames + assert 'value' in reader.fieldnames + for entry in reader: + disguise, value = entry['disguise'], entry['value'] + assert value not in forward_map + assert disguise not in reverse_map + forward_map[value] = disguise + reverse_map[disguise] = value + return forward_map, reverse_map + +def main(arguments, prog=None): + 'Main logic here' + parser = populate_parser() + if prog: parser.prog = prog + args = parser.parse_args(arguments) + + if args.generate: + generate_disguise_map(args.disguise_map) + return 0 + + if args.disguise_map == 'disguise.csv': + generate_disguise_map(args.disguise_map) + + forward_map, reverse_map = read_disguise_map(args.disguise_map) + mapping_to_use = reverse_map if args.undo else forward_map + + # choose the input stream + if args.file: + fin = open(args.file, 'r') + else: + fin = sys.stdin + + # choose the output stream + if args.output: + fout = open(args.output, 'w') + else: + fout = sys.stdout + + # like "grep -w" with a replace + for line in fin: + for key, val in mapping_to_use.items(): + pattern = re.escape(key) + if key[0].isalpha(): + pattern = r'\b' + pattern + if key[-1].isalpha(): + pattern = pattern + r'\b' + if re.search(pattern, line): + fout.write(re.sub(pattern, val, line)) + break + else: + fout.write(line) + + fout.flush() + if args.file: fin.close() + if args.output: fout.close() + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/scripts/flitcli/flitelf.py b/scripts/flitcli/flitelf.py index 5d04eaac..88a69587 100644 --- a/scripts/flitcli/flitelf.py +++ b/scripts/flitcli/flitelf.py @@ -1,6 +1,3 @@ -# Much of this is copied from the examples given in -# https://github.com/eliben/pyelftools.git - # -- LICENSE BEGIN -- # # Copyright (c) 2015-2020, Lawrence Livermore National Security, LLC. @@ -84,199 +81,123 @@ # -- LICENSE END -- ''' -Utility functions for dealing with ELF binary files. This file requires the -pyelftools package to be installed (i.e. module elftools). +Utility functions for dealing with ELF binary files. This file uses +alternative methods to do this functionality that does not require the +pyelftools package. + +Instead, this package uses binutils through subprocesses. The programs used +are "nm" and "c++filt" to perform the same functionality. ''' -from collections import namedtuple +from collections import namedtuple, defaultdict import subprocess as subp import os +import shutil -from elftools.elf.elffile import ELFFile -from elftools.elf.sections import SymbolTableSection +if not shutil.which('nm') or not shutil.which('c++filt'): + raise ImportError('Cannot find binaries "nm" and "c++filt"') SymbolTuple = namedtuple('SymbolTuple', - 'src, symbol, demangled, fname, lineno') + 'symbol, demangled, fname, lineno') SymbolTuple.__doc__ = ''' Tuple containing information about the symbols in a file. Has the following attributes: - src: source file that was compiled symbol: mangled symbol in the compiled version demangled: demangled version of symbol - fname: filename where the symbol is actually defined. This usually - will be equal to src, but may not be in some situations. + fname: filename where the symbol is defined. lineno: line number of definition within fname. ''' -def extract_symbols(objfile, srcfile): +def extract_symbols(objfile_or_list): ''' Extracts symbols for the given object file. - @param objfile: (str) path to object file + @param objfile_or_list: (str or list(str)) path to object file(s) @return two lists of SymbolTuple objects (funcsyms, remaining). The first is the list of exported functions that are strong symbols and have a filename and line number where they are defined. The second is all remaining symbols that are strong, exported, and defined. ''' - with open(objfile, 'rb') as fin: - elffile = ELFFile(fin) - - symtabs = [x for x in elffile.iter_sections() - if isinstance(x, SymbolTableSection)] - if len(symtabs) == 0: - raise RuntimeError('Object file {} does not have a symbol table' - .format(objfile)) - - # get globally exported defined symbols - syms = [sym for symtab in symtabs - for sym in symtab.iter_symbols() - if _is_symbol(sym) - and _is_extern(sym) - and _is_strong(sym) - and _is_defined(sym)] - - # split symbols into functions and non-functions - fsyms = [sym for sym in syms if _is_func(sym)] # functions - rsyms = list(set(syms).difference(fsyms)) # remaining - - # find filename and line numbers for each relevant func symbol - locs = _locate_symbols(elffile, fsyms) - - # demangle all symbols - fdemangled = _demangle([sym.name for sym in fsyms]) - rdemangled = _demangle([sym.name for sym in rsyms]) - - funcsym_tuples = [SymbolTuple(srcfile, fsyms[i].name, fdemangled[i], - locs[i][0], locs[i][1]) - for i in range(len(fsyms))] - remaining_tuples = [SymbolTuple(srcfile, rsyms[i].name, rdemangled[i], - None, None) - for i in range(len(rsyms))] - - return funcsym_tuples, remaining_tuples - -def _symbols(symtab): - 'Returns all symbols from the given symbol table' - return [sym for sym in symtab.iter_symbols() if _is_symbol(sym)] - -def _is_symbol(sym): - 'Returns True if elf.sections.Symbol object is a symbol' - return sym.name != '' and sym['st_info']['type'] != 'STT_FILE' - -def _is_extern(sym): - 'Returns True if elf.sections.Symbol is an extern symbol' - return sym['st_info']['bind'] != 'STB_LOCAL' - -def _is_weak(sym): - 'Returns True if elf.sections.Symbol is a weak symbol' - return sym['st_info']['bind'] == 'STB_WEAK' - -def _is_strong(sym): - 'Returns True if elf.sections.Symbol is a strong symbol' - return sym['st_info']['bind'] == 'STB_GLOBAL' - -def _is_defined(sym): - 'Returns True if elf.sections.Symbol is defined' - return sym['st_shndx'] != 'SHN_UNDEF' + funcsym_tuples = [] + remaining_tuples = [] + nm_args = [ + 'nm', + '--print-file-name', + '--extern-only', + '--defined-only', + ] + if isinstance(objfile_or_list, str): + nm_args.append(objfile_or_list) + else: + nm_args.extend(objfile_or_list) + symbol_strings = subp.check_output(nm_args).decode('utf-8').splitlines() + + obj_symbols = defaultdict(list) + symbols = [] + for symbol_string in symbol_strings: + loc, stype, symbol = symbol_string.split(maxsplit=2) + objfile, offset = loc.split(':') + symbols.append(symbol) + obj_symbols[objfile].append((offset, stype, symbol)) + + demangle_map = dict(zip(symbols, _demangle(symbols))) + + fileinfo_map = {} + linenumber_map = {} + for obj, symlist in obj_symbols.items(): + to_check = [] + for offset, stype, symbol in symlist: + if symbol in fileinfo_map and fileinfo_map[symbol]: + continue + elif stype.lower() != 't': + fileinfo_map[symbol] = (None, None) + else: + to_check.append((offset, symbol)) + fileinfo_map.update(_fnames_and_line_numbers(obj, to_check)) + + for symbol in symbols: + fnam, line = fileinfo_map[symbol] + symbol_tuple = SymbolTuple(symbol, demangle_map[symbol], fnam, line) + if fnam: + funcsym_tuples.append(symbol_tuple) + else: + remaining_tuples.append(symbol_tuple) -def _is_func(sym): - 'Returns True if elf.sections.Symbol is a function' - return sym['st_info']['type'] == 'STT_FUNC' + return funcsym_tuples, remaining_tuples def _demangle(symbol_list): 'Demangles each C++ name in the given list' + if not symbol_list: + return [] proc = subp.Popen(['c++filt'], stdin=subp.PIPE, stdout=subp.PIPE) out, _ = proc.communicate('\n'.join(symbol_list).encode()) demangled = out.decode('utf8').splitlines() assert len(demangled) == len(symbol_list) return demangled -def _locate_symbols(elffile, symbols): - ''' - Locates the filename and line number of each symbol in the elf file. - - @param elffile: (elf.elffile.ELFFile) The top-level elf file - @param symbols: (list(elf.sections.Symbol)) symbols to locate - - @return list(tuple(filename, lineno)) in the order of the given symbols - - If the file does not have DWARF info or a symbol is not found, an exception - is raised - - Test that even without a proper elffile, if there are no symbols to match, - then no error occurs and you can be on your merry way. - >>> _locate_symbols(object(), []) - [] +def _fnames_and_line_numbers(objfile, offset_symbol_tuples): ''' - if len(symbols) == 0: - return [] - - if not elffile.has_dwarf_info(): - raise RuntimeError('Elf file has no DWARF info') - - dwarfinfo = elffile.get_dwarf_info() - fltable = _gen_file_line_table(dwarfinfo) - - locations = [] - for sym in symbols: - for fname, lineno, start, end in fltable: - if start <= sym.entry['st_value'] < end: - locations.append((fname.decode('utf8'), lineno)) - break - else: - locations.append((None, None)) - - return locations - -def _gen_file_line_table(dwarfinfo): - ''' - Generates and returns a list of (filename, lineno, startaddr, endaddr). - - Tests that an empty dwarfinfo object will result in an empty return list - >>> class FakeDwarf: - ... def __init__(self): - ... pass - ... def iter_CUs(self): - ... return [] - >>> _gen_file_line_table(FakeDwarf()) - [] + Given a list of tuples of (offset, symbol), return a single dictionaries, a + mapping from symbol name to a tuple of (filename, line number). If the + filename and/or line number could not be determined, then both will be set + to None. ''' - # generate the table - table = [] - for unit in dwarfinfo.iter_CUs(): - lineprog = dwarfinfo.line_program_for_CU(unit) - prevstate = None - for entry in lineprog.get_entries(): - # We're interested in those entries where a new state is assigned - if entry.state is None or entry.state.end_sequence: - continue - # Looking for a range of addresses in two consecutive states that - # contain a required address. - if prevstate is not None: - filename = lineprog['file_entry'][prevstate.file - 1].name - dirno = lineprog['file_entry'][prevstate.file - 1].dir_index - filepath = os.path.join( - lineprog['include_directory'][dirno - 1], filename) - line = prevstate.line - fromaddr = prevstate.address - toaddr = max(fromaddr, entry.state.address) - table.append((filepath, line, fromaddr, toaddr)) - prevstate = entry.state - - # If there are no functions, then return an empty list - if len(table) == 0: - return [] - - # consolidate the table - consolidated = [] - prev = table[0] - for entry in table[1:]: - if prev[1] == entry[1] and prev[3] == entry[2]: - prev = (prev[0], prev[1], prev[2], entry[3]) - else: - consolidated.append(prev) - prev = entry - consolidated.append(prev) - - return consolidated + if not offset_symbol_tuples: + return {} + proc = subp.Popen(['addr2line', '-e', objfile], stdin=subp.PIPE, + stdout=subp.PIPE) + out, _ = proc.communicate('\n'.join(x[0] for x in offset_symbol_tuples) + .encode()) + info = out.decode('utf8').splitlines() + assert len(info) == len(offset_symbol_tuples), \ + 'len(info) = {}, len(offset_symbol_tuples) = {}'\ + .format(len(info), len(offset_symbol_tuples)) + mapping = {} + for line, symbol in zip(info, (x[1] for x in offset_symbol_tuples)): + filename, linenumber = line.strip().split(':') + if filename == '??' or linenumber == '0': + filename = None + linenumber = None + mapping[symbol] = (filename, linenumber) + return mapping diff --git a/tests/flit_cli/flit_bisect/Makefile b/tests/flit_cli/flit_bisect/Makefile index dccbc157..4ce0ee57 100644 --- a/tests/flit_cli/flit_bisect/Makefile +++ b/tests/flit_cli/flit_bisect/Makefile @@ -2,10 +2,6 @@ RUNNER := python3 SRC := $(wildcard tst_*.py) RUN_TARGETS := $(SRC:%.py=run_%) -IS_PYELF := $(shell if python3 -c 'import elftools' 2>/dev/null; then \ - echo true; \ - fi) - include ../../color_out.mk ifndef VERBOSE @@ -13,13 +9,7 @@ ifndef VERBOSE endif .PHONY: check help clean build run_% -ifeq ($(IS_PYELF),true) check: $(TARGETS) $(RUN_TARGETS) -else -check: - @$(call color_out,RED,Warning: pyelftools is not found on your system;\ - skipping bisect tests) -endif help: @echo "Makefile for running tests on FLiT framework" diff --git a/tests/flit_cli/flit_bisect/tst_bisect.py b/tests/flit_cli/flit_bisect/tst_bisect.py index 723001f5..500dd448 100644 --- a/tests/flit_cli/flit_bisect/tst_bisect.py +++ b/tests/flit_cli/flit_bisect/tst_bisect.py @@ -240,15 +240,15 @@ Test the All differing symbols section of the output >>> idx = bisect_out.index('All variability inducing symbols:') >>> print('\\n'.join(bisect_out[idx+1:])) # doctest:+ELLIPSIS - tests/BisectTest.cpp:96 ... -- real_problem_test(int, char**) (score 50.0) - tests/file4.cxx:110 ... -- file4_all() (score 30.0) - tests/file2.cpp:90 ... -- file2_func1_PROBLEM() (score 7.0) - tests/file1.cpp:92 ... -- file1_func2_PROBLEM() (score 5.0) - tests/file1.cpp:108 ... -- file1_func4_PROBLEM() (score 3.0) - tests/file3.cpp:103 ... -- file3_func5_PROBLEM() (score 3.0) - tests/A.cpp:95 ... -- A::fileA_method1_PROBLEM() (score 2.0) - tests/file1.cpp:100 ... -- file1_func3_PROBLEM() (score 2.0) - tests/file3.cpp:92 ... -- file3_func2_PROBLEM() (score 1.0) + /.../tests/BisectTest.cpp:96 ... -- real_problem_test(int, char**) (score 50.0) + /.../tests/file4.cxx:110 ... -- file4_all() (score 30.0) + /.../tests/file2.cpp:90 ... -- file2_func1_PROBLEM() (score 7.0) + /.../tests/file1.cpp:92 ... -- file1_func2_PROBLEM() (score 5.0) + /.../tests/file1.cpp:108 ... -- file1_func4_PROBLEM() (score 3.0) + /.../tests/file3.cpp:103 ... -- file3_func5_PROBLEM() (score 3.0) + /.../tests/A.cpp:95 ... -- A::fileA_method1_PROBLEM() (score 2.0) + /.../tests/file1.cpp:100 ... -- file1_func3_PROBLEM() (score 2.0) + /.../tests/file3.cpp:92 ... -- file3_func2_PROBLEM() (score 1.0) Test that the --compiler-type flag value made it into the bisect Makefile >>> troublecxx diff --git a/tests/flit_cli/flit_bisect/tst_bisect_biggest.py b/tests/flit_cli/flit_bisect/tst_bisect_biggest.py index 8745d6a8..63b9a009 100644 --- a/tests/flit_cli/flit_bisect/tst_bisect_biggest.py +++ b/tests/flit_cli/flit_bisect/tst_bisect_biggest.py @@ -100,7 +100,7 @@ >>> flit_bisect = th._path_import(th._script_dir, 'flit_bisect') >>> util = th._path_import(th._script_dir, 'flitutil') ->>> Sym = flit_bisect.elf.SymbolTuple +>>> Sym = flit_bisect.BisectSymbolTuple >>> def create_symbol(fileno, funcno, lineno, isproblem): ... prob_str = '_PROBLEM' if isproblem else '' ... filename = 'tests/file{}.cpp'.format(fileno) diff --git a/tests/flit_cli/flit_disguise/Makefile b/tests/flit_cli/flit_disguise/Makefile new file mode 100644 index 00000000..4ce0ee57 --- /dev/null +++ b/tests/flit_cli/flit_disguise/Makefile @@ -0,0 +1,27 @@ +RUNNER := python3 +SRC := $(wildcard tst_*.py) +RUN_TARGETS := $(SRC:%.py=run_%) + +include ../../color_out.mk + +ifndef VERBOSE +.SILENT: +endif + +.PHONY: check help clean build run_% +check: $(TARGETS) $(RUN_TARGETS) + +help: + @echo "Makefile for running tests on FLiT framework" + @echo " help print this help documentation and exit" + @echo " build just compile the targets" + @echo " check run tests and print results to the console" + @echo " clean remove all generated files" + +build: +clean: + +run_% : %.py + @$(call color_out_noline,BROWN, running) + @echo " $<" + $(RUNNER) $< diff --git a/tests/flit_cli/flit_disguise/tst_flit_disguise.py b/tests/flit_cli/flit_disguise/tst_flit_disguise.py new file mode 100755 index 00000000..a04855da --- /dev/null +++ b/tests/flit_cli/flit_disguise/tst_flit_disguise.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2020, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Tests FLiT's disguise subcommand as integration tests +''' + +import unittest as ut +import tempfile +from io import StringIO +import re + +import sys +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +NamedTempFile = lambda: tempfile.NamedTemporaryFile(mode='wt', buffering=1) + +class FlitTestBase(ut.TestCase): + + def capture_flit(self, args): + ''' + Runs the flit command-line tool with the given args. Returns the + standard output from the flit run as a list of lines. + ''' + with StringIO() as ostream: + retval = th.flit.main(args, outstream=ostream) + lines = ostream.getvalue().splitlines() + self.assertEqual(retval, 0) + return lines + + def run_flit(self, args): + 'Runs flit ignoring standard output' + self.capture_flit(args) + +class FlitDisguiseTest(FlitTestBase): + + def setup_flitdir(self, directory): + self.run_flit(['init', '--directory', directory]) + + def disguise_string(self, content, fields=None, mapping=None, undo=False): + 'Runs flit disguise on the content and returns the disguised version' + with NamedTempFile() as fcontent: + fcontent.write(content) + fcontent.flush() + args = ['disguise', fcontent.name] + + if fields is not None: + args.extend(['--fields', ','.join(fields)]) + + if undo: + args.append('--undo') + + if mapping is not None: + with NamedTempFile() as fout: + fout.write('disguise,value\n') + fout.file.writelines(['"{}","{}"\n'.format(key, value) + for key, value in mapping.items()]) + fout.flush() + args.extend(['--disguise-map', fout.name]) + return self.capture_flit(args) + + return self.capture_flit(args) + + def test_generate_map_default_flit_init(self): + with th.util.tempdir() as flitdir: + self.setup_flitdir(flitdir) + with th.util.pushd(flitdir): + output = self.capture_flit(['disguise', '--generate']) + self.assertEqual(output, ['Created disguise.csv']) + with open('disguise.csv') as disguise_in: + disguise_contents = disguise_in.readlines() + self.assertEqual('disguise,value\n', disguise_contents[0]) + self.assertEqual('objfile-1,Empty.cpp.o\n', disguise_contents[1]) + self.assertEqual('objfile-2,main.cpp.o\n', disguise_contents[2]) + self.assertEqual('filepath-1,main.cpp\n', disguise_contents[3]) + self.assertEqual('filepath-2,tests/Empty.cpp\n', disguise_contents[4]) + self.assertEqual('filename-1,Empty.cpp\n', disguise_contents[5]) + self.assertEqual('test-1,Empty\n', disguise_contents[-1]) + expected_symbols = ['main'] + symbol_lines = [x for x in disguise_contents if x.startswith('symbol')] + for symbol in expected_symbols: + self.assertTrue(any(re.match('symbol-\d*,{}\n'.format(symbol), line) + for line in disguise_contents)) + + def test_disguise_empty_map(self): + to_disguise = [ + ' Just some file contents.', + 'Nothing to worry about here.', + ] + disguised = self.disguise_string('\n'.join(to_disguise), mapping={}) + self.assertEqual(disguised, to_disguise) + + def test_disguise_normal(self): + disguise_mapping = { + 'disguise-01': 'map', + 'disguise-02': 'hi', + 'disguise-03': 'function(string, int, int)', + 'disguise-04': 'not found', + } + to_disguise = [ + 'hi there chico', + 'may mapping map file is not so good', + 'has a function called function(string, int, int).', + ] + expected_disguised = [ + 'disguise-02 there chico', + 'may mapping disguise-01 file is not so good', + 'has a function called disguise-03.', + ] + disguised = self.disguise_string( + '\n'.join(to_disguise), mapping=disguise_mapping) + self.assertEqual(disguised, expected_disguised) + + def test_disguise_normal_undo(self): + disguise_mapping = { + 'disguise-01': 'map', + 'disguise-02': 'hi', + 'disguise-03': 'function(string, int, int)', + 'disguise-04': 'not found', + } + disguised = [ + 'disguise-02 there chico', + 'may mapping disguise-01 file is not so good', + 'has a function called disguise-03.', + ] + expected_undisguised = [ + 'hi there chico', + 'may mapping map file is not so good', + 'has a function called function(string, int, int).', + ] + + undisguised = self.disguise_string( + '\n'.join(disguised), mapping=disguise_mapping, undo=True) + self.assertEqual(undisguised, expected_undisguised) + + def test_disguise_bad_map_file(self): + with NamedTempFile() as mapfile: + mapfile.write('not the correct header\n' + 'does not matter\n' + 'what the rest has...\n') + with self.assertRaises(AssertionError): + self.run_flit(['disguise', '--disguise-map', mapfile.name]) + +if __name__ == '__main__': + sys.exit(th.unittest_main()) diff --git a/tests/flit_install/tst_install_runthrough.py b/tests/flit_install/tst_install_runthrough.py index c03970a4..0eecb97d 100644 --- a/tests/flit_install/tst_install_runthrough.py +++ b/tests/flit_install/tst_install_runthrough.py @@ -294,6 +294,7 @@ 'share/flit/scripts/experimental/ninja_syntax.py', 'share/flit/scripts/flit.py', 'share/flit/scripts/flit_bisect.py', + 'share/flit/scripts/flit_disguise.py', 'share/flit/scripts/flit_experimental.py', 'share/flit/scripts/flit_import.py', 'share/flit/scripts/flit_init.py',