diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..cab70590 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,18 @@ +--- +name: 🐛 Bug report +about: Create a report to help us improve 🤔 + +--- + +## Bug Report + +**Describe the problem** +A clear and concise description of what the bug is. +Include screenshots if applicable. + +**Suggested Fix** +Do you have a suggestion for how to resole the issue? + +**Alternative approaches:** +A clear and concise description of any alternative solutions or features you've +considered. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..fb209aed --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,19 @@ +--- +name: 🚀 Feature Request +about: I have a suggestion (and may want to implement it 🙂)! + +--- + +## Feature Request + +**Describe the new feature:** +A clear and concise description of what your problem is. Ex. I have an issue +when [...] + +**Suggested change:** +A clear and concise description of what you want to happen. Add any considered +drawbacks. + +**Alternative approaches:** +A clear and concise description of any alternative solutions or features you've +considered. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..f2b40f0a --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +Fixes # + +**Description:** +Describe what you changed and why. + +- +- +- + +**Documentation:** +What documentation did you change? +If the change does not require updates to documentation, justify why. + +**Tests:** +What automated tests did you change? +If the change does not require updates to tests, justify why. diff --git a/.travis.yml b/.travis.yml index 36732aa3..5bb26a56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,39 +80,51 @@ # # -- LICENSE END -- -sudo: required language: cpp -dist: trusty -# addons: -# apt: -# packages: -# - gcc-5 -# - g++-5 +os: linux +addons: + apt: + packages: &native_deps + - python3 + - python3-pip -env: - global: - - LLVM_VERSION=3.9.0 - - LLVM_ARCHIVE_PATH=$HOME/clang+llvm.tar.xz - - export PATH=$HOME/usr/bin:$PATH - - export LD_LIBRARY_PATH=$HOME/usr/lib:$LD_LIBRARY_PATH +matrix: + include: + # Job 1: OpenMPI + - env: + - mpi_type=openmpi + - extra_pip=pyelftools + addons: + apt: + packages: + - *native_deps + - libopenmpi-dev + - openmpi-bin + # Job 2: MPICH + #- env: mpi_type=mpich + # addons: + # apt: + # packages: + # - *native_deps + # - libmpich-dev + # - mpich + # Job 3: No MPI + - env: + - mpi_type=none + - extra_pip= + addons: + apt: + packages: + - *native_deps -install: - #starts in $HOME/build/PRUNERS/FLiT - - cd $HOME/build - # Install LLVM/Clang 3.9 - - wget http://llvm.org/releases/$LLVM_VERSION/clang+llvm-$LLVM_VERSION-x86_64-linux-gnu-ubuntu-14.04.tar.xz -O $LLVM_ARCHIVE_PATH - - mkdir $HOME/usr - - tar xf $LLVM_ARCHIVE_PATH -C $HOME/usr --strip-components 1 - -# script: - - export CLANG_ONLY=True - - cd $HOME/build/PRUNERS/FLiT/src - - make -j 4 +before_install: pip3 install --user toml $extra_pip -notifications: - email: false - slack: - rooms: - - pruners:aXHVdiVFtqtMfzNW4IutZNDW - on_success: always - on_failure: always +script: make -j 4 && make check + +#notifications: +# email: false +# slack: +# rooms: +# - pruners:aXHVdiVFtqtMfzNW4IutZNDW +# on_success: always +# on_failure: always diff --git a/Makefile b/Makefile index 281634ca..17bb81f7 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PREFIX ?= /usr -#CC := clang++ -CC := g++ +#CXX ?= clang++ +CXX ?= g++ FFLAGS ?= LIBDIR := lib SRCDIR := src @@ -68,10 +68,10 @@ help: $(TARGET): $(OBJ) mkdir -p lib - $(CC) $(CPPFLAGS) -o $@ $^ $(LINKFLAGS) + $(CXX) $(CPPFLAGS) -o $@ $^ $(LINKFLAGS) $(SRCDIR)/%.o: $(SRCDIR)/%.cpp Makefile - $(CC) $(CPPFLAGS) $(DEPFLAGS) -c $< -o $@ + $(CXX) $(CPPFLAGS) $(DEPFLAGS) -c $< -o $@ .PRECIOUS: src/%.d -include $(SOURCE:%.cpp=%.d) @@ -109,6 +109,7 @@ install: $(TARGET) install -m 0755 $(SCRIPT_DIR)/flit.py $(PREFIX)/share/flit/scripts/ install -m 0755 $(SCRIPT_DIR)/flit_*.py $(PREFIX)/share/flit/scripts/ install -m 0644 $(SCRIPT_DIR)/flitutil.py $(PREFIX)/share/flit/scripts/ + install -m 0644 $(SCRIPT_DIR)/flitelf.py $(PREFIX)/share/flit/scripts/ install -m 0644 $(SCRIPT_DIR)/README.md $(PREFIX)/share/flit/scripts/ install -m 0644 $(DOC_DIR)/*.md $(PREFIX)/share/flit/doc/ install -m 0644 $(DATA_DIR)/Makefile.in $(PREFIX)/share/flit/data/ diff --git a/README.md b/README.md index e1a38e0d..15ea7eba 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,11 @@ +| branch | status | +|---------|---------| +| master | [![Build Status](https://travis-ci.org/PRUNERS/FLiT.svg?branch=master)](https://travis-ci.org/PRUNERS/FLiT) | +| devel | [![Build Status](https://travis-ci.org/PRUNERS/FLiT.svg?branch=devel)](https://travis-ci.org/PRUNERS/FLiT) | + +![PyPI - License](https://img.shields.io/pypi/l/Django.svg) +![PRs welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg) + # FLiT [![FLiT Bird](/images/flit-small.png)](https://github.com/PRUNERS/FLiT "FLiT") @@ -51,6 +59,7 @@ It consists of the following components: Contents: +* [Release Notes](documentation/release-notes.md) * [Installation](documentation/installation.md) * [Litmus Tests](documentation/litmus-tests.md) * [FLiT Command-Line](documentation/flit-command-line.md) diff --git a/benchmarks/polybench/flit-config.toml b/benchmarks/polybench/flit-config.toml index cbe16d21..314dced4 100644 --- a/benchmarks/polybench/flit-config.toml +++ b/benchmarks/polybench/flit-config.toml @@ -1,24 +1 @@ -# Autogenerated by "flit init" -# flit version v2.0-alpha.3 - -[database] - -type = 'sqlite' -filepath = 'results.sqlite' - - -[[hosts]] - -[hosts.dev_build] - -compiler_name = 'g++' -optimization_level = '-O2' -switches = '-funsafe-math-optimizations' - - -[hosts.ground_truth] - -compiler_name = 'g++' -optimization_level = '-O0' -switches = '' - +# Use the default values diff --git a/benchmarks/polybench/main.cpp b/benchmarks/polybench/main.cpp index 60a4a640..5b4dbcdb 100644 --- a/benchmarks/polybench/main.cpp +++ b/benchmarks/polybench/main.cpp @@ -1,3 +1,86 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + #include "flit.h" int main(int argCount, char* argList[]) { diff --git a/benchmarks/random/flit-config.toml b/benchmarks/random/flit-config.toml index 6fc21156..44a265ae 100644 --- a/benchmarks/random/flit-config.toml +++ b/benchmarks/random/flit-config.toml @@ -1,50 +1,4 @@ -# Autogenerated by "flit init" -# flit version v2.0-alpha.3 - -[database] - -# older versions of flit supported postgres. that has been removed. only -# sqlite is supported at the moment. -type = 'sqlite' - -# if relative path, it is relative to the directory containing this -# configuration file. -filepath = 'results.sqlite' - [run] timing = false -# For now, only one host is supported, all others are ignored -[[hosts]] - -name = 'yoga-manjaro' -flit_path = '/home/bentley/git/FLiT/scripts/flitcli/flit.py' -config_dir = '/home/bentley/git/FLiT/scripts/flitcli/config' - -# The settings for "make dev" -[hosts.dev_build] -# compiler_name must be found in [[hosts.compilers]] list under name attribute -# but the optimization level and switches do not need to be in the compiler list -compiler_name = 'g++' -optimization_level = '-O2' -switches = '-funsafe-math-optimizations' - -# The ground truth compilation to use in analysis, for "make gt" -[hosts.ground_truth] -# compiler_name must be found in [[hosts.compilers]] list under name attribute -# but the optimization level and switches do not need to be in the compiler list -compiler_name = 'g++' -optimization_level = '-O0' -switches = '' - - # This host's list of compilers. - # For now, only used for hosts.ground_truth and hosts.dev_build. - # TODO: use this list to generate the Makefile - [[hosts.compilers]] - - # binary can be an absolute path, relative path, or binary name (found in - # PATH). If you want to specify a compiler in the same directory as this - # config file, prepend with a "./" (e.g. "./my-compiler") - binary = 'g++' - name = 'g++' - +# Use default values for all other options diff --git a/benchmarks/random/main.cpp b/benchmarks/random/main.cpp index 60a4a640..5b4dbcdb 100644 --- a/benchmarks/random/main.cpp +++ b/benchmarks/random/main.cpp @@ -1,3 +1,86 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + #include "flit.h" int main(int argCount, char* argList[]) { diff --git a/data/Makefile.in b/data/Makefile.in index 426053d8..792a9c9c 100644 --- a/data/Makefile.in +++ b/data/Makefile.in @@ -87,18 +87,29 @@ DEV_TARGET ?= devrun GT_TARGET ?= gtrun GT_OUT := ground-truth.csv -UNAME_S := $(shell uname -s) +UNAME_S := {uname} + +# will be None if not specified in flit-config.toml +{compiler_defs} +CLANG_TYPE := clang +INTEL_TYPE := intel +GCC_TYPE := gcc + +# keep only the compilers that are not None and are in the path +COMPILERS := {compilers} FLIT_INC_DIR := {flit_include_dir} FLIT_LIB_DIR := {flit_lib_dir} FLIT_DATA_DIR := {flit_data_dir} FLIT_SCRIPT_DIR := {flit_script_dir} -DEV_CC ?= {dev_compiler} +DEV_CXX ?= {dev_compiler} +DEV_CXX_TYPE ?= {dev_type} DEV_OPTL ?= {dev_optl} DEV_SWITCHES ?= {dev_switches} -GT_CC := {ground_truth_compiler} +GT_CXX := {ground_truth_compiler} +GT_CXX_TYPE := {ground_truth_type} GT_OPTL := {ground_truth_optl} GT_SWITCHES := {ground_truth_switches} @@ -110,11 +121,11 @@ MPIRUN_ARGS := {mpirun_args} # initalize some variables to be appended later CC_REQUIRED := DEV_CFLAGS := +GT_CFLAGS := LD_REQUIRED := RUNWRAP = $(RUN_WRAPPER) ifeq ($(ENABLE_MPI),yes) # If we are using MPI - $(info MPI is enabled) RUNWRAP += mpirun $(MPIRUN_ARGS) CC_REQUIRED += -DFLIT_USE_MPI CC_REQUIRED += $(shell mpic++ --showme:compile) @@ -137,7 +148,9 @@ DEV_CFLAGS += -Wuninitialized DEV_CFLAGS += -Wno-shift-count-overflow # This flag specifies NOT to link as a position-independent executable -LD_REQUIRED += -no-pie +# Note: this flag does not exist in GCC 4 or GCC 5 so it will fail to compile +# we need extra logic to check for this and conditionally add this flag +#LD_REQUIRED += -no-pie LD_REQUIRED += -lm LD_REQUIRED += -lstdc++ ifeq ($(UNAME_S),Darwin) # If we are on a Mac OSX system @@ -147,9 +160,30 @@ else LD_REQUIRED += -Wl,-rpath=$(realpath $(FLIT_LIB_DIR)) endif -DEV_LDFLAGS += +# Helper functions to determine if the compiler is a particular version of GCC + +# Returns the compiler name as output by the --version flag +# @param 1: executable name or path to compiler +GET_COMPILER = $(shell $1 --version | head -n 1 | awk '{{ print $$1 }}') + +# Returns the version of the compiler as returned by -dumpversion +# @param 1: executable name or path to compiler +GET_COMPILER_VER = $(shell $1 -dumpversion) + +# Returns 1 if the major version matches else 0 +# @param 1: executable name or path to compiler +# @param 2: major version number as an integer +IS_MAJOR_VER = $(shell expr substr $(call GET_COMPILER_VER,$1) 1 1 = $2) -DEPFLAGS += -MMD -MF $(patsubst %.o,%.d,$@) +# Returns 1 if the compiler is GCC version 4 or version 5 +# @param 1: executable name or path to compiler +IS_VER_4_OR_5 = $(shell expr $(call IS_MAJOR_VER,$1,4) \| \ + $(call IS_MAJOR_VER,$1,5)) + +DEV_LDFLAGS = +GT_LDFLAGS = + +DEPFLAGS += -MMD -MP -MF $(patsubst %.o,%.d,$@) TESTS = $(wildcard tests/*.cpp) SOURCE = $(wildcard *.cpp) @@ -190,26 +224,26 @@ DEV_OBJ = $(addprefix $(OBJ_DIR)/,$(notdir $(SOURCE:%.cpp=%_dev.o))) DEV_DEPS = $(DEV_OBJ:%.o=%.d) GT_OBJ = $(addprefix $(OBJ_DIR)/,$(notdir $(SOURCE:%.cpp=%_gt.o))) GT_DEPS = $(GT_OBJ:%.o=%.d) +GT_OBJ_FPIC = $(GT_OBJ:%.o=%_fPIC.o) -CLANG := clang++ -INTEL := icpc -GCC := g++ - -ifdef CLANG_ONLY -COMPILERS := CLANG -else -COMPILERS := $(foreach c, GCC INTEL CLANG, \ - $(if $(shell which $($(c)) 2>/dev/null), $c,)) -endif - -HOSTNAME := $(shell hostname) +HOSTNAME := {hostname} RESULTS_DIR := results # on systems with non-standard gcc installations (such as module), clang may # be unable to determine the correct gcc toolchain +CLANG_REQUIRED := +ifeq ($(findstring GCC,$(COMPILERS)),GCC) GCC_TOOLCHAIN := $(dir $(shell which $(GCC) 2>/dev/null))/.. -CLANG_REQUIRED := --gcc-toolchain=$(GCC_TOOLCHAIN) +CLANG_REQUIRED += --gcc-toolchain=$(GCC_TOOLCHAIN) +endif + +ifeq ($(DEV_CXX_TYPE),$(CLANG_TYPE)) +DEV_CFLAGS += $(CLANG_REQUIRED) +endif +ifeq ($(GT_CXX_TYPE),$(CLANG_TYPE)) +GT_CFLAGS += $(CLANG_REQUIRED) +endif # Compiler setting targets # taken from: https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html @@ -217,128 +251,19 @@ CLANG_REQUIRED := --gcc-toolchain=$(GCC_TOOLCHAIN) # more comp settings, taken from here: # https://software.intel.com/sites/default/files/article/326703/fp-control-2012-08.pdf -#individual flags +# individual flags + ## optls -O0 := -O0 -O1 := -O1 -O2 := -O2 -O3 := -O3 - -#switches - -ASSOCMATH := -fassociative-math -AVX := -mavx -COMPTRANS := -mp1 -DEFFLAGS := -DISFMA := -no-fma -ENAFMA := -fma -FASTEXPREC := -fexcess-precision=fast -FASTM := -ffast-math -FINMATH := -ffinite-math-only -FLUSHDEN := -ftz -FMAGCC := -mavx2 -mfma -FMAICC := -march=core-avx2 -FORTRULES := -fcx-fortran-rules -FPCONT := -ffp-contract=on -FPMODDBL := -fp-model=double -FPMODEXC := -fp-model=except -FPMODEXT := -fp-model=extended -FPMODFST1 := -fp-model fast=1 -FPMODFST2 := -fp-model fast=2 -FPMODPRE := -fp-model=precise -FPMODSRC := -fp-model=source -FPMODSTR := -fp-model=strict -FPTRAP := -fp-trap=common -FSTORE := -ffloat-store -LIMITEDRANGE := -fcx-limited-range -MCONSTS := -fmerge-all-constants -NOFLUSHDEN := -no-ftz -NOPRECDIV := -no-prec-div -NOTRAP := -fno-trapping-math -PRECDIV := -prec-div -RECIPMATH := -freciprocal-math -ROUNDINGMATH := -frounding-math -ROUNDUSR := -fp-port -SIGNALNAN := -fsignaling-nans -SINGLEPRECCONST := -fsingle-precision-constant -SSE := -mfpmath=sse -mtune=native -STDEXPREC := -fexcess-precision=standard -UNSOPTS := -funsafe-math-optimizations -USEFASTM := --use_fast_math - -# Collections - -OPCODES := O0 O1 O2 O3 - -# NOTE: gcc disables ASSOCMATH @ O0 -SWITCHES_GCC += ASSOCMATH -SWITCHES_GCC += AVX -SWITCHES_GCC += DEFFLAGS -SWITCHES_GCC += FASTEXPREC -SWITCHES_GCC += FINMATH -SWITCHES_GCC += FMAGCC -SWITCHES_GCC += FORTRULES -SWITCHES_GCC += FPCONT -SWITCHES_GCC += FSTORE -SWITCHES_GCC += LIMITEDRANGE -SWITCHES_GCC += MCONSTS -SWITCHES_GCC += NOTRAP -SWITCHES_GCC += RECIPMATH -SWITCHES_GCC += ROUNDINGMATH -SWITCHES_GCC += SIGNALNAN -SWITCHES_GCC += SSE -SWITCHES_GCC += UNSOPTS - -#NOTE: Clang not honoring ASSOCMATH (issues warning with 3.9) -# see: https://llvm.org/bugs/show_bug.cgi?id=27372 - -SWITCHES_CLANG += ASSOCMATH -SWITCHES_CLANG += AVX -SWITCHES_CLANG += DEFFLAGS -SWITCHES_CLANG += FASTEXPREC -SWITCHES_CLANG += FINMATH -SWITCHES_CLANG += FMAGCC -SWITCHES_CLANG += FMAICC -SWITCHES_CLANG += FPCONT -SWITCHES_CLANG += FSTORE -SWITCHES_CLANG += MCONSTS -SWITCHES_CLANG += NOTRAP -SWITCHES_CLANG += RECIPMATH -SWITCHES_CLANG += ROUNDINGMATH -SWITCHES_CLANG += SIGNALNAN -SWITCHES_CLANG += SINGLEPRECCONST -SWITCHES_CLANG += SSE -SWITCHES_CLANG += STDEXPREC -SWITCHES_CLANG += UNSOPTS - -SWITCHES_INTEL += AVX -SWITCHES_INTEL += COMPTRANS -SWITCHES_INTEL += DEFFLAGS -SWITCHES_INTEL += DISFMA -SWITCHES_INTEL += ENAFMA -SWITCHES_INTEL += FLUSHDEN -SWITCHES_INTEL += FMAGCC -SWITCHES_INTEL += FMAICC -SWITCHES_INTEL += FPMODDBL -SWITCHES_INTEL += FPMODEXT -SWITCHES_INTEL += FPMODFST1 -SWITCHES_INTEL += FPMODFST2 -SWITCHES_INTEL += FPMODPRE -SWITCHES_INTEL += FPMODSRC -SWITCHES_INTEL += FPMODSTR -SWITCHES_INTEL += FSTORE -SWITCHES_INTEL += LIMITEDRANGE -SWITCHES_INTEL += MCONSTS -SWITCHES_INTEL += NOFLUSHDEN -SWITCHES_INTEL += NOPRECDIV -SWITCHES_INTEL += PRECDIV -SWITCHES_INTEL += ROUNDINGMATH -SWITCHES_INTEL += ROUNDUSR -SWITCHES_INTEL += SINGLEPRECCONST -SWITCHES_INTEL += SSE -SWITCHES_INTEL += USEFASTM +{opcodes_definitions} + +## switches +{switches_definitions} + +{compiler_opcodes} + +{compiler_switches} ########################################################## # @@ -365,6 +290,17 @@ R_DEP := $(R_OBJ:%.o=%.d) .PHONY: rec rec: $(R_TARGET) +# clang's flag is -nopie +# else if the current compiler is not GCC 4 or 5, then enable -no-pie +# GCC 4 and 5 do not need -no-pie since that is the default +ifeq ($($(R_CUR_COMPILER)_TYPE),$(CLANG_TYPE)) + LD_REQUIRED += -nopie +else ifeq ($($(R_CUR_COMPILER)_TYPE),$(GCC_TYPE)) +ifeq ($(call IS_VER_4_OR_5,$($(R_CUR_COMPILER))),0) + LD_REQUIRED += -no-pie +endif +endif + $(R_TARGET): $(R_OBJ) $($(R_CUR_COMPILER)) $($(R_CUR_OPTL)) $($(R_CUR_SWITCHES)) \ $($(R_CUR_COMPILER)_REQUIRED) $(CC_REQUIRED) \ @@ -385,6 +321,26 @@ $(OBJ_DIR)/%_$(R_ID).o: %.cpp Makefile custom.mk | $(OBJ_DIR) # Otherwise, we're not in a recursion. else # ifndef R_IS_RECURSED +# clang's flag is -nopie +# else if the current compiler is not GCC 4 or 5, then enable -no-pie +# GCC 4 and 5 do not need -no-pie since that is the default +ifeq ($(DEV_CXX_TYPE),$(CLANG_TYPE)) + DEV_LDFLAGS += -nopie +else ifeq ($(DEV_CXX_TYPE),$(GCC_TYPE)) +ifeq ($(call IS_VER_4_OR_5,$(DEV_CXX)),0) + DEV_LDFLAGS += -no-pie +endif +endif + +# same for the gt compiler +ifeq ($(GT_CXX_TYPE),$(CLANG_TYPE)) + GT_LDFLAGS += -nopie +else ifeq ($(GT_CXX_TYPE),$(GCC_TYPE)) +ifeq ($(call IS_VER_4_OR_5,$(GT_CXX)),0) + GT_LDFLAGS += -no-pie +endif +endif + $(OBJ_DIR): mkdir -p $(OBJ_DIR) $(RESULTS_DIR): @@ -396,13 +352,13 @@ $(RESULTS_DIR): # Set these as empty "simply-expanded variables". This affects the "+=" operator. # see https://ftp.gnu.org/old-gnu/Manuals/make-3.79.1/html_chapter/make_6.html -TARGET_OUTS := +TARGET_OUTS := # @param 1: compiler variable name (e.g. CLANG) # @param 2: optimization level variable name (e.g. O2) -# @param 3: switches variable name (e.g. USEFASTM) +# @param 3: switches variable name (e.g. USE_FAST_MATH) define RECURSION_RULE -TARGETS += $$(RESULTS_DIR)/$(strip $1)_$$(HOSTNAME)_$(strip $3)_$(strip $2) +TARGETS += $$(RESULTS_DIR)/$(strip $1)_$$(HOSTNAME)_$(strip $3)_$(strip $2) # TODO: use the variable $$(MAKECMDGOALS) to get the original make target # TODO- or see if it is even necessary @@ -423,7 +379,7 @@ endef $(foreach c, $(COMPILERS), \ $(foreach s, $(SWITCHES_$(strip $c)), \ - $(foreach o, $(OPCODES), \ + $(foreach o, $(OPCODES_$(strip $c)), \ $(eval $(call RECURSION_RULE, $c, $o, $s))))) TARGET_OUTS := $(TARGETS:%=%-out) TARGET_RESULTS := $(TARGET_OUTS:%=%-comparison.csv) @@ -444,9 +400,10 @@ TARGET_RESULTS := $(TARGET_OUTS:%=%-comparison.csv) OBJ_CLEAN = $(addprefix $(OBJ_DIR)/,$(notdir $(SOURCE:%.cpp=%_*.o))) DEP_CLEAN += $(OBJ_CLEAN:%.o=%.d) -.PHONY: dev gt groundtruth run runbuild +.PHONY: dev gt gt-fpic groundtruth run runbuild dev: $(DEV_TARGET) gt: groundtruth +gt-fpic: $(GT_OBJ_FPIC) groundtruth: $(GT_TARGET) run: $(TARGET_RESULTS) $(TARGET_OUTS) @@ -514,13 +471,13 @@ endif # ifeq ($(UNAME_S),Darwin): meaning, we are on a mac # Dev compilation rules first (easier to understand) $(DEV_TARGET): $(DEV_OBJ) Makefile custom.mk - $(DEV_CC) $(CC_REQUIRED) $(DEV_CFLAGS) \ + $(DEV_CXX) $(CC_REQUIRED) $(DEV_CFLAGS) \ -o $@ $(DEV_OBJ) $(LD_REQUIRED) $(DEV_LDFLAGS) $(OBJ_DIR)/%_dev.o: %.cpp Makefile custom.mk | $(OBJ_DIR) - $(DEV_CC) $(DEV_OPTL) $(DEV_SWITCHES) $(CC_REQUIRED) $(DEV_CFLAGS) $(DEPFLAGS) -c $< -o $@ \ + $(DEV_CXX) $(DEV_OPTL) $(DEV_SWITCHES) $(CC_REQUIRED) $(DEV_CFLAGS) $(DEPFLAGS) -c $< -o $@ \ -DFLIT_HOST='"$(HOSTNAME)"' \ - -DFLIT_COMPILER='"$(DEV_CC)"' \ + -DFLIT_COMPILER='"$(DEV_CXX)"' \ -DFLIT_OPTL='"$(DEV_OPTL)"' \ -DFLIT_SWITCHES='"$(DEV_SWITCHES)"' \ -DFLIT_FILENAME='"$(notdir $(DEV_TARGET))"' @@ -530,12 +487,20 @@ $(GT_OUT): $(GT_TARGET) $(RUNWRAP) ./$(GT_TARGET) --output $(GT_OUT) --no-timing $(GT_TARGET): $(GT_OBJ) Makefile custom.mk - $(GT_CC) $(CC_REQUIRED) -o $@ $(GT_OBJ) $(LD_REQUIRED) + $(GT_CXX) $(CC_REQUIRED) -o $@ $(GT_OBJ) $(LD_REQUIRED) $(GT_LDFLAGS) $(OBJ_DIR)/%_gt.o: %.cpp Makefile custom.mk | $(OBJ_DIR) - $(GT_CC) -g $(GT_OPTL) $(GT_SWITCHES) $(CC_REQUIRED) $(DEPFLAGS) -c $< -o $@ \ + $(GT_CXX) -g $(GT_OPTL) $(GT_SWITCHES) $(CC_REQUIRED) $(GT_CFLAGS) $(DEPFLAGS) -c $< -o $@ \ + -DFLIT_HOST='"$(HOSTNAME)"' \ + -DFLIT_COMPILER='"$(GT_CXX)"' \ + -DFLIT_OPTL='"$(GT_OPTL)"' \ + -DFLIT_SWITCHES='"$(GT_SWITCHES)"' \ + -DFLIT_FILENAME='"$(notdir $(GT_TARGET))"' + +$(OBJ_DIR)/%_gt_fPIC.o: %.cpp Makefile custom.mk | $(OBJ_DIR) + $(GT_CXX) -g $(GT_OPTL) $(GT_SWITCHES) $(CC_REQUIRED) $(GT_CFLAGS) $(DEPFLAGS) -fPIC -c $< -o $@ \ -DFLIT_HOST='"$(HOSTNAME)"' \ - -DFLIT_COMPILER='"$(GT_CC)"' \ + -DFLIT_COMPILER='"$(GT_CXX)"' \ -DFLIT_OPTL='"$(GT_OPTL)"' \ -DFLIT_SWITCHES='"$(GT_SWITCHES)"' \ -DFLIT_FILENAME='"$(notdir $(GT_TARGET))"' diff --git a/data/Makefile_bisect_binary.in b/data/Makefile_bisect_binary.in index aca9b2fb..5c9e6d65 100644 --- a/data/Makefile_bisect_binary.in +++ b/data/Makefile_bisect_binary.in @@ -97,10 +97,11 @@ PRECISION := {precision} TEST_CASE := {test_case} -TROUBLE_CC := {trouble_cc} +TROUBLE_CXX := {trouble_cxx} TROUBLE_OPTL := {trouble_optl} TROUBLE_SWITCHES := {trouble_switches} -BISECT_LINK := $(GT_CC) +TROUBLE_CXX_TYPE := {trouble_type} +BISECT_LINK := $(GT_CXX) TROUBLE_ID := {trouble_id} @@ -124,10 +125,23 @@ SPLIT_SRC := {EXTRA_CC_FLAGS} {EXTRA_LD_FLAGS} +TROUBLE_LDFLAGS := +TROUBLE_CFLAGS := + +ifeq ($(TROUBLE_CXX_TYPE),$(CLANG_TYPE)) + TROUBLE_LDFLAGS += -nopie + TROUBLE_CFLAGS += $(CLANG_REQUIRED) +else ifeq ($(TROUBLE_CXX_TYPE),$(GCC_TYPE)) +ifeq ($(call IS_VER_4_OR_5,$(TROUBLE_CXX)),0) + TROUBLE_LDFLAGS += -no-pie +endif +endif + BUILD_GT_LOCAL := {build_gt_local} TROUBLE_TARGET_OBJ := $(addprefix \ $(OBJ_DIR)/,$(notdir $(SOURCE:%.cpp=%_bisect_$(TROUBLE_ID).o))) +ALL_TROUBLE_FPIC := $(TROUBLE_TARGET_OBJ:%.o=%_fPIC.o) BISECT_GT_OBJ := $(addprefix \ $(OBJ_DIR)/,$(notdir $(BISECT_GT_SRC:%.cpp=%_gt.o))) TROUBLE_OBJ := $(addprefix \ @@ -186,7 +200,7 @@ $(GT_LIB_OUT): $(GT_LIB_TARGET) $(RUNWRAP) ./$(GT_LIB_TARGET) --output $(GT_LIB_OUT) --no-timing --precision "$(PRECISION)" $(TEST_CASE) $(GT_LIB_TARGET): $(GT_OBJ) Makefile custom.mk $(MAKEFILE) - $(GT_CC) $(CC_REQUIRED) -o $@ $(GT_OBJ) $(LD_REQUIRED) + $(GT_CXX) $(CC_REQUIRED) -o $@ $(GT_OBJ) $(LD_REQUIRED) $(BISECT_RESULT): $(BISECT_OUT) $(GT_LIB_OUT) $(GT_LIB_TARGET) $(RUNWRAP) ./$(GT_LIB_TARGET) --compare-mode --compare-gt $(GT_LIB_OUT) --suffix "-comparison.csv" $< -o /dev/null @@ -204,14 +218,15 @@ $(TROUBLE_TARGET_OUT): $(TROUBLE_TARGET) | $(BISECT_DIR) $(RUNWRAP) ./$< --precision "$(PRECISION)" --output $@ $(TEST_CASE) --no-timing $(BISECT_TARGET): $(BISECT_OBJ) $(SPLIT_OBJ) Makefile custom.mk | $(BISECT_DIR) - $(GT_CC) $(CC_REQUIRED) -o $@ $(BISECT_OBJ) $(SPLIT_OBJ) $(LD_REQUIRED) + $(GT_CXX) $(CC_REQUIRED) -o $@ $(BISECT_OBJ) $(SPLIT_OBJ) $(LD_REQUIRED) $(GT_LDFLAGS) $(TROUBLE_TARGET): $(TROUBLE_TARGET_OBJ) Makefile custom.mk | $(BISECT_DIR) - $(TROUBLE_CC) $(CC_REQUIRED) -o $@ $(TROUBLE_TARGET_OBJ) $(LD_REQUIRED) + $(TROUBLE_CXX) $(CC_REQUIRED) -o $@ $(TROUBLE_TARGET_OBJ) $(LD_REQUIRED) $(TROUBLE_LDFLAGS) -.PHONY: trouble trouble-out +.PHONY: trouble trouble-out trouble-fpic trouble: $(TROUBLE_TARGET) trouble-out: $(TROUBLE_TARGET_RESULT) $(TROUBLE_TARGET_OUT) +trouble-fpic: $(ALL_TROUBLE_FPIC) .PHONY: bisect bisect: $(BISECT_TARGET) $(GT_TARGET) @@ -258,28 +273,45 @@ $(BISECT_OBJ_DIR): # ground-truth files are already specified in Makefile # but need to specify how to do the fPIC variant $(BISECT_OBJ_DIR)/%_gt_fPIC.o: %.cpp Makefile custom.mk | $(BISECT_OBJ_DIR) - $(GT_CC) $(GT_OPTL) $(GT_SWITCHES) $(CC_REQUIRED) $(DEPFLAGS) -fPIC -c $< -o $@ \ - -DFLIT_HOST='"$(HOSTNAME)"' \ - -DFLIT_COMPILER='"$(GT_CC)"' \ - -DFLIT_OPTL='"$(GT_OPTL)"' \ - -DFLIT_SWITCHES='"$(GT_SWITCHES)"' \ - -DFLIT_FILENAME='"$(notdir $(GT_TARGET))"' + if [ -f "$(OBJ_DIR)/$*_gt_fPIC.o" ]; then \ + ln -s "../../$(OBJ_DIR)/$*_gt_fPIC.o" "$@"; \ + else \ + $(GT_CXX) $(GT_OPTL) $(GT_SWITCHES) $(CC_REQUIRED) $(GT_CFLAGS) $(DEPFLAGS) -fPIC -c $< -o $@ \ + -DFLIT_HOST='"$(HOSTNAME)"' \ + -DFLIT_COMPILER='"$(GT_CXX)"' \ + -DFLIT_OPTL='"$(GT_OPTL)"' \ + -DFLIT_SWITCHES='"$(GT_SWITCHES)"' \ + -DFLIT_FILENAME='"$(notdir $(GT_TARGET))"'; \ + fi # specify how to build the troublesome ones $(OBJ_DIR)/%_bisect_$(TROUBLE_ID).o: %.cpp Makefile custom.mk | $(OBJ_DIR) - $(TROUBLE_CC) $(TROUBLE_OPTL) $(TROUBLE_SWITCHES) $(CC_REQUIRED) $(DEPFLAGS) -c $< -o $@ \ + $(TROUBLE_CXX) $(TROUBLE_OPTL) $(TROUBLE_SWITCHES) $(CC_REQUIRED) $(TROUBLE_CFLAGS) $(DEPFLAGS) -c $< -o $@ \ -DFLIT_HOST='"$(HOSTNAME)"' \ - -DFLIT_COMPILER='"$(TROUBLE_CC)"' \ + -DFLIT_COMPILER='"$(TROUBLE_CXX)"' \ -DFLIT_OPTL='"$(TROUBLE_OPTL)"' \ -DFLIT_SWITCHES='"$(TROUBLE_SWITCHES)"' \ -DFLIT_FILENAME='"bisect-default-out"' # and the fPIC variant $(BISECT_OBJ_DIR)/%_bisect_$(TROUBLE_ID)_fPIC.o: %.cpp Makefile custom.mk | $(BISECT_OBJ_DIR) - $(TROUBLE_CC) $(TROUBLE_OPTL) $(TROUBLE_SWITCHES) $(CC_REQUIRED) $(DEPFLAGS) -fPIC -c $< -o $@ \ + if [ -f "$(OBJ_DIR)/$*_bisect_$(TROUBLE_ID)_fPIC.o" ]; then \ + ln -s "../../$(OBJ_DIR)/$*_bisect_$(TROUBLE_ID)_fPIC.o" "$@"; \ + else \ + $(TROUBLE_CXX) $(TROUBLE_OPTL) $(TROUBLE_SWITCHES) $(CC_REQUIRED) $(TROUBLE_CFLAGS) $(DEPFLAGS) -fPIC -c $< -o $@ \ + -DFLIT_HOST='"$(HOSTNAME)"' \ + -DFLIT_COMPILER='"$(TROUBLE_CXX)"' \ + -DFLIT_OPTL='"$(TROUBLE_OPTL)"' \ + -DFLIT_SWITCHES='"$(TROUBLE_SWITCHES)"' \ + -DFLIT_FILENAME='"bisect-default-out"'; \ + fi + +# and the fPIC variant +$(OBJ_DIR)/%_bisect_$(TROUBLE_ID)_fPIC.o: %.cpp Makefile custom.mk | $(OBJ_DIR) + $(TROUBLE_CXX) $(TROUBLE_OPTL) $(TROUBLE_SWITCHES) $(CC_REQUIRED) $(TROUBLE_CFLAGS) $(DEPFLAGS) -fPIC -c $< -o $@ \ -DFLIT_HOST='"$(HOSTNAME)"' \ - -DFLIT_COMPILER='"$(TROUBLE_CC)"' \ + -DFLIT_COMPILER='"$(TROUBLE_CXX)"' \ -DFLIT_OPTL='"$(TROUBLE_OPTL)"' \ -DFLIT_SWITCHES='"$(TROUBLE_SWITCHES)"' \ -DFLIT_FILENAME='"bisect-default-out"' diff --git a/documentation/README.md b/documentation/README.md index 0daf08a5..ed668c6b 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -2,6 +2,7 @@ [Return to top-level README file](../README.md) +* [Release Notes](release-notes.md) * [Installation](installation.md) * [Litmus Tests](litmus-tests.md) * [FLiT Command-Line](flit-command-line.md) diff --git a/documentation/available-compiler-flags.md b/documentation/available-compiler-flags.md index 9556f47f..a7084176 100644 --- a/documentation/available-compiler-flags.md +++ b/documentation/available-compiler-flags.md @@ -11,7 +11,9 @@ Convenient TOML lists: * [GCC](#gcc) * [Clang](#clang) * [Intel](#intel) -* [NVCC](#nvcc) + + +## All Flags In your configuration file `flit-config.toml` (see [FLiT Configuration File](flit-configuration-file.md)), you specify compiler flags for each of the @@ -20,60 +22,57 @@ If you want to have specific flag combinations, you can place it in the list, such as `"-mavx2 -mfma -funsafe-math-optimizations"`. Below is the original default list for the supported compilers: -| Flag | GCC | Clang | Intel | NVCC | -| ----------------------------- |:-----:|:-----:|:-----:|:----:| -| `-fassociative-math` | X | X | | | -| `-fcx-fortran-rules` | X | | | | -| `-fcx-limited-range` | X | | X | | -| `-fexcess-precision=fast` | X | X | | | -| `-fexcess-precision=standard` | | X | | | -| `-ffinite-math-only` | X | X | | | -| `-ffloat-store` | X | X | X | | -| `-ffp-contract=on` | X | X | | | -| `-fma` | | | X | | -| `-fmerge-all-constants` | X | X | X | | -| `-fno-trapping-math` | X | X | | | -| `-fp-model fast=1` | | | X | | -| `-fp-model fast=2` | | | X | | -| `-fp-model=double` | | | X | | -| `-fp-model=extended` | | | X | | -| `-fp-model=precise` | | | X | | -| `-fp-model=source` | | | X | | -| `-fp-model=strict` | | | X | | -| `-fp-port` | | | X | | -| `-freciprocal-math` | X | X | | | -| `-frounding-math` | X | X | X | | -| `-fsignaling-nans` | X | X | | | -| `-fsingle-precision-constant` | | X | X | | -| `-ftz` | | | X | | -| `-funsafe-math-optimizations` | X | X | | | -| `-march=core-avx2` | | X | X | | -| `-mavx` | X | X | X | | -| `-mavx2 -mfma` | X | X | X | | -| `-mfpmath=sse -mtune=native` | X | X | X | | -| `-mp1` | | | X | | -| `-no-fma` | | | X | | -| `-no-ftz` | | | X | | -| `-no-prec-div` | | | X | | -| `-prec-div` | | | X | | -| `--fmad=false` | | | | X | -| `--fmad=true` | | | | X | -| `--ftz=false` | | | | X | -| `--ftz=true` | | | | X | -| `--prec-div=false` | | | | X | -| `--prec-div=true` | | | | X | -| `--prec-sqrt=false` | | | | X | -| `--prec-sqrt=true` | | | | X | -| `--use_fast_math` | | | X | X | +| Flag | GCC | Clang | Intel | +| ----------------------------- |:-----:|:-----:|:-----:| +| `-fassociative-math` | X | X | | +| `-fcx-fortran-rules` | X | | | +| `-fcx-limited-range` | X | | X | +| `-fexcess-precision=fast` | X | X | | +| `-fexcess-precision=standard` | | X | | +| `-ffinite-math-only` | X | X | | +| `-ffloat-store` | X | X | X | +| `-ffp-contract=on` | X | X | | +| `-fma` | | | X | +| `-fmerge-all-constants` | X | X | X | +| `-fno-trapping-math` | X | X | | +| `-fp-model fast=1` | | | X | +| `-fp-model fast=2` | | | X | +| `-fp-model=double` | | | X | +| `-fp-model=extended` | | | X | +| `-fp-model=precise` | | | X | +| `-fp-model=source` | | | X | +| `-fp-model=strict` | | | X | +| `-fp-port` | | | X | +| `-freciprocal-math` | X | X | | +| `-frounding-math` | X | X | X | +| `-fsignaling-nans` | X | X | | +| `-fsingle-precision-constant` | | X | X | +| `-ftz` | | | X | +| `-funsafe-math-optimizations` | X | X | | +| `-march=core-avx2` | | X | X | +| `-mavx` | X | X | X | +| `-mavx2 -mfma` | X | X | X | +| `-mfpmath=sse -mtune=native` | X | X | X | +| `-mp1` | | | X | +| `-no-fma` | | | X | +| `-no-ftz` | | | X | +| `-no-prec-div` | | | X | +| `-prec-div` | | | X | For your convenience, here are toml-style lists that can be copied into your `flit-config.toml` file directly: ## GCC +**Note:** in some versions of python-toml, there is a parsing bug when a list +has an empty string in the middle. So simply put it at the end without a +comma. This has been fixed in the latest version of python-toml. + +You will likely want the empty string in your search space as it represents the +absence of any switches. + ```toml -switches = [ - '', +switches_list = [ '-fassociative-math', '-fcx-fortran-rules', '-fcx-limited-range', @@ -90,14 +89,14 @@ switches = [ '-mavx', '-mavx2 -mfma', '-mfpmath=sse -mtune=native', + '' ] ``` ## Clang ```toml -switches = [ - '', +switches_list = [ '-fassociative-math', '-fexcess-precision=fast', '-fexcess-precision=standard', @@ -115,14 +114,14 @@ switches = [ '-mavx', '-mavx2 -mfma', '-mfpmath=sse -mtune=native', + '' ] ``` ## Intel ```toml -switches = [ - '', +switches_list = [ '--use_fast_math', '-fcx-limited-range', '-ffloat-store', @@ -149,23 +148,7 @@ switches = [ '-no-ftz', '-no-prec-div', '-prec-div', -] -``` - -## NVCC - -```toml -switches = [ - '', - '--use_fast_math', - '--fmad=false', - '--fmad=true', - '--ftz=false', - '--ftz=true', - '--prec-div=false', - '--prec-div=true', - '--prec-sqrt=false', - '--prec-sqrt=true', + '' ] ``` diff --git a/documentation/flit-command-line.md b/documentation/flit-command-line.md index 75a895b6..d0f51713 100644 --- a/documentation/flit-command-line.md +++ b/documentation/flit-command-line.md @@ -119,8 +119,28 @@ the created `litmus-test-run` directory. You are free to examine the results using the `sqlite3` interactive tool or any other method you have for running queries on an SQLite3 database. +There is an option to specify a different database file than the one in +`flit-config.toml`. Here are three reasons you may want to use this option: + +1. run `flit import` outside of an initialized flit test directory +2. create a database file different from the one in `flit-config.toml` +3. use the import functionality on a system that does not have the python + `toml` package installed + +If one of those applies to you, then you can use the `--dbfile` argument to +`flit import`. + +```bash +flit import --dbfile temporary.sqlite backup/results/*.csv +``` + ## flit bisect +There is an additional optional dependency in order to run `flit bisect`. That +is [pyelftools](https://github.com/eliben/pyelftools) as discussed in [FLiT +Installation](installation.md). If `pyelftools` is not installed, then +`bisect` is disabled. + When FLiT runs identify compilations that cause some tests to exhibit variability, one may want to investigate further and understand where the compiler introduced overly aggressive optimizations. diff --git a/documentation/flit-configuration-file.md b/documentation/flit-configuration-file.md index 72b88e03..e169878b 100644 --- a/documentation/flit-configuration-file.md +++ b/documentation/flit-configuration-file.md @@ -21,20 +21,49 @@ to delete or comment out the values, which will cause the default values to be used._ ```toml -[database] +[host] +name = 'my.hostname.com' +flit_path = '/usr/bin/flit' +config_dir = '/usr/share/flit/config' +``` + +When you initialize your project with `flit init`, these fields will be +populated with values corresponding to the values on your machine: + +- `name`: hostname of the current machine +- `flit_path`: will be initialized to the path of the `flit.py` script or its + symbolic link +- `config_dir`: directory containing default configuration values. Either the + one installed (if using flit from an install) or the one from the flit + repository (if running directly from a git clone). + +**Note:** if these fields are omitted, then the values of the current machine +and `flit` executable will be used instead. This is useful if you intend to +run your tests on multiple machines and if you want to allow using different +paths for the flit executable and configuration. +The `flit_path` variable is put into the generated `Makefile` so that it can +update itself. + + +```toml +[database] type = 'sqlite3' filepath = 'results.sqlite' ``` -Above we specify the information for the database. Postgres used to be -supported, but has been replaced with SQLite3. For now, only `sqlite3` is -supported for the `type`. The only thing that needs to be specified is -`filepath`, which can be a relative or absolute path. +Above we specify the information for the database. We only support +[SQLite3](https://sqlite.org) databases currently. Therefore, there exists +only one valid value for `type`, which is `sqlite3`. If you have need of using +something other than SQLite3, please submit an +[issue](https://github.com/PRUNERS/FLiT/issues). + +The `filepath` field can be relative or absolute and specifies where to store +results for a full flit run. + ```toml [run] - timing = true timing_loops = -1 timing_repeats = 3 @@ -72,78 +101,194 @@ tests employ MPI, it is the test creator's responsibility to ensure that the test produces identical results every time. ```toml -[[hosts]] - -name = 'my.hostname.com' -flit_path = '/usr/bin/flit' -config_dir = 'project/flit-tests' -``` - -Now we start specifying information of the first host called `my.hostname.com`. -The `flit_path` is the path of the `flit` command-line tool at this host. The -`config_dir` is the directory where this configuration file is located on the -host. For now, it is expected that each machine already has a copy of the -tests with the same configuration file and that their location is documented -here. If needed, this may be revisited. - -For any variable under `hosts` that specifies a relative path, the relative -path is always with respect to the user's home directory. - -```toml -[hosts.dev_build] - +[dev_build] compiler_name = 'g++' optimization_level = '-O2' switches = '-funsafe-math-optimizations' ``` -Each host has a developer build that can be performed. The purpose of the -developer build is to try out the tests that you have developed without -committing yourself to a full FLiT run. +The `[dev_build]` section describes the developer build, or the configurations +for the `make dev` target from the autogenerated `Makefile` (which generates +the `devrun` test executable). + +- `compiler_name`: A name matching the `name` field from the `[[compiler]]` array + section. This will pull in the settings from that `compiler` for the + developer build. +- `optimization_level`: The optimization level to use in compiling `devrun` +- `switches`: Additional flags to use in compiling `devrun` -The `compiler` field here needs to match the `name` of one of the compilers -specified later in the `[[hosts.compilers]]` list. The `optimization_level` -and `switches` need not be in the `optimization_levels` and `switches` for the -compiler with the matching name. +The purpose of the developer build is to try out the tests that you have +developed without committing yourself to a full FLiT run. True, that can also +be done with the `gtrun` executable (seen below), but it may be helpful to +experiment with a build that contains higher optimizations, such as when you +are searching for inputs that demonstrate variability with a particular flag. -This does exactly one compilation with the compiler and flags that are -specified. This compilation is done using `make dev` and generates an -executable called `devrun`. You can call +For the generated `devrun` executable, you can call ```bash ./devrun --help ``` -to see the options available for the test executable, such as only running a -particular test or outputting debugging information. +to see the options available, such as only running a particular test or +outputting debugging information. ```toml -[hosts.ground_truth] - +[ground_truth] compiler_name = 'g++' optimization_level = '-O0' switches = '' ``` -Each host has a ground truth run to use as comparisons when doing analysis. -The `compiler` field here needs to match the `name` of one of the compilers specified -later in the `[[hosts.compilers]]` list. The `optimization_level` and -`switches` need not be in the `optimization_levels` and `switches` for -the compiler with the matching name. +The `[ground_truth]` build is intended to be the baseline for all other +compilations. Here, you should place the compilation that you trust, as it +will become a comparison for all other test results. + +The fields here are synonymous with those in the `[dev_build]` + +This configuration specifies the compilation of the `gtrun` executable created +from the `make gt` target in the autogenerated `Makefile`. The compiled +`gtrun` executable has the exact same command-line interface as the `devrun` +executable. ```toml - [[hosts.compilers]] +[[compiler]] +binary = 'g++' +name = 'g++' +type = 'gcc' +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] +switches_list = [ + '-fassociative-math', + '-fcx-fortran-rules', + '-fcx-limited-range', + '-fexcess-precision=fast', + '-ffinite-math-only', + '-ffloat-store', + '-ffp-contract=on', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-funsafe-math-optimizations', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '' +] +``` - binary = 'g++' - name = 'g++' +Here we specify the first compiler. + +- `binary`: The executable for this compiler. The one seen above is a simple + name and will use the executable `g++` from the system `PATH`. You can + alternatively specify an absolute path or a relative path. If you specify a + relative path, it will be relative to the directory containing + `flit-config.toml`, which is the top-level directory for your FLiT tests. +- `name`: A human-readable unique name for this compiler. This could be + something like `gcc-7.4`. It is used in the `[dev_build]` and + `[ground_truth]` sections to refer to this compiler. +- `type`: The type of compiler. The supported types are `gcc`, `clang` and + `intel`. If you need an additional type supported, please submit an + [issue](https://github.com/PRUNERS/FLiT/issues). +- `optimization_levels`: List of optimization levels to search over. You may + remove from or add to this list. If you omit this list, the default list + will be used (which is the same as the one shown here). The defaults are + specific to the type of compiler. See below for the default list for the + other supported compilers. +- `switches_list`: List of switches to search over. You may remove from or add + to this list. If you omit this list, the default list will be used (which is + the same as the one shown here). The defaults are specific to the type of + compiler. See below for the default list for the other supported compilers. + +Note that the section has two square brackets around it instead of one. This +indicates that it is an array and subsequent instances of `[[compiler]]` will +add to that array. + +If the `[[compiler]]` section is omitted, then the default values will be used +which includes the one from above named `g++` and the following two: + +```toml +[[compiler]] +binary = 'clang++' +name = 'clang++' +type = 'clang' +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] +switches_list = [ + '-fassociative-math', + '-fexcess-precision=fast', + '-fexcess-precision=standard', + '-ffinite-math-only', + '-ffloat-store', + '-ffp-contract=on', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-fsingle-precision-constant', + '-funsafe-math-optimizations', + '-march=core-avx2', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '' +] + +[[compiler]] +binary = 'icpc' +name = 'icpc' +type = 'intel' +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] +switches_list = [ + '--use_fast_math', + '-fcx-limited-range', + '-ffloat-store', + '-fma', + '-fmerge-all-constants', + '-fp-model fast=1', + '-fp-model fast=2', + '-fp-model=double', + '-fp-model=except', + '-fp-model=extended', + '-fp-model=precise', + '-fp-model=source', + '-fp-model=strict', + '-fp-port', + '-frounding-math', + '-fsingle-precision-constant', + '-ftz', + '-march=core-avx2', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '-mp1', + '-no-fma', + '-no-ftz', + '-no-prec-div', + '-prec-div', + '' +] ``` -Here we specify the first compiler for the first host. Since binary here is a -simple name, it will get the executable `g++` from the system `PATH`. If you -really mean you want to have a compiler that is located in your home directory, -you can do `./g++`. The name field is a human-readable unique name for this -compiler. For example, the binary field can have an absolute path where the -name can be something like `gcc-7.4`. +Do not worry if you do not have all of the compilers on your system, they will +be silently ignored if they are not found in the PATH. This is to allow you to +use the default values safely. + ## Full Configuration File @@ -151,47 +296,132 @@ Combining all of the above sections together, here is the full example (and default) configuration file: ```toml -[database] +[host] +name = 'my.hostname.com' +flit_path = '/usr/bin/flit' +config_dir = '/usr/share/flit/config' +[database] type = 'sqlite' filepath = 'results.sqlite' - [run] - timing = true timing_loops = -1 timing_repeats = 3 - enable_mpi = false mpirun_args = '' - -[[hosts]] - -name = 'my.hostname.com' -flit_path = '/usr/bin/flit' -config_dir = 'project/flit-tests' - - -[hosts.dev_build] - +[dev_build] compiler_name = 'g++' optimization_level = '-O2' switches = '-funsafe-math-optimizations' - -[hosts.ground_truth] - +[ground_truth] compiler_name = 'g++' optimization_level = '-O0' switches = '' - - [[hosts.compilers]] - - binary = 'g++' - name = 'g++' +[[compiler]] +binary = 'g++' +name = 'g++' +type = 'gcc' +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] +switches_list = [ + '-fassociative-math', + '-fcx-fortran-rules', + '-fcx-limited-range', + '-fexcess-precision=fast', + '-ffinite-math-only', + '-ffloat-store', + '-ffp-contract=on', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-funsafe-math-optimizations', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '' +] + +[[compiler]] +binary = 'clang++' +name = 'clang++' +type = 'clang' +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] +switches_list = [ + '-fassociative-math', + '-fexcess-precision=fast', + '-fexcess-precision=standard', + '-ffinite-math-only', + '-ffloat-store', + '-ffp-contract=on', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-fsingle-precision-constant', + '-funsafe-math-optimizations', + '-march=core-avx2', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '' +] + +[[compiler]] +binary = 'icpc' +name = 'icpc' +type = 'intel' +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] +switches_list = [ + '--use_fast_math', + '-fcx-limited-range', + '-ffloat-store', + '-fma', + '-fmerge-all-constants', + '-fp-model fast=1', + '-fp-model fast=2', + '-fp-model=double', + '-fp-model=except', + '-fp-model=extended', + '-fp-model=precise', + '-fp-model=source', + '-fp-model=strict', + '-fp-port', + '-frounding-math', + '-fsingle-precision-constant', + '-ftz', + '-march=core-avx2', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '-mp1', + '-no-fma', + '-no-ftz', + '-no-prec-div', + '-prec-div', + '' +] ``` diff --git a/documentation/installation.md b/documentation/installation.md index 1bcdd8b4..5f61b909 100644 --- a/documentation/installation.md +++ b/documentation/installation.md @@ -1,5 +1,7 @@ # FLiT Installation +[Prev](release-notes.md) +| [Table of Contents](README.md) | [Next](litmus-tests.md) @@ -8,6 +10,7 @@ Instruction Contents: * [Prerequisites](#prerequisites) * [Compilers](#compilers) + * [Clang Only](#clang-only) * [Optional Dependencies](#optional-dependencies) * [FLiT Setup](#flit-setup) * [Database Setup](#database-setup) @@ -18,7 +21,7 @@ Instruction Contents: Stuff you should already have: * [bash](https://www.gnu.org/software/bash) -* [binutils](https://www.gnu.org/software/binutils) +* [binutils](https://www.gnu.org/software/binutils) version 2.26 or higher * [coreutils](https://www.gnu.org/software/coreutils/coreutils.html) * hostname @@ -26,10 +29,14 @@ Stuff you may need to get * [git](https://git-scm.com) * [python3](https://www.python.org) - * The [toml](https://github.com/uiri/toml) module (for + * [toml](https://github.com/uiri/toml) module (for [TOML](https://github.com/toml-lang/toml) configuration files) + * (optional) [pyelftools](https://github.com/eliben/pyelftools) module for + parsing ELF files. This is used for `flit bisect`; all other functionality + will work without it. * [make](https://www.gnu.org/software/make) -* [gcc](https://gcc.gnu.org) version 4.9 or higher +* [gcc](https://gcc.gnu.org) version 4.9 or higher (or + [clang](https://clang.llvm.org), see section [Clang Only](#clang-only)) * [sqlite3](https://sqlite.org) version 3.0 or higher. You can use the one that comes with python, or install as a standalone. @@ -38,14 +45,27 @@ For Debian-based systems: ```bash sudo apt install \ bash binutils build-essential coreutils git hostname \ - python3 python3-toml + python3 +``` + +The python modules can be installed with `apt` + +```bash +sudo apt install python3-toml python3-pyelftools +``` + +or with `pip` + +```bash +sudo apt install python3-pip +pip3 install --user toml pyelftools ``` For homebrew on OSX (besides installing [Xcode](https://developer.apple.com/xcode)) ```bash brew install make python3 gcc git -pip3 install toml +pip3 install toml pyelftools ``` If you install python version 3.0 or later, then you will need to have a @@ -60,6 +80,44 @@ install another version of GCC, as well as Clang and the Intel compiler. If you are missing either Clang or the Intel compiler, FLiT will still work as expected. +The supported compiler versions are: + +| Compiler Type | Minimum Supported Version | +|:-------------:|:-------------------------:| +| gcc | 4.9.0 | +| clang | 3.4.0 | +| intel | 16.0 | + +If your compiler version is below those on this list and you want FLiT to +support it, please create an [issue](https://github.com/PRUNERS/FLiT/issues) +and we may be able to add support for you. Otherwise, you are on your own. + +Likewise, if you want support added for other types of compilers, such as the +PGI compiler or the IBM compiler, please create an +[issue](https://github.com/PRUNERS/FLiT/issues). + + +### Clang Only + +FLiT is mostly geared around having at least GCC around, however, users may +want to skip using GCC and use Clang instead. If this is your use case, this +can be done. + +To compile FLiT using Clang, set the `CXX` environment variable to the +executable for Clang you wish to use. For example: + +```bash +git clone https://github.com/PRUNERS/FLiT.git +cd FLiT +export CXX=clang +make +sudo make install +``` + +Then when creating your environment, simply provide only a Clang compiler. +This setup is largely untested, so if you have trouble, please submit an +[issue](https://github.com/PRUNERS/FLiT/issues). + ### Optional Dependencies FLiT has [MPI support](mpi-support.md) which you may want to use. To compile @@ -130,6 +188,8 @@ then you would uninstall with make uninstall PREFIX=~/my-installs/ ``` +[Prev](release-notes.md) +| [Table of Contents](README.md) | [Next](litmus-tests.md) diff --git a/documentation/mpi-support.md b/documentation/mpi-support.md index af4c7ad6..80df64d7 100644 --- a/documentation/mpi-support.md +++ b/documentation/mpi-support.md @@ -74,12 +74,34 @@ global pointer even if MPI is not enabled and you can have logic based on the rank and size of the MPI world. +## Test Return Values + +Under FLiT, you can write tests using MPI with multiple processes (as long as +the result is deterministic). To prevent confusion with multiple test return +values, one from each process, FLiT will ignore the test result value from all +processes except for the one with rank 0. Therefore, all calculated results +must be communicated to rank 0 in order to have it be captured by the FLiT +framework. + +If you find this does not fit with your use case, please submit an +[issue](https://github.com/PRUNERS/FLiT/issues/new?template=feature_request.md) +on GitHub. For example, it may be that users may want to configure whether +each result is captured individually and checked at test time, process by +process. However, That approach will not work in all cases, such as the +producer-consumer model where processes may end up getting different data and +tasks. For now, enforcing users to communicate with rank 0 is the most +straightforward way to ensure determinism. + + ## Conditional Compilation If you want your tests to conditionally use MPI, there is a macro-defined variable you can condition off of to disable or enable certain code. That variable is `FLIT_USE_MPI`. For example +Note: even when MPI is disabled, the global `flit::mpi` variable is still +available. + ```c++ T grid[1024][1024]; int nrows = 1024 / mpi->size; diff --git a/documentation/release-notes.md b/documentation/release-notes.md new file mode 100644 index 00000000..df32b20b --- /dev/null +++ b/documentation/release-notes.md @@ -0,0 +1,141 @@ +# Release Notes + +[Table of Contents](README.md) +| +[Next](installation.md) + +Release note versions: + +- [v2.0-beta.2](#v2.0-beta.2) _(10 April 2019)_ +- [v2.0-beta.1](#v2.0-beta.1) _(18 July 2018)_ +- [v2.0-alpha.3](#v2.0-alpha.3) _(29 January 2018)_ +- [v2.0-alpha.2](#v2.0-alpha.2) _(27 January 2018)_ +- [v2.0-alpha.1](#v2.0-alpha.1) _(25 September 2017)_ +- [v1.0.0](#v1.0.0) _(25 March 2017)_ + +## v2.0-beta.2 + +_Released on 10 April 2019_ + +**Highlights of Major Changes** + +- TravisCI continuous integration fixed and working +- `flit bisect` + - outputs results incrementally. + - optimizations, such as memoizing and search space reduction + - order findings by the magnitude of variability + - add flags `--compile-only` and `--precompile-fpic` + - additional bisect assertion for finding disjoint sets + - add dependency of `pyelftools` +- Users specify the FLiT search space in `flit-config.toml` instead of being hard-coded +- `flit import`: add `--dbfile` flag to create an import database without a `flit-config.toml` file +- Remove `hosts` section from `flit-config.toml` +- Many bug fixes + +**All Issues Addressed:** + +- [#174](https://github.com/PRUNERS/FLiT/issues/174) (PR [#179](https://github.com/PRUNERS/FLiT/pull/179)): Have bisect output results incrementally +- [#175](https://github.com/PRUNERS/FLiT/issues/175) (PR [#181](https://github.com/PRUNERS/FLiT/pull/181)): Have function passed to `bisect_search` only take one argument +- [#182](https://github.com/PRUNERS/FLiT/issues/182) (PR [#185](https://github.com/PRUNERS/FLiT/pull/185), [#186](https://github.com/PRUNERS/FLiT/pull/186), [#187](https://github.com/PRUNERS/FLiT/pull/187)): Fix TravisCI continuous integration +- [#180](https://github.com/PRUNERS/FLiT/issues/180) (PR [#190](https://github.com/PRUNERS/FLiT/pull/190)): Memoize the bisect test function +- [#143](https://github.com/PRUNERS/FLiT/issues/143) (PR [#195](https://github.com/PRUNERS/FLiT/pull/195)): Fix NaN return values from litmus tests +- [#172](https://github.com/PRUNERS/FLiT/issues/172) (PR [#197](https://github.com/PRUNERS/FLiT/pull/197)): Order bisect findings by magnitude of variability +- [#200](https://github.com/PRUNERS/FLiT/issues/200) (PR [#201](https://github.com/PRUNERS/FLiT/pull/201)): Fix CSV file parsing +- [#120](https://github.com/PRUNERS/FLiT/issues/120) (PR [#206](https://github.com/PRUNERS/FLiT/pull/206)): Allow user-provided compilers in `flit-config.toml` +- (PR [#209](https://github.com/PRUNERS/FLiT/pull/209)): Add `--compile-only` and `--precompile-fpic` to `flit bisect` +- [#211](https://github.com/PRUNERS/FLiT/issues/211) (PR [#215](https://github.com/PRUNERS/FLiT/pull/215), [#223](https://github.com/PRUNERS/FLiT/pull/223)): Add `--dbfile` to `flit import` to import without a `flit-config.toml` file +- [#220](https://github.com/PRUNERS/FLiT/issues/220) (PR [#221](https://github.com/PRUNERS/FLiT/pull/221)): Have the test harness delete temporary directories even when exceptions are thrown +- [#217](https://github.com/PRUNERS/FLiT/issues/217) (PR [#219](https://github.com/PRUNERS/FLiT/pull/219)): Create a template for issues and pull requests (a GitHub-specific update) +- [#212](https://github.com/PRUNERS/FLiT/issues/212) (PR [#213](https://github.com/PRUNERS/FLiT/pull/213)): For bisect, if the Makefile errors out, print out the Makefile output for debugging +- [#225](https://github.com/PRUNERS/FLiT/issues/225) (PR [#226](https://github.com/PRUNERS/FLiT/pull/226)): For bisect, add assertion for finding disjoint sets (cannot continue if they overlap) +- [#230](https://github.com/PRUNERS/FLiT/issues/230) (PR [#232](https://github.com/PRUNERS/FLiT/pull/232)): Fix compilation if header files go missing +- [#229](https://github.com/PRUNERS/FLiT/issues/229) (PR [#231](https://github.com/PRUNERS/FLiT/pull/231)): Fix bisect deadlock where multiple updates to `ground-truth.csv` were attempted +- [#194](https://github.com/PRUNERS/FLiT/issues/194) (PR [#236](https://github.com/PRUNERS/FLiT/pull/236)): Fix the slow bisect search due to an explosion of templated function definitions +- [#238](https://github.com/PRUNERS/FLiT/issues/238) (PR [#241](https://github.com/PRUNERS/FLiT/pull/241)): Fix bisect search space problems. Specifically, only search over strong and publicly exported function symbols. This pull request added the dependency of `pyelftools`. +- [#233](https://github.com/PRUNERS/FLiT/issues/233) (PR [#243](https://github.com/PRUNERS/FLiT/pull/243)): Update the required version of `binutils` in the documentation +- [#239](https://github.com/PRUNERS/FLiT/issues/239) (PR [#242](https://github.com/PRUNERS/FLiT/pull/242)): Fix support for older versions of the Clang compiler (version 4) +- [#244](https://github.com/PRUNERS/FLiT/issues/244) (PR [#246](https://github.com/PRUNERS/FLiT/pull/246)); Remove the `hosts` section from `flit-config.toml`. It was not used. +- [#249](https://github.com/PRUNERS/FLiT/issues/249) (PR [#250](https://github.com/PRUNERS/FLiT/pull/250)): Update documentation about how FLiT handles MPI tests. Specifically how the test return values are handled. +- [#119](https://github.com/PRUNERS/FLiT/issues/119) (PR [#251](https://github.com/PRUNERS/FLiT/pull/251)): Allow the user to specify the list of optimization levels and flags in `flit-config.toml` for the FLiT search space. No longer hard-coded. +- [#256](https://github.com/PRUNERS/FLiT/issues/256) (PR [#258](https://github.com/PRUNERS/FLiT/pull/258)): Fix bisect crash when an object file has an empty dwarf info +- [#257](https://github.com/PRUNERS/FLiT/issues/257) (PR [#259](https://github.com/PRUNERS/FLiT/pull/259)): Fix bisect crash when an object file has no dwarf info section + + +## v2.0-beta.1 + +_Released on 18 July 2018_ + +Since FLiT has been stable in the alpha phase for some time and some other things have stabilized more (such as documentation and interface), we are happy to announce the move to beta status. + +In this release, we have included the following pull requests: + +[#115](https://github.com/PRUNERS/FLiT/pull/115), [#116](https://github.com/PRUNERS/FLiT/pull/116), [#128](https://github.com/PRUNERS/FLiT/pull/128), [#130](https://github.com/PRUNERS/FLiT/pull/130), [#131](https://github.com/PRUNERS/FLiT/pull/131), [#132](https://github.com/PRUNERS/FLiT/pull/132), [#135](https://github.com/PRUNERS/FLiT/pull/135), [#139](https://github.com/PRUNERS/FLiT/pull/139), [#142](https://github.com/PRUNERS/FLiT/pull/142), [#145](https://github.com/PRUNERS/FLiT/pull/145), [#147](https://github.com/PRUNERS/FLiT/pull/147), [#150](https://github.com/PRUNERS/FLiT/pull/150), [#154](https://github.com/PRUNERS/FLiT/pull/154), [#155](https://github.com/PRUNERS/FLiT/pull/155), [#156](https://github.com/PRUNERS/FLiT/pull/156), [#157](https://github.com/PRUNERS/FLiT/pull/157), [#158](https://github.com/PRUNERS/FLiT/pull/158), [#159](https://github.com/PRUNERS/FLiT/pull/159), [#160](https://github.com/PRUNERS/FLiT/pull/160), [#161](https://github.com/PRUNERS/FLiT/pull/161), [#167](https://github.com/PRUNERS/FLiT/pull/167), [#170](https://github.com/PRUNERS/FLiT/pull/170), [#176](https://github.com/PRUNERS/FLiT/pull/176) + +This includes the following features: + +- Bisect + - Fully implemented and supported - assigns variability blame to individual source files and functions + - Works with Intel linker problems + - Autodelete intermediate generated files to reduce disk usage from going out of control + - Autorun all differences found in a given database file + - Unfortunately, this functionality is not adequately documented (see issue [#136](https://github.com/PRUNERS/FLiT/pull/136)) +- Add MPI support within `flit-config.toml` +- Add timing parameters to `flit-config.toml` +- Add `--info` flag to test executable to show how it was compiled +- Rename results and database columns to be more understandable +- Add `plot_timing_histogram.py` for auto-generating histogram plots and update `plot_timing.py` +- Add benchmarks: polybench and random - benchmark examples of FLiT usage +- Add `uninstall` target to the top-level `Makefile` +- Remove CUDA completely, it was broken and cumbering the space +- Remove old unused files. The repository is more lean with all files relevant +- License notices on files +- Fix support for GCC 4.8 (had a compiler error in FLiT) + + +## v2.0-alpha.3 + +_Released on 29 January 2018_ + +There was reported a bug in the installation making `flit import` not work. This has been fixed and a new alpha release is ready. + + +## v2.0-alpha.2 + +_Released on 27 January 2018_ + +Many fixes in this update: + +- Remove `TestInput` and `CUTestInput` types, replace with arrays and `std::vector` +- Disable unfinished flit command-line commands +- Remove sections of the `flit-config.toml` settings file that are unused +- Fix incremental build +- Add `--version` argument to `flit` +- Recursive generated `Makefile` allows for more scalability with project size and number of tests +- Add some tests for FLiT's infrastructure - not comprehensive, but some automated tests +- Fix timing functionality +- Add a time plot + + +## v2.0-alpha.1 + +_Released on 25 September 2017_ + +Not everything for this new release is finished. The tool has been rearchitected to be easier to use within existing code bases. Many things have changed. Here are a few of them: + +- A command-line tool called `flit` with subcommands +- Storage is into an SQLite database rather than Postgres (for simplicity and ease of installation) +- A configuration file, `flit-config.toml`, where you can specify your ground-truth compilation and your developer compilation. Some of the values in this configuration file are not yet implemented to be used. +- An included `custom.mk` makefile to help you specify how to build your source files with the tests +- Turn FLiT into a shared library to be linked + + +## v.1.0.0 + +_Released on 25 March 2017_ + +This release provides litmus tests, a reproducibility test framework, and analysis tools for the results of the tests. + + +[Table of Contents](README.md) +| +[Next](installation.md) diff --git a/gensrc/expression.py b/gensrc/expression.py index 0d55d27a..754f55b4 100644 --- a/gensrc/expression.py +++ b/gensrc/expression.py @@ -158,10 +158,6 @@ def __repr__(self): def random_expression(env, length, vars_only=False): ''' Generates a random mathematical expression as a string - - >>> env = Environment({'x'; Variable('x', 'int'}) - >>> random_expression(env, 3) - Expression((x + 3.25124) * x) ''' # Populate the expression with operations expr = Expression() diff --git a/images/bisect-search.pdf b/images/bisect-search.pdf new file mode 100644 index 00000000..cfb2a179 Binary files /dev/null and b/images/bisect-search.pdf differ diff --git a/images/bisect-search.svg b/images/bisect-search.svg new file mode 100644 index 00000000..cc641e97 --- /dev/null +++ b/images/bisect-search.svg @@ -0,0 +1,1452 @@ + + + +image/svg+xml= +Found second bad file +X +Mark passing obj, assume other half fails and split +✓ +Mark passing obj, assume other half fails and split +✓ +Split in half +X +Restart search using unmarked files +Found first bad file += +X +Mark passing obj, assume other half fails and split +✓ +Split in half +X +Split in half +X +Check that the test fails + \ No newline at end of file diff --git a/litmus-tests/tests/DistributivityOfMultiplication.cpp b/litmus-tests/tests/DistributivityOfMultiplication.cpp index d416c9ff..e80714f1 100644 --- a/litmus-tests/tests/DistributivityOfMultiplication.cpp +++ b/litmus-tests/tests/DistributivityOfMultiplication.cpp @@ -243,52 +243,8 @@ DistributivityOfMultiplication::getDefaultInput() { return flit::as_float(val); }; - // Put in canned values of previously found diverging inputs - // These are entered as hex values to maintain the exact value instead of trying - // to specify enough decimal digits to get the same floating-point value - std::vector ti = { - convert(0x2b99, 0x2bb4d082ca2e7ec7), // 3.586714e-1573 - convert(0x725a, 0x14c0a0cd445b52d5), // 6.131032e+3879 - convert(0x075d, 0x0bc91b713fc2fba5), // 4.278225e-4366 - - convert(0x3408, 0xd98776d83be541b8), // 1.497721e-922 - convert(0x7da5, 0x32daa5df77e78b5e), // 2.847787e+4750 - convert(0x376a, 0xfa52e8946985dab4), // 8.479921e-662 - - convert(0x2355, 0xb32ca57fbcc68a6c), // 1.541551e-2209 - convert(0x7337, 0x4855e1d4f174504d), // 7.201858e+3946 - convert(0x736a, 0xac4f338d852e88cd), // 3.863064e+3962 - - convert(0x4727, 0x9c8f934e1cc682d2), // 3.753403e+551 - convert(0x02e7, 0x3b753c2d81c6bf78), // 3.612998e-4709 - convert(0x485b, 0xeae81af41947d10a), // 2.936812e+644 - - convert(0x1b91, 0x9e18ddbb66670e9f), // 4.852600e-2808 - convert(0x205f, 0x58ab17d3e5309234), // 5.031688e-2438 - convert(0x4a4a, 0x1b8a0c6541700676), // 3.521930e+792 - - convert(0x4178, 0xd2cee20bba5c6843), // 5.069741e+113 - convert(0x3564, 0x58406d82dd970b8e), // 3.483973e-818 - convert(0x150a, 0x92cde10402bc42ef), // 4.292064e-3311 - - convert(0x1965, 0x630847ac1ecd253c), // 1.288694e-2975 - convert(0x004c, 0x6e2b4c6a070d3835), // 1.093228e-4909 - convert(0x380f, 0x92b14ca6d81b5a24), // 2.324058e-612 - - convert(0x4492, 0x870e870425dcb0cf), // 3.384007e+352 - convert(0x71dd, 0x159330946cecd9a8), // 1.498527e+3842 - convert(0x586a, 0xfc38e15fe5d604a5), // 1.079136e+1882 - - convert(0x240d, 0xae73609d2bf51b7d), // 3.680220e-2154 - convert(0x2a67, 0x89b93255d3362c94), // 8.669256e-1665 - convert(0x2462, 0x79d020dd3c308e90), // 9.941326e-2129 - - convert(0x6703, 0x6455d50eb2825cf7), // 3.818039e+3006 - convert(0x1f77, 0x70b75c7169817349), // 9.267715e-2508 - convert(0x4ab4, 0x27faa40914dad6a6), // 4.148019e+824 - }; - - return ti; + /// TODO: find values that demonstrate variability with long double + return {}; } REGISTER_TYPE(DistributivityOfMultiplication) diff --git a/litmus-tests/tests/InliningProblem.cpp b/litmus-tests/tests/InliningProblem.cpp index a3b8cc05..66aa5258 100644 --- a/litmus-tests/tests/InliningProblem.cpp +++ b/litmus-tests/tests/InliningProblem.cpp @@ -93,7 +93,7 @@ class InliningProblem : public flit::TestBase { virtual size_t getInputsPerRun() override { return 1; } virtual std::vector getDefaultInput() override { - return { .1, 1.1e3, -.1, -1.1e3, 1/3 }; + return { .1, 1.1e3, 0.5, 3.14159256, 1/3 }; } protected: diff --git a/litmus-tests/tests/ReciprocalMath.cpp b/litmus-tests/tests/ReciprocalMath.cpp index b0b31204..0ebf2733 100644 --- a/litmus-tests/tests/ReciprocalMath.cpp +++ b/litmus-tests/tests/ReciprocalMath.cpp @@ -89,7 +89,59 @@ class ReciprocalMath : public flit::TestBase { virtual size_t getInputsPerRun() override { return 5; } virtual std::vector getDefaultInput() override { - return { .1, 1.1e3, -.1, -1.1e3, 1/3 }; + return { 7505932213.155222, + -0.0003670095914968359, + 8780.668938367293, + -0.0006155790233503232, + 1.0530559711831378, + + 8338244879.47946, + -26.46659826627008, + -46.167772633949596, + -8.01886332371321, + 9.389477622336432, + + 87.0485837234231, + 814.095091206393, + 376.6986397719769, + 21.457991635073714, + -34.26603687459364, + + 31.594514417427042, + 0.0007624103417064105, + -0.02577922636404706, + -0.0009569015524931334, + -0.0003699749232482463, + + 5902221514.636204, + -0.00021708934100337198, + 9749.762436162175, + 0.8607435476176816, + -94.28304433351171, + + -49.70966603739113, + 7238595461.899788, + 985.3112995369329, + -707.7407922965596, + -671.0617089400928, + + -3699.38938030149, + -7073528488.719316, + -164.55786123623116, + -0.9364155629331614, + 7707.943761459553, + + -62.53791853138796, + -199.1618358455679, + -52.036903076794985, + -2.8204333352680377, + -2930.340776062591, + + -2852.2836425016662, + 72.33604399822339, + 378.80829294847405, + 9.71642682532883, + -2.6077212300950436}; } protected: diff --git a/pylint.rc b/pylint.rc new file mode 100644 index 00000000..5c6e95f6 --- /dev/null +++ b/pylint.rc @@ -0,0 +1,548 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + invalid-unicode-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + locally-enabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape, + C1801 + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio).You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=optparse.Values,sys.exit + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,io + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[BASIC] + +# Naming style matching correct argument names +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style +#argument-rgx= + +# Naming style matching correct attribute names +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style +#class-attribute-rgx= + +# Naming style matching correct class names +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming-style +#class-rgx= + +# Naming style matching correct constant names +#const-naming-style=UPPER_CASE +const-naming-style=any + +# Regular expression matching correct constant names. Overrides const-naming- +# style +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma +good-names=i, + j, + k, + ex, + Run, + _ + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming style matching correct inline iteration names +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style +#inlinevar-rgx= + +# Naming style matching correct method names +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style +#method-rgx= + +# Naming style matching correct module names +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style +#variable-rgx= + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/scripts/flitcli/config/flit-default.toml.in b/scripts/flitcli/config/flit-default.toml.in index c7c5d054..5e44f117 100644 --- a/scripts/flitcli/config/flit-default.toml.in +++ b/scripts/flitcli/config/flit-default.toml.in @@ -82,13 +82,25 @@ # Autogenerated by "flit init" # flit version {flit_version} +# General information +[host] + +# if you omit name, the system hostname will be used +name = '{hostname}' + +# if you omit flit_path, the current flit executable will be used +flit_path = '{flit_path}' + +# if you omit config_dir, it will be taken from the current flit executable +config_dir = '{config_dir}' + + [database] -# older versions of flit supported postgres. that has been removed. only -# sqlite is supported at the moment. +# only sqlite is supported, so this field can be safely omitted type = 'sqlite' -# if relative path, it is relative to the directory containing this +# if this is a relative path, it is relative to the directory containing this # configuration file. filepath = 'results.sqlite' @@ -116,37 +128,155 @@ enable_mpi = false mpirun_args = '' -# For now, only the first host is supported, all others are ignored -[[hosts]] +# The settings for "make dev", which generates the "devrun" executable +[dev_build] -name = '{hostname}' -flit_path = '{flit_path}' -config_dir = '{config_dir}' - -# The settings for "make dev" -[hosts.dev_build] -# compiler_name must be found in [[hosts.compilers]] list under name attribute -# but the optimization level and switches do not need to be in the compiler list +# compiler_name must be found in the [[compiler]] list under the name attribute +# The optimization level and switches need not match those found in the +# associated [[compiler]] compiler_name = 'g++' optimization_level = '-O2' switches = '-funsafe-math-optimizations' -# The ground truth compilation to use in analysis, for "make gt" -[hosts.ground_truth] -# compiler_name must be found in [[hosts.compilers]] list under name attribute -# but the optimization level and switches do not need to be in the compiler list +# The ground truth compilation to use in analysis, for "make gt", which +# generates the "gtrun" executable +[ground_truth] + +# compiler_name must be found in the [[compiler]] list under the name attribute +# The optimization level and switches need not match those found in the +# associated [[compiler]] compiler_name = 'g++' optimization_level = '-O0' switches = '' - # This host's list of compilers. - # For now, only used for hosts.ground_truth and hosts.dev_build. - # TODO: use this list to generate the Makefile - [[hosts.compilers]] +# This host's list of compilers. +# - binary: can be an absolute path, relative path, or binary name (found in +# PATH). If you want to specify a compiler in the same directory as this +# config file, prepend with a "./" (e.g. "./my-compiler") +# - name: can be any string. Used to recognize in the other options such as +# dev_build and ground_truth +# - type: the brand of the compiler. We support ('gcc', 'clang', 'intel') +# Currently, only one of each type may be specified. +# Note that these are all defaulted to use g++, clang++, and icpc from the +# PATH. +# If you specify any one compiler, then these defaults are erased. If you +# specify no compiler, then these defaults take effect. + +[[compiler]] +binary = 'g++' +name = 'g++' +type = 'gcc' +# search space of optimization levels +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] +# search space of compiler switches. +# Note: in some versions of python-toml, there is a parsing bug when a list +# has an empty string in the middle. So simply put it at the end without +# a comma. This has been fixed in the latest version of python-toml. +# Note: gcc disables -fassociative-math @ O0 +switches_list = [ + '-fassociative-math', + '-fcx-fortran-rules', + '-fcx-limited-range', + '-fexcess-precision=fast', + '-ffinite-math-only', + '-ffloat-store', + '-ffp-contract=on', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-funsafe-math-optimizations', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '' +] - # binary can be an absolute path, relative path, or binary name (found in - # PATH). If you want to specify a compiler in the same directory as this - # config file, prepend with a "./" (e.g. "./my-compiler") - binary = 'g++' - name = 'g++' +[[compiler]] +binary = 'clang++' +name = 'clang++' +type = 'clang' +# search space of optimization levels +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] +# search space of compiler switches. +# Note: in some versions of python-toml, there is a parsing bug when a list +# has an empty string in the middle. So simply put it at the end without +# a comma. This has been fixed in the latest version of python-toml. +# Note: Clang not honoring -fassociative-math (issues warning with 3.9) +# see: https://llvm.org/bugs/show_bug.cgi?id=27372 +switches_list = [ + '-fassociative-math', + '-fexcess-precision=fast', + '-fexcess-precision=standard', + '-ffinite-math-only', + '-ffloat-store', + '-ffp-contract=on', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-fsingle-precision-constant', + '-funsafe-math-optimizations', + '-march=core-avx2', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '' +] +[[compiler]] +binary = 'icpc' +name = 'icpc' +type = 'intel' +# search space of optimization levels +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] +# search space of compiler switches. +# Note: in some versions of python-toml, there is a parsing bug when a list +# has an empty string in the middle. So simply put it at the end without +# a comma. This has been fixed in the latest version of python-toml. +switches_list = [ + '--use_fast_math', + '-fcx-limited-range', + '-ffloat-store', + '-fma', + '-fmerge-all-constants', + '-fp-model fast=1', + '-fp-model fast=2', + '-fp-model=double', + '-fp-model=except', + '-fp-model=extended', + '-fp-model=precise', + '-fp-model=source', + '-fp-model=strict', + '-fp-port', + '-frounding-math', + '-fsingle-precision-constant', + '-ftz', + '-march=core-avx2', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '-mp1', + '-no-fma', + '-no-ftz', + '-no-prec-div', + '-prec-div', + '' +] diff --git a/scripts/flitcli/config/version.txt b/scripts/flitcli/config/version.txt index 1c5979fc..8a80b658 100644 --- a/scripts/flitcli/config/version.txt +++ b/scripts/flitcli/config/version.txt @@ -1 +1 @@ -v2.0-beta.1 +v2.0-beta.2 diff --git a/scripts/flitcli/flit.py b/scripts/flitcli/flit.py index 399aaaf6..8317158a 100755 --- a/scripts/flitcli/flit.py +++ b/scripts/flitcli/flit.py @@ -129,28 +129,28 @@ def generate_help_documentation(subcom_map): >>> help_str, help_subcom_str = generate_help_documentation(dict()) ''' parser = argparse.ArgumentParser( - description=''' - The flit command-line tool allows for users to write - portability test cases. One can test primarily for - compiler effects on reproducibility of floating-point - algorithms. That at least is the main use case for this - tool, although you may come up with some other uses. - ''', - ) + description=''' + The flit command-line tool allows for users to write + portability test cases. One can test primarily for + compiler effects on reproducibility of floating-point + algorithms. That at least is the main use case for this + tool, although you may come up with some other uses. + ''', + ) parser.add_argument('-v', '--version', action='store_true', help='Print version and exit') subparsers = parser.add_subparsers(metavar='subcommand', dest='subcommand') help_subparser = subparsers.add_parser( - 'help', help='display help for a specific subcommand') + 'help', help='display help for a specific subcommand') help_subparser.add_argument( - metavar='subcommand', - dest='help_subcommand', - choices=subcom_map.keys(), - help=''' - display the help documentation for a specific subcommand. - choices are {0}. - '''.format(', '.join(sorted(subcom_map.keys()))), - ) + metavar='subcommand', + dest='help_subcommand', + choices=subcom_map.keys(), + help=''' + display the help documentation for a specific subcommand. + choices are {0}. + '''.format(', '.join(sorted(subcom_map.keys()))), + ) for name, module in sorted(subcom_map.items()): subparsers.add_parser(name, help=module.brief_description) @@ -160,7 +160,30 @@ def generate_help_documentation(subcom_map): return (parser.format_help(), help_subparser.format_help()) -def main(arguments): +def main(arguments, outstream=None, errstream=None): + ''' + Main logic here. + + For ease of use when within python, the stdout can be captured using the + optional outstream parameter. You can use this to capture the stdout that + would go to the console and put it into a StringStream or maybe a file. + ''' + if outstream is None and errstream is None: + return _main_impl(arguments) + oldout = sys.stdout + olderr = sys.stderr + try: + if outstream is not None: + sys.stdout = outstream + if errstream is not None: + sys.stderr = errstream + return _main_impl(arguments) + finally: + sys.stdout = oldout + sys.stderr = olderr + +def _main_impl(arguments): + 'Implementation of main' script_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, script_dir) import flitconfig as conf @@ -195,21 +218,19 @@ def main(arguments): print(help_subcommand_str) return 0 - elif help_subcommand not in all_subcommands: + if help_subcommand not in all_subcommands: sys.stderr.write('Error: invalid subcommand: {0}.\n' \ .format(subcommand)) sys.stderr.write('Call with --help for more information\n') return 1 - else: - # just forward to the documentation from the submodule - return subcom_map[help_subcommand].main( - ['--help'], prog='{0} {1}'.format(sys.argv[0], help_subcommand)) - else: - # it is one of the other subcommands. Just forward the request on - return subcom_map[subcommand].main( - arguments, prog='{0} {1}'.format(sys.argv[0], subcommand)) + # just forward to the documentation from the submodule + return subcom_map[help_subcommand].main( + ['--help'], prog='{0} {1}'.format(sys.argv[0], help_subcommand)) + + # it is one of the other subcommands. Just forward the request on + return subcom_map[subcommand].main( + arguments, prog='{0} {1}'.format(sys.argv[0], subcommand)) if __name__ == '__main__': sys.exit(main(sys.argv[1:])) - diff --git a/scripts/flitcli/flit_bisect.py b/scripts/flitcli/flit_bisect.py index fbcffa45..1d378209 100644 --- a/scripts/flitcli/flit_bisect.py +++ b/scripts/flitcli/flit_bisect.py @@ -17,7 +17,7 @@ # All rights reserved. # # This file is part of FLiT. For details, see -# https://pruners.github.io/flit +# https://pruners.github.io/flitK # Please also read # https://github.com/PRUNERS/FLiT/blob/master/LICENSE # @@ -85,16 +85,16 @@ files that cause the variability. ''' -from collections import namedtuple +from tempfile import NamedTemporaryFile import argparse import csv import datetime +import heapq import glob import hashlib import logging import multiprocessing as mp import os -import re import shutil import sqlite3 import subprocess as subp @@ -102,6 +102,11 @@ import flitconfig as conf import flitutil as util +try: + import flitelf as elf +except ImportError: + elf = None + brief_description = 'Bisect compilation to identify problematic source code' @@ -174,9 +179,9 @@ def create_bisect_makefile(directory, replacements, gt_src, @param trouble_src: (list) which source files would be compiled with the trouble compilation within the resulting binary. @param split_symbol_map: - (dict fname -> list [list good symbols, list bad symbols]) - Files to compile as a split between good and bad, specifying good and - bad symbols for each file. + (dict fname -> list [list baseline symbols, list differing symbols]) + Files to compile as a split between baseline and differing, specifying + baseline and differing symbols for each file. Within replacements, there are some optional fields: - cpp_flags: (list) (optional) List of c++ compiler flags to give to @@ -254,10 +259,97 @@ def create_bisect_makefile(directory, replacements, gt_src, return makefile +def run_make(makefilename='Makefile', directory='.', verbose=False, + jobs=mp.cpu_count(), target=None): + ''' + Runs a make command. If the build fails, then stdout and stderr will be + output into the log. + + @param makefilename: Name of the Makefile (default 'Makefile') + @param directory: Path to the directory to run in (default '.') + @param verbose: True means echo the output to the console (default False) + @param jobs: number of parallel jobs (default #cpus) + @param target: Makefile target to build (defaults to GNU Make's default) + + @return None + + @throws subprocess.CalledProcessError on failure + + Configure the logger to spit to stdout instead of stderr + >>> import logging + >>> logger = logging.getLogger() + >>> for handler in logger.handlers[:]: + ... logger.removeHandler(handler) + >>> import sys + >>> logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + Make sure the exception is thrown + >>> with NamedTemporaryFile(mode='w') as tmpmakefile: + ... print('.PHONY: default\\n', file=tmpmakefile) + ... print('default:\\n', file=tmpmakefile) + ... print('\\t@echo hello\\n', file=tmpmakefile) + ... print('\\t@false\\n', file=tmpmakefile) + ... tmpmakefile.flush() + ... + ... run_make(makefilename=tmpmakefile.name) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + subprocess.CalledProcessError: Command ... returned non-zero exit status 2. + + See what was output to the logger + >>> with NamedTemporaryFile(mode='w') as tmpmakefile: + ... print('.PHONY: default\\n', file=tmpmakefile) + ... print('default:\\n', file=tmpmakefile) + ... print('\\t@echo hello\\n', file=tmpmakefile) + ... print('\\t@false\\n', file=tmpmakefile) + ... tmpmakefile.flush() + ... + ... try: + ... run_make(makefilename=tmpmakefile.name) #doctest: +ELLIPSIS + ... except: + ... pass + ERROR:root:make error occurred. Here is the output: + make: Entering directory ... + hello + make: *** [...default] Error 1 + make: Leaving directory ... + + + Undo the logger configurations + >>> for handler in logger.handlers[:]: + ... logger.removeHandler(handler) + ''' + command = [ + 'make', + '-C', directory, + '-f', makefilename, + '-j', str(jobs), + ] + if target is not None: + command.append(target) + + with NamedTemporaryFile() as tmpout: + try: + if not verbose: + subp.check_call(command, stdout=tmpout, stderr=subp.STDOUT) + else: + ps = subp.Popen(command, stdout=subp.PIPE, stderr=subp.STDOUT) + subp.check_call(['tee', tmpout.name], stdin=ps.stdout) + ps.communicate() + if ps.returncode != 0: + raise subp.CalledProcessError(ps.returncode, command) + except: + tmpout.flush() + with open(tmpout.name, 'r') as tmpin: + msg = 'make error occurred. Here is the output:\n' \ + + tmpin.read() + logging.error('%s', msg) + raise + def build_bisect(makefilename, directory, target='bisect', verbose=False, - jobs=None): + jobs=mp.cpu_count()): ''' Creates the bisect executable by executing a parallel make. @@ -276,68 +368,145 @@ def build_bisect(makefilename, directory, @return None ''' logging.info('Building the bisect executable') - if jobs is None: - jobs = mp.cpu_count() - kwargs = dict() - if not verbose: - kwargs['stdout'] = subp.DEVNULL - kwargs['stderr'] = subp.DEVNULL - subp.check_call( - ['make', '-C', directory, '-f', makefilename, '-j', str(jobs), target], - **kwargs) + run_make( + makefilename=makefilename, + directory=directory, + verbose=verbose, + jobs=jobs, + target=target) def update_gt_results(directory, verbose=False, - jobs=mp.cpu_count()): + jobs=mp.cpu_count(), fpic=False): ''' Update the ground-truth.csv results file for FLiT tests within the given directory. @param directory: where to execute make @param verbose: False means block output from GNU make and running + @param jobs: number of parallel jobs (default #cpus) + @param fpic: True means compile the gt-fpic files too (default False) + + @return None ''' - kwargs = dict() - if not verbose: - kwargs['stdout'] = subp.DEVNULL - kwargs['stderr'] = subp.DEVNULL gt_resultfile = util.extract_make_var( 'GT_OUT', os.path.join(directory, 'Makefile'))[0] logging.info('Updating ground-truth results - %s', gt_resultfile) print('Updating ground-truth results -', gt_resultfile, end='', flush=True) - subp.check_call( - ['make', '-j', str(jobs), '-C', directory, gt_resultfile], **kwargs) + run_make( + directory=directory, + verbose=verbose, + jobs=jobs, + target=gt_resultfile) + if fpic: + run_make( + directory=directory, + verbose=verbose, + jobs=jobs, + target='gt-fpic') print(' - done') logging.info('Finished Updating ground-truth results') -def is_result_bad(resultfile): +def get_comparison_result(resultfile): ''' - Returns True if the results from the resultfile is considered 'bad', - meaning it is a different answer from the ground-truth. + Returns the floating-point comparison value stored in the flit comparison + csv file (resultfile). + + The result is pulled simply from the comparison column (first row). + + >>> from tempfile import NamedTemporaryFile as TFile + >>> with TFile(mode='w', delete=False) as fout: + ... _ = fout.write('name,host,compiler,...,comparison,...\\n' + ... 'test,name,clang++,...,15.342,...\\n') + ... fname = fout.name + >>> get_comparison_result(fname) + 15.342 + + Make sure NULL values are handled appropriately + >>> with open(fname, 'w') as fout: + ... _ = fout.write('comparison\\n' + ... 'NULL\\n') + >>> print(get_comparison_result(fname)) + None + + Try out a value that is not a number and not NULL + >>> with open(fname, 'w') as fout: + ... _ = fout.write('comparison\\n' + ... 'coconut\\n') + >>> get_comparison_result(fname) + Traceback (most recent call last): + ... + ValueError: could not convert string to float: 'coconut' - @param resultfile: path to the results csv file after comparison - @return True if the result is different from ground-truth + Delete the file now that we're done with it + >>> import os + >>> os.remove(fname) ''' with open(resultfile, 'r') as fin: parser = csv.DictReader(fin) # should only have one row for row in parser: - # identical to ground truth means comparison is zero - return float(row['comparison']) != 0.0 - -SymbolTuple = namedtuple('SymbolTuple', 'src, symbol, demangled, fname, lineno') -SymbolTuple.__doc__ = ''' -Tuple containing information about the symbols in a file. Has the following -attributes: - src: source file that was compiled - symbol: mangled symbol in the compiled version - demangled: demangled version of symbol - fname: filename where the symbol is actually defined. This usually - will be equal to src, but may not be in some situations. - lineno: line number of definition within fname. -''' + val = row['comparison'] + return float(val) if val != 'NULL' else None + +def is_result_differing(resultfile): + ''' + Returns True if the results from the resultfile is considered 'differing', + meaning it is a different answer from the ground-truth. + + @param resultfile: path to the results csv file after comparison + @return True if the result is different from ground-truth + + Try out a positive value + >>> from tempfile import NamedTemporaryFile as TFile + >>> with TFile(mode='w', delete=False) as fout: + ... _ = fout.write('name,host,compiler,...,comparison,...\\n' + ... 'test,name,clang++,...,15.342,...\\n') + ... fname = fout.name + >>> is_result_differing(fname) + True + + Try out a value that is less than zero + >>> with open(fname, 'w') as fout: + ... _ = fout.write('comparison\\n' + ... '-1e-34\\n') + >>> is_result_differing(fname) + True + + Try out a value that is identically zero + >>> with open(fname, 'w') as fout: + ... _ = fout.write('comparison\\n' + ... '0.0\\n') + >>> is_result_differing(fname) + False + + Make sure NULL values are handled appropriately + >>> with open(fname, 'w') as fout: + ... _ = fout.write('comparison\\n' + ... 'NULL\\n') + >>> is_result_differing(fname) + Traceback (most recent call last): + ... + TypeError: float() argument must be a string or a number, not 'NoneType' + + Try out a value that is not a number and not NULL + >>> with open(fname, 'w') as fout: + ... _ = fout.write('comparison\\n' + ... 'coconut\\n') + >>> is_result_differing(fname) + Traceback (most recent call last): + ... + ValueError: could not convert string to float: 'coconut' + + Delete the file now that we're done with it + >>> import os + >>> os.remove(fname) + ''' + return float(get_comparison_result(resultfile)) != 0.0 +_extract_symbols_memos = {} def extract_symbols(file_or_filelist, objdir): ''' - Extracts symbols for the given file(s) given. The corresponding object is + Extracts symbols for the given source file(s). The corresponding object is assumed to be in the objdir with the filename replaced with the GNU Make pattern %.cpp=%_gt.o. @@ -346,171 +515,380 @@ def extract_symbols(file_or_filelist, objdir): @param objdir: (str) directory where object files are compiled for the given files. - @return a list of SymbolTuple objects + @return two lists of SymbolTuple objects (funcsyms, remaining). + The first is the list of exported functions that are strong symbols and + have a filename and line number where they are defined. The second is + all remaining symbols that are strong, exported, and defined. ''' - symbol_tuples = [] # if it is not a string, then assume it is a list of strings if not isinstance(file_or_filelist, str): + funcsym_tuples = [] + remainingsym_tuples = [] for fname in file_or_filelist: - symbol_tuples.extend(extract_symbols(fname, objdir)) - return symbol_tuples + funcsyms, remaining = extract_symbols(fname, objdir) + funcsym_tuples.extend(funcsyms) + remainingsym_tuples.extend(remaining) + return (funcsym_tuples, remainingsym_tuples) # now we know it is a string, so assume it is a filename fname = file_or_filelist fbase = os.path.splitext(os.path.basename(fname))[0] fobj = os.path.join(objdir, fbase + '_gt.o') - # use nm and objdump to get the binary information we need - symbol_strings = subp.check_output([ - 'nm', - '--extern-only', - '--defined-only', - fobj, - ]).decode('utf-8').splitlines() - demangled_symbol_strings = subp.check_output([ - 'nm', - '--extern-only', - '--defined-only', - '--demangle', - fobj, - ]).decode('utf-8').splitlines() - objdump_strings = subp.check_output([ - 'objdump', '--disassemble-all', '--line-numbers', fobj, - ]).decode('utf-8').splitlines() - - # create the symbol -> (fname, lineno) map - symbol_line_mapping = dict() - symbol = None - for line in objdump_strings: - if len(line.strip()) == 0: # skip empty lines - continue - if line[0].isdigit(): # we are at a symbol - symbol = line.split()[1][1:-2] - continue - if symbol is None: # if we don't have an active symbol - continue # then skip - srcmatch = re.search(':[0-9]+$', line) - if srcmatch is not None: - deffile = line[:srcmatch.start()] - defline = int(line[srcmatch.start()+1:]) - symbol_line_mapping[symbol] = (deffile, defline) - symbol = None # deactivate the symbol to not overwrite - - - # generate the symbol tuples - for symbol_string, demangled_string in zip(symbol_strings, - demangled_symbol_strings): - symbol = symbol_string.split(maxsplit=2)[2] - demangled = demangled_string.split(maxsplit=2)[2] - try: - deffile, defline = symbol_line_mapping[symbol] - except KeyError: - deffile, defline = None, None - symbol_tuples.append( - SymbolTuple(fname, symbol, demangled, deffile, defline)) + if fobj in _extract_symbols_memos: + return _extract_symbols_memos[fobj] - return symbol_tuples + _extract_symbols_memos[fobj] = elf.extract_symbols(fobj, fname) + return _extract_symbols_memos[fobj] -def bisect_search(is_bad, elements): +def memoize_strlist_func(func): ''' - Performs the bisect search, attempting to minimize the bad list. We could - go through the list one at a time, but that would cause us to call is_bad() - more than necessary. Here we assume that calling is_bad() is expensive, so - we want to minimize calls to is_bad(). This function has - O(k*log(n))*O(is_bad) - where n is the size of the questionable_list and k is - the number of bad elements in questionable_list. - - Note: A key assumption to this algorithm is that all bad elements are + Memoize a function that takes a list of strings and returns a value. This + function returns the memoized version. It is expected that the list + passed in will be in the same order. This memoization will not + work if for instance the input is first shuffled. + + >>> def to_memoize(strlist): + ... print(strlist) + ... return strlist[0] + >>> memoized = memoize_strlist_func(to_memoize) + >>> memoized(['a', 'b', 'c']) + ['a', 'b', 'c'] + 'a' + >>> memoized(['a', 'b', 'c']) + 'a' + >>> memoized(['e', 'a']) + ['e', 'a'] + 'e' + >>> memoized(['a', 'b', 'c']) + 'a' + >>> memoized(['e', 'a']) + 'e' + ''' + memo = {} + def memoized_func(strlist): + 'func but memoized' + idx = tuple(sorted(strlist)) + if idx in memo: + return memo[idx] + value = func(strlist) + memo[idx] = value + return value + return memoized_func + +def bisect_biggest(score_func, elements, found_callback=None, k=1, + skip_verification=False): + ''' + Performs the bisect search, attempting to find the biggest offenders. This + is different from bisect_search() in that this function only tries to + identify the top k offenders, not all of them. If k is greater than or equal + to the total number of offenders, then bisect_biggest() is more expensive + than bisect_search(). + + We do not want to call score_func() very much. We assume the score_func() + is is an expensive operation. We could go throught the list one at a time, + but that would cause us to potentially call score_func() more than + necessary. + + Note: The same assumption as bisect_search() is in place. That is that all + differing elements are independent. This means if an element contributes + to a differing score, then it would contribute to a differing score by + itself as well. This is not always true, and this function does not verify + this assumption. Instead, it will only return the largest singleton + offenders. + + @param score_func: a function that takes one argument (to_test) and returns + a number greater than zero if one of the elements in to_test causes the + result to be differing. This value returned is used to compare the + elements so that the largest k differing elements are found and + returned. If all differing elements return the same numerical value, + then this will be less efficient than bisect_search. + Note: if the set of elements is not differing, then either return 0 or + a negative value. + @param elements: the elements to search over. Subsets of this list will be + given to score_func(). + @param found_callback: a callback function to be called on every found + differing element. Will be given two arguments, the element, and the + score from score_func(). + @param k: number of biggest elements to return. The default is to return + the one biggest offender. If there are less than k elements that + return positive scores, then only the found offenders will be returned. + @param skip_verification: skip the verification assertions for performance + reasons. + + @return list of the biggest offenders with their scores + [(elem, score), ...] + + >>> def score_func(x): + ... print('scoring:', x) + ... return -2*min(x) if x else 0 + + >>> bisect_biggest(score_func, [1, 3, 4, 5, -1, 10, 0, -15, 3], k=3, + ... skip_verification=True) + scoring: [1, 3, 4, 5, -1, 10, 0, -15, 3] + scoring: [1, 3, 4, 5] + scoring: [-1, 10, 0, -15, 3] + scoring: [-1, 10] + scoring: [0, -15, 3] + scoring: [0] + scoring: [-15, 3] + scoring: [-15] + scoring: [3] + scoring: [-1] + scoring: [10] + [(-15, 30), (-1, 2)] + + >>> bisect_biggest(score_func, [-1, -2, -3, -4, -5], k=3, + ... found_callback=print) + scoring: [-1, -2, -3, -4, -5] + scoring: [-1, -2] + scoring: [-3, -4, -5] + scoring: [-3] + scoring: [-4, -5] + scoring: [-4] + scoring: [-5] + -5 10 + -4 8 + -3 6 + [(-5, 10), (-4, 8), (-3, 6)] + + >>> bisect_biggest(score_func, []) + [] + + >>> bisect_biggest(score_func, [-1, -2, -3, -4, -5], k=1) + scoring: [-1, -2, -3, -4, -5] + scoring: [-1, -2] + scoring: [-3, -4, -5] + scoring: [-3] + scoring: [-4, -5] + scoring: [-4] + scoring: [-5] + [(-5, 10)] + + Test that verification is performed + >>> def score_func(x): + ... return -2*min(x) if len(x) > 1 else 0 + >>> bisect_biggest(score_func, [-1, -2], k=2) + Traceback (most recent call last): + ... + AssertionError: Assumption that differing elements are independent was wrong + + >>> def score_func(x): + ... score = 0 + ... if len(x) == 0: return 0 + ... if min(x) < -1: score += max(0, -2*min(x)) + ... if len(x) > 1: score += max(0, -2*max(x)) + ... return score + >>> bisect_biggest(score_func, [-1, -2], k=2) + Traceback (most recent call last): + ... + AssertionError: Assumption that minimal sets are non-overlapping was wrong + ''' + if len(elements) == 0: + return [] + found_list = [] + frontier = [] + push = lambda x: heapq.heappush(frontier, (-score_func(x), x)) + pop = lambda: heapq.heappop(frontier) + push(elements) + while len(frontier) > 0 and frontier[0][0] < 0 and len(found_list) < k: + score, elems = pop() + if len(elems) == 1: + found_list.append((elems[0], -score)) + if found_callback is not None: + found_callback(elems[0], -score) + else: + push(elems[:len(elems) // 2]) + push(elems[len(elems) // 2:]) + + if not skip_verification: + if len(frontier) == 0 or frontier[0][0] >= 0: + # found everything, so do the traditional assertions + non_differing_list = \ + list(set(elements).difference(x[0] for x in found_list)) + assert score_func(non_differing_list) <= 0, \ + 'Assumption that differing elements are independent was wrong' + assert score_func(elements) == \ + score_func([x[0] for x in found_list]), \ + 'Assumption that minimal sets are non-overlapping was wrong' + + return found_list + +def bisect_search(score_func, elements, found_callback=None, + skip_verification=False): + ''' + Performs the bisect search, attempting to find all elements contributing to + a positive score. This score_func() function is intended to identify when + there are "differing" elements by returning a positive score. This is + different from bisect_biggest() in that we find all offenders and can + therefore do some optimization that bisect_biggest() cannot do. + + We do not want to call score_func() very much. We assume the score_func() + is an expensive operation. We could go throught the list one at a time, + but that would cause us to potentially call score_func() more than + necessary. + + This function has complexity + O(k*log(n))*O(score_func) + where n is the size of the elements and k is the number of differing + elements to find. + + Note: A key assumption to this algorithm is that all differing elements are independent. That may not always be true, so there are redundant checks within the algorithm to verify that this assumption is not vialoated. If the assumption is found to be violated, then an AssertionError is raised. - @param is_bad: a function that takes two arguments (maybe_bad_list, - maybe_good_list) and returns True if the maybe_bad_list has a bad - element - @param elements: contains bad elements, but potentially good elements too - - @return minimal bad list of all elements that cause is_bad() to return True + @param score_func: a function that takes one argument, the list of elements + to test if they are differing. The function then returns a positive + value if the given list has a differing element. If the given list + does not have a differing element, this can return zero or a negative + value. Note: this function must be able to handle empty lists. An + empty list should instantly return a non-positive value, as there + cannot possibly be a differing element passed to it. + It is expected that this function is memoized because it may be called + more than once on the same input during the execution of this + algorithm. + @param elements: contains differing elements, but potentially non-differing + elements too + @param found_callback: a callback function to be called on every found + differing element. Will be given two arguments, the element, and the + score from score_func(). + @param skip_verification: skip the verification assertions for performance + reasons. This will run the score_func() two fewer times. + + @return minimal differing list of all elements that cause score_func() to + return positive values, along with their scores, sorted descending by + score. + [(elem, score), ...] Here's an example of finding all negative numbers in a list. Not very useful for this particular task, but it is demonstrative of how to use it. >>> call_count = 0 - >>> def is_bad(x,y): - ... global call_count - ... call_count += 1 - ... return min(x) < 0 - >>> x = bisect_search(is_bad, [1, 3, 4, 5, -1, 10, 0, -15, 3]) - >>> sorted(x) - [-15, -1] + >>> memo = {} + >>> def score_func(x): + ... idx = tuple(sorted(x)) + ... if idx not in memo: + ... global call_count + ... call_count += 1 + ... memo[idx] = -2*min(x) if x else 0 + ... return memo[idx] + >>> bisect_search(score_func, [1, 3, 4, 5, -1, 10, 0, -15, 3]) + [(-15, 30), (-1, 2)] as a rough performance metric, we want to be sure our call count remains - low for the is_bad() function. + low for the score_func() function. Note, we implemented memoization in + score_func(), so we are only counting unique calls and not duplicate calls + to score_func(). >>> call_count - 9 + 10 + + Test out the found_callback() functionality. + >>> s = set() + >>> bisect_search(score_func, [-1, -2, -3, -4], + ... found_callback=lambda x, y: s.add(x)) + [(-4, 8), (-3, 6), (-2, 4), (-1, 2)] + >>> sorted(s) + [-4, -3, -2, -1] See what happens when it has a pair that only show up together and not - alone. Only if -6 and 5 are in the list, then is_bad returns true. - The assumption of this algorithm is that bad elements are independent, - so this should throw an exception. - >>> def is_bad(x,y): - ... return max(x) - min(x) > 10 - >>> x = bisect_search(is_bad, [-6, 2, 3, -3, -1, 0, 0, -5, 5]) + alone. Only if -6 and 5 are in the list, then score_func() returns a + positive value. The assumption of this algorithm is that differing + elements are independent, so this should throw an exception. + >>> def score_func(x): + ... return max(x) - min(x) - 10 if x else 0 + >>> bisect_search(score_func, [-6, 2, 3, -3, -1, 0, 0, -5, 5]) Traceback (most recent call last): ... - AssertionError: Assumption that bad elements are independent was wrong + AssertionError: Found element does not cause variability: 5 + + Check that the assertion for found element is not turned off with + skip_verification. + >>> bisect_search(score_func, [-6, 2, 3, -3, -1, 0, 0, -5, 5], + ... skip_verification=True) + Traceback (most recent call last): + ... + AssertionError: Found element does not cause variability: 5 + + Check that the found_callback is not called on false positives. Here I + expect no output since no single element can be found. + >>> try: + ... bisect_search(score_func, [-6, 2, 3, -3, -1, 0, 0, -5, 5], + ... found_callback=print) + ... except AssertionError: + ... pass + + Check that the verification can catch the other case of overlapping minimal + sets. + >>> def score_func(x): + ... score = 0 + ... if len(x) == 0: return score + ... if min(x) < -5: score += 15 - min(x) + ... return score + max(x) - min(x) - 10 + >>> bisect_search(score_func, [-6, 2, 3, -3, -1, 0, -5, 5]) + Traceback (most recent call last): + ... + AssertionError: Assumption that minimal sets are non-overlapping was wrong + + Check that this is skipped with skip_verification + >>> bisect_search(score_func, [-6, 2, 3, -3, -1, 0, -5, 5], + ... skip_verification=True) + [(-6, 11)] ''' + if len(elements) == 0: + return [] + # copy the incoming list so that we don't modify it quest_list = list(elements) - known_list = [] - bad_list = [] - while len(quest_list) > 0 and is_bad(quest_list, known_list): + differing_list = [] + while len(quest_list) > 0 and score_func(quest_list) > 0: - # find one bad element + # find one differing element quest_copy = quest_list - no_test = list(known_list) - last_result = False while len(quest_copy) > 1: - # split the questionable list into two lists half_1 = quest_copy[:len(quest_copy) // 2] - half_2 = quest_copy[len(quest_copy) // 2:] - last_result = is_bad(half_1, no_test + half_2) - if last_result: + if score_func(half_1) > 0: quest_copy = half_1 - no_test.extend(half_2) - # TODO: possible optimization. - # TODO- if the length of half_2 is big enough, test - # TODO- is_bad(half_2, no_test + half_1) - # TODO- and if that returns False, then mark half_2 as known so - # TODO- that we don't need to search it again. else: # optimization: mark half_1 as known, so that we don't need to # search it again quest_list = quest_list[len(half_1):] - known_list.extend(half_1) # update the local search - quest_copy = half_2 - no_test.extend(half_1) - - bad_element = quest_list.pop(0) - - # double check that we found a bad element before declaring it bad - if last_result or is_bad([bad_element], known_list + quest_list): - bad_list.append(bad_element) - - # add to the known list to not search it again - known_list.append(bad_element) - - # Perform a sanity check. If we have found all of the bad items, then - # compiling with all but these bad items will cause a good build. - # This will fail if our hypothesis class is wrong - good_list = list(set(elements).difference(bad_list)) - assert not is_bad(good_list, bad_list), \ - 'Assumption that bad elements are independent was wrong' - - return bad_list + quest_copy = quest_copy[len(half_1):] + + # since we remove known non-differing elements as we find them, the + # differing element will be at the beginning of quest_list. + differing_element = quest_list.pop(0) + + # double check that we found a differing element before declaring it + # differing + score = score_func([differing_element]) + assert score > 0, \ + 'Found element does not cause variability: {}' \ + .format(differing_element) + differing_list.append((differing_element, score)) + # inform caller that a differing element was found + if found_callback is not None: + found_callback(differing_element, score) + + if not skip_verification: + # Perform a sanity check. If we have found all of the differing items, + # then compiling with all but these differing items will cause a + # non-differing build. + # This will fail if our hypothesis class is wrong + non_differing_list = \ + list(set(elements).difference(x[0] for x in differing_list)) + assert score_func(non_differing_list) <= 0, \ + 'Assumption that differing elements are independent was wrong' + assert score_func(elements) == \ + score_func([x[0] for x in differing_list]),\ + 'Assumption that minimal sets are non-overlapping was wrong' + + # sort descending by score + differing_list.sort(key=lambda x: -x[1]) + + return differing_list def parse_args(arguments, prog=sys.argv[0]): ''' @@ -632,6 +1010,51 @@ def parse_args(arguments, prog=sys.argv[0]): runbusect-01-out), or the object files (e.g. bisect-01/obj/*). ''') + parser.add_argument('-k', '--biggest', metavar='K', type=int, default=None, + help=''' + Instead of finding and returning all symbols that + cause variability, only return the largest K + contributors, with their contribution to + variability. If K is close to the total number of + total contributing functions, then this is a much + slower approach in general than the full algorithm. + It is best if K is small. This value used comes + from the custom comparison function you provide for + your flit test. + + Note: many files (perhaps more than K, although + possibly less than K) may be found during the + search. This is done to ensure that the symbols + identified are the largest contributors that are + identifiable through this approach. + + Also note that using the --biggest flag, you + restrict the search space to only singleton + contributors. That means that if there is a pair + of contributors that only contribute when they are + both compiled with the given compilation, then they + produce a measurable variation. + ''') + parser.add_argument('--compile-only', action='store_true', + help=''' + Only applicable with the --auto-sqlite-run option. + Only goes through the precompile step and then + exits. + ''') + parser.add_argument('--precompile-fpic', action='store_true', + help=''' + Only applicable with the --auto-sqlite-run option. + In the precompile phase, also precompiles the fPIC + object files into the top-level obj directory. + ''') + parser.add_argument('--skip-verification', action='store_true', + help=''' + By default, bisect will run some assertions + verifying that the assumptions made to have a + performant algorithm are valid. This turns off + those assertions so as to not run the tests more + often than necessary. + ''') args = parser.parse_args(arguments) @@ -647,139 +1070,106 @@ def parse_args(arguments, prog=sys.argv[0]): return args -def search_for_linker_problems(args, bisect_path, replacements, sources, libs): +def test_makefile(args, makepath, testing_list, indent=' '): ''' - Performs the search over the space of statically linked libraries for - problems. + Runs the compilation in the makefile and returns the generated comparison + result. - Linking will be done with the ground-truth compiler, but with the static - libraries specified. During this search, all source files will be compiled - with the ground-truth compilation, but the static libraries will be - included in the linking. + @param args: parsed command-line arguments (see parse_args()) + @param makepath (str): absolute or relative path to the Makefile + @param testing_list (list(str)): list of items being tested (for logging + purposes) - Doing a binary search here actually breaks things since including some - static libraries will require including others to resolve the symbols in - the included static libraries. So, instead this function just runs with - the libraries included, and checks to see if there are reproducibility - problems. + @return (float) generated comparison result ''' - def bisect_libs_build_and_check(trouble_libs, dummy_libs): + print('{}Created {} - compiling and running'.format(indent, makepath), + end='', flush=True) + logging.info('%sCreated %s', indent, makepath) + logging.info('%sChecking:', indent) + for src in testing_list: + logging.info(' %s%s', indent, src) + + try: + build_bisect(makepath, args.directory, verbose=args.verbose, + jobs=args.jobs) + finally: + if args.delete: + build_bisect(makepath, args.directory, verbose=args.verbose, + jobs=args.jobs, target='bisect-smallclean') + resultfile = util.extract_make_var('BISECT_RESULT', makepath, + args.directory)[0] + resultpath = os.path.join(args.directory, resultfile) + result = get_comparison_result(resultpath) + result_str = str(result) + + sys.stdout.write(' - score {0}\n'.format(result_str)) + logging.info('%sResult was %s', indent, result_str) + + return result + +def _gen_bisect_lib_checker(args, bisect_path, replacements, sources, + indent=' '): + ''' + Generates and returns the function that builds and check a list of + libraries for showing variability. The returned function is memoized, so + no need to be careful to not call it more than once with the same + arguments. + ''' + def builder_and_checker(libs): ''' Compiles all source files under the ground truth compilation and - statically links in the trouble_libs. + statically links in the libs. - @param trouble_libs: static libraries to compile in - @param dummy_libs: static libraries to ignore and not include - This variable is not used, but necessary for the interface. + @param libs: static libraries to compile in - @return True if the compilation has a non-zero comparison between this - mixed compilation and the full ground-truth compilation. + @return The comparison value between this mixed compilation and the + full baseline compilation. ''' repl_copy = dict(replacements) repl_copy['link_flags'] = list(repl_copy['link_flags']) - repl_copy['link_flags'].extend(trouble_libs) + repl_copy['link_flags'].extend(libs) makefile = create_bisect_makefile(bisect_path, repl_copy, sources, [], dict()) makepath = os.path.join(bisect_path, makefile) + return test_makefile(args, makepath, libs, indent=indent) - print(' Create {0} - compiling and running'.format(makepath), - end='', flush=True) - logging.info('Created %s', makepath) - logging.info('Checking:') - for lib in trouble_libs: - logging.info(' %s', lib) - - try: - build_bisect(makepath, args.directory, verbose=args.verbose, - jobs=args.jobs) - finally: - if args.delete: - build_bisect(makepath, args.directory, verbose=args.verbose, - jobs=args.jobs, target='bisect-smallclean') - resultfile = util.extract_make_var('BISECT_RESULT', makepath, - args.directory)[0] - resultpath = os.path.join(args.directory, resultfile) - result_is_bad = is_result_bad(resultpath) - - result_str = 'bad' if result_is_bad else 'good' - sys.stdout.write(' - {0}\n'.format(result_str)) - logging.info('Result was %s', result_str) - - return result_is_bad - - print('Searching for bad intel static libraries:') - logging.info('Searching for bad static libraries included by intel linker:') - #bad_libs = bisect_search(bisect_libs_build_and_check, libs) - #return bad_libs - if bisect_libs_build_and_check(libs, []): - return libs - return [] + return memoize_strlist_func(builder_and_checker) -def search_for_source_problems(args, bisect_path, replacements, sources): +def _gen_bisect_source_checker(args, bisect_path, replacements, sources, + indent=' '): ''' - Performs the search over the space of source files for problems. + Generates and returns the function that builds and check a list of sources + for showing variability. The returned function is memoized, so no need to + be careful to not call it more than once with the same arguments. ''' - def bisect_build_and_check(trouble_src, gt_src): + def builder_and_checker(sources_to_optimize): ''' - Compiles the compilation with trouble_src compiled with the trouble - compilation and with gt_src compiled with the ground truth compilation. + Compiles the compilation with sources_to_optimize compiled with the + optimized compilation and with everything else compiled with the + baseline compilation. - @param trouble_src: source files to compile with trouble compilation - @param gt_src: source files to compile with ground truth compilation + @param sources_to_optimize: source files to compile with the + variability-inducing optimizations - @return True if the compilation has a non-zero comparison between this - mixed compilation and the full ground truth compilation. + @return The comparison value between this mixed compilation and the + full baseline compilation. ''' + gt_src = list(set(sources).difference(sources_to_optimize)) makefile = create_bisect_makefile(bisect_path, replacements, gt_src, - trouble_src, dict()) + sources_to_optimize, dict()) makepath = os.path.join(bisect_path, makefile) + return test_makefile(args, makepath, sources_to_optimize, indent=indent) - print(' Created {0} - compiling and running'.format(makepath), end='', - flush=True) - logging.info('Created %s', makepath) - logging.info('Checking:') - for src in trouble_src: - logging.info(' %s', src) - - try: - build_bisect(makepath, args.directory, verbose=args.verbose, - jobs=args.jobs) - finally: - if args.delete: - build_bisect(makepath, args.directory, verbose=args.verbose, - jobs=args.jobs, target='bisect-smallclean') - resultfile = util.extract_make_var('BISECT_RESULT', makepath, - args.directory)[0] - resultpath = os.path.join(args.directory, resultfile) - result_is_bad = is_result_bad(resultpath) - - result_str = 'bad' if result_is_bad else 'good' - sys.stdout.write(' - {0}\n'.format(result_str)) - logging.info('Result was %s', result_str) - - return result_is_bad - - print('Searching for bad source files:') - logging.info('Searching for bad source files under the trouble' - ' compilation') - bad_sources = bisect_search(bisect_build_and_check, sources) - return bad_sources + return memoize_strlist_func(builder_and_checker) -def search_for_symbol_problems(args, bisect_path, replacements, sources, - bad_source): +def _gen_bisect_symbol_checker(args, bisect_path, replacements, sources, + fsymbols, remainingsymbols, indent=' '): ''' - Performs the search over the space of symbols within bad source files for - problems. - - @param args: parsed command-line arguments - @param bisect_path: directory where bisect is being performed - @param replacements: dictionary of values to use in generating the Makefile - @param sources: all source files - @param bad_source: the one bad source file to search for bad symbols - - @return a list of identified bad symbols (if any) + Generates and returns the function that builds and check a list of sources + for showing variability. The returned function is memoized, so no need to + be careful to not call it more than once with the same arguments. ''' - def bisect_symbol_build_and_check(trouble_symbols, gt_symbols): + def builder_and_checker(symbols_to_optimize): ''' Compiles the compilation with all files compiled under the ground truth compilation except for the given symbols for the given files. @@ -787,83 +1177,250 @@ def bisect_symbol_build_and_check(trouble_symbols, gt_symbols): In order to be able to isolate these symbols, the files will need to be compiled with -fPIC, but that is handled by the generated Makefile. - @param trouble_symbols: (list of SymbolTuple) symbols to use - from the trouble compilation - @param gt_symbols: (list of SymbolTuple) symbols to use from - the ground truth compilation + @param symbols_to_optimize: (list of SymbolTuple) symbols to use + from the variability-inducing optimization compilation - @return True if the compilation has a non-zero comparison between this - mixed compilation and the full ground truth compilation. + @return The comparison value between this mixed compilation and the + full baseline compilation. ''' + gt_symbols = list(set(fsymbols + remainingsymbols) + .difference(symbols_to_optimize)) all_sources = list(sources) # copy the list of all source files - symbol_sources = [x.src for x in trouble_symbols + gt_symbols] + symbol_sources = [x.src for x in symbols_to_optimize + gt_symbols] trouble_src = [] gt_src = list(set(all_sources).difference(symbol_sources)) symbol_map = {x: [ [y.symbol for y in gt_symbols if y.src == x], - [z.symbol for z in trouble_symbols if z.src == x], + [z.symbol for z in symbols_to_optimize if z.src == x], ] for x in symbol_sources} makefile = create_bisect_makefile(bisect_path, replacements, gt_src, trouble_src, symbol_map) makepath = os.path.join(bisect_path, makefile) + symbol_strings = [ + ' {sym.fname}:{sym.lineno} {sym.symbol} -- {sym.demangled}' + .format(sym=sym) for sym in symbols_to_optimize + ] + return test_makefile(args, makepath, symbol_strings, indent=indent) - print(' Created {0} - compiling and running'.format(makepath), end='', - flush=True) - logging.info('Created %s', makepath) - logging.info('Checking:') - for sym in trouble_symbols: - logging.info( - '%s', - ' {sym.fname}:{sym.lineno} {sym.symbol} -- {sym.demangled}' - .format(sym=sym)) + return memoize_strlist_func(builder_and_checker) - try: - build_bisect(makepath, args.directory, verbose=args.verbose, - jobs=args.jobs) - finally: - if args.delete: - build_bisect(makepath, args.directory, verbose=args.verbose, - jobs=args.jobs, target='bisect-smallclean') - resultfile = util.extract_make_var('BISECT_RESULT', makepath, - args.directory)[0] - resultpath = os.path.join(args.directory, resultfile) - result_is_bad = is_result_bad(resultpath) - - result_str = 'bad' if result_is_bad else 'good' - sys.stdout.write(' - {0}\n'.format(result_str)) - logging.info('Result was %s', result_str) - - return result_is_bad - - print('Searching for bad symbols in:', bad_source) - logging.info('Searching for bad symbols in: %s', bad_source) - logging.info('Note: inlining disabled to isolate functions') - logging.info('Note: only searching over globally exported functions') - logging.debug('Symbols:') - symbol_tuples = extract_symbols(bad_source, - os.path.join(args.directory, 'obj')) - for sym in symbol_tuples: - message = ' {sym.fname}:{sym.lineno} {sym.symbol} -- {sym.demangled}' \ - .format(sym=sym) - logging.info('%s', message) +def search_for_linker_problems(args, bisect_path, replacements, sources, libs): + ''' + Performs the search over the space of statically linked libraries for + problems. - # Check to see if -fPIC destroyed any chance of finding any bad symbols - if not bisect_symbol_build_and_check(symbol_tuples, []): - message_1 = ' Warning: -fPIC compilation destroyed the optimization' - message_2 = ' Cannot find any trouble symbols' + Linking will be done with the ground-truth compiler, but with the static + libraries specified. During this search, all source files will be compiled + with the ground-truth compilation, but the static libraries will be + included in the linking. + + Doing a binary search here actually breaks things since including some + static libraries will require including others to resolve the symbols in + the included static libraries. So, instead this function just runs with + the libraries included, and checks to see if there are reproducibility + problems. + ''' + memoized_checker = _gen_bisect_lib_checker(args, bisect_path, replacements, + sources) + + print('Searching for differing intel static libraries:') + logging.info('Searching for differing static libraries included by intel ' + 'linker:') + #differing_library_msg = ' Found differing library {} (score {})' + #differing_library_callback = lambda filename, score: \ + # util.printlog(differing_library_msg.format(filename, score)) + #if args.biggest is None: + # differing_libs = bisect_search( + # memoized_checker, libs, found_callback=differing_library_callback, + # skip_verification=args.skip_verification) + #else: + # differing_libs = bisect_biggest( + # memoized_checker, libs, found_callback=differing_library_callback, + # k=args.biggest, skip_verification=args.skip_verification) + #return differing_libs + if memoized_checker(libs): + return libs + return [] + +def search_for_source_problems(args, bisect_path, replacements, sources): + ''' + Performs the search over the space of source files for problems. + ''' + + memoized_checker = _gen_bisect_source_checker(args, bisect_path, + replacements, sources) + + print('Searching for differing source files:') + logging.info('Searching for differing source files under the trouble' + ' compilation') + differing_source_msg = ' Found differing source file {}: score {}' + differing_source_callback = lambda filename, score: \ + util.printlog(differing_source_msg.format(filename, score)) + differing_sources = bisect_search(memoized_checker, sources, + found_callback=differing_source_callback, + skip_verification=args.skip_verification) + return differing_sources + +def search_for_symbol_problems(args, bisect_path, replacements, sources, + differing_source, found_callback=None, + indent=''): + ''' + Performs the search over the space of symbols within differing source files + for problems. + + @param args: parsed command-line arguments + @param bisect_path: directory where bisect is being performed + @param replacements: dictionary of values to use in generating the Makefile + @param sources: all source files + @param differing_source: the one differing source file to search for + differing symbols + @param found_callback: (optional) a callback to be called on each found + symbol + @param indent: (optional) indentation to use for the logging and printing + messages + + @return a list of identified differing symbols (if any), with their + associated scores + [(symbol, score), ...] + ''' + print('{}Searching for differing symbols in: {}'.format( + indent, differing_source)) + logging.info('%sSearching for differing symbols in: %s', differing_source, + indent) + logging.info('%sNote: inlining disabled to isolate functions', indent) + logging.info('%sNote: only searching over globally exported functions', + indent) + logging.debug('%sSymbols:', indent) + fsymbol_tuples, remaining_symbols = \ + extract_symbols(differing_source, os.path.join(args.directory, 'obj')) + for sym in fsymbol_tuples: + message = '{indent} {sym.fname}:{sym.lineno} {sym.symbol} ' \ + '-- {sym.demangled}'.format(indent=indent, sym=sym) + logging.debug('%s', message) + + memoized_checker = _gen_bisect_symbol_checker( + args, bisect_path, replacements, sources, fsymbol_tuples, + remaining_symbols, indent=indent + ' ') + + # Check to see if -fPIC destroyed any chance of finding any differing + # symbols + if memoized_checker(fsymbol_tuples) <= 0.0: + message_1 = '{} Warning: -fPIC compilation destroyed the ' \ + 'optimization'.format(indent) + message_2 = '{} Cannot find any trouble symbols'.format(indent) print(message_1) print(message_2) logging.warning('%s', message_1) logging.warning('%s', message_2) return [] - bad_symbols = bisect_search(bisect_symbol_build_and_check, symbol_tuples) - return bad_symbols + differing_symbol_msg = \ + '{indent} Found differing symbol on line {sym.lineno} -- ' \ + '{sym.demangled} (score {score})' + + def differing_symbol_callback(sym, score): + 'Prints the finding and calls the registered callback' + util.printlog(differing_symbol_msg.format(sym=sym, score=score, + indent=indent)) + if found_callback is not None: + found_callback(sym, score) + + if args.biggest is None: + differing_symbols = bisect_search( + memoized_checker, fsymbol_tuples, + found_callback=differing_symbol_callback, + skip_verification=args.skip_verification) + else: + differing_symbols = bisect_biggest( + memoized_checker, fsymbol_tuples, + found_callback=differing_symbol_callback, k=args.biggest, + skip_verification=args.skip_verification) + return differing_symbols + +def search_for_k_most_diff_symbols(args, bisect_path, replacements, sources): + ''' + This function is similar to both search_for_source_problems() and + search_for_symbol_problems(). This function will search for source + problems AND also symbol problems such that the top k differing functions + between the baseline compilation and the variability-inducing optimized + compilation. + + @param args: parsed command-line arguments + @param bisect_path: directory where bisect is being performed + @param replacements: dictionary of values to use in generating the Makefile + @param sources: all source files + + @return a list of identified differing symbols (if any), with their + associated scores + [(symbol, score), ...] + ''' + assert args.biggest is not None + assert args.biggest > 0 + + util.printlog('Looking for the top {} different symbol(s) by starting with ' + 'files'.format(args.biggest)) + + differing_symbols = [] + differing_source_msg = ' Found differing source file {}: score {}' + differing_sources = [] + + class ExitEarlyException(Exception): + 'Exception used to exit early from bisect search' + pass + + def differing_symbol_callback(symbol, score): + 'captures the symbol and checks for early termination' + assert len(differing_symbols) <= args.biggest + assert score > 0 + + if len(differing_symbols) >= args.biggest and \ + score < differing_symbols[-1][1]: + # exit early because we're done with this file + raise ExitEarlyException + differing_symbols.append((symbol, score)) + differing_symbols.sort(key=lambda x: -x[1]) + differing_symbols[:] = differing_symbols[:args.biggest] + + symbol_search = lambda differing_source: \ + search_for_symbol_problems( + args, bisect_path, replacements, sources, differing_source, + found_callback=differing_symbol_callback, indent=' ') + + def differing_source_callback(filename, score): + ''' + prints and captures the found source file, and checks for early + termination. + ''' + assert len(differing_symbols) <= args.biggest + assert score > 0 + + util.printlog(differing_source_msg.format(filename, score)) + differing_sources.append((filename, score)) + if len(differing_symbols) >= args.biggest and \ + score < differing_symbols[-1][1]: + # exit early because we're done with this file + raise ExitEarlyException + + try: + symbol_search(filename) + except ExitEarlyException: + pass + + memoized_source_checker = _gen_bisect_source_checker( + args, bisect_path, replacements, sources) + try: + # Note: ignore return because we already capture found sources + bisect_biggest(memoized_source_checker, sources, k=len(sources), + found_callback=differing_source_callback) + except ExitEarlyException: + pass + + return differing_sources, differing_symbols def compile_trouble(directory, compiler, optl, switches, verbose=False, - jobs=mp.cpu_count(), delete=True): + jobs=mp.cpu_count(), delete=True, fpic=False): ''' Compiles the trouble executable for the given arguments. This is useful to compile the trouble executable as it will force the creation of all needed @@ -875,8 +1432,11 @@ def compile_trouble(directory, compiler, optl, switches, verbose=False, # see if the Makefile needs to be regenerated # we use the Makefile to check for itself, sweet - subp.check_call(['make', '-C', directory, 'Makefile'], - stdout=subp.DEVNULL, stderr=subp.DEVNULL) + run_make( + directory=directory, + verbose=verbose, + jobs=1, + target='Makefile') # trouble compilations all happen in the same directory trouble_path = os.path.join(directory, 'bisect-precompile') @@ -891,9 +1451,10 @@ def compile_trouble(directory, compiler, optl, switches, verbose=False, 'flit_version': conf.version, 'precision': '', 'test_case': '', - 'trouble_cc': compiler, + 'trouble_cxx': compiler, 'trouble_optl': optl, 'trouble_switches': switches, + 'trouble_type': None, 'trouble_id': trouble_hash, 'link_flags': [], 'cpp_flags': [], @@ -905,6 +1466,9 @@ def compile_trouble(directory, compiler, optl, switches, verbose=False, # Compile the trouble executable simply so that we have the object files build_bisect(makepath, directory, verbose=verbose, jobs=jobs, target='trouble') + if fpic: + build_bisect(makepath, directory, verbose=verbose, + jobs=jobs, target='trouble-fpic') # Remove this prebuild temporary directory now if delete: @@ -933,8 +1497,8 @@ def run_bisect(arguments, prog=sys.argv[0]): # see if the Makefile needs to be regenerated # we use the Makefile to check for itself, sweet - subp.check_call(['make', '-C', args.directory, 'Makefile'], - stdout=subp.DEVNULL, stderr=subp.DEVNULL) + run_make(directory=args.directory, verbose=args.verbose, jobs=1, + target='Makefile') # create a unique directory for this bisect run bisect_dir = create_bisect_dir(args.directory) @@ -973,9 +1537,10 @@ def run_bisect(arguments, prog=sys.argv[0]): 'flit_version': conf.version, 'precision': args.precision, 'test_case': args.testcase, - 'trouble_cc': args.compiler, + 'trouble_cxx': args.compiler, 'trouble_optl': args.optl, 'trouble_switches': args.switches, + 'trouble_type': None, 'trouble_id': trouble_hash, 'link_flags': [], 'cpp_flags': [], @@ -985,7 +1550,7 @@ def run_bisect(arguments, prog=sys.argv[0]): update_gt_results(args.directory, verbose=args.verbose, jobs=args.jobs) # Find out if the linker is to blame (e.g. intel linker linking mkl libs) - bad_libs = [] + differing_libs = [] if os.path.basename(args.compiler) in ('icc', 'icpc'): warning_message = 'Warning: The intel compiler may not work with bisect' logging.info('%s', warning_message) @@ -1015,26 +1580,27 @@ def run_bisect(arguments, prog=sys.argv[0]): os.path.join(intel_lib_dir, 'libsvml.a'), ] try: - bad_libs = search_for_linker_problems(args, bisect_path, - replacements, sources, libs) + differing_libs = search_for_linker_problems( + args, bisect_path, replacements, sources, libs) except subp.CalledProcessError: print() print(' Executable failed to run.') - print('Failed to search for bad libraries -- cannot continue.') + print('Failed to search for differing libraries' + ' -- cannot continue.') return bisect_num, None, None, None, 1 - print(' bad static libraries:') + print(' differing static libraries:') logging.info('BAD STATIC LIBRARIES:') - for lib in bad_libs: + for lib in differing_libs: print(' ' + lib) logging.info(' %s', lib) - if len(bad_libs) == 0: + if len(differing_libs) == 0: print(' None') logging.info(' None') # For now, if the linker was to blame, then say there may be nothing # else we can do. - if len(bad_libs) > 0: + if len(differing_libs) > 0: message = 'May not be able to search further, because of intel ' \ 'optimizations' print(message) @@ -1047,66 +1613,117 @@ def run_bisect(arguments, prog=sys.argv[0]): # If the libraries were a problem, then reset what the baseline # ground-truth is, especially since we updated the LINK_FLAGS in the # generated Makefiles. - if len(bad_libs) > 0: + if len(differing_libs) > 0: replacements['build_gt_local'] = 'true' try: - bad_sources = search_for_source_problems(args, bisect_path, - replacements, sources) + if args.biggest is not None: + differing_sources, differing_symbols = \ + search_for_k_most_diff_symbols(args, bisect_path, + replacements, sources) + else: + differing_sources = search_for_source_problems( + args, bisect_path, replacements, sources) except subp.CalledProcessError: print() print(' Executable failed to run.') - print('Failed to search for bad sources -- cannot continue.') - logging.exception('Failed to search for bad sources.') - return bisect_num, bad_libs, None, None, 1 - - print(' bad sources:') - logging.info('BAD SOURCES:') - for src in bad_sources: - print(' ' + src) - logging.info(' %s', src) - if len(bad_sources) == 0: - print(' None') - logging.info(' None') - - - # Search for bad symbols one bad file at a time + print('Failed to search for differing sources -- cannot continue.') + logging.exception('Failed to search for differing sources.') + return bisect_num, differing_libs, None, None, 1 + + if args.biggest is None: + print('all variability inducing source file(s):') + logging.info('ALL VARIABILITY INCUDING SOURCE FILE(S):') + else: + print('The found highest variability inducing source file{}:'.format( + 's' if len(differing_sources) > 1 else '')) + logging.info('%d HIGHEST VARIABILITY SOURCE FILE%s:', + args.biggest, 'S' if args.biggest > 1 else '') + + for src in differing_sources: + util.printlog(' {} (score {})'.format(src[0], src[1])) + if len(differing_sources) == 0: + util.printlog(' None') + + # Search for differing symbols one differing file at a time # This will allow us to maybe find some symbols where crashes before would # cause problems and no symbols would be identified - bad_symbols = [] - for bad_source in bad_sources: - try: - file_bad_symbols = search_for_symbol_problems( - args, bisect_path, replacements, sources, bad_source) - except subp.CalledProcessError: - print() - print(' Executable failed to run.') - print('Failed to search for bad symbols in {} -- cannot continue' \ - .format(bad_source)) - logging.exception('Failed to search for bad symbols in %s', - bad_source) - bad_symbols.extend(file_bad_symbols) - if len(file_bad_symbols) > 0: - print(' bad symbols in {}:'.format(bad_source)) - logging.info(' bad symbols in %s:', bad_source) - for sym in file_bad_symbols: - message = ' line {sym.lineno} -- {sym.demangled}' \ - .format(sym=sym) + # + # Only do this if we didn't already perform the search above with + # search_for_k_most_diff_symbols() + if args.biggest is None: + differing_symbols = [] + for differing_source, _ in differing_sources: + try: + file_differing_symbols = search_for_symbol_problems( + args, bisect_path, replacements, sources, differing_source) + except subp.CalledProcessError: + print() + print(' Executable failed to run.') + print('Failed to search for differing symbols in {}' + '-- cannot continue'.format(differing_source)) + logging.exception( + 'Failed to search for differing symbols in %s', + differing_source) + return bisect_num, differing_libs, differing_sources, None, 1 + differing_symbols.extend(file_differing_symbols) + if len(file_differing_symbols) > 0: + if args.biggest is None: + message = ' All differing symbols in {}:'\ + .format(differing_source) + else: + message = ' {} differing symbol{} in {}:'.format( + args.biggest, 's' if args.biggest > 1 else '', + differing_source) print(message) - logging.info('%s', message) - - print('All bad symbols:') - logging.info('BAD SYMBOLS:') - for sym in bad_symbols: - message = ' {sym.fname}:{sym.lineno} {sym.symbol} -- {sym.demangled}' \ - .format(sym=sym) + logging.info(message) + for sym, score in file_differing_symbols: + message = \ + ' line {sym.lineno} -- {sym.demangled} ' \ + '(score {score})'.format(sym=sym, score=score) + print(message) + logging.info('%s', message) + + if not args.skip_verification and len(differing_sources) > 1: + # Verify that there are no missed files, i.e. those that are more + # than singletons and that are to be grouped with one of the found + # symbols. + message = 'Verifying assumption about independent symbols' + print(message) + logging.info('%s', message) + fsymbol_tuples, remaining_symbols = \ + extract_symbols([x[0] for x in differing_sources], + os.path.join(args.directory, 'obj')) + checker = _gen_bisect_symbol_checker( + args, bisect_path, replacements, sources, fsymbol_tuples, + remaining_symbols) + assert checker(fsymbol_tuples) == \ + checker([x[0] for x in differing_symbols]), \ + 'Assumption about independent symbols is False, ' \ + 'false negative results are possible' + + differing_symbols.sort(key=lambda x: (-x[1], x[0])) + + if args.biggest is None: + print('All variability inducing symbols:') + logging.info('ALL VARIABILITY INCUDING SYMBOLS:') + else: + print('The {} highest variability symbol{}:' + .format(args.biggest, 's' if args.biggest > 1 else '')) + logging.info('THE %d HIGHEST VARIABILITY INDUCING SYMBOL%s:', + args.biggest, 'S' if args.biggest > 1 else '') + + for sym, score in differing_symbols: + message = \ + ' {sym.fname}:{sym.lineno} {sym.symbol} -- {sym.demangled} ' \ + '(score {score})'.format(sym=sym, score=score) print(message) logging.info('%s', message) - if len(bad_symbols) == 0: + if len(differing_symbols) == 0: print(' None') logging.info(' None') - return bisect_num, bad_libs, bad_sources, bad_symbols, 0 + return bisect_num, differing_libs, differing_sources, differing_symbols, 0 def auto_bisect_worker(arg_queue, result_queue): ''' @@ -1128,9 +1745,9 @@ def auto_bisect_worker(arg_queue, result_queue): - compiler: (str) compiler used - optl: (str) optimization level - switches: (str) switches - - libs: (list of str) bad libraries found - - srcs: (list of str) bad source files found - - syms: (list of SymbolTuple) bad symbols found + - libs: (list of str) differing libraries found + - srcs: (list of str) differing source files found + - syms: (list of SymbolTuple) differing symbols found - ret: (int) return code of running @return None @@ -1169,6 +1786,11 @@ def auto_bisect_worker(arg_queue, result_queue): # exit the function pass + except: + # without putting something onto the result_queue, the parent process will deadlock + result_queue.put((row, -1, None, None, None, 1)) + raise + def parallel_auto_bisect(arguments, prog=sys.argv[0]): ''' Runs bisect in parallel under the auto mode. This is only applicable if @@ -1236,8 +1858,8 @@ def parallel_auto_bisect(arguments, prog=sys.argv[0]): # see if the Makefile needs to be regenerated # we use the Makefile to check for itself, sweet - subp.check_call(['make', '-C', args.directory, 'Makefile'], - stdout=subp.DEVNULL, stderr=subp.DEVNULL) + run_make(directory=args.directory, verbose=args.verbose, jobs=1, + target='Makefile') print('Before parallel bisect run, compile all object files') for i, compilation in enumerate(sorted(compilation_set)): @@ -1248,11 +1870,16 @@ def parallel_auto_bisect(arguments, prog=sys.argv[0]): flush=True) compile_trouble(args.directory, compiler, optl, switches, verbose=args.verbose, jobs=args.jobs, - delete=args.delete) + delete=args.delete, fpic=args.precompile_fpic) print(' done', flush=True) # Update ground-truth results before launching workers - update_gt_results(args.directory, verbose=args.verbose, jobs=args.jobs) + update_gt_results(args.directory, verbose=args.verbose, jobs=args.jobs, + fpic=args.precompile_fpic) + + if args.compile_only: + print('Done with precompilation -- exiting') + return 0 # Generate the worker queue arg_queue = mp.Queue() @@ -1340,12 +1967,16 @@ def main(arguments, prog=sys.argv[0]): stuff and runs the run_bisect multiple times if so. ''' + if elf is None: + print('Error: pyelftools is not installed, bisect disabled', + file=sys.stderr) + return 1 + if '-a' in arguments or '--auto-sqlite-run' in arguments: return parallel_auto_bisect(arguments, prog) _, _, _, _, ret = run_bisect(arguments, prog) return ret - if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/scripts/flitcli/flit_import.py b/scripts/flitcli/flit_import.py index 4b078dbc..510b3acc 100644 --- a/scripts/flitcli/flit_import.py +++ b/scripts/flitcli/flit_import.py @@ -82,42 +82,56 @@ 'Implements the import subcommand, importing results into a database' -import flitutil as util - -import toml - import argparse import csv import datetime import os -import sqlite3 import sys +import flitutil as util + brief_description = 'Import flit results into the configured database' def _file_check(filename): + 'Check that a file exists or raise an exception' if not os.path.isfile(filename): raise argparse.ArgumentTypeError('File does not exist: {0}'.format(filename)) return filename +def get_dbfile_from_toml(tomlfile): + 'get and return the database.filepath field' + import toml + try: + projconf = toml.load(tomlfile) + except FileNotFoundError: + print('Error: {0} not found. Run "flit init"'.format(tomlfile), + file=sys.stderr) + raise + util.fill_defaults(projconf) + + assert projconf['database']['type'] == 'sqlite', \ + 'Only sqlite database supported' + return projconf['database']['filepath'] + def main(arguments, prog=sys.argv[0]): + 'Main logic here' parser = argparse.ArgumentParser( - prog=prog, - description=''' - Import flit results into the configured database. The - configured database is found from the settings in - flit-config.toml. You can import either exported results or - results from manually running the tests. Note that importing - the same thing twice will result in having two copies of it - in the database. - ''', - ) + prog=prog, + description=''' + Import flit results into the configured database. The configured + database is found from the settings in flit-config.toml. You can + import either exported results or results from manually running the + tests. Note that importing the same thing twice will result in + having two copies of it in the database. + ''', + ) parser.add_argument('importfile', nargs='+', type=_file_check, help=''' File(s) to import into the database. These files may be csv files or sqlite3 databases. ''') - parser.add_argument('-a', '--append', type=int, default=None, metavar='RUN_ID', + parser.add_argument('-a', '--append', type=int, default=None, + metavar='RUN_ID', help=''' Append the import to the specified run id. The default behavior is to add a new run to include the @@ -142,27 +156,36 @@ def main(arguments, prog=sys.argv[0]): call this program multiple times, each one with --run specified to the next run you want to import. ''') + parser.add_argument('-D', '--dbfile', default=None, + help=''' + Use this database file rather than the one + specified in flit-config.toml. This option is + especially useful when you want to import results + but do not have the flit-config.toml file + available, as that is currently the only reason for + flit-config.toml to be read by this command. It + can also be used when you do not have the toml + python package installed (goodie!). + ''') args = parser.parse_args(arguments) - try: - projconf = toml.load('flit-config.toml') - except FileNotFoundError: - print('Error: {0} not found. Run "flit init"'.format(tomlfile), - file=sys.stderr) - return 1 - util.fill_defaults(projconf) + if args.dbfile is None: + args.dbfile = get_dbfile_from_toml('flit-config.toml') - assert projconf['database']['type'] == 'sqlite', \ - 'Only sqlite database supported' - db = util.sqlite_open(projconf['database']['filepath']) + if os.path.isfile(args.dbfile): + print('Appending', args.dbfile) + else: + print('Creating', args.dbfile) + db = util.sqlite_open(args.dbfile) # create a new run and set the args.append run id if args.append is None: # Create a new run to use in import db.execute('insert into runs(rdate,label) values (?,?)', - (datetime.datetime.now(), args.label)) + (datetime.datetime.now(), args.label)) db.commit() - args.append = db.execute('select id from runs order by id').fetchall()[-1]['id'] + args.append = \ + db.execute('select id from runs order by id').fetchall()[-1]['id'] # Make sure the run id exists. run_ids = [x['id'] for x in db.execute('select id from runs')] @@ -229,5 +252,7 @@ def main(arguments, prog=sys.argv[0]): ''', to_insert) db.commit() + return 0 + if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/scripts/flitcli/flit_init.py b/scripts/flitcli/flit_init.py index 125b48ba..0e5071f1 100644 --- a/scripts/flitcli/flit_init.py +++ b/scripts/flitcli/flit_init.py @@ -85,7 +85,6 @@ import argparse import os import shutil -import socket import sys import flitconfig as conf @@ -95,6 +94,7 @@ brief_description = 'Initializes a flit test directory for use' def main(arguments, prog=sys.argv[0]): + 'Main logic here' parser = argparse.ArgumentParser( prog=prog, description=''' @@ -161,7 +161,7 @@ def copy_files(dest_to_src, remove_license=True): litmus_to_copy[os.path.join('tests', srcfile)] = srcpath copy_files(litmus_to_copy) - flit_update.main(['--directory', args.directory]) + return flit_update.main(['--directory', args.directory]) if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/scripts/flitcli/flit_make.py b/scripts/flitcli/flit_make.py index 90deb059..83e56a17 100644 --- a/scripts/flitcli/flit_make.py +++ b/scripts/flitcli/flit_make.py @@ -93,13 +93,14 @@ brief_description = 'Runs the make and adds to the database' def main(arguments, prog=sys.argv[0]): + 'Main logic here' parser = argparse.ArgumentParser( - prog=prog, - description=''' - This command runs the full set of tests and adds the results - to the configured database. - ''', - ) + prog=prog, + description=''' + This command runs the full set of tests and adds the results + to the configured database. + ''', + ) processors = multiprocessing.cpu_count() parser.add_argument('-j', '--jobs', type=int, default=processors, help=''' @@ -142,20 +143,16 @@ def main(arguments, prog=sys.argv[0]): # TODO: can we make a progress bar here? print('Calling GNU Make for the runbuild') subprocess.check_call([ - 'make', - 'runbuild', - '-j{0}'.format(args.jobs), - ] + make_args, - **check_call_kwargs - ) + 'make', + 'runbuild', + '-j{0}'.format(args.jobs), + ] + make_args, **check_call_kwargs) print('Calling GNU Make to execute the tests') subprocess.check_call([ - 'make', - 'run', - '-j{0}'.format(args.exec_jobs), - ] + make_args, - **check_call_kwargs - ) + 'make', + 'run', + '-j{0}'.format(args.exec_jobs), + ] + make_args, **check_call_kwargs) print('Importing into the database') # TODO: find a way to not import over again if called multiple times status = flit_import.main(['--label', args.label] + @@ -163,5 +160,7 @@ def main(arguments, prog=sys.argv[0]): if status != 0: return status + return 0 + if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/scripts/flitcli/flit_update.py b/scripts/flitcli/flit_update.py index f3582db3..183caf77 100644 --- a/scripts/flitcli/flit_update.py +++ b/scripts/flitcli/flit_update.py @@ -84,6 +84,7 @@ import argparse import os +import re import sys import toml @@ -91,30 +92,207 @@ import flitutil brief_description = 'Updates the Makefile based on flit-config.toml' +_supported_compiler_types = ('clang', 'gcc', 'intel') -def main(arguments, prog=sys.argv[0]): +def parse_args(arguments, prog=sys.argv[0]): + 'Return parsed arugments' parser = argparse.ArgumentParser( - prog=prog, - description=''' - Updates the Makefile based on flit-config.toml. The Makefile - is autogenerated and should not be modified manually. If there - are things you want to replace or add, you can use custom.mk - which is included at the end of the Makefile. So, you may add - rules, add to variables, or override variables. - ''', - ) + prog=prog, + description=''' + Updates the Makefile based on flit-config.toml. The Makefile + is autogenerated and should not be modified manually. If there + are things you want to replace or add, you can use custom.mk + which is included at the end of the Makefile. So, you may add + rules, add to variables, or override variables. + ''', + ) parser.add_argument('-C', '--directory', default='.', help='The directory to initialize') args = parser.parse_args(arguments) + return args + +def load_projconf(directory): + ''' + Loads and returns the project configuration found in the given tomlfile. + This function checks for validity of that tomlfile and fills it with + default values. + + @param directory: directory containing 'flit-config.toml'. - tomlfile = os.path.join(args.directory, 'flit-config.toml') + @return project configuration as a struct of dicts and lists depending on + the structure of the given tomlfile. + ''' + tomlfile = os.path.join(directory, 'flit-config.toml') try: projconf = toml.load(tomlfile) except FileNotFoundError: print('Error: {0} not found. Run "flit init"'.format(tomlfile), file=sys.stderr) + raise + + defaults = flitutil.get_default_toml() + + if 'compiler' in projconf: + assert isinstance(projconf['compiler'], list), \ + 'flit-config.toml improperly configured, ' \ + 'needs [[compiler]] section' + + default_type_map = {c['type']: c for c in defaults['compiler']} + type_map = {} # type -> compiler + name_map = {} # name -> compiler + for compiler in projconf['compiler']: + + # make sure each compiler has a name, type, and binary + for field in ('name', 'type', 'binary'): + assert field in compiler, \ + 'flit-config.toml: compiler "{0}"'.format(compiler) + \ + ' is missing the "{0}" field'.format(field) + + # check that the type is valid + assert compiler['type'] in _supported_compiler_types, \ + 'flit-config.toml: unsupported compiler type "{0}"' \ + .format(compiler['type']) + + # check that we only have one of each type specified + assert compiler['type'] not in type_map, \ + 'flit-config.toml: cannot have multiple compilers of the ' \ + 'same type ({0})'.format(compiler['type']) + type_map[compiler['type']] = compiler + + # check that we only have one of each name specified + assert compiler['name'] not in name_map, \ + 'flit-config.toml: cannot have multiple compilers of the ' \ + 'same name ({0})'.format(compiler['name']) + name_map[compiler['name']] = compiler + + # if optimization_levels or switches_list are missing for any + # compiler, put in the default flags for that compiler + default = default_type_map[compiler['type']] + for field in ('optimization_levels', 'switches_list'): + if field not in compiler: + compiler[field] = default[field] + + # Fill in the rest of the default values + flitutil.fill_defaults(projconf, defaults) + + return projconf + +def flag_name(flag): + ''' + Returns an associated Makefile variable name for the given compiler flag + + @param flag: (str) switches for the compiler + + @return (str) a valid Makefile variable unique to the given flag + + >>> flag_name('') + 'NO_FLAGS' + + >>> flag_name('-') + Traceback (most recent call last): + ... + AssertionError: Error: cannot handle flag only made of dashes + + >>> flag_name('----') + Traceback (most recent call last): + ... + AssertionError: Error: cannot handle flag only made of dashes + + >>> flag_name('-funsafe-math-optimizations') + 'FUNSAFE_MATH_OPTIMIZATIONS' + + >>> flag_name('-Ofast -march=32bit') + 'OFAST__MARCH_32BIT' + ''' + if flag == '': + return 'NO_FLAGS' + name = re.sub('[^0-9A-Za-z]', '_', flag.upper().strip('-')) + assert re.match('^[0-9]', name) is None, \ + 'Error: cannot handle flag that starts with a number' + assert len(name) > 0, 'Error: cannot handle flag only made of dashes' + return name + +def gen_assignments(flag_map): + ''' + Given a mapping of Makefile variable name to value, create a single string + of assignments suitable for placing within a Makefile + + @note no checking is performed on the keys of the map. They are assumed to + be valid Makefile variables + + @param flag_map: ({str: str}) mapping from Makefile variable name to + Makefile value. + @return (str) The string to insert into a Makefile to create the + assignments + + >>> gen_assignments({}) + '' + + >>> gen_assignments({'single_name': 'single_value'}) + 'single_name := single_value' + + Here we use an OrderedDict for the test to be robust. If we used a normal + dict, then the output lines could show up in a different order. + >>> from collections import OrderedDict + >>> print(gen_assignments( + ... OrderedDict([('hello', 'there'), ('my', 'friend')]))) + hello := there + my := friend + + >>> print(gen_assignments(OrderedDict([ + ... ('REALLY_A_VERY_LONG_VARIABLE_NAME_HERE', 'bob'), + ... ('not_so_long_32', 'harry'), + ... ('short', 'very long value here'), + ... ]))) + REALLY_A_VERY_LONG_VARIABLE_NAME_HERE := bob + not_so_long_32 := harry + short := very long value here + ''' + name_assignments = ['{} := {}'.format(name.ljust(15), flag) + for name, flag in flag_map.items()] + return '\n'.join(name_assignments) + +def gen_multi_assignment(name, values): + ''' + Generates a multi-line assignment string for a Makefile + + @note no checking is done on the name or values to see if they are valid to + place within a Makefile. + + @param name: (str) Makefile variable name + @param values: (iter(str)) iterable of values to assign, one per line + + @return (str) a single string with the multi-line assignment suitable for a + Makefile. + + >>> gen_multi_assignment('CLANG', None) + 'CLANG :=' + + >>> gen_multi_assignment('CLANG', []) + 'CLANG :=' + + >>> print(gen_multi_assignment('hello_there', ['my friend', 'my enemy'])) + hello_there := + hello_there += my friend + hello_there += my enemy + ''' + values = values or tuple() # if None, set to an empty tuple + justified = name.ljust(15) + beginning = justified + ' :=' + return '\n'.join( + [beginning] + ['{} += {}'.format(justified, x) for x in values]) + +def main(arguments, prog=sys.argv[0]): + 'Main logic here' + args = parse_args(arguments, prog=prog) + + try: + projconf = load_projconf(args.directory) + except FileNotFoundError: + return 1 + except AssertionError as ex: + print('Error: ' + ex.args[0], file=sys.stderr) return 1 - flitutil.fill_defaults(projconf) makefile = os.path.join(args.directory, 'Makefile') if os.path.exists(makefile): @@ -122,35 +300,21 @@ def main(arguments, prog=sys.argv[0]): else: print('Creating {0}'.format(makefile)) - host = projconf['hosts'][0] - dev_build = host['dev_build'] - dev_compiler_name = dev_build['compiler_name'] - dev_optl = dev_build['optimization_level'] - dev_switches = dev_build['switches'] - matching_dev_compilers = [x for x in host['compilers'] - if x['name'] == dev_compiler_name] - assert len(matching_dev_compilers) > 0, \ - 'Compiler name {0} not found'.format(dev_compiler_name) - assert len(matching_dev_compilers) < 2, \ - 'Multiple compilers with name {0} found'.format(dev_compiler_name) - dev_compiler_bin = matching_dev_compilers[0]['binary'] - if '/' in dev_compiler_bin: - dev_compiler_bin = os.path.realpath(dev_compiler_bin) - - ground_truth = host['ground_truth'] - gt_compiler_name = ground_truth['compiler_name'] - gt_optl = ground_truth['optimization_level'] - gt_switches = ground_truth['switches'] - matching_gt_compilers = [x for x in host['compilers'] - if x['name'] == gt_compiler_name] + dev_build = projconf['dev_build'] + matching_dev_compilers = [x for x in projconf['compiler'] + if x['name'] == dev_build['compiler_name']] assert len(matching_dev_compilers) > 0, \ - 'Compiler name {0} not found'.format(gt_compiler_name) - assert len(matching_dev_compilers) < 2, \ - 'Multiple compilers with name {0} found'.format(gt_compiler_names) - # TODO: use the compiler mnemonic rather than the path - gt_compiler_bin = matching_gt_compilers[0]['binary'] - if '/' in dev_compiler_bin: - gt_compiler_bin = os.path.realpath(gt_compiler_bin) + 'Compiler name {0} not found'.format(dev_build['compiler_name']) + + ground_truth = projconf['ground_truth'] + matching_gt_compilers = [x for x in projconf['compiler'] + if x['name'] == ground_truth['compiler_name']] + assert len(matching_gt_compilers) > 0, \ + 'Compiler name {0} not found'.format(ground_truth['compiler_name']) + + base_compilers = {x.upper(): None for x in _supported_compiler_types} + base_compilers.update({compiler['type'].upper(): compiler['binary'] + for compiler in projconf['compiler']}) test_run_args = '' if not projconf['run']['timing']: @@ -160,26 +324,54 @@ def main(arguments, prog=sys.argv[0]): '--timing-loops', str(projconf['run']['timing_loops']), '--timing-repeats', str(projconf['run']['timing_repeats']), ]) - flitutil.process_in_file( - os.path.join(conf.data_dir, 'Makefile.in'), - makefile, - { - 'dev_compiler': dev_compiler_bin, - 'dev_optl': dev_optl, - 'dev_switches': dev_switches, - 'ground_truth_compiler': gt_compiler_bin, - 'ground_truth_optl': gt_optl, - 'ground_truth_switches': gt_switches, - 'flit_include_dir': conf.include_dir, - 'flit_lib_dir': conf.lib_dir, - 'flit_data_dir': conf.data_dir, - 'flit_script_dir': conf.script_dir, - 'flit_version': conf.version, - 'test_run_args': test_run_args, - 'enable_mpi': 'yes' if projconf['run']['enable_mpi'] else 'no', - 'mpirun_args': projconf['run']['mpirun_args'], - }, - overwrite=True) + + replacements = { + 'uname': os.uname().sysname, + 'hostname': os.uname().nodename, + 'dev_compiler': matching_dev_compilers[0]['binary'], + 'dev_type': matching_dev_compilers[0]['type'], + 'dev_optl': dev_build['optimization_level'], + 'dev_switches': dev_build['switches'], + 'ground_truth_compiler': matching_gt_compilers[0]['binary'], + 'ground_truth_type': matching_gt_compilers[0]['type'], + 'ground_truth_optl': ground_truth['optimization_level'], + 'ground_truth_switches': ground_truth['switches'], + 'flit_include_dir': conf.include_dir, + 'flit_lib_dir': conf.lib_dir, + 'flit_data_dir': conf.data_dir, + 'flit_script_dir': conf.script_dir, + 'flit_version': conf.version, + 'test_run_args': test_run_args, + 'enable_mpi': 'yes' if projconf['run']['enable_mpi'] else 'no', + 'mpirun_args': projconf['run']['mpirun_args'], + 'compiler_defs': gen_assignments({ + key: val for key, val in base_compilers.items()}), + 'compilers': ' '.join([compiler['type'].upper() + for compiler in projconf['compiler']]), + 'opcodes_definitions': gen_assignments({ + flag_name(x): x + for compiler in projconf['compiler'] + for x in compiler['optimization_levels']}), + 'switches_definitions': gen_assignments({ + flag_name(x): x + for compiler in projconf['compiler'] + for x in compiler['switches_list']}), + 'compiler_opcodes': '\n\n'.join([ + gen_multi_assignment( + 'OPCODES_' + compiler['type'].upper(), + [flag_name(x) for x in compiler['optimization_levels']]) + for compiler in projconf['compiler']]), + 'compiler_switches': '\n\n'.join([ + gen_multi_assignment( + 'SWITCHES_' + compiler['type'].upper(), + [flag_name(x) for x in compiler['switches_list']]) + for compiler in projconf['compiler']]), + } + + flitutil.process_in_file(os.path.join(conf.data_dir, 'Makefile.in'), + makefile, replacements, overwrite=True) + + return 0 if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/scripts/flitcli/flitconfig.py b/scripts/flitcli/flitconfig.py index d9a66c4e..add10553 100644 --- a/scripts/flitcli/flitconfig.py +++ b/scripts/flitcli/flitconfig.py @@ -89,7 +89,7 @@ import os -all = [ +__all__ = [ 'version', 'script_dir', 'doc_dir', @@ -123,4 +123,3 @@ # directory containing litmus tests litmus_test_dir = os.path.realpath(os.path.join(script_dir, '..', '..', 'litmus-tests', 'tests')) - diff --git a/scripts/flitcli/flitelf.py b/scripts/flitcli/flitelf.py new file mode 100644 index 00000000..3d534ff5 --- /dev/null +++ b/scripts/flitcli/flitelf.py @@ -0,0 +1,282 @@ +# Much of this is copied from the examples given in +# https://github.com/eliben/pyelftools.git + +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Utility functions for dealing with ELF binary files. This file requires the +pyelftools package to be installed (i.e. module elftools). +''' + +from collections import namedtuple +import subprocess as subp +import os + +from elftools.elf.elffile import ELFFile +from elftools.elf.sections import SymbolTableSection + +SymbolTuple = namedtuple('SymbolTuple', + 'src, symbol, demangled, fname, lineno') +SymbolTuple.__doc__ = ''' +Tuple containing information about the symbols in a file. Has the following +attributes: + src: source file that was compiled + symbol: mangled symbol in the compiled version + demangled: demangled version of symbol + fname: filename where the symbol is actually defined. This usually + will be equal to src, but may not be in some situations. + lineno: line number of definition within fname. +''' + +def extract_symbols(objfile, srcfile): + ''' + Extracts symbols for the given object file. + + @param objfile: (str) path to object file + + @return two lists of SymbolTuple objects (funcsyms, remaining). + The first is the list of exported functions that are strong symbols and + have a filename and line number where they are defined. The second is + all remaining symbols that are strong, exported, and defined. + ''' + with open(objfile, 'rb') as fin: + elffile = ELFFile(fin) + + symtabs = [x for x in elffile.iter_sections() + if isinstance(x, SymbolTableSection)] + if len(symtabs) == 0: + raise RuntimeError('Object file {} does not have a symbol table' + .format(objfile)) + + # get globally exported defined symbols + syms = [sym for symtab in symtabs + for sym in symtab.iter_symbols() + if _is_symbol(sym) + and _is_extern(sym) + and _is_strong(sym) + and _is_defined(sym)] + + # split symbols into functions and non-functions + fsyms = [sym for sym in syms if _is_func(sym)] # functions + rsyms = list(set(syms).difference(fsyms)) # remaining + + # find filename and line numbers for each relevant func symbol + locs = _locate_symbols(elffile, fsyms) + + # demangle all symbols + fdemangled = _demangle([sym.name for sym in fsyms]) + rdemangled = _demangle([sym.name for sym in rsyms]) + + funcsym_tuples = [SymbolTuple(srcfile, fsyms[i].name, fdemangled[i], + locs[i][0], locs[i][1]) + for i in range(len(fsyms))] + remaining_tuples = [SymbolTuple(srcfile, rsyms[i].name, rdemangled[i], + None, None) + for i in range(len(rsyms))] + + return funcsym_tuples, remaining_tuples + +def _symbols(symtab): + 'Returns all symbols from the given symbol table' + return [sym for sym in symtab.iter_symbols() if _is_symbol(sym)] + +def _is_symbol(sym): + 'Returns True if elf.sections.Symbol object is a symbol' + return sym.name != '' and sym['st_info']['type'] != 'STT_FILE' + +def _is_extern(sym): + 'Returns True if elf.sections.Symbol is an extern symbol' + return sym['st_info']['bind'] != 'STB_LOCAL' + +def _is_weak(sym): + 'Returns True if elf.sections.Symbol is a weak symbol' + return sym['st_info']['bind'] == 'STB_WEAK' + +def _is_strong(sym): + 'Returns True if elf.sections.Symbol is a strong symbol' + return sym['st_info']['bind'] == 'STB_GLOBAL' + +def _is_defined(sym): + 'Returns True if elf.sections.Symbol is defined' + return sym['st_shndx'] != 'SHN_UNDEF' + +def _is_func(sym): + 'Returns True if elf.sections.Symbol is a function' + return sym['st_info']['type'] == 'STT_FUNC' + +def _demangle(symbol_list): + 'Demangles each C++ name in the given list' + proc = subp.Popen(['c++filt'], stdin=subp.PIPE, stdout=subp.PIPE) + out, _ = proc.communicate('\n'.join(symbol_list).encode()) + demangled = out.decode('utf8').splitlines() + assert len(demangled) == len(symbol_list) + return demangled + +def _locate_symbols(elffile, symbols): + ''' + Locates the filename and line number of each symbol in the elf file. + + @param elffile: (elf.elffile.ELFFile) The top-level elf file + @param symbols: (list(elf.sections.Symbol)) symbols to locate + + @return list(tuple(filename, lineno)) in the order of the given symbols + + If the file does not have DWARF info or a symbol is not found, an exception + is raised + + Test that even without a proper elffile, if there are no symbols to match, + then no error occurs and you can be on your merry way. + >>> _locate_symbols(object(), []) + [] + ''' + if len(symbols) == 0: + return [] + + if not elffile.has_dwarf_info(): + raise RuntimeError('Elf file has no DWARF info') + + dwarfinfo = elffile.get_dwarf_info() + fltable = _gen_file_line_table(dwarfinfo) + + locations = [] + for sym in symbols: + for fname, lineno, start, end in fltable: + if start <= sym.entry['st_value'] < end: + locations.append((fname.decode('utf8'), lineno)) + break + else: + locations.append((None, None)) + + return locations + +def _gen_file_line_table(dwarfinfo): + ''' + Generates and returns a list of (filename, lineno, startaddr, endaddr). + + Tests that an empty dwarfinfo object will result in an empty return list + >>> class FakeDwarf: + ... def __init__(self): + ... pass + ... def iter_CUs(self): + ... return [] + >>> _gen_file_line_table(FakeDwarf()) + [] + ''' + # generate the table + table = [] + for unit in dwarfinfo.iter_CUs(): + lineprog = dwarfinfo.line_program_for_CU(unit) + prevstate = None + for entry in lineprog.get_entries(): + # We're interested in those entries where a new state is assigned + if entry.state is None or entry.state.end_sequence: + continue + # Looking for a range of addresses in two consecutive states that + # contain a required address. + if prevstate is not None: + filename = lineprog['file_entry'][prevstate.file - 1].name + dirno = lineprog['file_entry'][prevstate.file - 1].dir_index + filepath = os.path.join( + lineprog['include_directory'][dirno - 1], filename) + line = prevstate.line + fromaddr = prevstate.address + toaddr = max(fromaddr, entry.state.address) + table.append((filepath, line, fromaddr, toaddr)) + prevstate = entry.state + + # If there are no functions, then return an empty list + if len(table) == 0: + return [] + + # consolidate the table + consolidated = [] + prev = table[0] + for entry in table[1:]: + if prev[1] == entry[1] and prev[3] == entry[2]: + prev = (prev[0], prev[1], prev[2], entry[3]) + else: + consolidated.append(prev) + prev = entry + consolidated.append(prev) + + return consolidated diff --git a/scripts/flitcli/flitutil.py b/scripts/flitcli/flitutil.py index d9ebc7f4..2adeff64 100644 --- a/scripts/flitcli/flitutil.py +++ b/scripts/flitcli/flitutil.py @@ -87,13 +87,13 @@ import flitconfig as conf import copy +import logging import os import socket import sqlite3 import subprocess as subp import sys import tempfile -import toml # cached values _default_toml = None @@ -120,6 +120,7 @@ def get_default_toml(): Gets the default toml configuration file for FLIT and returns the configuration object. ''' + import toml global _default_toml if _default_toml is None: _default_toml = toml.loads(get_default_toml_string()) @@ -349,3 +350,79 @@ def extract_make_var(var, makefile='Makefile', directory='.'): var_values = output.split('=', maxsplit=1)[1].split() return var_values +def extract_make_vars(makefile='Makefile', directory='.'): + ''' + Extracts all GNU Make variables from the given Makefile, except for those + that are built-in. It is returned as a dictionary of + {'var': ['val', ...]} + + @note, all variables are returned, including internal Makefile + variables. + + >>> from tempfile import NamedTemporaryFile as NTF + >>> with NTF(mode='w+') as fout: + ... print('A := hello there sweetheart\\n' + ... 'B = \\n', + ... file=fout, flush=True) + ... allvars = extract_make_vars(fout.name) + + >>> allvars['A'] + ['hello', 'there', 'sweetheart'] + >>> allvars['B'] + [] + + What if the file does not exist? It throws an exception: + + >>> extract_make_var('A', 'file-should-not-exist.mk') # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + subprocess.CalledProcessError: Command ... returned non-zero exit status 2. + ''' + with tempfile.NamedTemporaryFile(mode='w+') as fout: + print('$(foreach v,$(.VARIABLES),$(info $(v)=$($(v))**==**))\n' + '.PHONY: empty-target\n' + 'empty-target:\n' + "\t@true\n", file=fout, flush=True) + output = subp.check_output( + ['make', '-f', makefile, '-f', fout.name, 'empty-target', + '--directory', directory, '--no-print-directory'], + stderr=subp.DEVNULL) + lines = output.strip().decode('utf-8').splitlines() + var_values = {} + prevkey = None + for line in lines: + if prevkey is None: + split = line.split('=', maxsplit=1) + prevkey = split[0] + values = split[1] + var_values[prevkey] = [] + else: + values = line + + key = prevkey + + if values.endswith('**==**'): + values = values.replace('**==**', '') + prevkey = None + + var_values[key].extend(values.split()) + + return var_values + +def printlog(message): + ''' + Prints the message to stdout and then logs the message at the info level. + It is expected that the logging module has already been configured using + the root logger. + + >>> logger = logging.getLogger() + >>> for handler in logger.handlers[:]: + ... logger.removeHandler(handler) + >>> import sys + >>> logging.basicConfig(stream=sys.stdout, level=logging.INFO) + >>> printlog('Danger Will Robinson!') + Danger Will Robinson! + INFO:root:Danger Will Robinson! + ''' + print(message) + logging.info(message) diff --git a/scripts/watch-progress.sh b/scripts/watch-progress.sh index 062213f4..bdaad8c0 100755 --- a/scripts/watch-progress.sh +++ b/scripts/watch-progress.sh @@ -1,4 +1,86 @@ #!/bin/bash +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + cd results watch -n 2 \ diff --git a/src/FlitCsv.cpp b/src/FlitCsv.cpp new file mode 100644 index 00000000..f6da743e --- /dev/null +++ b/src/FlitCsv.cpp @@ -0,0 +1,174 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#include "FlitCsv.h" + +#include +#include +#include + +namespace flit { + +std::string const& CsvRow::operator[](std::string col) const { + if (m_header == nullptr) { + throw std::logic_error("No header defined"); + } + auto iter = std::find(m_header->begin(), m_header->end(), col); + if (iter == m_header->end()) { + std::stringstream message; + message << "No column named " << col; + throw std::invalid_argument(message.str()); + } + auto idx = iter - m_header->begin(); + return this->at(idx); +} + +CsvRow CsvReader::parseRow(std::istream &in) { + enum class State { + DEFAULT, + IN_STRING, + }; + State state = State::DEFAULT; + + char quote_char = '"'; + char separator = ','; + char line_end = '\n'; + + auto transition_state = [&state](State newstate) { + //std::cout << "State transitioned from " + // << static_cast(state) + // << " to " + // << static_cast(newstate) + // << std::endl; + state = newstate; + }; + + CsvRow row; + char current; + std::ostringstream running; + int running_size = 0; + while (in.get(current)) { + if (state == State::DEFAULT) { + if (running_size == 0 && current == quote_char) { + transition_state(State::IN_STRING); + } else if (current == separator) { + row.emplace_back(running.str()); + running.str(""); + running_size = 0; + } else if (current == line_end) { + row.emplace_back(running.str()); + running.str(""); + running_size = 0; + break; // break out of the while loop + } else { + running << current; + running_size++; + } + } else if (state == State::IN_STRING) { + if (current == quote_char) { + transition_state(State::DEFAULT); + } else { + running << current; + running_size++; + } + } else { + throw std::runtime_error( + "Please contact Michael Bentley, this shouldn't happen..."); + } + } + + // We should not be within a STRING when we exit the while loop + if (state != State::DEFAULT) { + throw std::runtime_error("Error parsing CSV file"); + } + + // If we stopped because we reached the end of file... + // (i.e. ignore empty last rows) + if (!in && !row.empty()) { + row.emplace_back(running.str()); + } + + return row; +} + +} // end of namespace flit diff --git a/src/FlitCsv.h b/src/FlitCsv.h new file mode 100644 index 00000000..dcb1ade4 --- /dev/null +++ b/src/FlitCsv.h @@ -0,0 +1,244 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#ifndef FLIT_CSV_H +#define FLIT_CSV_H + +#include "flitHelpers.h" + +#include +#include +#include + +namespace flit { + +/** Helper class for CsvReader. + * + * Represents a single row either indexed by number or by column name. + */ +class CsvRow : public std::vector { +public: + // Inherit base class constructors + using std::vector::vector; + + const CsvRow* header() const { return m_header; } + void setHeader(CsvRow* head) { m_header = head; } + + using std::vector::operator[]; + std::string const& operator[](std::string col) const; + + bool operator!=(const CsvRow &other) { + // Check the header + if (this->header() == nullptr && other.header() != nullptr) { + return true; + } + if (this->header() != nullptr && other.header() == nullptr) { + return true; + } + if (this->header() != nullptr && + other.header() != nullptr && + *this->header() != *other.header()) + { + return true; + } + + // Check row contents + if (this->size() != other.size()) { + return true; + } + for (CsvRow::size_type i = 0; i < this->size(); i++) { + if (this->at(i) != other.at(i)) { + return true; + } + } + + return false; + } + + bool operator==(const CsvRow &other) { return !(*this != other); } + +private: + CsvRow* m_header {nullptr}; // not owned by this class +}; + +/** Class for parsing csv files */ +class CsvReader { +public: + CsvReader(std::istream &in) + : m_header(CsvReader::parseRow(in)), m_in(in), m_is_done(!bool(in)) {} + + operator bool() const { return !m_is_done; } + CsvRow* header() { return &m_header; } + std::istream& stream() { return m_in; } + + CsvReader& operator>> (CsvRow& row) { + row = CsvReader::parseRow(m_in); + row.setHeader(this->header()); + if (row.empty()) { + m_is_done = true; + } + return *this; + } + +private: + static CsvRow parseRow(std::istream &in); + +private: + CsvRow m_header; + std::istream &m_in; + bool m_is_done; +}; + +class CsvWriter { +public: + explicit CsvWriter(std::ostream &out) + : m_out(out), m_is_line_beginning(true) {} + virtual ~CsvWriter() {} + + void new_row() { + this->m_out << std::endl; + this->m_is_line_beginning = true; + } + + template + void write_row (const std::vector& row) { + if (!this->m_is_line_beginning) { + throw std::runtime_error("Cannot write a row to a partially created " + "row. Call CsvWriter::new_row() first"); + } + for (const T &elem : row) { + *this << elem; + } + this->new_row(); + } + +#define FLIT_CSVWRITER_OPERATOR(type) \ + CsvWriter& operator<< (const type val) { \ + this->append(val); \ + return *this; \ + } + + FLIT_CSVWRITER_OPERATOR(int); + FLIT_CSVWRITER_OPERATOR(long); + FLIT_CSVWRITER_OPERATOR(long long); + FLIT_CSVWRITER_OPERATOR(unsigned int); + FLIT_CSVWRITER_OPERATOR(unsigned long); + FLIT_CSVWRITER_OPERATOR(unsigned long long); + FLIT_CSVWRITER_OPERATOR(unsigned __int128); + FLIT_CSVWRITER_OPERATOR(float); + FLIT_CSVWRITER_OPERATOR(double); + FLIT_CSVWRITER_OPERATOR(long double); + +#undef FLIT_CSVWRITER_OPERATOR + + // handle std::string separately + CsvWriter& operator<< (const std::string &val) { + // if there are offending characters in val, then quote the field + if (val.find_first_of(",\r\n") != std::string::npos) { + this->append('"' + val + '"'); + } else { + this->append(val); + } + return *this; + } + +private: + template + void append(const T &val) { + if (!this->m_is_line_beginning) { + this->m_out << ','; + } + this->m_is_line_beginning = false; + this->m_out << val; + } + +private: + std::ostream &m_out; + bool m_is_line_beginning; +}; + + + +} // end of namespace flit + +#endif // FLIT_CSV_H diff --git a/src/InfoStream.cpp b/src/InfoStream.cpp index 0b5c90fc..03c9b936 100644 --- a/src/InfoStream.cpp +++ b/src/InfoStream.cpp @@ -95,8 +95,8 @@ namespace { public: InfoStreamBackend() : std::ostream(), _null() { hide(); } ~InfoStreamBackend() { hide(); } - void show() { init(std::cout.rdbuf()); } - void hide() { init(&_null); } + void show() { rdbuf(std::cout.rdbuf()); } + void hide() { rdbuf(&_null); } private: NullBuffer _null; @@ -110,7 +110,7 @@ InfoStream::InfoStream() : std::ostream() , _threadbuf() { - init(_threadbuf.rdbuf()); + rdbuf(_threadbuf.rdbuf()); } InfoStream::~InfoStream() { diff --git a/src/flit.cpp b/src/flit.cpp index f70543c6..019e138d 100644 --- a/src/flit.cpp +++ b/src/flit.cpp @@ -101,78 +101,6 @@ namespace { -/** Helper class for Csv. - * - * Represents a single row either indexed by number or by column name. - */ -class CsvRow : public std::vector { -public: - // Inherit base class constructors - using std::vector::vector; - - const CsvRow* header() const { return m_header; } - void setHeader(CsvRow* head) { m_header = head; } - - using std::vector::operator[]; - std::string const& operator[](std::string col) const { - if (m_header == nullptr) { - throw std::logic_error("No header defined"); - } - auto iter = std::find(m_header->begin(), m_header->end(), col); - if (iter == m_header->end()) { - std::stringstream message; - message << "No column named " << col; - throw std::invalid_argument(message.str()); - } - auto idx = iter - m_header->begin(); - return this->at(idx); - } - -private: - CsvRow* m_header {nullptr}; // not owned by this class -}; - -/** Class for parsing csv files */ -class Csv { -public: - Csv(std::istream &in) : m_header(Csv::parseRow(in)), m_in(in) { - m_header.setHeader(&m_header); - } - - Csv& operator>> (CsvRow& row) { - row = Csv::parseRow(m_in); - row.setHeader(&m_header); - return *this; - } - - operator bool() const { return static_cast(m_in); } - -private: - static CsvRow parseRow(std::istream &in) { - std::string line; - std::getline(in, line); - - std::stringstream lineStream(line); - std::string token; - - // tokenize on ',' - CsvRow row; - while(std::getline(lineStream, token, ',')) { - row.emplace_back(token); - } - - // check for trailing comma with no data after it - if (!lineStream && token.empty()) { - row.emplace_back(""); - } - - return row; - } - -private: - CsvRow m_header; - std::istream &m_in; -}; /** Returns true if the element is in the container */ template @@ -466,9 +394,9 @@ std::string readFile(const std::string &filename) { std::vector parseResults(std::istream &in) { std::vector results; - Csv csv(in); + CsvReader reader(in); CsvRow row; - while (csv >> row) { + for (CsvRow row; reader >> row; ) { auto nanosec = std::stol(row["nanosec"]); Variant value; std::string resultfile; @@ -501,9 +429,10 @@ std::unordered_map parseMetadata(std::istream &in) { "file" }; - Csv csv(in); + CsvReader csv(in); CsvRow row; - if (csv >> row) { + csv >> row; + if (!row.empty()) { for (auto key : metadataKeys) { metadata.emplace(key, row[key]); } diff --git a/src/flit.h b/src/flit.h index bb668bd4..22f11ba6 100644 --- a/src/flit.h +++ b/src/flit.h @@ -87,6 +87,7 @@ #ifndef FLIT_H #define FLIT_H 0 +#include "FlitCsv.h" #include "MpiEnvironment.h" #include "TestBase.h" #include "flitHelpers.h" @@ -287,64 +288,67 @@ inline void outputResults ( std::string executableFilename = FLIT_FILENAME) { // Output the column headers - out << "name," - "host," - "compiler," - "optl," - "switches," - "precision," - "score_hex," - "score," - "resultfile," - "comparison_hex," - "comparison," - "file," - "nanosec" - << std::endl; - for(const auto& result: results){ - out - << result.name() << "," // test case name - << hostname << "," // hostname - << compiler << "," // compiler - << optimization_level << "," // optimization level - << switches << "," // compiler flags - << result.precision() << "," // precision + CsvWriter writer(out); + writer + << "name" + << "host" + << "compiler" + << "optl" + << "switches" + << "precision" + << "score_hex" + << "score" + << "resultfile" + << "comparison_hex" + << "comparison" + << "file" + << "nanosec"; + writer.new_row(); + for (const auto& result: results) { + writer + << result.name() // test case name + << hostname // hostname + << compiler // compiler + << optimization_level // optimization level + << switches // compiler flags + << result.precision() // precision ; if (result.result().type() == Variant::Type::LongDouble) { - out - << as_int(result.result().longDouble()) << "," // score_hex - << result.result().longDouble() << "," // score + writer + << as_int(result.result().longDouble()) // score_hex + << result.result().longDouble() // score ; } else { - out - << FLIT_NULL << "," // score_hex - << FLIT_NULL << "," // score + writer + << FLIT_NULL // score_hex + << FLIT_NULL // score ; } if (result.resultfile().empty()) { - out << FLIT_NULL << ","; // resultfile + writer << FLIT_NULL; // resultfile } else { - out << result.resultfile() << ","; // resultfile + writer << result.resultfile(); // resultfile } if (result.is_comparison_null()) { - out - << FLIT_NULL << "," // comparison_hex - << FLIT_NULL << "," // comparison + writer + << FLIT_NULL // comparison_hex + << FLIT_NULL // comparison ; } else { - out - << as_int(result.comparison()) << "," // comparison_hex - << result.comparison() << "," // comparison + writer + << as_int(result.comparison()) // comparison_hex + << result.comparison() // comparison ; } - out - << executableFilename << "," // executable filename + writer + << executableFilename // executable filename << result.nanosecs() // nanoseconds - << std::endl; + ; + writer.new_row(); } } diff --git a/tests/.gitignore b/tests/.gitignore index 8d152b31..7469162f 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,3 +1,4 @@ harness_tester flit_src/tst_flit_cpp flit_src/tst_flitHelpers_h +flit_src/tst_FlitCsv diff --git a/tests/Makefile b/tests/Makefile index 000a3fed..728d16c8 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -4,6 +4,7 @@ CFLAGS += -std=c++11 LDFLAGS := TESTER_SRC := harness_tester.cpp HARNESS := test_harness.h +HARNESS_PY := test_harness.py TESTER := $(TESTER_SRC:%.cpp=%) SUB_MAKES := $(wildcard */Makefile */makefile) SUB_DIRS := $(patsubst %/,%,$(dir $(SUB_MAKES))) @@ -16,7 +17,7 @@ build: $(TESTER) $(BUILD_TARGETS) include color_out.mk -check: run__$(TESTER) $(CHECK_TARGETS) +check: run__$(TESTER) run_tester_py $(CHECK_TARGETS) @$(call color_out,GREEN,All tests pass) help: @@ -29,8 +30,16 @@ help: clean: $(CLEAN_TARGETS) rm -f $(TESTER) +.PHONY: run_tester_py +run_tester_py: $(HARNESS_PY) + @$(call color_out_noline,BROWN, testing) + @echo " $(HARNESS_PY)" + @python3 -m doctest $(HARNESS_PY) + .PHONY: run__% build__% check__% clean__% run__$(TESTER): $(TESTER) + @$(call color_out_noline,BROWN, testing) + @echo " $(HARNESS)" @./$(TESTER) | grep "failures: *10" >/dev/null build__%: diff --git a/tests/color_out.mk b/tests/color_out.mk index 03c446b8..533749ee 100644 --- a/tests/color_out.mk +++ b/tests/color_out.mk @@ -1,7 +1,14 @@ # A Makefile function # @param 1: color (e.g. BLUE or GREEN) # @param 2: message to be printed in the color -color_out = @if [ -t 1 ]; then \ +color_out_noline = \ + @if [ -t 1 ]; then \ + /bin/echo -ne "$(BASH_$1)$2$(BASH_CLEAR)"; \ + else \ + /bin/echo -n "$2"; \ + fi +color_out = \ + @if [ -t 1 ]; then \ /bin/echo -e "$(BASH_$1)$2$(BASH_CLEAR)"; \ else \ /bin/echo "$2"; \ @@ -11,7 +18,9 @@ BASH_CLEAR := \e[0m BASH_BLACK := \e[0;30m BASH_BROWN := \e[0;33m BASH_GRAY := \e[0;37m +BASH_GREY := \e[0;37m BASH_DARKGRAY := \e[1;30m +BASH_DARKGREY := \e[1;30m BASH_RED := \e[1;31m BASH_GREEN := \e[1;32m BASH_YELLOW := \e[1;33m diff --git a/tests/flit_cli/Makefile b/tests/flit_cli/Makefile index 242995ab..2edb693f 100644 --- a/tests/flit_cli/Makefile +++ b/tests/flit_cli/Makefile @@ -1,19 +1,29 @@ -RUNNER := python3 -SRC := $(wildcard tst_*.py) -RUN_TARGETS := $(SRC:%.py=run_%) +SUB_MAKES := $(wildcard */Makefile */makefile) +SUB_DIRS := $(patsubst %/,%,$(dir $(SUB_MAKES))) +CHECK_TARGETS := $(addprefix check__,$(SUB_DIRS)) +CLEAN_TARGETS := $(addprefix clean__,$(SUB_DIRS)) +BUILD_TARGETS := $(addprefix build__,$(SUB_DIRS)) -.PHONY: check help clean build run_% -check: $(TARGETS) $(RUN_TARGETS) +.PHONY: build check help clean +build: $(BUILD_TARGETS) + +check: $(CHECK_TARGETS) help: @echo "Makefile for running tests on FLiT framework" @echo " help print this help documentation and exit" - @echo " build just compile the targets" + @echo " build just compile all subdirectories" @echo " check run tests and print results to the console" @echo " clean remove all generated files" -build: -clean: +.PHONY: build__% check__% clean__% + +build__%: + @$(MAKE) build --directory $* + +check__%: + @$(MAKE) check --directory $* + +clean__%: + @$(MAKE) clean --directory $* -run_% : %.py - @$(RUNNER) $< diff --git a/tests/flit_cli/flit_bisect/Makefile b/tests/flit_cli/flit_bisect/Makefile new file mode 100644 index 00000000..b7fb7853 --- /dev/null +++ b/tests/flit_cli/flit_bisect/Makefile @@ -0,0 +1,33 @@ +RUNNER := python3 +SRC := $(wildcard tst_*.py) +RUN_TARGETS := $(SRC:%.py=run_%) + +IS_PYELF := $(shell if python3 -c 'import elftools' 2>/dev/null; then \ + echo true; \ + fi) + +include ../../color_out.mk + +.PHONY: check help clean build run_% +ifeq ($(IS_PYELF),true) +check: $(TARGETS) $(RUN_TARGETS) +else +check: + $(call color_out,RED,Warning: pyelftools is not found on your system;\ + skipping bisect tests) +endif + +help: + @echo "Makefile for running tests on FLiT framework" + @echo " help print this help documentation and exit" + @echo " build just compile the targets" + @echo " check run tests and print results to the console" + @echo " clean remove all generated files" + +build: +clean: + +run_% : %.py + @$(call color_out_noline,BROWN, running) + @echo " $<" + @$(RUNNER) $< diff --git a/tests/flit_cli/flit_bisect/data/tests/A.cpp b/tests/flit_cli/flit_bisect/data/tests/A.cpp new file mode 100644 index 00000000..2d1c4164 --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/A.cpp @@ -0,0 +1,106 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#include "A.h" + +#include + +#include + +int A::offset = 2; + +A::A() {} +A::~A() {} + +int A::fileA_method1_PROBLEM() { + if (std::string(FLIT_OPTL) == "-O3") { + return 3 + offset; // variability introduced = offset = 2 + } else { + return 3; + } +} + +int fileA_all() { + A a; + return a.fileA_method1_PROBLEM(); +} diff --git a/tests/flit_cli/flit_bisect/data/tests/A.h b/tests/flit_cli/flit_bisect/data/tests/A.h new file mode 100644 index 00000000..a4fb888c --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/A.h @@ -0,0 +1,97 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#ifndef A_H +#define A_H + +class A { +public: + static int offset; // is a global strong symbol that is not a function + A(); + virtual ~A(); + virtual int fileA_method1_PROBLEM(); +}; + +int fileA_all(); + +#endif // A_H diff --git a/tests/flit_cli/flit_bisect/data/tests/BisectTest.cpp b/tests/flit_cli/flit_bisect/data/tests/BisectTest.cpp new file mode 100644 index 00000000..b999332c --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/BisectTest.cpp @@ -0,0 +1,117 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#include "file1.h" +#include "file2.h" +#include "file3.h" +#include "file4.h" +#include "A.h" + +#include + +#include + +#include + +template +class BisectTest : public flit::TestBase { +public: + BisectTest(std::string id) : flit::TestBase(std::move(id)) {} + virtual size_t getInputsPerRun() override { return 0; } + virtual std::vector getDefaultInput() override { return {}; } + virtual long double compare(long double ground_truth, + long double test_results) const override { + return std::abs(test_results - ground_truth); + } + +protected: + virtual flit::Variant run_impl(const std::vector &ti) override { + FLIT_UNUSED(ti); + return file1_all() + file2_all() + file3_all() + file4_all() + fileA_all(); + } + +protected: + using flit::TestBase::id; +}; + +REGISTER_TYPE(BisectTest) diff --git a/tests/flit_cli/flit_bisect/data/tests/file1.cpp b/tests/flit_cli/flit_bisect/data/tests/file1.cpp new file mode 100644 index 00000000..5e37a6a8 --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/file1.cpp @@ -0,0 +1,124 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#include "file1.h" + +#include + +#include + +int file1_func1() { return 1; } + +int file1_func2_PROBLEM() { + if (std::string(FLIT_OPTL) == "-O3") { + return 2 + 5; // variability introduced = 5 + } else { + return 2; + } +} + +int file1_func3_PROBLEM() { + if (std::string(FLIT_OPTL) == "-O3") { + return 3 + 2; // variability introduced = 2 + } else { + return 3; + } +} + +int file1_func4_PROBLEM() { + if (std::string(FLIT_OPTL) == "-O3") { + return 4 + 3; // variability introduced = 3 + } else { + return 4; + } +} + +int file1_func5() { return 5; } + +int file1_all() { + return file1_func1() + + file1_func2_PROBLEM() + + file1_func3_PROBLEM() + + file1_func4_PROBLEM() + + file1_func5(); +} diff --git a/tests/flit_cli/flit_bisect/data/tests/file1.h b/tests/flit_cli/flit_bisect/data/tests/file1.h new file mode 100644 index 00000000..e426f2ca --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/file1.h @@ -0,0 +1,94 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#ifndef FILE1_H +#define FILE1_H + +int file1_func1(); +int file1_func2_PROBLEM(); // variability = 5 +int file1_func3_PROBLEM(); // variability = 2 +int file1_func4_PROBLEM(); // variability = 3 +int file1_func5(); +int file1_all(); + +#endif // FILE1_H diff --git a/tests/flit_cli/flit_bisect/data/tests/file2.cpp b/tests/flit_cli/flit_bisect/data/tests/file2.cpp new file mode 100644 index 00000000..45d84bcf --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/file2.cpp @@ -0,0 +1,109 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#include "file2.h" + +#include + +#include + +int file2_func1_PROBLEM() { + if (std::string(FLIT_OPTL) == "-O3") { + return 1 + 7; // variability introduced = 7 + } else { + return 1; + } +} + +int file2_func2() { return 2; } +int file2_func3() { return 3; } +int file2_func4() { return 4; } +int file2_func5() { return 5; } + +int file2_all() { + return file2_func1_PROBLEM() + + file2_func2() + + file2_func3() + + file2_func4() + + file2_func5(); +} diff --git a/tests/flit_cli/flit_bisect/data/tests/file2.h b/tests/flit_cli/flit_bisect/data/tests/file2.h new file mode 100644 index 00000000..087b0ba0 --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/file2.h @@ -0,0 +1,94 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#ifndef FILE2_H +#define FILE2_H + +int file2_func1_PROBLEM(); // variability = 7 +int file2_func2(); +int file2_func3(); +int file2_func4(); +int file2_func5(); +int file2_all(); + +#endif // FILE2_H diff --git a/tests/flit_cli/flit_bisect/data/tests/file3.cpp b/tests/flit_cli/flit_bisect/data/tests/file3.cpp new file mode 100644 index 00000000..22fcfea5 --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/file3.cpp @@ -0,0 +1,117 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#include "file1.h" + +#include + +#include + +int file3_func1() { return 1; } + +int file3_func2_PROBLEM() { + if (std::string(FLIT_OPTL) == "-O3") { + return 2 + 1; // variability introduced = 1 + } else { + return 2; + } +} + +int file3_func3() { return 3; } +int file3_func4() { return 4; } + +int file3_func5_PROBLEM() { + if (std::string(FLIT_OPTL) == "-O3") { + return 5 + 3; // variability introduced = 3 + } else { + return 5; + } +} + +int file3_all() { + return file3_func1() + + file3_func2_PROBLEM() + + file3_func3() + + file3_func4() + + file3_func5_PROBLEM(); +} diff --git a/tests/flit_cli/flit_bisect/data/tests/file3.h b/tests/flit_cli/flit_bisect/data/tests/file3.h new file mode 100644 index 00000000..70334e98 --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/file3.h @@ -0,0 +1,94 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#ifndef FILE3_H +#define FILE3_H + +int file3_func1(); +int file3_func2_PROBLEM(); // variability = 1 +int file3_func3(); +int file3_func4(); +int file3_func5_PROBLEM(); // variability = 3 +int file3_all(); + +#endif // FILE3_H diff --git a/tests/flit_cli/flit_bisect/data/tests/file4.cpp b/tests/flit_cli/flit_bisect/data/tests/file4.cpp new file mode 100644 index 00000000..c9acb952 --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/file4.cpp @@ -0,0 +1,114 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#include "file4.h" + +#include + +#include + +int file4_func1() { return 1; } +int file4_func2() { return 2; } +int file4_func3() { return 3; } +int file4_func4() { return 4; } + +template +int file4_func5_TEMPLATE_PROBLEM() { + if (std::string(FLIT_OPTL) == "-O3") { + return 5 + 10; // variability introduced = 10 + } else { + return 5; + } +} + + + +int file4_all() { + return file4_func1() + + file4_func2() + + file4_func3() + + file4_func4() + + file4_func5_TEMPLATE_PROBLEM<0>() + + file4_func5_TEMPLATE_PROBLEM<1>() + + file4_func5_TEMPLATE_PROBLEM<2>(); +} diff --git a/tests/flit_cli/flit_bisect/data/tests/file4.h b/tests/flit_cli/flit_bisect/data/tests/file4.h new file mode 100644 index 00000000..fc9baf15 --- /dev/null +++ b/tests/flit_cli/flit_bisect/data/tests/file4.h @@ -0,0 +1,95 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#ifndef FILE4_H +#define FILE4_H + +int file4_func1(); +int file4_func2(); +int file4_func3(); +int file4_func4(); +template +int file4_func5_TEMPLATE_PTOBLEM(); // variability = 10 +int file4_all(); + +#endif // FILE4_H diff --git a/tests/flit_cli/flit_bisect/tst_bisect.py b/tests/flit_cli/flit_bisect/tst_bisect.py new file mode 100644 index 00000000..fb149bdc --- /dev/null +++ b/tests/flit_cli/flit_bisect/tst_bisect.py @@ -0,0 +1,237 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Tests FLiT's capabilities to run bisect and identify the problem files and +functions + +The tests are below using doctest + +Let's now make a temporary directory and test that we can successfully compile +and run FLiT bisect + +>>> import glob +>>> import os +>>> import shutil +>>> import subprocess as subp +>>> from io import StringIO + +>>> class BisectTestError(RuntimeError): pass + +>>> with th.tempdir() as temp_dir: +... with StringIO() as ostream: +... retval = th.flit.main(['init', '-C', temp_dir], outstream=ostream) +... if retval != 0: +... raise BisectTestError( +... 'Could not initialize (retval={0}):\\n'.format(retval) + +... ostream.getvalue()) +... init_out = ostream.getvalue().splitlines() +... shutil.rmtree(os.path.join(temp_dir, 'tests')) +... _ = shutil.copytree(os.path.join('data', 'tests'), +... os.path.join(temp_dir, 'tests')) +... with StringIO() as ostream: +... retval = th.flit.main(['bisect', '-C', temp_dir, +... '--precision', 'double', +... 'g++ -O3', 'BisectTest'], +... outstream=ostream) +... if retval != 0: +... raise BisectTestError( +... 'Could not bisect (retval={0}):\\n'.format(retval) + +... ostream.getvalue()) +... bisect_out = ostream.getvalue().splitlines() +... with open(os.path.join(temp_dir, 'bisect-01', 'bisect.log')) as fin: +... raw_log = fin.readlines() +... stripped_log = [line[line.index(' bisect:')+8:].rstrip() +... for line in raw_log] +... log_contents = [line for line in stripped_log if line.strip() != ''] + +Verify the output of flit init +>>> print('\\n'.join(init_out)) # doctest:+ELLIPSIS +Creating /.../flit-config.toml +Creating /.../custom.mk +Creating /.../main.cpp +Creating /.../tests/Empty.cpp +Creating /.../Makefile + +Let's see that the ground truth results are updated first +>>> bisect_out[0] +'Updating ground-truth results - ground-truth.csv - done' + +Verify that all source files were found and output during the search +>>> sorted([x.split(':')[0].split()[-1] for x in bisect_out +... if x.startswith(' Found differing source file')]) +['tests/A.cpp', 'tests/file1.cpp', 'tests/file2.cpp', 'tests/file3.cpp', 'tests/file4.cpp'] + +Verify that the four differing sources were output in the "differing sources:" +section +>>> idx = bisect_out.index('all variability inducing source file(s):') +>>> print('\\n'.join(bisect_out[idx+1:idx+6])) + tests/file4.cpp (score 30.0) + tests/file1.cpp (score 10.0) + tests/file2.cpp (score 7.0) + tests/file3.cpp (score 4.0) + tests/A.cpp (score 2.0) +>>> bisect_out[idx+6].startswith('Searching for differing symbols in:') +True + +Verify that all four files were searched individually +>>> sorted([x.split()[-1] for x in bisect_out +... if x.startswith('Searching for differing symbols in:')]) +['tests/A.cpp', 'tests/file1.cpp', 'tests/file2.cpp', 'tests/file3.cpp', 'tests/file4.cpp'] + +Verify all non-templated functions were identified during the symbol searches +>>> print('\\n'.join( +... sorted([' '.join(x.split()[-6:]) for x in bisect_out +... if x.startswith(' Found differing symbol on line')]))) +line 100 -- file1_func3_PROBLEM() (score 2.0) +line 103 -- file3_func5_PROBLEM() (score 3.0) +line 106 -- file4_all() (score 30.0) +line 108 -- file1_func4_PROBLEM() (score 3.0) +line 90 -- file2_func1_PROBLEM() (score 7.0) +line 92 -- file1_func2_PROBLEM() (score 5.0) +line 92 -- file3_func2_PROBLEM() (score 1.0) +line 95 -- A::fileA_method1_PROBLEM() (score 2.0) + +Verify the differing symbols section for file1.cpp +>>> idx = bisect_out.index(' All differing symbols in tests/file1.cpp:') +>>> print('\\n'.join(bisect_out[idx+1:idx+4])) + line 92 -- file1_func2_PROBLEM() (score 5.0) + line 108 -- file1_func4_PROBLEM() (score 3.0) + line 100 -- file1_func3_PROBLEM() (score 2.0) +>>> bisect_out[idx+4].startswith(' ') +False + +Verify the differing symbols section for file2.cpp +>>> idx = bisect_out.index(' All differing symbols in tests/file2.cpp:') +>>> bisect_out[idx+1] +' line 90 -- file2_func1_PROBLEM() (score 7.0)' +>>> bisect_out[idx+2].startswith(' ') +False + +Verify the differing symbols section for file3.cpp +>>> idx = bisect_out.index(' All differing symbols in tests/file3.cpp:') +>>> print('\\n'.join(bisect_out[idx+1:idx+3])) + line 103 -- file3_func5_PROBLEM() (score 3.0) + line 92 -- file3_func2_PROBLEM() (score 1.0) +>>> bisect_out[idx+3].startswith(' ') +False + +Verify the bad symbols section for file4.cpp +>>> idx = bisect_out.index(' All differing symbols in tests/file4.cpp:') +>>> print('\\n'.join(sorted(bisect_out[idx+1:idx+2]))) + line 106 -- file4_all() (score 30.0) +>>> bisect_out[idx+2].startswith(' ') +False + +Verify the bad symbols section for fileA.cpp +>>> idx = bisect_out.index(' All differing symbols in tests/A.cpp:') +>>> print('\\n'.join(sorted(bisect_out[idx+1:idx+2]))) + line 95 -- A::fileA_method1_PROBLEM() (score 2.0) +>>> bisect_out[idx+2].startswith(' ') +False + +Test the All differing symbols section of the output +>>> idx = bisect_out.index('All variability inducing symbols:') +>>> print('\\n'.join(bisect_out[idx+1:])) # doctest:+ELLIPSIS + tests/file4.cpp:106 ... -- file4_all() (score 30.0) + tests/file2.cpp:90 ... -- file2_func1_PROBLEM() (score 7.0) + tests/file1.cpp:92 ... -- file1_func2_PROBLEM() (score 5.0) + tests/file1.cpp:108 ... -- file1_func4_PROBLEM() (score 3.0) + tests/file3.cpp:103 ... -- file3_func5_PROBLEM() (score 3.0) + tests/A.cpp:95 ... -- A::fileA_method1_PROBLEM() (score 2.0) + tests/file1.cpp:100 ... -- file1_func3_PROBLEM() (score 2.0) + tests/file3.cpp:92 ... -- file3_func2_PROBLEM() (score 1.0) + +TODO: test the log_contents variable +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/tests/flit_cli/flit_bisect/tst_bisect_biggest.py b/tests/flit_cli/flit_bisect/tst_bisect_biggest.py new file mode 100644 index 00000000..07d630fe --- /dev/null +++ b/tests/flit_cli/flit_bisect/tst_bisect_biggest.py @@ -0,0 +1,294 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Tests FLiT's capabilities to run bisect and identify the problem files and +functions + +The tests are below using doctest + +Let's now make a temporary directory and test that we can successfully compile +and run FLiT bisect + +>>> import glob +>>> import os +>>> import shutil +>>> import subprocess as subp +>>> from io import StringIO + +let's stub out some functions that actually confer with the compiler. These +make the test take way too long and that interaction has already been tested in +tst_bisect.py. + +>>> flit_bisect = th._path_import(th._script_dir, 'flit_bisect') +>>> util = th._path_import(th._script_dir, 'flitutil') +>>> Sym = flit_bisect.elf.SymbolTuple +>>> def create_symbol(fileno, funcno, lineno, isproblem): +... prob_str = '_PROBLEM' if isproblem else '' +... filename = 'tests/file{}.cpp'.format(fileno) +... funcname = 'file{}_func{}{}'.format(fileno, funcno, prob_str) +... return Sym(filename, +... '_Z19{}v'.format(funcname), +... funcname + '()', +... filename, +... lineno) + +>>> all_file_scores = { +... 'main.cpp': 0.0, +... 'tests/BisectTest.cpp': 0.0, +... 'tests/file1.cpp': 10.0, +... 'tests/file2.cpp': 7.0, +... 'tests/file3.cpp': 4.0, +... 'tests/file4.cpp': 0.0, +... 'tests/A.cpp': 2.0, +... } + +>>> all_symbol_scores = { +... create_symbol(1, 1, 90, False): 0.0, +... create_symbol(1, 2, 92, True): 5.0, +... create_symbol(1, 3, 100, True): 2.0, +... create_symbol(1, 4, 108, True): 3.0, +... create_symbol(1, 5, 116, False): 0.0, +... create_symbol(2, 1, 90, True): 7.0, +... create_symbol(2, 2, 98, False): 0.0, +... create_symbol(2, 3, 99, False): 0.0, +... create_symbol(2, 4, 100, False): 0.0, +... create_symbol(2, 5, 101, False): 0.0, +... create_symbol(3, 1, 90, False): 0.0, +... create_symbol(3, 2, 92, True): 1.0, +... create_symbol(3, 3, 100, False): 0.0, +... create_symbol(3, 4, 101, False): 0.0, +... create_symbol(3, 5, 103, True): 3.0, +... create_symbol(4, 1, 103, False): 0.0, +... Sym('tests/A.cpp', '_ZN1A21fileA_method1_PROBLEMEv', +... 'A::fileA_method1_PROBLEM()', 'tests/A.cpp', 95): 2.0, +... } + +>>> def build_bisect_stub(makepath, directory, target='bisect', verbose=False, +... jobs=None): +... assert target == 'bisect' +... assert verbose == False +... # Get the files being tested out of the makepath, and the BISECT_RESULT +... # variable, and generate the expected results. +... sources = util.extract_make_var('TROUBLE_SRC', makepath) +... symbol_files = util.extract_make_var('TROUBLE_SYMBOLS', makepath) +... symbols = [] +... if symbol_files: +... with open(os.path.join(directory, symbol_files[0]), 'r') as fin: +... symbols = [x.strip() for x in fin.readlines()] +... source_score = sum([all_file_scores[x] for x in sources]) +... symbol_score = sum([score for symbol, score in all_symbol_scores.items() +... if symbol.symbol in symbols]) +... result_file = util.extract_make_var('BISECT_RESULT', makepath)[0] +... with open(os.path.join(directory, result_file), 'w') as rout: +... rout.write('comparison\\n') +... rout.write('{}\\n'.format(source_score + symbol_score)) + +>>> def update_gt_results_stub(directory, verbose=False, jobs=4): +... # do nothing +... print('Updating ground-truth results - ground-truth.csv - done') +... pass + +>>> def extract_symbols_stub(file_or_filelist, objdir): +... symbol_tuples = [] +... if not isinstance(file_or_filelist, str): +... for fname in file_or_filelist: +... symbol_tuples.extend(extract_symbols_stub(fname, objdir)) +... return symbol_tuples, [] +... +... return sorted([x for x in all_symbol_scores +... if x.fname == file_or_filelist], +... key=lambda x: x.symbol), [] + +>>> flit_bisect.build_bisect = build_bisect_stub +>>> flit_bisect.update_gt_results = update_gt_results_stub +>>> flit_bisect.extract_symbols = extract_symbols_stub + +Now for the test after we stubbed a single file + +>>> with th.tempdir() as temp_dir: +... with StringIO() as ostream: +... _ = th.flit.main(['init', '-C', temp_dir], outstream=ostream) +... init_out = ostream.getvalue().splitlines() +... shutil.rmtree(os.path.join(temp_dir, 'tests')) +... _ = shutil.copytree(os.path.join('data', 'tests'), +... os.path.join(temp_dir, 'tests')) +... with StringIO() as ostream: +... try: +... oldout = sys.stdout +... sys.stdout = ostream +... _ = flit_bisect.main(['--directory', temp_dir, +... '--precision', 'double', +... 'g++ -O3', 'BisectTest', +... '--biggest', '1']) +... finally: +... sys.stdout = oldout +... bisect_out_1 = ostream.getvalue().splitlines() +... with StringIO() as ostream: +... try: +... oldout = sys.stdout +... sys.stdout = ostream +... _ = flit_bisect.main(['--directory', temp_dir, +... '--precision', 'double', +... 'g++ -O3', 'BisectTest', +... '--biggest', '2']) +... finally: +... sys.stdout = oldout +... bisect_out_2 = ostream.getvalue().splitlines() +... with open(os.path.join(temp_dir, 'bisect-01', 'bisect.log')) as fin: +... log_contents = fin.readlines() + +Verify the output of flit init +>>> print('\\n'.join(init_out)) # doctest:+ELLIPSIS +Creating /.../flit-config.toml +Creating /.../custom.mk +Creating /.../main.cpp +Creating /.../tests/Empty.cpp +Creating /.../Makefile + +>>> print('\\n'.join(bisect_out_1)) # doctest:+ELLIPSIS +Updating ground-truth results - ground-truth.csv - done +Looking for the top 1 different symbol(s) by starting with files + Created /.../bisect-01/bisect-make-01.mk - compiling and running - score 23.0 + ... + Found differing source file tests/file1.cpp: score 10.0 + Searching for differing symbols in: tests/file1.cpp + Created ... + ... + Found differing symbol on line 92 -- file1_func2_PROBLEM() (score 5.0) + ...Found differing source file tests/file2.cpp: score 7.0 + Searching for differing symbols in: tests/file2.cpp + Created ... + ... + Found differing symbol on line 90 -- file2_func1_PROBLEM() (score 7.0) + ...Found differing source file tests/file3.cpp: score 4.0 +The found highest variability inducing source files: + tests/file1.cpp (score 10.0) + tests/file2.cpp (score 7.0) + tests/file3.cpp (score 4.0) +The 1 highest variability symbol: + tests/file2.cpp:90 _Z19file2_func1_PROBLEMv -- file2_func1_PROBLEM() (score 7.0) + +>>> print('\\n'.join(bisect_out_2)) # doctest:+ELLIPSIS +Updating ground-truth results - ground-truth.csv - done +Looking for the top 2 different symbol(s) by starting with files + Created /.../bisect-02/bisect-make-01.mk - compiling and running - score 23.0 + ... + Found differing source file tests/file1.cpp: score 10.0 + Searching for differing symbols in: tests/file1.cpp + Created ... + ... + Found differing symbol on line 92 -- file1_func2_PROBLEM() (score 5.0) + ... Found differing symbol on line 108 -- file1_func4_PROBLEM() (score 3.0) + ...Found differing source file tests/file2.cpp: score 7.0 + Searching for differing symbols in: tests/file2.cpp + Created ... + ... + Found differing symbol on line 90 -- file2_func1_PROBLEM() (score 7.0) + ...Found differing source file tests/file3.cpp: score 4.0 +The found highest variability inducing source files: + tests/file1.cpp (score 10.0) + tests/file2.cpp (score 7.0) + tests/file3.cpp (score 4.0) +The 2 highest variability symbols: + tests/file2.cpp:90 _Z19file2_func1_PROBLEMv -- file2_func1_PROBLEM() (score 7.0) + tests/file1.cpp:92 _Z19file1_func2_PROBLEMv -- file1_func2_PROBLEM() (score 5.0) + +TODO: Test the log_contents variable + +TODO: This test takes about a whole minute on my laptop. Let's create mocks +TODO- and stubs to eliminate the dependence on the compiler. The interaction +TODO- with the real compiler is already tested in tst_bisect.py, so we can +TODO- safely stub it out here. +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/tests/flit_cli/flit_import/Makefile b/tests/flit_cli/flit_import/Makefile new file mode 100644 index 00000000..ce0c8e04 --- /dev/null +++ b/tests/flit_cli/flit_import/Makefile @@ -0,0 +1,23 @@ +RUNNER := python3 +SRC := $(wildcard tst_*.py) +RUN_TARGETS := $(SRC:%.py=run_%) + +include ../../color_out.mk + +.PHONY: check help clean build run_% +check: $(TARGETS) $(RUN_TARGETS) + +help: + @echo "Makefile for running tests on FLiT framework" + @echo " help print this help documentation and exit" + @echo " build just compile the targets" + @echo " check run tests and print results to the console" + @echo " clean remove all generated files" + +build: +clean: + +run_% : %.py + @$(call color_out_noline,BROWN, running) + @echo " $<" + @$(RUNNER) $< diff --git a/tests/flit_cli/flit_import/data/data1.csv b/tests/flit_cli/flit_import/data/data1.csv new file mode 100644 index 00000000..06444c02 --- /dev/null +++ b/tests/flit_cli/flit_import/data/data1.csv @@ -0,0 +1,7 @@ +name,host,compiler,optl,switches,precision,score_hex,score,resultfile,comparison_hex,comparison,file,nanosec +Test-name 1,My-host,g++,-O3,-funsafe-math-optimizations,d,0x3fc1f3bf9fbd47d9a800,4.13E-19,NULL,0xbf8d8000000000000000,-4.81E-35,Executable-1,0 +Test-name 1,My-host,g++,-O3,-funsafe-math-optimizations,e,0x39b1f4945ea2bda83e1d,2.622993437820000000049493126424993646159392840860537999954199990192081383442238818022440754628591392062370104918700079633947091308461839161171640941846366999487116897033647392640059787776192308046148485442313923637505619864489663342532546868165054067822553526472224056250818846029404834715074970010619585201127975867937128686584859879235956730154796047568484754028218349365890624790982353393885557838362503601335004366804753200077871854785718735900172999572949926325063118472484723814495993721500770550490477690570702216083488816431615025776587959223632347438896354452271830717685450631135729485941832893902859041046485120727804367381754781456875315095074113343122617644273653171652021857112889615423937484463835787809320022112937369856636506646976768460185212153428152275393010129262643490977972613221204038554898919875314600364413668735286620701405148737003869390251211450716301354137742031331212078879640318656013417890315294552993898491795938960646946762588295746816126420665952524682531599334464e-486,NULL,0x00000000000000000,0,Executable-1,0 +Test-name 1,My-host,g++,-O3,-funsafe-math-optimizations,f,0x406bfbc7780000000000,6.38336044132261E+032,NULL,0x40548000000000000000,3.86856262276681E+025,Executable-1,0 +Test-name 2,My-host,g++,-O3,-funsafe-math-optimizations,d,0x43f3f78060de921d6800,8.48639885075809E+304,NULL,0x43bf8000000000000000,9.74531401139999E+288,Executable-1,0 +Test-name 2,My-host,g++,-O3,-funsafe-math-optimizations,e,0x750fc6e7f87ea219b5f0,2.414900878482699999795508402316024758692093524806230899470711317155208208917239351004592145700838964809713539689950081780652142004017701820489109185237739731776069652775475587364235041317881184647657737514056077406660197340530427402720810614023982185997501558174758654488852938662786077427428151283412205029413321065359541219873748339197031818496674331872259620434905462771175297875629859443933488190292705817203369288279859977237950632350306150894955353470107246953620032474441533425556138127561529626825449889989460593037745787637371551643659888695656111848431709385771441755211411455955350026612450283221437557870777655391290501233122703416744333570839252203524526590014751409815036338943626475847611276171463663976894962144501520730899078171939790820650957206041614439841880474407327983121273223057622974837366966306292949739537499800325401059186202791792679518011908836657422244571139693819475514296103878538735858327875567104379905240638787073442073798378865190480293376910192972788631712551243e+4089,NULL,0x00000000000000000,0,Executable-1,0 +Test-name 2,My-host,g++,-O3,-funsafe-math-optimizations,f,0x3fa7f4dbb50000000000,6.18E-27,NULL,0x3f908000000000000000,3.85E-34,Executable-1,0 diff --git a/tests/flit_cli/flit_import/tst_dbfile.py b/tests/flit_cli/flit_import/tst_dbfile.py new file mode 100644 index 00000000..c63b9d38 --- /dev/null +++ b/tests/flit_cli/flit_import/tst_dbfile.py @@ -0,0 +1,143 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Tests FLiT's --version argument + +The tests are below using doctest + +Let's now make a temporary directory and test that flit init populates the +correct files + +TODO: how do I test its behavior when I do not have toml installed? + +>>> from io import StringIO +>>> import os +>>> import sqlite3 +>>> import subprocess as subp +>>> import tempfile + +>>> datafile = os.path.join('data', 'data1.csv') + +Test that if flit-config.toml does not exist, then an error occurs +But here we suppress stderr +>>> try: +... olderr = sys.stderr +... with StringIO() as ostream: +... sys.stderr = ostream +... th.flit.main(['import', datafile]) +... finally: +... sys.stderr = olderr +Traceback (most recent call last): + ... +FileNotFoundError: [Errno 2] No such file or directory: 'flit-config.toml' + +Test that with --dbfile , that it creates and imports into that file +>>> with th.tempdir() as temp_dir: +... fname = os.path.join(temp_dir, 'my-results.sqlite') +... with StringIO() as ostream: +... _ = th.flit.main(['import', '--dbfile', fname, datafile], +... outstream=ostream) +... import_out = ostream.getvalue().splitlines() +... with sqlite3.connect(fname) as connection: +... cursor = connection.execute('select * from tests;') +... rows = [row for row in cursor] + +>>> print('\\n'.join(import_out)) # doctest:+ELLIPSIS +Creating /.../my-results.sqlite +Importing data/data1.csv + +>>> len(rows) +6 +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/tests/flit_cli/flit_update/Makefile b/tests/flit_cli/flit_update/Makefile new file mode 100644 index 00000000..ce0c8e04 --- /dev/null +++ b/tests/flit_cli/flit_update/Makefile @@ -0,0 +1,23 @@ +RUNNER := python3 +SRC := $(wildcard tst_*.py) +RUN_TARGETS := $(SRC:%.py=run_%) + +include ../../color_out.mk + +.PHONY: check help clean build run_% +check: $(TARGETS) $(RUN_TARGETS) + +help: + @echo "Makefile for running tests on FLiT framework" + @echo " help print this help documentation and exit" + @echo " build just compile the targets" + @echo " check run tests and print results to the console" + @echo " clean remove all generated files" + +build: +clean: + +run_% : %.py + @$(call color_out_noline,BROWN, running) + @echo " $<" + @$(RUNNER) $< diff --git a/tests/flit_cli/flit_update/README.md b/tests/flit_cli/flit_update/README.md new file mode 100644 index 00000000..e699da98 --- /dev/null +++ b/tests/flit_cli/flit_update/README.md @@ -0,0 +1,29 @@ +# Tests + +## Tests covering `[[compiler]]` section + +1. `tst_nocompilers.py`: Test that not specifying the `[[compiler]]` section + gives the default values +2. `tst_nooptl.py`: Test that specifying a compiler, but not specifying the + optimization levels gives the default values +3. `tst_noswitches.py`: Test that specifying a compiler, but not specifying the + switches list gives the default values +4. `tst_onlyprovidedoptlswitches.py`: Test that the provided list of + optimization levels and switches are used, and nothing more. +5. `tst_onlyprovidedcompilers.py`: Test that by only specifying one compiler, + only that specified compiler is used +6. `tst_badconfig.py`: Tests error cases in the configuration file, such as + specifying more than one of a certain type of compiler + +# Needed tests + +1. `FLIT_INC_DIR`, `FLIT_LIB_DIR`, `FLIT_DATA_DIR`, and `FLIT_SCRIPT_DIR`, for + both from git repository and for an installed FLiT +2. `DEV_CC`, `DEV_OPTL`, and `DEV_SWITCHES` for both provided and non-provided +3. `GT_CC`, `GT_OPTL`, and `GT_SWITCHES` for both provided and non-provided +4. `TEST_RUN_ARGS` from `timing`, `timing_loops`, and `timing_repeats` for both + provided and non-provided +5. `ENABLE_MPI` and `MPIRUN_ARGS` for both provided and non-provided +6. `HOSTNAME` +7. `UNAME_S` + diff --git a/tests/flit_cli/flit_update/data/nocompilers.toml b/tests/flit_cli/flit_update/data/nocompilers.toml new file mode 100644 index 00000000..f0d03294 --- /dev/null +++ b/tests/flit_cli/flit_update/data/nocompilers.toml @@ -0,0 +1,2 @@ +# Test that with no compilers specified the default three compilers with their +# default flags are used. diff --git a/tests/flit_cli/flit_update/data/nooptl.toml b/tests/flit_cli/flit_update/data/nooptl.toml new file mode 100644 index 00000000..445b031e --- /dev/null +++ b/tests/flit_cli/flit_update/data/nooptl.toml @@ -0,0 +1,93 @@ +# Test that missing optimization levels causes the defaults to be used + +[[compiler]] +binary = 'g++' +name = 'g++' +type = 'gcc' + +# missing optimization levels + +switches_list = [ + '-fassociative-math', + '-fcx-fortran-rules', + '-fcx-limited-range', + '-fexcess-precision=fast', + '-ffinite-math-only', + '-ffloat-store', + '-ffp-contract=on', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-funsafe-math-optimizations', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '' +] + +[[compiler]] +binary = 'clang++' +name = 'clang++' +type = 'clang' + +# missing optimization levels + +switches_list = [ + '-fassociative-math', + '-fexcess-precision=fast', + '-fexcess-precision=standard', + '-ffinite-math-only', + '-ffloat-store', + '-ffp-contract=on', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-fsingle-precision-constant', + '-funsafe-math-optimizations', + '-march=core-avx2', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '' +] + +[[compiler]] +binary = 'icpc' +name = 'icpc' +type = 'intel' + +# missing optimization levels + +switches_list = [ + '--use_fast_math', + '-fcx-limited-range', + '-ffloat-store', + '-fma', + '-fmerge-all-constants', + '-fp-model fast=1', + '-fp-model fast=2', + '-fp-model=double', + '-fp-model=except', + '-fp-model=extended', + '-fp-model=precise', + '-fp-model=source', + '-fp-model=strict', + '-fp-port', + '-frounding-math', + '-fsingle-precision-constant', + '-ftz', + '-march=core-avx2', + '-mavx', + '-mavx2 -mfma', + '-mfpmath=sse -mtune=native', + '-mp1', + '-no-fma', + '-no-ftz', + '-no-prec-div', + '-prec-div', + '' +] diff --git a/tests/flit_cli/flit_update/data/noswitches.toml b/tests/flit_cli/flit_update/data/noswitches.toml new file mode 100644 index 00000000..da8e8f02 --- /dev/null +++ b/tests/flit_cli/flit_update/data/noswitches.toml @@ -0,0 +1,40 @@ +# Test that missing switches causes the compiler-specific defaults to be used + +[[compiler]] +binary = 'g++' +name = 'g++' +type = 'gcc' +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] + +# Missing switches + +[[compiler]] +binary = 'clang++' +name = 'clang++' +type = 'clang' +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] + +# Missing switches + +[[compiler]] +binary = 'icpc' +name = 'icpc' +type = 'intel' +optimization_levels = [ + '-O0', + '-O1', + '-O2', + '-O3', +] + +# Missing switches diff --git a/tests/flit_cli/flit_update/data/onlyprovidedcompilers.toml b/tests/flit_cli/flit_update/data/onlyprovidedcompilers.toml new file mode 100644 index 00000000..131b55a9 --- /dev/null +++ b/tests/flit_cli/flit_update/data/onlyprovidedcompilers.toml @@ -0,0 +1,6 @@ +# Test only providing g++ + +[[compiler]] +binary = '/usr/bin/my-g++' +name = 'g++' +type = 'gcc' diff --git a/tests/flit_cli/flit_update/data/onlyprovidedoptlswitches.toml b/tests/flit_cli/flit_update/data/onlyprovidedoptlswitches.toml new file mode 100644 index 00000000..838b770e --- /dev/null +++ b/tests/flit_cli/flit_update/data/onlyprovidedoptlswitches.toml @@ -0,0 +1,46 @@ +# Test that only the provided optimization levels and switches are used + +[[compiler]] +binary = 'g++' +name = 'g++' +type = 'gcc' + +optimization_levels = [ + '-O2', + '-O3', +] +switches_list = [ + '-funsafe-math-optimizations', + '-mavx', + '' +] + +[[compiler]] +binary = 'clang++' +name = 'clang++' +type = 'clang' + +optimization_levels = [ + '-O0', + '-O1', +] +switches_list = [ + '-ffinite-math-only', + '-ffloat-store', +] + +[[compiler]] +binary = 'icpc' +name = 'icpc' +type = 'intel' + +optimization_levels = [ + '-Ofast', + '-Og', +] +switches_list = [ + '-fmerge-all-constants', + '-fp-model fast=1', + '-fp-model fast=2', + '-DUSE_MPI' +] diff --git a/tests/flit_cli/flit_update/tst_badconfig.py b/tests/flit_cli/flit_update/tst_badconfig.py new file mode 100644 index 00000000..495501f0 --- /dev/null +++ b/tests/flit_cli/flit_update/tst_badconfig.py @@ -0,0 +1,178 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Tests error cases in the configuration file, such as specifying more than one of a certain type of compiler. + +>>> from io import StringIO +>>> import os +>>> import shutil + +>>> from tst_common_funcs import runconfig + +>>> configstr = \\ +... '[dev_build]\\n' \\ +... 'compiler_name = \\'name-does-not-exist\\'\\n' +>>> runconfig(configstr) +Traceback (most recent call last): +... +AssertionError: Compiler name name-does-not-exist not found + +>>> configstr = \\ +... '[ground_truth]\\n' \\ +... 'compiler_name = \\'another-name-that-does-not-exist\\'\\n' +>>> runconfig(configstr) +Traceback (most recent call last): +... +AssertionError: Compiler name another-name-that-does-not-exist not found + +>>> runconfig('[compiler]\\n') +Traceback (most recent call last): +... +tst_common_funcs.UpdateTestError: Failed to update Makefile: Error: flit-config.toml improperly configured, needs [[compiler]] section + +>>> runconfig('[[compiler]]\\n') +Traceback (most recent call last): +... +tst_common_funcs.UpdateTestError: Failed to update Makefile: Error: flit-config.toml: compiler "{}" is missing the "name" field + +>>> runconfig('[[compiler]]\\n' +... 'name = \\'hello\\'\\n') +Traceback (most recent call last): +... +tst_common_funcs.UpdateTestError: Failed to update Makefile: Error: flit-config.toml: compiler "{'name': 'hello'}" is missing the "type" field + +>>> runconfig('[[compiler]]\\n' +... 'name = \\'hello\\'\\n' +... 'type = \\'gcc\\'\\n') # doctest:+ELLIPSIS +Traceback (most recent call last): +... +tst_common_funcs.UpdateTestError: Failed to update Makefile: Error: flit-config.toml: compiler "{...}" is missing the "binary" field + +>>> runconfig('[[compiler]]\\n' +... 'binary = \\'my-special-compiler\\'\\n' +... 'name = \\'hello\\'\\n' +... 'type = \\'my-unsupported-type\\'\\n') +Traceback (most recent call last): +... +tst_common_funcs.UpdateTestError: Failed to update Makefile: Error: flit-config.toml: unsupported compiler type "my-unsupported-type" + +>>> runconfig('[[compiler]]\\n' +... 'binary = \\'gcc\\'\\n' +... 'name = \\'gcc\\'\\n' +... 'type = \\'gcc\\'\\n' +... '\\n' +... '[[compiler]]\\n' +... 'binary = \\'gcc-2\\'\\n' +... 'name = \\'gcc-2\\'\\n' +... 'type = \\'gcc\\'\\n' +... ) +Traceback (most recent call last): +... +tst_common_funcs.UpdateTestError: Failed to update Makefile: Error: flit-config.toml: cannot have multiple compilers of the same type (gcc) + +>>> runconfig('[[compiler]]\\n' +... 'binary = \\'gcc\\'\\n' +... 'name = \\'gcc\\'\\n' +... 'type = \\'gcc\\'\\n' +... '\\n' +... '[[compiler]]\\n' +... 'binary = \\'gcc-2\\'\\n' +... 'name = \\'gcc\\'\\n' +... 'type = \\'clang\\'\\n' +... ) +Traceback (most recent call last): +... +tst_common_funcs.UpdateTestError: Failed to update Makefile: Error: flit-config.toml: cannot have multiple compilers of the same name (gcc) +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/tests/flit_cli/flit_update/tst_common_funcs.py b/tests/flit_cli/flit_update/tst_common_funcs.py new file mode 100644 index 00000000..f1cffa70 --- /dev/null +++ b/tests/flit_cli/flit_update/tst_common_funcs.py @@ -0,0 +1,195 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- +''' +This module holds common functions used in all of the tests for +`flit_update.py` +''' + +from io import StringIO +import os +import sys + +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +class UpdateTestError(RuntimeError): + 'Error used in runconfig()' + pass + +def deref_makelist(name, makevars): + ''' + The makevars are a dictionary of key list pairs of strings in the + form of a dictionary. This function is useful when a variable within + makevars contains a list of other variables within makevars. This + dereferences all of them and returns a sorted list of them all. + + @param name (str) variable name from makevars + @param makevars (dict{str: list(str)}) all makefile variables + + See some examples to see exactly what to expect: + + >>> makevars = { + ... 'A': ['one', 'first'], 'B': ['two', 'second'], + ... 'C': ['three', 'third'], 'D': ['four', 'fourth'], + ... 'evens': ['D', 'B'], 'odds': ['A', 'C'] + ... } + + >>> deref_makelist('evens', makevars) + ['four fourth', 'two second'] + + >>> deref_makelist('odds', makevars) + ['one first', 'three third'] + ''' + return sorted([' '.join(makevars[x]) for x in makevars[name]]) + +def get_default_compiler(typename): + ''' + Returns the default compiler from the default toml configuration. + + @param typename (str) name of the type of compiler to extract + @return (dict) default compiler values from the default flit config + + >>> gcc = get_default_compiler('gcc') + >>> gcc['binary'] + 'g++' + >>> gcc['type'] + 'gcc' + >>> gcc['name'] + 'g++' + + >>> clang = get_default_compiler('clang') + >>> clang['optimization_levels'] + ['-O0', '-O1', '-O2', '-O3'] + >>> clang['binary'] + 'clang++' + + >>> intel = get_default_compiler('intel') + >>> intel['binary'] + 'icpc' + ''' + defaults = th.util.get_default_toml() + default_compiler = [x for x in defaults['compiler'] + if x['type'] == typename] + assert len(default_compiler) == 1 + return default_compiler[0] + +def runconfig(configstr): + ''' + Runs `flit init`, then writes the given configuration string into + `flit-config.toml`, and then runs `flit update`. + + @param configstr (str) contents to put into `flit-config.toml` + @return (tuple(list(str), list(str), dict{str: list(str)}) + Three things are returned: + 1. init_out (list(str)): the lines of output from `flit init` + 2. update_out (list(str)): the lines of output from `flit update` + 3. makevars (dict{str: list(str)}): all Makefile variables from the + `Makefile` generated from `flit update` + + I will not put tests for this function here since this is tested by its use + in the other test functions. + ''' + with th.tempdir() as temp_dir: + with StringIO() as ostream: + retval = th.flit.main(['init', '-C', temp_dir], + outstream=ostream, errstream=ostream) + init_out = ostream.getvalue().splitlines() + if retval != 0: + raise UpdateTestError('Failed to initialize flit directory') + with open(os.path.join(temp_dir, 'flit-config.toml'), 'w') as fout: + print(configstr, file=fout, flush=True) + with StringIO() as ostream: + retval = th.flit.main(['update', '-C', temp_dir], + outstream=ostream, errstream=ostream) + update_out = ostream.getvalue().splitlines() + if retval != 0: + raise UpdateTestError('Failed to update Makefile: ' + + ' '.join(update_out)) + makevars = th.util.extract_make_vars(directory=temp_dir) + return (init_out, update_out, makevars) + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/scripts/flitcli/config/flit-default-future.toml.in b/tests/flit_cli/flit_update/tst_nocompilers.py similarity index 55% rename from scripts/flitcli/config/flit-default-future.toml.in rename to tests/flit_cli/flit_update/tst_nocompilers.py index a0873550..838623f3 100644 --- a/scripts/flitcli/config/flit-default-future.toml.in +++ b/tests/flit_cli/flit_update/tst_nocompilers.py @@ -79,97 +79,93 @@ # purposes. # # -- LICENSE END -- -# Autogenerated by "flit init" -# flit version {flit_version} - -[database] - -# older versions of flit supported postgres. that has been removed. only -# sqlite is supported at the moment. -type = 'sqlite' - -# if relative path, it is relative to the directory containing this -# configuration file. -filepath = 'results.sqlite' - - -[run] - -# Set this to false to not do any timing at all -timing = true - -# The number of loops to run with the timing. For values < 0, the number of -# loops to run will be determined automatically. -timing_loops = -1 - -# How many times to repeat the timing. The minimum of the repeated timings -# will be kept. -timing_repeats = 3 - - -# For now, only the first host is supported, all others are ignored -[[hosts]] - -# TODO: add documentation here for each element. - -name = '{hostname}' -flit_path = '{flit_path}' -config_dir = '{config_dir}' - -# The settings for "make dev" -[hosts.dev_build] -# compiler_name must be found in [[hosts.compilers]] list under name attribute -# but the optimization level and switches do not need to be in the compiler list -compiler_name = 'g++' -optimization_level = '-O2' -switches = '-funsafe-math-optimizations' - -# The ground truth compilation to use in analysis, for "make gt" -[hosts.ground_truth] -# compiler_name must be found in [[hosts.compilers]] list under name attribute -# but the optimization level and switches do not need to be in the compiler list -compiler_name = 'g++' -optimization_level = '-O0' -switches = '' - - # This host's list of compilers. - # For now, only used for hosts.ground_truth and hosts.dev_build. - # TODO: use this list to generate the Makefile - [[hosts.compilers]] - - # TODO: figure out how to specify path for each host machine - # TODO: what if the compilers list is part of the host? - - # binary can be an absolute path, relative path, or binary name (found in - # PATH). If you want to specify a compiler in the same directory as this - # config file, prepend with a "./" (e.g. "./my-compiler") - binary = 'g++' - # TODO: this is not yet used... - # It is recommended to include version number in the name. This is how the - # compiler will be recognized in the results, and subsequently in the - # database and analysis. - name = 'g++' - # TODO: implement these supported types - # There are a few supported types: [ gcc, intel, clang ] - type = 'gcc' - optimization_levels = [ - '-O0', - '-O1', - '-O2', - '-O3', - #'-Ofast', - #'-O...' ? - ] - # Note: in some versions of python-toml, there is a parsing bug when a list - # has an empty string in the middle. So simply put it at the end without a - # comma. This has been fixed in the latest version of python-toml. - switches_list = [ - '-fassociative-math', - '-mavx', - '-mp1', - '-mavx2 -mfma', - '' - # ... - ] - +''' +Tests that specifying no compilers results in the default compilers with the +default flags. + +>>> from io import StringIO +>>> import os +>>> import shutil + +>>> from tst_common_funcs import ( +... deref_makelist, get_default_compiler, runconfig) + +>>> testconf = 'data/nocompilers.toml' +>>> with open(testconf, 'r') as fin: +... init_out, update_out, makevars = runconfig(fin.read()) + +Get default values for each compiler from the default configuration + +>>> default_gcc = get_default_compiler('gcc') +>>> default_clang = get_default_compiler('clang') +>>> default_intel = get_default_compiler('intel') + +>>> print('\\n'.join(init_out)) # doctest:+ELLIPSIS +Creating .../flit-config.toml +Creating .../custom.mk +Creating .../main.cpp +Creating .../tests/Empty.cpp +Creating .../Makefile + +>>> print('\\n'.join(update_out)) # doctest:+ELLIPSIS +Updating .../Makefile + +Check that the compiler binaries are the default values +>>> gcc = makevars['GCC'] +>>> len(gcc) == 1 +True +>>> gcc[0] == default_gcc['binary'] +True + +>>> clang = makevars['CLANG'] +>>> len(clang) == 1 +True +>>> clang[0] == default_clang['binary'] +True + +>>> intel = makevars['INTEL'] +>>> len(intel) == 1 +True +>>> intel[0] == default_intel['binary'] +True + +>>> sorted(makevars['COMPILERS']) +['CLANG', 'GCC', 'INTEL'] + +>>> deref_makelist('OPCODES_GCC', makevars) == \\ +... sorted(default_gcc['optimization_levels']) +True + +>>> deref_makelist('OPCODES_CLANG', makevars) == \\ +... sorted(default_clang['optimization_levels']) +True + +>>> deref_makelist('OPCODES_INTEL', makevars) == \\ +... sorted(default_intel['optimization_levels']) +True + +>>> deref_makelist('SWITCHES_GCC', makevars) == \\ +... sorted(default_gcc['switches_list']) +True + +>>> deref_makelist('SWITCHES_CLANG', makevars) == \\ +... sorted(default_clang['switches_list']) +True + +>>> deref_makelist('SWITCHES_INTEL', makevars) == \\ +... sorted(default_intel['switches_list']) +True +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/tests/flit_cli/flit_update/tst_nooptl.py b/tests/flit_cli/flit_update/tst_nooptl.py new file mode 100644 index 00000000..2570d89e --- /dev/null +++ b/tests/flit_cli/flit_update/tst_nooptl.py @@ -0,0 +1,135 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Test that missing optimization levels causes the defaults to be used + +>>> from io import StringIO +>>> import os +>>> import shutil + +>>> from tst_common_funcs import ( +... deref_makelist, get_default_compiler, runconfig) + +>>> testconf = 'data/nooptl.toml' +>>> with open(testconf, 'r') as fin: +... init_out, update_out, makevars = runconfig(fin.read()) + +Get default values for each compiler from the default configuration +>>> default_gcc = get_default_compiler('gcc') +>>> default_clang = get_default_compiler('clang') +>>> default_intel = get_default_compiler('intel') + +>>> print('\\n'.join(init_out)) # doctest:+ELLIPSIS +Creating .../flit-config.toml +Creating .../custom.mk +Creating .../main.cpp +Creating .../tests/Empty.cpp +Creating .../Makefile + +>>> print('\\n'.join(update_out)) # doctest:+ELLIPSIS +Updating .../Makefile + +>>> deref_makelist('OPCODES_GCC', makevars) == \\ +... sorted(default_gcc['optimization_levels']) +True + +>>> deref_makelist('OPCODES_CLANG', makevars) == \\ +... sorted(default_clang['optimization_levels']) +True + +>>> deref_makelist('OPCODES_INTEL', makevars) == \\ +... sorted(default_intel['optimization_levels']) +True +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/tests/flit_cli/tst_bisect.py b/tests/flit_cli/flit_update/tst_noswitches.py similarity index 74% rename from tests/flit_cli/tst_bisect.py rename to tests/flit_cli/flit_update/tst_noswitches.py index 1052aba5..b07091c2 100644 --- a/tests/flit_cli/tst_bisect.py +++ b/tests/flit_cli/flit_update/tst_noswitches.py @@ -81,17 +81,55 @@ # -- LICENSE END -- ''' -Tests flit bisect functionality. +Test that missing switches causes the compiler-specific defaults to be used + +>>> from io import StringIO +>>> import shutil + +>>> from tst_common_funcs import ( +... deref_makelist, get_default_compiler, runconfig) + +>>> testconf = 'data/noswitches.toml' +>>> with open(testconf, 'r') as fin: +... init_out, update_out, makevars = runconfig(fin.read()) + +Get default values for each compiler from the default configuration + +>>> default_gcc = get_default_compiler('gcc') +>>> default_clang = get_default_compiler('clang') +>>> default_intel = get_default_compiler('intel') + +>>> print('\\n'.join(init_out)) # doctest:+ELLIPSIS +Creating .../flit-config.toml +Creating .../custom.mk +Creating .../main.cpp +Creating .../tests/Empty.cpp +Creating .../Makefile + +>>> print('\\n'.join(update_out)) # doctest:+ELLIPSIS +Updating .../Makefile + +>>> deref_makelist('SWITCHES_GCC', makevars) == \\ +... sorted(default_gcc['switches_list']) +True + +>>> deref_makelist('SWITCHES_CLANG', makevars) == \\ +... sorted(default_clang['switches_list']) +True + +>>> deref_makelist('SWITCHES_INTEL', makevars) == \\ +... sorted(default_intel['switches_list']) +True ''' # Test setup before the docstring is run. import sys before_path = sys.path[:] -sys.path.append('..') +sys.path.append('../..') import test_harness as th sys.path = before_path if __name__ == '__main__': - import doctest - doctest.testmod() - + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/tests/flit_cli/flit_update/tst_onlyprovidedcompilers.py b/tests/flit_cli/flit_update/tst_onlyprovidedcompilers.py new file mode 100644 index 00000000..f0f48af7 --- /dev/null +++ b/tests/flit_cli/flit_update/tst_onlyprovidedcompilers.py @@ -0,0 +1,148 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Tests only providing gcc and no other compiler. Tests that +- the binary is taken from the configuration file +- the other two compilers are not present (i.e. clang and intel) + +>>> from io import StringIO +>>> import os +>>> import shutil + +>>> from tst_common_funcs import ( +... deref_makelist, get_default_compiler, runconfig) + +>>> testconf = 'data/onlyprovidedcompilers.toml' +>>> with open(testconf, 'r') as fin: +... init_out, update_out, makevars = runconfig(fin.read()) + +Get default values for each compiler from the default configuration + +>>> print('\\n'.join(init_out)) # doctest:+ELLIPSIS +Creating .../flit-config.toml +Creating .../custom.mk +Creating .../main.cpp +Creating .../tests/Empty.cpp +Creating .../Makefile + +>>> print('\\n'.join(update_out)) # doctest:+ELLIPSIS +Updating .../Makefile + +Check that the compiler binaries are the default values +>>> makevars['GCC'] +['/usr/bin/my-g++'] + +>>> makevars['CLANG'] +['None'] + +>>> makevars['INTEL'] +['None'] + +>>> makevars['COMPILERS'] +['GCC'] + +>>> 'OPCODES_GCC' in makevars +True +>>> 'SWITCHES_GCC' in makevars +True +>>> 'OPCODES_CLANG' in makevars +False +>>> 'SWITCHES_CLANG' in makevars +False +>>> 'OPCODES_INTEL' in makevars +False +>>> 'SWITCHES_INTEL' in makevars +False +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/tests/flit_cli/flit_update/tst_onlyprovidedoptlswitches.py b/tests/flit_cli/flit_update/tst_onlyprovidedoptlswitches.py new file mode 100644 index 00000000..8299b1f9 --- /dev/null +++ b/tests/flit_cli/flit_update/tst_onlyprovidedoptlswitches.py @@ -0,0 +1,142 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Test that only the provided optimization levels and switches are used + +>>> from io import StringIO +>>> import shutil +>>> import os + +>>> from tst_common_funcs import ( +... deref_makelist, get_default_compiler, runconfig) + +>>> testconf = 'data/onlyprovidedoptlswitches.toml' +>>> with open(testconf, 'r') as fin: +... init_out, update_out, makevars = runconfig(fin.read()) + +Get default values for each compiler from the default configuration + +>>> default_gcc = get_default_compiler('gcc') +>>> default_clang = get_default_compiler('clang') +>>> default_intel = get_default_compiler('intel') + +>>> print('\\n'.join(init_out)) # doctest:+ELLIPSIS +Creating .../flit-config.toml +Creating .../custom.mk +Creating .../main.cpp +Creating .../tests/Empty.cpp +Creating .../Makefile + +>>> print('\\n'.join(update_out)) # doctest:+ELLIPSIS +Updating .../Makefile + +>>> deref_makelist('OPCODES_GCC', makevars) +['-O2', '-O3'] + +>>> deref_makelist('OPCODES_CLANG', makevars) +['-O0', '-O1'] + +>>> deref_makelist('OPCODES_INTEL', makevars) +['-Ofast', '-Og'] + +>>> deref_makelist('SWITCHES_GCC', makevars) +['', '-funsafe-math-optimizations', '-mavx'] + +>>> deref_makelist('SWITCHES_CLANG', makevars) +['-ffinite-math-only', '-ffloat-store'] + +>>> deref_makelist('SWITCHES_INTEL', makevars) +['-DUSE_MPI', '-fmerge-all-constants', '-fp-model fast=1', '-fp-model fast=2'] +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('../..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) diff --git a/tests/flit_cli/flit_version/Makefile b/tests/flit_cli/flit_version/Makefile new file mode 100644 index 00000000..ce0c8e04 --- /dev/null +++ b/tests/flit_cli/flit_version/Makefile @@ -0,0 +1,23 @@ +RUNNER := python3 +SRC := $(wildcard tst_*.py) +RUN_TARGETS := $(SRC:%.py=run_%) + +include ../../color_out.mk + +.PHONY: check help clean build run_% +check: $(TARGETS) $(RUN_TARGETS) + +help: + @echo "Makefile for running tests on FLiT framework" + @echo " help print this help documentation and exit" + @echo " build just compile the targets" + @echo " check run tests and print results to the console" + @echo " clean remove all generated files" + +build: +clean: + +run_% : %.py + @$(call color_out_noline,BROWN, running) + @echo " $<" + @$(RUNNER) $< diff --git a/tests/flit_cli/tst_version.py b/tests/flit_cli/flit_version/tst_version.py similarity index 99% rename from tests/flit_cli/tst_version.py rename to tests/flit_cli/flit_version/tst_version.py index afb041f0..cdb86d20 100644 --- a/tests/flit_cli/tst_version.py +++ b/tests/flit_cli/flit_version/tst_version.py @@ -128,7 +128,7 @@ # Test setup before the docstring is run. import sys before_path = sys.path[:] -sys.path.append('..') +sys.path.append('../..') import test_harness as th sys.path = before_path diff --git a/tests/flit_install/Makefile b/tests/flit_install/Makefile index 242995ab..6cc755e5 100644 --- a/tests/flit_install/Makefile +++ b/tests/flit_install/Makefile @@ -2,6 +2,8 @@ RUNNER := python3 SRC := $(wildcard tst_*.py) RUN_TARGETS := $(SRC:%.py=run_%) +include ../color_out.mk + .PHONY: check help clean build run_% check: $(TARGETS) $(RUN_TARGETS) @@ -16,4 +18,6 @@ build: clean: run_% : %.py + @$(call color_out_noline,BROWN, running) + @echo " $<" @$(RUNNER) $< diff --git a/tests/flit_install/tst_uninstall_runthrough.py b/tests/flit_install/tst_uninstall_runthrough.py index b442035a..9f4a2670 100644 --- a/tests/flit_install/tst_uninstall_runthrough.py +++ b/tests/flit_install/tst_uninstall_runthrough.py @@ -121,10 +121,12 @@ ... stdout=subp.DEVNULL, stderr=subp.DEVNULL) ... prevdir = os.path.realpath(os.curdir) ... os.chdir(temp_dir) -... all_files = glob.glob('**', recursive=True) +... all_files = [os.path.join(base, name) +... for base, folders, filenames in os.walk('.') +... for name in folders + filenames] ... os.chdir(prevdir) >>> sorted(all_files) -['lib', 'lib/otherlib.so'] +['./lib', './lib/otherlib.so'] ''' # Test setup before the docstring is run. diff --git a/tests/flit_makefile/Makefile b/tests/flit_makefile/Makefile index 242995ab..6cc755e5 100644 --- a/tests/flit_makefile/Makefile +++ b/tests/flit_makefile/Makefile @@ -2,6 +2,8 @@ RUNNER := python3 SRC := $(wildcard tst_*.py) RUN_TARGETS := $(SRC:%.py=run_%) +include ../color_out.mk + .PHONY: check help clean build run_% check: $(TARGETS) $(RUN_TARGETS) @@ -16,4 +18,6 @@ build: clean: run_% : %.py + @$(call color_out_noline,BROWN, running) + @echo " $<" @$(RUNNER) $< diff --git a/tests/flit_makefile/fake_clang34.py b/tests/flit_makefile/fake_clang34.py new file mode 100755 index 00000000..bc586bc3 --- /dev/null +++ b/tests/flit_makefile/fake_clang34.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- +'Pretend to be clang 3.4, specifically checking for unsupported flags' + +import sys + +def print_version(): + 'Print fake version information' + print('clang version 3.4.1 (tags/RELEASE_341/final)') + print('Target: x86_64-pc-linux-gnu') + print('Thread model: posix') + print('InstalledDir: /usr/bin') + +def main(arguments): + 'Main logic here' + + recognized_arguments = [ + '-fno-pie', + '-std', + '-g', + '-o', + '-nopie', + '-fassociative-math', + '-mavx', + '-fexcess-precision', + '-ffinite-math-only', + '-mavx2', + '-mfma', + '-march', + '-ffp-contract', + '-ffloat-store', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-fsingle-precision-constant', + '-mfpmath', + '-mtune', + '-funsafe-math-optimizations', + '-MMD', + '-MP', + '-MF', + '-c' + ] + + recognized_beginnings = [ + '-W', + '-D', + '-I', + '-l', + '-L', + '-O', + ] + + if '--version' in arguments: + print_version() + return 0 + + if '-dumpversion' in arguments: + print('3.4.0') + return 0 + + for arg in arguments: + canonical = arg.split('=', maxsplit=1)[0] + if canonical.startswith('-'): + recognized = canonical in recognized_arguments or \ + any(canonical.startswith(x) for x in recognized_beginnings) + if not recognized: + print('Error: unrecognized argument "{0}"'.format(arg), + file=sys.stderr) + return 1 + + if '-o' in arguments or '--output' in arguments: + idx = arguments.index('-o' if '-o' in arguments else '--output') + outfile = arguments[idx + 1] + open(outfile, 'a').close() # create an empty file if it does not exist + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tests/flit_makefile/fake_gcc4.py b/tests/flit_makefile/fake_gcc4.py new file mode 100755 index 00000000..1f7cd6f0 --- /dev/null +++ b/tests/flit_makefile/fake_gcc4.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- +'Pretend to be gcc 4.8.4, specifically checking for unsupported flags' + +import sys + +def print_version(): + 'Print fake version information' + print('Ubuntu g++ (GCC) 4.8.4') + print('Copyright (C) 2018 Free Software Foundation, Inc.') + print('This is a fake GCC compiler that does not actually do anything') + +def main(arguments): + 'Main logic here' + + recognized_arguments = [ + '-fno-pie', + '-std', + '-g', + '-o', + '-fassociative-math', + '-mavx', + '-fexcess-precision', + '-ffinite-math-only', + '-mavx2', + '-fcx-fortran-rules', + '-ffp-contract', + '-ffloat-store', + '-fcx-limited-range', + '-fmerge-all-constants', + '-fno-trapping-math', + '-freciprocal-math', + '-frounding-math', + '-fsignaling-nans', + '-mfpmath', + '-funsafe-math-optimizations', + '-MMD', + '-MP', + '-MF', + '-c' + ] + + recognized_beginnings = [ + '-W', + '-D', + '-I', + '-l', + '-L', + '-O', + ] + + if '--version' in arguments: + print_version() + return 0 + + if '-dumpversion' in arguments: + print('4.8.4') + return 0 + + + for arg in arguments: + canonical = arg.split('=', maxsplit=1)[0] + if canonical.startswith('-'): + recognized = canonical in recognized_arguments or \ + any(canonical.startswith(x) for x in recognized_beginnings) + if not recognized: + print('Error: unrecognized argument "{0}"'.format(arg), + file=sys.stderr) + return 1 + + if '-o' in arguments or '--output' in arguments: + idx = arguments.index('-o' if '-o' in arguments else '--output') + outfile = arguments[idx + 1] + open(outfile, 'a').close() # create an empty file if it does not exist + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tests/flit_makefile/tst_clang34.py b/tests/flit_makefile/tst_clang34.py new file mode 100644 index 00000000..3366c785 --- /dev/null +++ b/tests/flit_makefile/tst_clang34.py @@ -0,0 +1,140 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Tests FLiT's capabilities with compiling against older versions of Clang + +Let's now make a temporary directory and test that using the fake clang, we can +verify correct usage. + +>>> from io import StringIO +>>> import glob +>>> import os +>>> import shutil +>>> import subprocess as subp + +>>> with th.tempdir() as temp_dir: +... with StringIO() as ostream: +... _ = th.flit.main(['init', '-C', temp_dir], outstream=ostream) +... init_out = ostream.getvalue().splitlines() +... os.remove(os.path.join(temp_dir, 'flit-config.toml')) +... with open(os.path.join(temp_dir, 'flit-config.toml'), 'w') as conf: +... _ = conf.write('[dev_build]\\n') +... _ = conf.write("compiler_name = 'fake-clang'\\n") +... _ = conf.write('[ground_truth]\\n') +... _ = conf.write("compiler_name = 'fake-clang'\\n") +... _ = conf.write('[[compiler]]\\n') +... _ = conf.write("binary = './fake_clang34.py'\\n") +... _ = conf.write("name = 'fake-clang'\\n") +... _ = conf.write("type = 'clang'\\n") +... _ = shutil.copy('fake_clang34.py', temp_dir) +... _ = subp.check_output(['make', '--always-make', 'Makefile', '-C', temp_dir]) +... make_out = subp.check_output(['make', 'gt', '-C', temp_dir]) +... make_out = make_out.decode('utf8').splitlines() + +Verify the output of flit init +>>> print('\\n'.join(init_out)) # doctest:+ELLIPSIS +Creating .../flit-config.toml +Creating .../custom.mk +Creating .../main.cpp +Creating .../tests/Empty.cpp +Creating .../Makefile + +Check the output of Make +>>> any(['-no-pie' in x for x in make_out]) +False +>>> len([1 for x in make_out if '-nopie' in x]) +1 +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) + diff --git a/tests/flit_makefile/tst_empty_project.py b/tests/flit_makefile/tst_empty_project.py index 10240ce2..320af4c5 100644 --- a/tests/flit_makefile/tst_empty_project.py +++ b/tests/flit_makefile/tst_empty_project.py @@ -87,10 +87,15 @@ Let's now make a temporary directory and test that flit init populates the correct files + +>>> class TestError(RuntimeError): pass + >>> import glob >>> import os >>> with th.tempdir() as temp_dir: -... th.flit.main(['init', '-C', temp_dir]) # doctest:+ELLIPSIS +... retval = th.flit.main(['init', '-C', temp_dir]) # doctest:+ELLIPSIS +... if retval != 0: +... raise TestError('Main returned {}'.format(retval)) ... files = os.listdir(temp_dir) Creating /.../flit-config.toml Creating /.../custom.mk diff --git a/tests/flit_makefile/tst_gcc4.py b/tests/flit_makefile/tst_gcc4.py new file mode 100644 index 00000000..fb783ffa --- /dev/null +++ b/tests/flit_makefile/tst_gcc4.py @@ -0,0 +1,140 @@ +# -- LICENSE BEGIN -- +# +# Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# Written by +# Michael Bentley (mikebentley15@gmail.com), +# Geof Sawaya (fredricflinstone@gmail.com), +# and Ian Briggs (ian.briggs@utah.edu) +# under the direction of +# Ganesh Gopalakrishnan +# and Dong H. Ahn. +# +# LLNL-CODE-743137 +# +# All rights reserved. +# +# This file is part of FLiT. For details, see +# https://pruners.github.io/flit +# Please also read +# https://github.com/PRUNERS/FLiT/blob/master/LICENSE +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the disclaimer below. +# +# - Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the disclaimer +# (as noted below) in the documentation and/or other materials +# provided with the distribution. +# +# - Neither the name of the LLNS/LLNL nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# +# Additional BSD Notice +# +# 1. This notice is required to be provided under our contract +# with the U.S. Department of Energy (DOE). This work was +# produced at Lawrence Livermore National Laboratory under +# Contract No. DE-AC52-07NA27344 with the DOE. +# +# 2. Neither the United States Government nor Lawrence Livermore +# National Security, LLC nor any of their employees, makes any +# warranty, express or implied, or assumes any liability or +# responsibility for the accuracy, completeness, or usefulness of +# any information, apparatus, product, or process disclosed, or +# represents that its use would not infringe privately-owned +# rights. +# +# 3. Also, reference herein to any specific commercial products, +# process, or services by trade name, trademark, manufacturer or +# otherwise does not necessarily constitute or imply its +# endorsement, recommendation, or favoring by the United States +# Government or Lawrence Livermore National Security, LLC. The +# views and opinions of authors expressed herein do not +# necessarily state or reflect those of the United States +# Government or Lawrence Livermore National Security, LLC, and +# shall not be used for advertising or product endorsement +# purposes. +# +# -- LICENSE END -- + +''' +Tests FLiT's capabilities with compiling against older versions of GCC + +Let's now make a temporary directory and test that using the fake gcc, we can +verify correct usage. + +>>> from io import StringIO +>>> import glob +>>> import os +>>> import shutil +>>> import subprocess as subp + +>>> with th.tempdir() as temp_dir: +... with StringIO() as ostream: +... _ = th.flit.main(['init', '-C', temp_dir], outstream=ostream) +... init_out = ostream.getvalue().splitlines() +... os.remove(os.path.join(temp_dir, 'flit-config.toml')) +... with open(os.path.join(temp_dir, 'flit-config.toml'), 'w') as conf: +... _ = conf.write('[dev_build]\\n') +... _ = conf.write("compiler_name = 'fake-gcc'\\n") +... _ = conf.write('[ground_truth]\\n') +... _ = conf.write("compiler_name = 'fake-gcc'\\n") +... _ = conf.write('[[compiler]]\\n') +... _ = conf.write("binary = './fake_gcc4.py'\\n") +... _ = conf.write("name = 'fake-gcc'\\n") +... _ = conf.write("type = 'gcc'\\n") +... _ = shutil.copy('fake_gcc4.py', temp_dir) +... _ = subp.check_output(['make', '--always-make', 'Makefile', '-C', temp_dir]) +... make_out = subp.check_output(['make', 'gt', '-C', temp_dir]) +... make_out = make_out.decode('utf8').splitlines() + +Verify the output of flit init +>>> print('\\n'.join(init_out)) # doctest:+ELLIPSIS +Creating .../flit-config.toml +Creating .../custom.mk +Creating .../main.cpp +Creating .../tests/Empty.cpp +Creating .../Makefile + +Check the output of Make +>>> any(['-no-pie' in x for x in make_out]) +False +>>> any(['-nopie' in x for x in make_out]) +False +''' + +# Test setup before the docstring is run. +import sys +before_path = sys.path[:] +sys.path.append('..') +import test_harness as th +sys.path = before_path + +if __name__ == '__main__': + from doctest import testmod + failures, tests = testmod() + sys.exit(failures) + diff --git a/tests/flit_mpi/Makefile b/tests/flit_mpi/Makefile index 1d04452e..0a7cd3c3 100644 --- a/tests/flit_mpi/Makefile +++ b/tests/flit_mpi/Makefile @@ -1,8 +1,8 @@ -RUNNER := python3 -SRC := $(wildcard tst_*.py) -RUN_TARGETS := $(SRC:%.py=run_%) -MPIRUN := $(shell command -v mpirun 2>/dev/null) -MPICXX := $(shell command -v mpic++ 2>/dev/null) +RUNNER := python3 +SRC := $(wildcard tst_*.py) +RUN_TARGETS := $(SRC:%.py=run_%) +MPIRUN := $(shell command -v mpirun 2>/dev/null) +MPICXX := $(shell command -v mpic++ 2>/dev/null) include ../color_out.mk @@ -28,4 +28,6 @@ build: clean: run_% : %.py + @$(call color_out_noline,BROWN, running) + @echo " $<" @$(RUNNER) $< diff --git a/tests/flit_mpi/data/flit-config.toml b/tests/flit_mpi/data/flit-config.toml index 63aeaf74..686e8e0b 100644 --- a/tests/flit_mpi/data/flit-config.toml +++ b/tests/flit_mpi/data/flit-config.toml @@ -1,3 +1,5 @@ [run] enable_mpi = true mpirun_args = '-n 2' + +# Use default for all other options diff --git a/tests/flit_mpi/tst_run_mpi.py b/tests/flit_mpi/tst_run_mpi.py index 10d38069..13308c46 100644 --- a/tests/flit_mpi/tst_run_mpi.py +++ b/tests/flit_mpi/tst_run_mpi.py @@ -93,12 +93,20 @@ >>> import shutil >>> import subprocess as subp +>>> class TestError(RuntimeError): pass + >>> with th.tempdir() as temp_dir: -... th.flit.main(['init', '-C', temp_dir]) # doctest:+ELLIPSIS +... retval = th.flit.main(['init', '-C', temp_dir]) # doctest:+ELLIPSIS +... if retval != 0: +... raise TestError('Main #1 returned with {}, failed to initialize' +... .format(retval)) ... _ = shutil.copy(os.path.join('data', 'MpiHello.cpp'), ... os.path.join(temp_dir, 'tests')) ... _ = shutil.copy(os.path.join('data', 'flit-config.toml'), temp_dir) -... th.flit.main(['update', '-C', temp_dir]) +... retval = th.flit.main(['update', '-C', temp_dir]) +... if retval != 0: +... raise TestError('Main #2 returned with {}, failed to update' +... .format(retval)) ... compile_str = subp.check_output(['make', '-C', temp_dir, 'gt'], ... stderr=subp.STDOUT) ... run_str = subp.check_output(['make', '-C', temp_dir, 'ground-truth.csv'], @@ -135,17 +143,10 @@ >>> run_str_2 = run_str_2.decode('utf-8').strip().splitlines() >>> run_str_3 = run_str_3.decode('utf-8').strip().splitlines() -Make sure the info statement about MPI being enabled is done at each make call - ->>> compile_str[1] -'MPI is enabled' ->>> run_str[1] -'MPI is enabled' - Make sure the correct arguments are passed to mpirun ->>> run_str[2] -'mpirun -n 2 ./gtrun --output ground-truth.csv --no-timing' +>>> 'mpirun -n 2 ./gtrun --output ground-truth.csv --no-timing' in run_str +True Make sure the console messages are there, but they can be out of order diff --git a/tests/flit_src/Makefile b/tests/flit_src/Makefile index 1ab42844..19377a72 100644 --- a/tests/flit_src/Makefile +++ b/tests/flit_src/Makefile @@ -8,12 +8,17 @@ CFLAGS += -std=c++11 LDFLAGS := -L$(FLITLIBDIR) -lflit LDFLAGS += -Wl,-rpath=$(FLITLIBDIR) SRC := $(wildcard *.cpp) +DEPS := $(SRC:%.cpp=%.d) TARGETS := $(SRC:%.cpp=%) RUN_TARGETS := $(TARGETS:%=run_%) HARNESS := ../test_harness.h CFLAGS += $(addprefix -I,$(dir $(HARNESS))) CFLAGS += -I../../src +DEPFLAGS = -MD -MF $*.d + +include ../color_out.mk + .PHONY: build check help clean run_% build: $(TARGETS) @@ -27,13 +32,19 @@ help: @echo " clean remove all generated files" clean: + rm -f $(DEPS) rm -f $(TARGETS) run_% : % + @$(call color_out_noline,BROWN, running) + @echo " $<" @./$< --quiet % : %.cpp $(HARNESS) Makefile - $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(DEPFLAGS) $< -o $@ $(LDFLAGS) + +.PRECIOUS: %.d +-include $(DEPS) $(TARGETS): $(FLITLIB) diff --git a/tests/flit_src/tst_FlitCsv.cpp b/tests/flit_src/tst_FlitCsv.cpp new file mode 100644 index 00000000..167a0b4d --- /dev/null +++ b/tests/flit_src/tst_FlitCsv.cpp @@ -0,0 +1,297 @@ +/* -- LICENSE BEGIN -- + * + * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * Written by + * Michael Bentley (mikebentley15@gmail.com), + * Geof Sawaya (fredricflinstone@gmail.com), + * and Ian Briggs (ian.briggs@utah.edu) + * under the direction of + * Ganesh Gopalakrishnan + * and Dong H. Ahn. + * + * LLNL-CODE-743137 + * + * All rights reserved. + * + * This file is part of FLiT. For details, see + * https://pruners.github.io/flit + * Please also read + * https://github.com/PRUNERS/FLiT/blob/master/LICENSE + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the disclaimer below. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the disclaimer + * (as noted below) in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the LLNS/LLNL nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL + * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * Additional BSD Notice + * + * 1. This notice is required to be provided under our contract + * with the U.S. Department of Energy (DOE). This work was + * produced at Lawrence Livermore National Laboratory under + * Contract No. DE-AC52-07NA27344 with the DOE. + * + * 2. Neither the United States Government nor Lawrence Livermore + * National Security, LLC nor any of their employees, makes any + * warranty, express or implied, or assumes any liability or + * responsibility for the accuracy, completeness, or usefulness of + * any information, apparatus, product, or process disclosed, or + * represents that its use would not infringe privately-owned + * rights. + * + * 3. Also, reference herein to any specific commercial products, + * process, or services by trade name, trademark, manufacturer or + * otherwise does not necessarily constitute or imply its + * endorsement, recommendation, or favoring by the United States + * Government or Lawrence Livermore National Security, LLC. The + * views and opinions of authors expressed herein do not + * necessarily state or reflect those of the United States + * Government or Lawrence Livermore National Security, LLC, and + * shall not be used for advertising or product endorsement + * purposes. + * + * -- LICENSE END -- + */ + +#include "test_harness.h" + +#include "FlitCsv.h" + +namespace { + +std::ostream& operator<<(std::ostream& out, std::vector vec) { + bool first = true; + out << "["; + for (std::string &val : vec) { + if (!first) { + out << ", "; + } + first = false; + out << '"' << val << '"'; + } + out << "]"; + return out; +} + +std::ostream& operator<<(std::ostream& out, flit::CsvRow& row) { + out << "CsvRow:\n"; + if (row.header() == nullptr) { + out << " Header: NULL\n"; + } else { + out << " Header: " + << static_cast>(*row.header()) + << std::endl; + } + out << " Values: " << static_cast>(row) + << std::endl; + return out; +} + +} // end of unnamed namespace + +namespace tst_CsvRow { +void tst_CsvRow_header() { + flit::CsvRow row {"1", "2", "3", "4"}; + TH_EQUAL(row.header(), nullptr); + row.setHeader(&row); + TH_EQUAL(row.header(), &row); +} +TH_REGISTER(tst_CsvRow_header); + +void tst_CsvRow_operator_brackets_string() { + flit::CsvRow row {"1", "2", "3", "4"}; + flit::CsvRow header {"a", "b", "c", "d"}; + row.setHeader(&header); + TH_EQUAL(row["a"], "1"); + TH_EQUAL(row["b"], "2"); + TH_EQUAL(row["c"], "3"); + TH_EQUAL(row["d"], "4"); + TH_THROWS(row["Mike"], std::invalid_argument); + + // Row missing elements + header.emplace_back("e"); + TH_THROWS(row["e"], std::out_of_range); + + // null header + row.setHeader(nullptr); + TH_THROWS(row["a"], std::logic_error); +} +TH_REGISTER(tst_CsvRow_operator_brackets_string); + +void tst_CsvRow_operator_brackets_int() { + flit::CsvRow row {"1", "2", "3", "4"}; + flit::CsvRow header {"a", "b", "c", "d"}; + row.setHeader(&header); + TH_EQUAL(row[0], "1"); + TH_EQUAL(row[1], "2"); + TH_EQUAL(row[2], "3"); + TH_EQUAL(row[3], "4"); + TH_THROWS(row.at(4), std::out_of_range); + + // Row missing elements + header.emplace_back("e"); + TH_EQUAL(row.at(3), "4"); + TH_THROWS(row.at(4), std::out_of_range); + + // null header + row.setHeader(nullptr); + TH_EQUAL(row.at(2), "3"); + TH_THROWS(row.at(4), std::out_of_range); +} +TH_REGISTER(tst_CsvRow_operator_brackets_int); +} // end of namespace tst_CsvRow + +namespace tst_CsvReader { +void tst_CsvReader_oneRowAtATime() { + std::istringstream in( + "first,second,third,fourth\n" // header row + "a, b,c,\n" + "1,2,3,4,5,6,7\n" + "\n" + "hello,\"there,my\",friends,\"newline \n" + "in quotes\"," + ); + + flit::CsvReader reader(in); + flit::CsvRow expected_header {"first", "second", "third", "fourth"}; + TH_EQUAL(*reader.header(), expected_header); + + flit::CsvRow row; + reader >> row; + flit::CsvRow expected_row; + expected_row = {"a", " b", "c", ""}; + expected_row.setHeader(&expected_header); + TH_EQUAL(row, expected_row); + TH_VERIFY(reader); + + reader >> row; + expected_row = {"1", "2", "3", "4", "5", "6", "7"}; + expected_row.setHeader(&expected_header); + TH_EQUAL(row, expected_row); + TH_VERIFY(reader); + + reader >> row; + expected_row = {""}; + expected_row.setHeader(&expected_header); + TH_EQUAL(row, expected_row); + TH_VERIFY(reader); + + reader >> row; + expected_row = {"hello", "there,my", "friends", "newline \nin quotes", ""}; + expected_row.setHeader(&expected_header); + TH_EQUAL(row, expected_row); + TH_VERIFY(reader); + + reader >> row; + expected_row = {}; + expected_row.setHeader(&expected_header); + TH_VERIFY(row.empty()); + TH_EQUAL(row, expected_row); + TH_VERIFY(!reader); +} +TH_REGISTER(tst_CsvReader_oneRowAtATime); + +void tst_CsvReader_createRowVector() { + std::istringstream in( + "first,second,third,fourth\n" // header row + "a, b,c,\n" + "1,2,3,4,5,6,7\n" + "\n" + "hello,\"there,my\",friends,\"newline \n" + "in quotes\"," + ); + + flit::CsvReader reader(in); + flit::CsvRow expected_header {"first", "second", "third", "fourth"}; + TH_EQUAL(*reader.header(), expected_header); + + std::vector expected_rows = { + {"a", " b", "c", ""}, + {"1", "2", "3", "4", "5", "6", "7"}, + {""}, + {"hello", "there,my", "friends", "newline \nin quotes", ""}, + }; + for (auto &row : expected_rows) { row.setHeader(&expected_header); } + decltype(expected_rows) rows; + for (flit::CsvRow row; reader >> row;) { rows.emplace_back(row); } + TH_EQUAL(rows, expected_rows); +} + +} // end of namespace tst_CsvReader + + +namespace tst_CsvWriter { + +/// Tests that the CsvWriter will end the file in a newline (if writing rows) +void tst_CsvWriter_write_row_addsNewline() { + std::istringstream in( + "first,second,third,fourth\n" // header row + "a, b,c," + ); + std::ostringstream out; + + flit::CsvReader reader(in); + flit::CsvWriter writer(out); + + writer.write_row(*reader.header()); + flit::CsvRow row; + while (reader >> row) { writer.write_row(row); } + + TH_EQUAL(out.str().back(), '\n'); + TH_EQUAL(in.str() + '\n', out.str()); +} +TH_REGISTER(tst_CsvWriter_write_row_addsNewline); + +/// Tests that CsvWriter can write out the exact same csv read in +void tst_CsvWriter_write_row_exactly() { + std::istringstream in( + "first,second,third,fourth\n" // header row + "a, b,c,\n" + "1,2,3,4,5,6,7\n" + "\n" + "hello,\"there,my\",friends,\"newline \n" + "in quotes\",\n" + ); + std::ostringstream out; + + flit::CsvReader reader(in); + flit::CsvWriter writer(out); + + writer.write_row(*reader.header()); + flit::CsvRow row; + while (reader >> row) { writer.write_row(row); } + + TH_EQUAL(in.str(), out.str()); +} +TH_REGISTER(tst_CsvWriter_write_row_exactly); + +} // end of namespace tst_CsvWriter + diff --git a/tests/flit_src/tst_flit_cpp.cpp b/tests/flit_src/tst_flit_cpp.cpp index 8971e0e1..1ca81d3f 100644 --- a/tests/flit_src/tst_flit_cpp.cpp +++ b/tests/flit_src/tst_flit_cpp.cpp @@ -90,6 +90,7 @@ #include #include +#include #include #include #include @@ -133,58 +134,6 @@ std::ostream& operator<<(std::ostream& out, const std::vector &v) { } } // end of unnamed namespace -namespace tst_CsvRow { -void tst_CsvRow_header() { - CsvRow row {"1", "2", "3", "4"}; - TH_EQUAL(row.header(), nullptr); - row.setHeader(&row); - TH_EQUAL(row.header(), &row); -} -TH_REGISTER(tst_CsvRow_header); - -void tst_CsvRow_operator_brackets() { - CsvRow row {"1", "2", "3", "4"}; - CsvRow header {"a", "b", "c", "d"}; - row.setHeader(&header); - TH_EQUAL(row["a"], "1"); - TH_EQUAL(row["b"], "2"); - TH_EQUAL(row["c"], "3"); - TH_EQUAL(row["d"], "4"); - TH_THROWS(row["Mike"], std::invalid_argument); - - // Row missing elements - header.emplace_back("e"); - TH_THROWS(row["e"], std::out_of_range); - - // null header - row.setHeader(nullptr); - TH_THROWS(row["a"], std::logic_error); -} -TH_REGISTER(tst_CsvRow_operator_brackets); -} // end of namespace tst_CsvRow - -void tst_Csv() { - std::istringstream in( - "first,second,third,fourth\n" - "a, b,c,\n" - "1,2,3,4,5,6,7\n" - "\n" - ); - Csv csv(in); - CsvRow row; - csv >> row; - auto &header = *row.header(); - TH_EQUAL(header, CsvRow({"first", "second", "third", "fourth"})); - TH_EQUAL(row, CsvRow({"a", " b", "c", ""})); - - csv >> row; - TH_EQUAL(row, CsvRow({"1", "2", "3", "4", "5", "6", "7"})); - - csv >> row; - TH_EQUAL(row, CsvRow({""})); -} -TH_REGISTER(tst_Csv); - void tst_isIn() { // an empty vector TH_VERIFY(!isIn(std::vector{}, "")); @@ -577,7 +526,7 @@ TH_REGISTER(tst_readFile_exists); void tst_readFile_doesnt_exist() { TH_THROWS(flit::readFile("/this/file/should/not/exist"), - std::system_error); + std::ios_base::failure); } TH_REGISTER(tst_readFile_doesnt_exist); diff --git a/tests/test_harness.h b/tests/test_harness.h index 0a03ab7c..77c37ad6 100644 --- a/tests/test_harness.h +++ b/tests/test_harness.h @@ -130,14 +130,14 @@ // Assertion definitions #define TH_VERIFY_MSG(x, msg) \ if (!(x)) { \ - throw th::AssertionError(__func__, __LINE__, msg);\ + throw th::AssertionError(__FILE__, __func__, __LINE__, msg);\ } #define TH_VERIFY(x) TH_VERIFY_MSG(x, "TH_VERIFY("#x")") #define TH_EQUAL(a, b) TH_VERIFY_MSG((a) == (b), "TH_EQUAL("#a", "#b")") #define TH_NOT_EQUAL(a, b) TH_VERIFY_MSG((a) != (b), "TH_NOT_EQUAL("#a", "#b")") #define TH_FAIL(msg) \ TH_VERIFY_MSG(false, std::string("TH_FAIL(\"") + msg + "\")") -#define TH_SKIP(msg) throw th::SkipError(__func__, __LINE__, msg) +#define TH_SKIP(msg) throw th::SkipError(__FILE__, __func__, __LINE__, msg) #define TH_THROWS(exp, exception) \ try { \ exp; \ @@ -170,18 +170,21 @@ namespace th { // Signals an assertion failure class AssertionError : public std::exception { public: - AssertionError(std::string func, long line, std::string msg) - : _func(func), _line(line), _msg(msg) + AssertionError(std::string file, std::string func, long line, + std::string msg) + : _file(file), _func(func), _line(line), _msg(msg) { std::ostringstream msg_stream; msg_stream - << "Assertion failure: " << _func << ":" << _line << " - " << _msg; + << "Assertion failure: " << _file << ":" + << _line << " in " << _func << " - " << _msg; _what = msg_stream.str(); } virtual const char* what() const noexcept override { return _what.c_str(); } protected: + std::string _file; std::string _func; long _line; std::string _msg; @@ -191,12 +194,13 @@ namespace th { // Signals a test that is skipped class SkipError : public AssertionError { public: - SkipError(std::string func, long line, std::string msg) - : AssertionError(func, line, msg) + SkipError(std::string file, std::string func, long line, std::string msg) + : AssertionError(file, func, line, msg) { std::ostringstream msg_stream; msg_stream - << "Test skipped: " << _func << ":" << _line << " - " << _msg; + << "Test skipped: " << _file << ":" + << _line << " in " << _func << " - " << _msg; _what = msg_stream.str(); } }; @@ -226,10 +230,10 @@ int main(int argCount, char *argList[]) { } skipped_tests.emplace_back(test_name); } catch (const th::AssertionError &err) { - std::cout << test_name << ": " << err.what() << "\n"; + std::cout << test_name << ":\n " << err.what() << "\n"; failed_tests.emplace_back(test_name); } catch (const std::exception &err) { - std::cout << test_name << ": Uncought exception - " + std::cout << test_name << ": Uncought exception\n " << typeid(err).name() << ": " << err.what() << "\n"; failed_tests.emplace_back(test_name); } catch (...) { @@ -272,4 +276,3 @@ int main(int argCount, char *argList[]) { } #endif // TEST_HARNESS_H - diff --git a/tests/test_harness.py b/tests/test_harness.py index 73a4b354..4df30665 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -185,24 +185,37 @@ def tempdir(*args, **kwargs): ... print(os.path.isdir(temporary_directory)) ... True - >>> print(os.path.isdir(temporary_directory)) + >>> os.path.isdir(temporary_directory) False - >>> print(os.path.exists(temporary_directory)) + >>> os.path.exists(temporary_directory) False Test that an exception is not thrown if it was already deleted >>> import shutil >>> with tempdir() as new_dir: ... shutil.rmtree(new_dir) + + Test that the directory is still deleted even if an exception is thrown + within the with statement. + >>> try: + ... with tempdir() as new_dir: + ... temporary_directory = new_dir + ... raise RuntimeError() + ... except: + ... pass + >>> os.path.isdir(temporary_directory) + False ''' import tempfile import shutil new_dir = tempfile.mkdtemp(*args, **kwargs) - yield new_dir try: - shutil.rmtree(new_dir) - except FileNotFoundError: - pass + yield new_dir + finally: + try: + shutil.rmtree(new_dir) + except FileNotFoundError: + pass def touch(filename): ''' @@ -214,6 +227,7 @@ def touch(filename): flit = _path_import(_script_dir, 'flit') config = _path_import(_script_dir, 'flitconfig') +util = _path_import(_script_dir, 'flitutil') # Remove the things that are no longer necessary del contextmanager