From 1dbb938139fbdab45218cea2a13d383f9eeaa98e Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Mon, 3 Feb 2020 22:46:27 -0700 Subject: [PATCH 01/21] Add bash-completion to the installation --- Makefile | 32 ++-- scripts/bash-completion/flit | 278 +++++++++++++++++++++++++++++++++++ 2 files changed, 296 insertions(+), 14 deletions(-) create mode 100644 scripts/bash-completion/flit diff --git a/Makefile b/Makefile index b06a94af..b16ffe1f 100644 --- a/Makefile +++ b/Makefile @@ -11,21 +11,23 @@ FLIT_HEADERS += $(wildcard src/flit/*.h) # Install variables -SCRIPT_DIR := scripts/flitcli -DATA_DIR := data -CONFIG_DIR := $(SCRIPT_DIR)/config -DOC_DIR := documentation -LITMUS_TESTS += $(wildcard litmus-tests/tests/*.cpp) -LITMUS_TESTS += $(wildcard litmus-tests/tests/*.h) +SCRIPT_DIR := scripts/flitcli +BASH_COMPLETE_DIR := scripts/bash-completion +DATA_DIR := data +CONFIG_DIR := $(SCRIPT_DIR)/config +DOC_DIR := documentation +LITMUS_TESTS += $(wildcard litmus-tests/tests/*.cpp) +LITMUS_TESTS += $(wildcard litmus-tests/tests/*.h) -EFFECTIVE_PREFIX := $(DESTDIR)$(PREFIX) -INST_BINDIR := $(EFFECTIVE_PREFIX)/bin -INST_LIBDIR := $(EFFECTIVE_PREFIX)/lib -INST_INCLUDEDIR := $(EFFECTIVE_PREFIX)/include/flit -INST_SHAREDIR := $(EFFECTIVE_PREFIX)/share/flit -INST_SRCDIR := $(EFFECTIVE_PREFIX)/share/flit/src -INST_LICENSEDIR := $(EFFECTIVE_PREFIX)/share/licenses/flit -INST_FLIT_CONFIG := $(EFFECTIVE_PREFIX)/share/flit/scripts/flitconfig.py +EFFECTIVE_PREFIX := $(DESTDIR)$(PREFIX) +INST_BINDIR := $(EFFECTIVE_PREFIX)/bin +INST_LIBDIR := $(EFFECTIVE_PREFIX)/lib +INST_INCLUDEDIR := $(EFFECTIVE_PREFIX)/include/flit +INST_SHAREDIR := $(EFFECTIVE_PREFIX)/share/flit +INST_SRCDIR := $(EFFECTIVE_PREFIX)/share/flit/src +INST_LICENSEDIR := $(EFFECTIVE_PREFIX)/share/licenses/flit +INST_FLIT_CONFIG := $(EFFECTIVE_PREFIX)/share/flit/scripts/flitconfig.py +INST_BASH_COMPLETE_DIR := $(EFFECTIVE_PREFIX)/share/bash-completion/completions -include tests/color_out.mk @@ -67,6 +69,7 @@ install: @$(call color_out,BLUE,Installing: DESTDIR=$(DESTDIR) PREFIX=$(PREFIX)) $(MKDIR) $(INST_BINDIR) $(MKDIR) $(INST_INCLUDEDIR) + $(MKDIR) $(INST_BASH_COMPLETE_DIR) $(MKDIR) $(INST_SHAREDIR)/scripts/experimental $(MKDIR) $(INST_SHAREDIR)/doc $(MKDIR) $(INST_SHAREDIR)/data/tests @@ -88,6 +91,7 @@ install: install -m 0644 $(SCRIPT_DIR)/flitutil.py $(INST_SHAREDIR)/scripts/ install -m 0644 $(SCRIPT_DIR)/flitelf.py $(INST_SHAREDIR)/scripts/ install -m 0644 $(SCRIPT_DIR)/README.md $(INST_SHAREDIR)/scripts/ + install -m 0644 $(BASH_COMPLETE_DIR)/* $(INST_BASH_COMPLETE_DIR)/ install -m 0644 $(DOC_DIR)/*.md $(INST_SHAREDIR)/doc/ install -m 0644 $(DATA_DIR)/Makefile.in $(INST_SHAREDIR)/data/ install -m 0644 $(DATA_DIR)/Makefile_bisect_binary.in $(INST_SHAREDIR)/data/ diff --git a/scripts/bash-completion/flit b/scripts/bash-completion/flit new file mode 100644 index 00000000..59ebaedc --- /dev/null +++ b/scripts/bash-completion/flit @@ -0,0 +1,278 @@ +# +# bash completion for the flit command +# + +_flit__sqlite_files() +{ + # match files ending in one of + # .db, .sdb, .sqlite, .db3, .s3db, .sqlite3, .sl3 + # * First call _filedir to populate everything into ${COMPREPLY[]} + # * Then filter out by file ending + _filedir + mapfile -t COMPREPLY < <( for entry in "${COMPREPLY[@]}"; do + if [ -f "${entry}" ]; then + echo "${entry}" | egrep ".*\.(db|sdb|sqlite|db3|s3db|sqlite3|sl3)$"; + else + echo "${entry}"; + fi + done + ) + return 0 +} + +_flit_help() +{ + local cur available_subcommands + available_subcommands="bisect init make update import" + cur="${COMP_WORDS[COMP_CWORD]}" + COMPREPLY=( $(compgen -W "${available_subcommands}" -- ${cur}) ) +} + +_flit_bisect() +{ + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-h --help + -C --directory + -p --precision + -a --auto-sqlite-run + --parallel + -v --verbose + -j --jobs + -d --delete + -k --biggest + --compile-only + --precompile-fpic + --skip-verification + -t --compiler-type" + + case "${prev}" in + + -C|--directory) + _filedir -d + return + ;; + + -p|--precision) + # Because one of the options has a space, we use the + # "mapfile -t COMPREPLY" + # approach instead + #COMPREPLY=( $(compgen -W "float double \"long double\"" -- ${cur}) ) + mapfile -t COMPREPLY < <( compgen -W "float double 'long\\ double'" -- "${cur}") + return 0 + ;; + + -a|--auto-sqlite-run) + _flit__sqlite_files + return 0 + ;; + + --parallel|-j|--jobs|-k|--biggest) + # these take integers. What options can we give? + return 0 + ;; + + -t|--compiler-type) + COMPREPLY=( $(compgen -W "clang gcc intel misc auto" -- "${cur}") ) + return 0 + ;; + + esac + + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + + # TODO: help construct the compilation string + # TODO: If there has already been one positional argument, then give a list + # of test names + + return 0 +} + +_flit_init() +{ + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-h --help + -C --directory + --overwrite + -L --litmus-tests" + + case "${prev}" in + -C|--directory) + _filedir -d + return 0 + ;; + esac + + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 +} + +_flit_make() +{ + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-h --help + -j --jobs + --exec-jobs + -q --quiet + --make-args + -l --label" + + case "${prev}" in + + -j|--jobs|--exec-jobs|--make-args|-l|--label) + # do no completion -- numbers, make arguments, and label + return 0 + ;; + + esac + + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 +} + +_flit_update() +{ + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-h --help + -C --directory" + + case "${prev}" in + + -C|--directory) + _filedir -d + return 0 + ;; + + esac + + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 +} + +_flit_import() +{ + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-h --help + -a --append + -l --label + -r --run + -D --dbfile" + + case "${prev}" in + + -a|--append|-l|--label|-r|--run) + # no completion -- numbers and labels + return 0 + ;; + + -D|--dbfile) + _flit__sqlite_files + return 0 + ;; + + esac + + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + _filedir # as positional arguments, we can specify files + + return 0 +} + +_flit_experimental_help() +{ + local cur available_subcommands + available_subcommands="ninja" + cur="${COMP_WORDS[COMP_CWORD]}" + COMPREPLY=( $(compgen -W "${available_subcommands}" -- ${cur}) ) +} + +_flit_experimental_ninja() +{ + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-h --help + -C --directory + -q --quiet" + + case "${prev}" in + + -C|--directory) + _filedir -d + return 0 + ;; + + esac + + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 +} + +_flit_experimental() +{ + local cur prev subcommand available_subcommands + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + subcommand="${COMP_WORDS[2]}" + available_subcommands="--help --version help ninja" + + # subcommand completion + if [ ${COMP_CWORD} -eq 2 ]; then + COMPREPLY=( $(compgen -W "${available_subcommands}" -- "${cur}") ) + return 0 + fi + + # subcommand subcompletion + case "${subcommand}" in + help) _flit_experimental_help ;; + ninja) _flit_experimental_ninja ;; + esac + + return 0 +} + +_flit() +{ + local cur prev subcommand available_subcommands + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + subcommand="${COMP_WORDS[1]}" + available_subcommands="--help --version + experimental help bisect init make update import" + + # subcommand completion + if [ ${COMP_CWORD} -le 1 ]; then + COMPREPLY=( $(compgen -W "${available_subcommands}" -- "${cur}") ) + return 0 + fi + + # subcommand subcompletion + case "${subcommand}" in + help) _flit_help ;; + bisect) _flit_bisect ;; + init) _flit_init ;; + make) _flit_make ;; + update) _flit_update ;; + import) _flit_import ;; + experimental) _flit_experimental ;; + esac + + return 0 +} +complete -o bashdefault -F _flit flit 2>/dev/null || \ + complete -F _flit flit From 5a45f2b7efaf472cd23303ce2fa1957a68500d40 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 4 Feb 2020 08:00:20 -0700 Subject: [PATCH 02/21] Makefile: implement uninstall of bash-completion --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index b16ffe1f..d884ad98 100644 --- a/Makefile +++ b/Makefile @@ -165,7 +165,9 @@ uninstall: $(RMDIR) $(INST_LICENSEDIR) $(RM) $(INST_BINDIR)/flit $(RM) $(EFFECTIVE_PREFIX)/include/flit.h + $(RM) $(INST_BASH_COMPLETE_DIR)/flit -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/include 2>/dev/null + -rmdir --ignore-fail-on-non-empty $(INST_BASH_COMPLETE_DIR) 2>/dev/null -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/share/licenses 2>/dev/null -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/share 2>/dev/null -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/bin 2>/dev/null From 0ad6ea7f726242aa50819daace63744461ed14bd Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 4 Feb 2020 08:00:41 -0700 Subject: [PATCH 03/21] Makefile: add to installation output --- Makefile | 108 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/Makefile b/Makefile index d884ad98..7d5e62ce 100644 --- a/Makefile +++ b/Makefile @@ -81,9 +81,12 @@ install: $(MKDIR) $(INST_LICENSEDIR) @$(call color_out,CYAN, Creating command-line symbolic link $(INST_BINDIR)/flit) ln -sf ../share/flit/scripts/flit.py $(INST_BINDIR)/flit + @$(call color_out,BROWN, Installing includes into $(INST_INCLUDEDIR)) install -m 0644 $(HEADERS) $(EFFECTIVE_PREFIX)/include install -m 0644 $(FLIT_HEADERS) $(INST_INCLUDEDIR) + @$(call color_out,BROWN, Installing source into $(INST_SRCDIR)) install -m 0644 $(SOURCE) $(INST_SRCDIR) + @$(call color_out,BROWN, Installing scripts into $(INST_SHAREDIR)/scripts) install -m 0755 $(SCRIPT_DIR)/flit.py $(INST_SHAREDIR)/scripts/ install -m 0755 $(SCRIPT_DIR)/flit_*.py $(INST_SHAREDIR)/scripts/ install -m 0755 $(SCRIPT_DIR)/experimental/flit_*.py $(INST_SHAREDIR)/scripts/experimental/ @@ -91,7 +94,10 @@ install: install -m 0644 $(SCRIPT_DIR)/flitutil.py $(INST_SHAREDIR)/scripts/ install -m 0644 $(SCRIPT_DIR)/flitelf.py $(INST_SHAREDIR)/scripts/ install -m 0644 $(SCRIPT_DIR)/README.md $(INST_SHAREDIR)/scripts/ + @$(call color_out,BROWN, Intalling bash-completion script into $(INST_BASH_COMPLETE_DIR)) + @$(call color_out,GREEN, You can source it in your ~/.bashrc or copy it to /etc/bash_completion.d/) install -m 0644 $(BASH_COMPLETE_DIR)/* $(INST_BASH_COMPLETE_DIR)/ + @$(call color_out,BROWN, Installing documentation into $(INST_SHAREDIR)/doc) install -m 0644 $(DOC_DIR)/*.md $(INST_SHAREDIR)/doc/ install -m 0644 $(DATA_DIR)/Makefile.in $(INST_SHAREDIR)/data/ install -m 0644 $(DATA_DIR)/Makefile_bisect_binary.in $(INST_SHAREDIR)/data/ @@ -101,60 +107,62 @@ install: install -m 0644 $(DATA_DIR)/db/tables-sqlite.sql $(INST_SHAREDIR)/data/db/ install -m 0644 $(CONFIG_DIR)/version.txt $(INST_SHAREDIR)/config/ install -m 0644 $(CONFIG_DIR)/flit-default.toml.in $(INST_SHAREDIR)/config/ + @$(call color_out,BROWN, Installing litmus tests into $(INST_SHAREDIR)/litmus-tests) install -m 0644 $(LITMUS_TESTS) $(INST_SHAREDIR)/litmus-tests/ install -m 0644 LICENSE $(INST_LICENSEDIR) + @$(call color_out,BROWN, Intsalling benchmarks into $(INST_SHAREDIR)/benchmarks) cp -r benchmarks/* $(INST_SHAREDIR)/benchmarks/ @$(call color_out,CYAN, Generating $(INST_FLIT_CONFIG)) - @# Make the flitconfig.py script specifying this installation information - @echo "'''" > $(INST_FLIT_CONFIG) - @echo "Contains paths and other configurations for the flit installation." >> $(INST_FLIT_CONFIG) - @echo "This particular file was autogenerated at the time of installation." >> $(INST_FLIT_CONFIG) - @echo "This is the file that allows installations to work from any prefix." >> $(INST_FLIT_CONFIG) - @echo "'''" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "import os" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "all = [" >> $(INST_FLIT_CONFIG) - @echo " 'version'," >> $(INST_FLIT_CONFIG) - @echo " 'script_dir'," >> $(INST_FLIT_CONFIG) - @echo " 'doc_dir'," >> $(INST_FLIT_CONFIG) - @echo " 'src_dir'," >> $(INST_FLIT_CONFIG) - @echo " 'include_dir'," >> $(INST_FLIT_CONFIG) - @echo " 'config_dir'," >> $(INST_FLIT_CONFIG) - @echo " 'data_dir'," >> $(INST_FLIT_CONFIG) - @echo " 'litmus_test_dir'," >> $(INST_FLIT_CONFIG) - @echo " ]" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "_scriptpath = os.path.dirname(os.path.abspath(__file__))" >> $(INST_FLIT_CONFIG) - @echo "_prefix = os.path.realpath(" >> $(INST_FLIT_CONFIG) - @echo " os.path.join(_scriptpath, '..', '..', '..'))" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "# flit scripts" >> $(INST_FLIT_CONFIG) - @echo "script_dir = os.path.join(_prefix, 'share', 'flit', 'scripts')" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "# flit documentation" >> $(INST_FLIT_CONFIG) - @echo "doc_dir = os.path.join(_prefix, 'share', 'flit', 'doc')" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "# flit C++ include files, primarily flit/flit.h" >> $(INST_FLIT_CONFIG) - @echo "include_dir = os.path.join(_prefix, 'include')" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "# flit C++ source files" >> $(INST_FLIT_CONFIG) - @echo "src_dir = os.path.join(_prefix, 'share', 'flit', 'src')" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "# default configuration for flit init" >> $(INST_FLIT_CONFIG) - @echo "config_dir = os.path.join(_prefix, 'share', 'flit', 'config')" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "# current version" >> $(INST_FLIT_CONFIG) - @echo "version_file = os.path.join(config_dir, 'version.txt')" >> $(INST_FLIT_CONFIG) - @echo "with open(version_file, 'r') as _version_file_opened:" >> $(INST_FLIT_CONFIG) - @echo " version = _version_file_opened.read().strip()" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "# default data files such as Makefile.in and main.cpp" >> $(INST_FLIT_CONFIG) - @echo "data_dir = os.path.join(_prefix, 'share', 'flit', 'data')" >> $(INST_FLIT_CONFIG) - @echo >> $(INST_FLIT_CONFIG) - @echo "# directory containing litmus tests" >> $(INST_FLIT_CONFIG) - @echo "litmus_test_dir = os.path.join(" >> $(INST_FLIT_CONFIG) - @echo " _prefix, 'share', 'flit', 'litmus-tests')" >> $(INST_FLIT_CONFIG) + # Make the flitconfig.py script specifying this installation information + echo "'''" > $(INST_FLIT_CONFIG) + echo "Contains paths and other configurations for the flit installation." >> $(INST_FLIT_CONFIG) + echo "This particular file was autogenerated at the time of installation." >> $(INST_FLIT_CONFIG) + echo "This is the file that allows installations to work from any prefix." >> $(INST_FLIT_CONFIG) + echo "'''" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "import os" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "all = [" >> $(INST_FLIT_CONFIG) + echo " 'version'," >> $(INST_FLIT_CONFIG) + echo " 'script_dir'," >> $(INST_FLIT_CONFIG) + echo " 'doc_dir'," >> $(INST_FLIT_CONFIG) + echo " 'src_dir'," >> $(INST_FLIT_CONFIG) + echo " 'include_dir'," >> $(INST_FLIT_CONFIG) + echo " 'config_dir'," >> $(INST_FLIT_CONFIG) + echo " 'data_dir'," >> $(INST_FLIT_CONFIG) + echo " 'litmus_test_dir'," >> $(INST_FLIT_CONFIG) + echo " ]" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "_scriptpath = os.path.dirname(os.path.abspath(__file__))" >> $(INST_FLIT_CONFIG) + echo "_prefix = os.path.realpath(" >> $(INST_FLIT_CONFIG) + echo " os.path.join(_scriptpath, '..', '..', '..'))" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "# flit scripts" >> $(INST_FLIT_CONFIG) + echo "script_dir = os.path.join(_prefix, 'share', 'flit', 'scripts')" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "# flit documentation" >> $(INST_FLIT_CONFIG) + echo "doc_dir = os.path.join(_prefix, 'share', 'flit', 'doc')" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "# flit C++ include files, primarily flit/flit.h" >> $(INST_FLIT_CONFIG) + echo "include_dir = os.path.join(_prefix, 'include')" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "# flit C++ source files" >> $(INST_FLIT_CONFIG) + echo "src_dir = os.path.join(_prefix, 'share', 'flit', 'src')" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "# default configuration for flit init" >> $(INST_FLIT_CONFIG) + echo "config_dir = os.path.join(_prefix, 'share', 'flit', 'config')" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "# current version" >> $(INST_FLIT_CONFIG) + echo "version_file = os.path.join(config_dir, 'version.txt')" >> $(INST_FLIT_CONFIG) + echo "with open(version_file, 'r') as _version_file_opened:" >> $(INST_FLIT_CONFIG) + echo " version = _version_file_opened.read().strip()" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "# default data files such as Makefile.in and main.cpp" >> $(INST_FLIT_CONFIG) + echo "data_dir = os.path.join(_prefix, 'share', 'flit', 'data')" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) + echo "# directory containing litmus tests" >> $(INST_FLIT_CONFIG) + echo "litmus_test_dir = os.path.join(" >> $(INST_FLIT_CONFIG) + echo " _prefix, 'share', 'flit', 'litmus-tests')" >> $(INST_FLIT_CONFIG) .PHONY: uninstall uninstall: From 41854ce547834fde088ad9401c7d5d63c0a7d059 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 4 Feb 2020 08:05:58 -0700 Subject: [PATCH 04/21] tests: fix install and uninstall tests --- Makefile | 3 ++- tests/flit_install/tst_install_runthrough.py | 3 +++ tests/flit_install/tst_uninstall_runthrough.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7d5e62ce..15cbd5af 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,8 @@ uninstall: $(RM) $(EFFECTIVE_PREFIX)/include/flit.h $(RM) $(INST_BASH_COMPLETE_DIR)/flit -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/include 2>/dev/null - -rmdir --ignore-fail-on-non-empty $(INST_BASH_COMPLETE_DIR) 2>/dev/null + -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/share/bash-completion/completions 2>/dev/null + -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/share/bash-completion 2>/dev/null -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/share/licenses 2>/dev/null -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/share 2>/dev/null -rmdir --ignore-fail-on-non-empty $(EFFECTIVE_PREFIX)/bin 2>/dev/null diff --git a/tests/flit_install/tst_install_runthrough.py b/tests/flit_install/tst_install_runthrough.py index 8c26f070..cade3460 100644 --- a/tests/flit_install/tst_install_runthrough.py +++ b/tests/flit_install/tst_install_runthrough.py @@ -168,6 +168,9 @@ 'include/flit/timeFunction.h', 'include/flit/tinydir.h', 'share', + 'share/bash-completion', + 'share/bash-completion/completions', + 'share/bash-completion/completions/flit', 'share/flit', 'share/flit/benchmarks', 'share/flit/benchmarks/README.md', diff --git a/tests/flit_install/tst_uninstall_runthrough.py b/tests/flit_install/tst_uninstall_runthrough.py index 4a2f263a..5f14da43 100644 --- a/tests/flit_install/tst_uninstall_runthrough.py +++ b/tests/flit_install/tst_uninstall_runthrough.py @@ -113,7 +113,7 @@ >>> sorted(dirs1) ['bin', 'include', 'share'] >>> sorted(dirs2) -['bin/flit', 'include/flit', 'include/flit.h', 'share/flit', 'share/licenses'] +['bin/flit', 'include/flit', 'include/flit.h', 'share/bash-completion', 'share/flit', 'share/licenses'] >>> tempdir_exists False From c57880fece8927ed7b25da6d205c17b5daa2c545 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Wed, 5 Feb 2020 01:08:13 -0700 Subject: [PATCH 05/21] flitconfig: add bash_completion_dir --- Makefile | 5 +++++ scripts/flitcli/flitconfig.py | 4 ++++ tests/flit_install/tst_install_runthrough.py | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/Makefile b/Makefile index 15cbd5af..f41b33da 100644 --- a/Makefile +++ b/Makefile @@ -129,6 +129,7 @@ install: echo " 'src_dir'," >> $(INST_FLIT_CONFIG) echo " 'include_dir'," >> $(INST_FLIT_CONFIG) echo " 'config_dir'," >> $(INST_FLIT_CONFIG) + echo " 'bash_completion_dir'," >> $(INST_FLIT_CONFIG) echo " 'data_dir'," >> $(INST_FLIT_CONFIG) echo " 'litmus_test_dir'," >> $(INST_FLIT_CONFIG) echo " ]" >> $(INST_FLIT_CONFIG) @@ -152,6 +153,10 @@ install: echo "# default configuration for flit init" >> $(INST_FLIT_CONFIG) echo "config_dir = os.path.join(_prefix, 'share', 'flit', 'config')" >> $(INST_FLIT_CONFIG) echo >> $(INST_FLIT_CONFIG) + echo "# where bash completion scripts for flit reside" >> $(INST_FLIT_CONFIG) + echo "bash_completion_dir = os.path.join(" >> $(INST_FLIT_CONFIG) + echo " _prefix, 'share', 'bash-completion', 'completions')" >> $(INST_FLIT_CONFIG) + echo >> $(INST_FLIT_CONFIG) echo "# current version" >> $(INST_FLIT_CONFIG) echo "version_file = os.path.join(config_dir, 'version.txt')" >> $(INST_FLIT_CONFIG) echo "with open(version_file, 'r') as _version_file_opened:" >> $(INST_FLIT_CONFIG) diff --git a/scripts/flitcli/flitconfig.py b/scripts/flitcli/flitconfig.py index a7b10f5b..ecf94a7e 100644 --- a/scripts/flitcli/flitconfig.py +++ b/scripts/flitcli/flitconfig.py @@ -96,6 +96,7 @@ 'src_dir', 'include_dir', 'config_dir', + 'bash_completion_dir', 'data_dir', 'litmus_test_dir', ] @@ -115,6 +116,9 @@ # default configuration for flit init config_dir = os.path.join(script_dir, 'config') +# where bash completion scripts for flit reside +bash_completion_dir = os.path.join(script_dir, '..', 'bash-completion') + with open(os.path.join(config_dir, 'version.txt'), 'r') as version_file: version = version_file.read().strip() diff --git a/tests/flit_install/tst_install_runthrough.py b/tests/flit_install/tst_install_runthrough.py index cade3460..c03970a4 100644 --- a/tests/flit_install/tst_install_runthrough.py +++ b/tests/flit_install/tst_install_runthrough.py @@ -138,6 +138,10 @@ >>> flitconfig.config_dir == os.path.join(prefix, 'share', 'flit', 'config') True +>>> flitconfig.bash_completion_dir == os.path.join( +... prefix, 'share', 'bash-completion', 'completions') +True + >>> flitconfig.data_dir == os.path.join(prefix, 'share', 'flit', 'data') True From 88838a2884902163d3f8612d78c5d56d181a7d22 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Wed, 5 Feb 2020 08:44:50 -0700 Subject: [PATCH 06/21] tests/bash-completion: add test for top-level flit command --- scripts/bash-completion/flit | 4 +- tests/bash_completion/tst_completion_flit.py | 122 +++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100755 tests/bash_completion/tst_completion_flit.py diff --git a/scripts/bash-completion/flit b/scripts/bash-completion/flit index 59ebaedc..34c90158 100644 --- a/scripts/bash-completion/flit +++ b/scripts/bash-completion/flit @@ -252,7 +252,9 @@ _flit() COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" subcommand="${COMP_WORDS[1]}" - available_subcommands="--help --version + available_subcommands=" + -h --help + -v --version experimental help bisect init make update import" # subcommand completion diff --git a/tests/bash_completion/tst_completion_flit.py b/tests/bash_completion/tst_completion_flit.py new file mode 100755 index 00000000..14d4af3c --- /dev/null +++ b/tests/bash_completion/tst_completion_flit.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +import os +import subprocess as subp +import sys +import unittest + +before_path = sys.path[:] +sys.path.append('..') +import test_harness as th +sys.path = before_path + +def _completion_vars(program, command): + ''' + Returns a dictionary of bash varibles to trigger bash-completion. + + >>> vals = _completion_vars('foo', 'bar --he') + >>> sorted(vals.keys()) + ['COMP_CWORD', 'COMP_LINE', 'COMP_POINT', 'COMP_WORDS'] + >>> [vals[x] for x in sorted(vals.keys())] + [2, 'foo bar --he', 12, 'foo bar --he'] + + >>> vals = _completion_vars('foo', 'bar --help ') + >>> sorted(vals.keys()) + ['COMP_CWORD', 'COMP_LINE', 'COMP_POINT', 'COMP_WORDS'] + >>> [vals[x] for x in sorted(vals.keys())] + [3, 'foo bar --help ', 15, 'foo bar --help '] + ''' + line = program + ' ' + command + words = line.rstrip() + args = command.split() + cword = len(args) + point = len(line) + if line[-1] == ' ': + words += ' ' + cword += 1 + return { + 'COMP_LINE': line, + 'COMP_WORDS': words, + 'COMP_CWORD': cword, + 'COMP_POINT': point, + } + +def get_completion(completion_file, program, command): + ''' + Returns the potential completions from a bash-completion + + @param completion_file (str): file containing bash-completion definition + @param program (str): program being invoked and completed (e.g., 'cp') + @param command (str): rest of the command-line args (e.g., '-r ./') + @return (list(str)): All potential completions if the user were to have + entered program + ' ' + command into an interactive terminal and hit + tab twice. + ''' + replacements = _completion_vars(program, command) + replacements['completion_file'] = completion_file + replacements['program'] = program + full_cmdline = ( + r'source {completion_file};' + 'COMP_LINE="{COMP_LINE}"' + ' COMP_WORDS=({COMP_WORDS})' + ' COMP_CWORD={COMP_CWORD}' + ' COMP_POINT={COMP_POINT};' + '$(complete -p {program} |' + ' sed "s/.*-F \\([^ ]*\\) .*/\\1/") &&' + ' for elem in "${{COMPREPLY[@]}}"; do' + ' echo "${{elem}}";' + ' done'.format(**replacements)) + out = subp.check_output(['bash', '-i', '-c', full_cmdline]) + return out.decode('utf-8').splitlines() + +class TestFlitBaseCompletion(unittest.TestCase): + def assertEqualCompletion(self, args, expected): + 'Asserts that the expected completions are obtained' + completion_file = os.path.join(th.config.bash_completion_dir, 'flit') + actual = get_completion(completion_file, 'flit', args) + self.assertEqual(sorted(actual), sorted(expected)) + + def test_no_args(self): + self.assertEqualCompletion('', [ + '-h', '--help', + '-v', '--version', + 'help', + 'bisect', + 'experimental', + 'import', + 'init', + 'make', + 'update', + ]) + + def test_dash(self): + self.assertEqualCompletion('-', ['-h', '--help', '-v', '--version']) + + def test_dash_help(self): + self.assertEqualCompletion('-h', ['-h']) + self.assertEqualCompletion('-h ', []) + self.assertEqualCompletion('--help', ['--help']) + self.assertEqualCompletion('--help ', []) + + def test_dash_version(self): + self.assertEqualCompletion('-v', ['-v']) + self.assertEqualCompletion('-v ', []) + self.assertEqualCompletion('--version', ['--version']) + self.assertEqualCompletion('--version ', []) + + def test_nonmatching(self): + self.assertEqualCompletion('-a', []) + self.assertEqualCompletion('a', []) + self.assertEqualCompletion('c', []) + self.assertEqualCompletion('bisecj', []) + + def test_matching_subcommand(self): + self.assertEqualCompletion('h', ['help']) + self.assertEqualCompletion('bi', ['bisect']) + self.assertEqualCompletion('exp', ['experimental']) + self.assertEqualCompletion('i', ['import', 'init']) + self.assertEqualCompletion('m', ['make']) + self.assertEqualCompletion('up', ['update']) + +if __name__ == '__main__': + unittest.main() From 70a2aa91fb28b07f4a6fd1ec039e3e5e14fa4af9 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Wed, 5 Feb 2020 08:45:25 -0700 Subject: [PATCH 07/21] tests/bash-completion: only output on failure --- tests/bash_completion/Makefile | 27 ++++++++++++++++++++ tests/bash_completion/tst_completion_flit.py | 17 +++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tests/bash_completion/Makefile diff --git a/tests/bash_completion/Makefile b/tests/bash_completion/Makefile new file mode 100644 index 00000000..2b279835 --- /dev/null +++ b/tests/bash_completion/Makefile @@ -0,0 +1,27 @@ +RUNNER := python3 +SRC := $(wildcard tst_*.py) +RUN_TARGETS := $(SRC:%.py=run_%) + +include ../color_out.mk + +ifndef VERBOSE +.SILENT: +endif + +.PHONY: check help clean build run_% +check: $(TARGETS) $(RUN_TARGETS) + +help: + @echo "Makefile for running tests on FLiT framework" + @echo " help print this help documentation and exit" + @echo " build just compile the targets" + @echo " check run tests and print results to the console" + @echo " clean remove all generated files" + +build: +clean: + +run_% : %.py + @$(call color_out_noline,BROWN, running) + @echo " $<" + $(RUNNER) $< diff --git a/tests/bash_completion/tst_completion_flit.py b/tests/bash_completion/tst_completion_flit.py index 14d4af3c..82a632c8 100755 --- a/tests/bash_completion/tst_completion_flit.py +++ b/tests/bash_completion/tst_completion_flit.py @@ -118,5 +118,20 @@ def test_matching_subcommand(self): self.assertEqualCompletion('m', ['make']) self.assertEqualCompletion('up', ['update']) +def main(): + 'Calls unittest.main(), only printing if the tests failed' + from io import StringIO + captured = StringIO() + old_stderr = sys.stderr + try: + sys.stderr = captured + result = unittest.main(exit=False) + finally: + sys.stderr = old_stderr + if not result.result.wasSuccessful(): + print(captured.getvalue()) + return 1 + return 0 + if __name__ == '__main__': - unittest.main() + sys.exit(main()) From 1d2c171b259f507ec37ba948dcbebbd76779668e Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Sat, 8 Feb 2020 23:11:58 -0700 Subject: [PATCH 08/21] Small fixes from code review --- Makefile | 2 +- scripts/flitcli/flitutil.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f41b33da..f6dccd6c 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ MKDIR ?= mkdir -m 0755 -p SOURCE := $(wildcard src/flit/*.cpp) HEADERS := src/flit.h -FLIT_HEADERS += $(wildcard src/flit/*.h) +FLIT_HEADERS := $(wildcard src/flit/*.h) # Install variables diff --git a/scripts/flitcli/flitutil.py b/scripts/flitcli/flitutil.py index 35a076c7..17deb162 100644 --- a/scripts/flitcli/flitutil.py +++ b/scripts/flitcli/flitutil.py @@ -665,6 +665,7 @@ def check_output(*args, **kwargs): Output to standard error will be suppressed >>> check_output(['python', '-c', 'import sys; print("hi", file=sys.stderr)']) + '' ''' output = subp.check_output(stderr=subp.DEVNULL, *args, **kwargs) return output.decode(encoding='utf-8') From 49f783023158f7827ea723f218663ef887bbb715 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Sat, 8 Feb 2020 23:14:26 -0700 Subject: [PATCH 09/21] tests: move bash_completion into flit_cli/ --- tests/{ => flit_cli}/bash_completion/Makefile | 0 tests/{ => flit_cli}/bash_completion/tst_completion_flit.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{ => flit_cli}/bash_completion/Makefile (100%) rename tests/{ => flit_cli}/bash_completion/tst_completion_flit.py (100%) diff --git a/tests/bash_completion/Makefile b/tests/flit_cli/bash_completion/Makefile similarity index 100% rename from tests/bash_completion/Makefile rename to tests/flit_cli/bash_completion/Makefile diff --git a/tests/bash_completion/tst_completion_flit.py b/tests/flit_cli/bash_completion/tst_completion_flit.py similarity index 100% rename from tests/bash_completion/tst_completion_flit.py rename to tests/flit_cli/bash_completion/tst_completion_flit.py From fe29859268e0a02366da192d4a3233985b492b77 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Mon, 10 Feb 2020 16:11:01 -0700 Subject: [PATCH 10/21] tests/bash-completion: fix paths since dir move --- tests/flit_cli/bash_completion/Makefile | 2 +- tests/flit_cli/bash_completion/tst_completion_flit.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/flit_cli/bash_completion/Makefile b/tests/flit_cli/bash_completion/Makefile index 2b279835..3d50838d 100644 --- a/tests/flit_cli/bash_completion/Makefile +++ b/tests/flit_cli/bash_completion/Makefile @@ -2,7 +2,7 @@ RUNNER := python3 SRC := $(wildcard tst_*.py) RUN_TARGETS := $(SRC:%.py=run_%) -include ../color_out.mk +include ../../color_out.mk ifndef VERBOSE .SILENT: diff --git a/tests/flit_cli/bash_completion/tst_completion_flit.py b/tests/flit_cli/bash_completion/tst_completion_flit.py index 82a632c8..2b70caa0 100755 --- a/tests/flit_cli/bash_completion/tst_completion_flit.py +++ b/tests/flit_cli/bash_completion/tst_completion_flit.py @@ -6,7 +6,7 @@ import unittest before_path = sys.path[:] -sys.path.append('..') +sys.path.append('../..') import test_harness as th sys.path = before_path From 2f71f5aa713cfbcd91c0921d5341ef41bde06f2b Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 18 Feb 2020 18:25:48 -0700 Subject: [PATCH 11/21] flit-cli: rearchitect argument parsing All argument parsing is done on the top-level using the populate_parser() function from all subcommand scripts. This is done to be able to generate automatic bash- completion tests and perhaps automatic bash-completion scripts. --- documentation/experimental-features.md | 2 +- scripts/flitcli/experimental/flit_ninja.py | 31 ++-- scripts/flitcli/flit.py | 173 +++++++++--------- scripts/flitcli/flit_bisect.py | 27 +-- scripts/flitcli/flit_experimental.py | 16 +- scripts/flitcli/flit_import.py | 48 ++--- scripts/flitcli/flit_init.py | 18 +- scripts/flitcli/flit_make.py | 22 ++- scripts/flitcli/flit_update.py | 21 +-- scripts/flitcli/unfinished/flit_analyze.py | 23 ++- scripts/flitcli/unfinished/flit_check.py | 31 ++-- scripts/flitcli/unfinished/flit_run.py | 24 ++- src/flit.h | 2 +- src/flit/subprocess.h | 2 +- .../flit_bisect/data/tests/BisectTest.cpp | 2 +- .../tst_build_with_compiler_specific_flags.py | 2 +- tests/flit_makefile/tst_clang34.py | 2 +- tests/test_harness.py | 2 +- 18 files changed, 244 insertions(+), 204 deletions(-) diff --git a/documentation/experimental-features.md b/documentation/experimental-features.md index 6e06631b..845f45f1 100644 --- a/documentation/experimental-features.md +++ b/documentation/experimental-features.md @@ -72,7 +72,7 @@ read variables if they are present. The variables read from `custom.mk` are This `custom.mk` file is in GNU Makefile format. Any additional variables and rules defined inside are completely ignored. Feel free to use anything from -GNU Make including `if` statements and things like `$(wildcard ../src/*.cpp)`. +GNU Make including `if` statements and things like `$(wildcard ../src/*.cpp)`. diff --git a/scripts/flitcli/experimental/flit_ninja.py b/scripts/flitcli/experimental/flit_ninja.py index 22b5cdbf..4312efba 100755 --- a/scripts/flitcli/experimental/flit_ninja.py +++ b/scripts/flitcli/experimental/flit_ninja.py @@ -102,31 +102,29 @@ BUILD_FILENAME = 'build.ninja' -def parse_args(arguments, prog=sys.argv[0]): +def populate_parser(parser=None): ''' Parse command-line arguments - >>> parse_args([]) + >>> populate_parser().parse_args([]) Namespace(directory='.', quiet=False) - >>> parse_args(['-C', 'my/dir', '-q']) + >>> populate_parser().parse_args(['-C', 'my/dir', '-q']) Namespace(directory='my/dir', quiet=True) - >>> parse_args(['--directory', 'another/dir', '--quiet']) + >>> populate_parser().parse_args(['--directory', 'another/dir', '--quiet']) Namespace(directory='another/dir', quiet=True) ''' - parser = argparse.ArgumentParser( - prog=prog, - description=''' - Generates a Ninja build file instead of a GNU Makefile for - performing the FLiT build in a FLiT test directory. - ''', - ) + if parser is None: + parser = argparse.ArgumentParser() + parser.description = ''' + Generates a Ninja build file instead of a GNU Makefile for + performing the FLiT build in a FLiT test directory. + ''' parser.add_argument('-C', '--directory', default='.', help='The directory to genreate build.ninja') parser.add_argument('-q', '--quiet', action='store_true') - args = parser.parse_args(arguments) - return args + return parser def check_output(*args, **kwargs): ''' @@ -311,7 +309,7 @@ def load_makefile(self, makefile): >>> cxxflags_orig = list(w.cxxflags) >>> ldflags_orig = list(w.ldflags) - + >>> with NamedTemporaryFile() as makefile_out: ... w.load_makefile(makefile_out.name) @@ -666,9 +664,10 @@ def write(self): n.newline() n.build('run', 'phony', [x + comparison_suffix for x in results_files]) -def main(arguments, prog=sys.argv[0]): +def main(arguments): 'Main logic here' - args = parse_args(arguments, prog=prog) + parser = populate_parser() + args = parser.parse_args(arguments) arguments = [x for x in arguments if x not in ('-q', '--quiet')] if not args.quiet: diff --git a/scripts/flitcli/flit.py b/scripts/flitcli/flit.py index 9f272850..d5b7d075 100755 --- a/scripts/flitcli/flit.py +++ b/scripts/flitcli/flit.py @@ -97,73 +97,82 @@ import importlib import os import sys +from collections import namedtuple import flitconfig as conf -def import_helper_modules(directory): - 'Imports the modules found in the given directory.' +Subcommand = namedtuple('Subcommand', + 'name, brief_description, main, populate_parser') + +def load_subcommands(directory): + ''' + Creates subcommands from modules found in the given directory. + + From that directory, all subcommands will be loaded with the pattern + flit_*.py. + + Returns a list of Subcommand tuples. + ''' if directory not in sys.path: sys.path.insert(0, directory) subcommand_files = glob.glob(os.path.join(directory, 'flit_*.py')) - subcommands = [os.path.basename(x)[5:-3] for x in subcommand_files] + subcommand_names = [os.path.basename(x)[5:-3] for x in subcommand_files] subcom_modules = [importlib.import_module(os.path.basename(x)[:-3]) for x in subcommand_files] - subcom_map = dict(zip(subcommands, subcom_modules)) + module_map = dict(zip(subcommand_names, subcom_modules)) # Make sure each module has the expected interface - for name, module in subcom_map.items(): + for name, module in module_map.items(): expected_attributes = [ 'brief_description', + 'populate_parser', 'main', ] for attr in expected_attributes: assert hasattr(module, attr), \ - 'Module {0} is missing attribute {1}'.format(name, attr) + 'Module flit_{0} is missing attribute {1}'.format(name, attr) + + subcommands = [ + Subcommand(name, m.brief_description, m.main, m.populate_parser) + for name, m in module_map.items()] - return subcom_map + return subcommands -def generate_help_documentation(subcom_map, prog=sys.argv[0], description=None): +def populate_parser(parser=None, subcommands=None): ''' - Generates and returns both the formatted help for the general flit - executable, but also for the help subcommand. They are returned as a - tuple. + Populate and return the ArgumentParser. If not given, a new one is made. + All arguments are optional. - >>> help_str, help_subcom_str = generate_help_documentation(dict()) + @param subcommands (list(Subcommand)): list of subcommands (default=None) + @param parser (argparse.ArgumentParser): parser to populate or None to + generate a new one. (default=None) ''' - if description is None: - description = ''' + if parser is None: + parser = argparse.ArgumentParser() + parser.description = ''' The flit command-line tool allows for users to write portability test cases. One can test primarily for compiler effects on reproducibility of floating-point algorithms. That at least is the main use case for this tool, although you may come up with some other uses. ''' - parser = argparse.ArgumentParser(prog=prog, description=description) - parser.add_argument('-v', '--version', action='store_true', - help='Print version and exit') - subparsers = parser.add_subparsers(metavar='subcommand', dest='subcommand') - help_subparser = subparsers.add_parser( - 'help', help='display help for a specific subcommand') - help_subparser.add_argument( - metavar='subcommand', - dest='help_subcommand', - choices=subcom_map.keys(), - help=''' - display the help documentation for a specific subcommand. - choices are {0}. - '''.format(', '.join(sorted(subcom_map.keys()))), - ) - for name, module in sorted(subcom_map.items()): - subparsers.add_parser(name, help=module.brief_description) - - # Note: we do not use parser for the actual parsing, because we want the - # arguments for each subcommand to be handled by the associated module. - # That does not seem to be well supported by the argparse module. - - return (parser.format_help(), help_subparser.format_help()) - -def main(arguments, prog=sys.argv[0], module_dir=conf.script_dir, - description=None, outstream=None, errstream=None): + parser.add_argument('-v', '--version', action='version', + version='flit version ' + conf.version, + help='print the version and exit') + if subcommands: + subparsers = parser.add_subparsers( + title='Subcommands', + dest='subcommand', + metavar='subcommand', + required=True) + for subcommand in subcommands: + subparser = subparsers.add_parser( + subcommand.name, help=subcommand.brief_description) + subcommand.populate_parser(subparser) + return parser + +def main(arguments, module_dir=conf.script_dir, outstream=None, + errstream=None): ''' Main logic here. @@ -172,7 +181,7 @@ def main(arguments, prog=sys.argv[0], module_dir=conf.script_dir, would go to the console and put it into a StringStream or maybe a file. ''' if outstream is None and errstream is None: - return _main_impl(arguments, prog, module_dir) + return _main_impl(arguments, module_dir) oldout = sys.stdout olderr = sys.stderr try: @@ -180,57 +189,55 @@ def main(arguments, prog=sys.argv[0], module_dir=conf.script_dir, sys.stdout = outstream if errstream is not None: sys.stderr = errstream - return _main_impl(arguments, prog, module_dir) + return _main_impl(arguments, module_dir) finally: sys.stdout = oldout sys.stderr = olderr -def _main_impl(arguments, prog, module_dir, description=None): - 'Implementation of main' - subcom_map = import_helper_modules(module_dir) - - help_str, help_subcommand_str = generate_help_documentation( - subcom_map, prog=prog, description=description) - if len(arguments) == 0 or arguments[0] in ('-h', '--help'): - print(help_str) - return 0 - - if arguments[0] in ('-v', '--version'): - print('flit version', conf.version) - return 0 - - all_subcommands = ['help'] + list(subcom_map.keys()) - subcommand = arguments.pop(0) - - if subcommand not in all_subcommands: - sys.stderr.write('Error: invalid subcommand: {0}.\n' \ - .format(subcommand)) - sys.stderr.write('Call with --help for more information\n') - return 1 - - if subcommand == 'help': - if len(arguments) == 0: - help_subcommand = 'help' +def create_help_subcommand(subcommands): + 'Create and return the help subcommand' + subcommand_map = {sub.name: sub for sub in subcommands} + + help_description = 'display help for a specific subcommand' + + def help_populate_parser(parser=None): + if parser is None: + parser = argparse.ArgumentParser() + parser.description = help_description + subparsers = parser.add_subparsers( + title='Subcommands', + dest='help_subcommand', + metavar='subcommand', + required=False) + for subcommand in subcommands: + subparsers.add_parser(subcommand.name, + help=subcommand.brief_description) + return parser + + def help_main(arguments): + parser = help_populate_parser(argparse.ArgumentParser()) + args = parser.parse_args(arguments) + if args.help_subcommand: + sub = subcommand_map[args.help_subcommand].populate_parser() + sub.print_help() else: - help_subcommand = arguments.pop(0) + parser.print_help() + return 0 - if help_subcommand in ('-h', '--help', 'help'): - print(help_subcommand_str) - return 0 + return Subcommand('help', help_description, help_main, + help_populate_parser) - if help_subcommand not in all_subcommands: - sys.stderr.write('Error: invalid subcommand: {0}.\n' \ - .format(subcommand)) - sys.stderr.write('Call with --help for more information\n') - return 1 +def _main_impl(arguments, module_dir): + 'Implementation of main' + subcommands = load_subcommands(module_dir) + subcommands.append(create_help_subcommand(subcommands)) - # just forward to the documentation from the submodule - return subcom_map[help_subcommand].main( - ['--help'], prog='{0} {1}'.format(sys.argv[0], help_subcommand)) + parser = populate_parser(subcommands=subcommands) + args = parser.parse_args(arguments) - # it is one of the other subcommands. Just forward the request on - return subcom_map[subcommand].main( - arguments, prog='{0} {1}'.format(prog, subcommand)) + subcommand_map = {sub.name: sub for sub in subcommands} + subcommand = subcommand_map[args.subcommand] + return subcommand.main(arguments[1:]) if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/scripts/flitcli/flit_bisect.py b/scripts/flitcli/flit_bisect.py index 374fdfca..2550bcd4 100644 --- a/scripts/flitcli/flit_bisect.py +++ b/scripts/flitcli/flit_bisect.py @@ -328,7 +328,7 @@ def create_bisect_makefile(directory, replacements, gt_src, for x in repl_copy['link_flags']]) del repl_copy['link_flags'] - + # Find the next unique file name available in directory num = 0 @@ -1056,16 +1056,16 @@ def bisect_search(score_func, elements, found_callback=None, return differing_list -def parse_args(arguments, prog=sys.argv[0]): +def populate_parser(parser=None): ''' Builds a parser, parses the arguments, and returns the parsed arguments. @param arguments: (list of str) arguments given to the program @param prog: (str) name of the program ''' - parser = argparse.ArgumentParser( - prog=prog, - description=''' + if parser is None: + parser = argparse.ArgumentParser() + parser.description = ''' Compiles the source code under both the ground-truth compilation and a given problematic compilation. This tool then finds the minimal set of source files needed to be @@ -1075,8 +1075,7 @@ def parse_args(arguments, prog=sys.argv[0]): The log of the procedure will be kept in bisect.log. Note that this file is overwritten if you call flit bisect again. - ''', - ) + ''' # These positional arguments only make sense if not doing an auto run parser.add_argument('compilation', @@ -1237,7 +1236,13 @@ def parse_args(arguments, prog=sys.argv[0]): given compiler is the same as that used by the ground-truth compilation. ''') + return parser +def parse_args(arguments): + ''' + Builds a parser, parses the arguments, and returns the parsed arguments. + ''' + parser = populate_parser() args = parser.parse_args(arguments) # Split the compilation into separate components @@ -1658,7 +1663,7 @@ def compile_trouble(directory, compiler, optl, switches, compiler_type, if delete: shutil.rmtree(trouble_path) -def run_bisect(arguments, prog=sys.argv[0]): +def run_bisect(arguments): ''' The actual function for running the bisect command-line tool. @@ -1675,7 +1680,7 @@ def run_bisect(arguments, prog=sys.argv[0]): return value for sources and symbols are both None. If the search fails in the symbols part, then only the symbols return value is None. ''' - args = parse_args(arguments, prog) + args = parse_args(arguments) if args.compiler_type == 'auto': projconf = util.load_projconf(args.directory) @@ -2021,8 +2026,8 @@ def parallel_auto_bisect(arguments, prog=sys.argv[0]): # prepend a compilation and test case so that if the user provided # some, then an error will occur. args = parse_args( - ['--precision', 'double', 'compilation', 'testcase'] + arguments, - prog) + ['--precision', 'double', 'compilation', 'testcase'] + arguments) + sqlitefile = args.auto_sqlite_run projconf = util.load_projconf(args.directory) diff --git a/scripts/flitcli/flit_experimental.py b/scripts/flitcli/flit_experimental.py index f4c37474..513351c2 100644 --- a/scripts/flitcli/flit_experimental.py +++ b/scripts/flitcli/flit_experimental.py @@ -92,17 +92,25 @@ import flitconfig as conf brief_description = 'Allows access to experimental features' +experimental_dir = os.path.join(conf.script_dir, 'experimental') -def main(arguments, prog=sys.argv[0]): - 'Main logic here' - description = ''' +def populate_parser(parser=None, subcommands=None): + if subcommands is None: + subcommands = flit.load_subcommands(experimental_dir) + parser = flit.populate_parser(subcommands=subcommands, parser=parser) + parser.description = ''' The experimental command gives access to more subcommands that are not yet fully mature or are under evaluation for adoption as a true FLiT command. ''' + return parser + +def main(arguments): + 'Main logic here' + subcommands = flit.load_subcommands(experimental_dir) + parser = populate_parser(subcommands=subcommands) return flit._main_impl( arguments, - prog=prog, module_dir=os.path.join(conf.script_dir, 'experimental'), description=description) diff --git a/scripts/flitcli/flit_import.py b/scripts/flitcli/flit_import.py index f2871b1a..49677e4f 100644 --- a/scripts/flitcli/flit_import.py +++ b/scripts/flitcli/flit_import.py @@ -98,33 +98,17 @@ def _file_check(filename): raise argparse.ArgumentTypeError('File does not exist: {0}'.format(filename)) return filename -def get_dbfile_from_toml(tomlfile): - 'get and return the database.filepath field' - import toml - try: - projconf = toml.load(tomlfile) - except FileNotFoundError: - print('Error: {0} not found. Run "flit init"'.format(tomlfile), - file=sys.stderr) - raise - util.fill_defaults(projconf) - - assert projconf['database']['type'] == 'sqlite', \ - 'Only sqlite database supported' - return projconf['database']['filepath'] - -def main(arguments, prog=sys.argv[0]): - 'Main logic here' - parser = argparse.ArgumentParser( - prog=prog, - description=''' +def populate_parser(parser=None): + 'Populate or create an ArgumentParser' + if parser is None: + parser = argparse.ArgumentParser() + parser.description=''' Import flit results into the configured database. The configured database is found from the settings in flit-config.toml. You can import either exported results or results from manually running the tests. Note that importing the same thing twice will result in having two copies of it in the database. - ''', - ) + ''' parser.add_argument('importfile', nargs='+', type=_file_check, help=''' File(s) to import into the database. These files @@ -167,6 +151,26 @@ def main(arguments, prog=sys.argv[0]): can also be used when you do not have the toml python package installed (goodie!). ''') + return parser + +def get_dbfile_from_toml(tomlfile): + 'get and return the database.filepath field' + import toml + try: + projconf = toml.load(tomlfile) + except FileNotFoundError: + print('Error: {0} not found. Run "flit init"'.format(tomlfile), + file=sys.stderr) + raise + util.fill_defaults(projconf) + + assert projconf['database']['type'] == 'sqlite', \ + 'Only sqlite database supported' + return projconf['database']['filepath'] + +def main(arguments, prog=sys.argv[0]): + 'Main logic here' + parser = populate_parser() args = parser.parse_args(arguments) if args.dbfile is None: diff --git a/scripts/flitcli/flit_init.py b/scripts/flitcli/flit_init.py index 8f5c951a..47aa6740 100644 --- a/scripts/flitcli/flit_init.py +++ b/scripts/flitcli/flit_init.py @@ -93,24 +93,28 @@ brief_description = 'Initializes a flit test directory for use' -def main(arguments, prog=sys.argv[0]): - 'Main logic here' - parser = argparse.ArgumentParser( - prog=prog, - description=''' +def populate_parser(parser=None): + 'Populate or create an ArgumentParser' + if parser is None: + parser = argparse.ArgumentParser() + parser.description=''' Initializes a flit test directory for use. It will initialize the directory by copying the default configuration file into the given directory. If a configuration file already exists, this command does nothing. The config file is called flit-config.toml. - ''', - ) + ''' parser.add_argument('-C', '--directory', default='.', help='The directory to initialize') parser.add_argument('--overwrite', action='store_true', help='Overwrite init files if they are already there') parser.add_argument('-L', '--litmus-tests', action='store_true', help='Copy over litmus tests too') + return parser + +def main(arguments): + 'Main logic here' + parser = populate_parser() args = parser.parse_args(arguments) os.makedirs(args.directory, exist_ok=True) diff --git a/scripts/flitcli/flit_make.py b/scripts/flitcli/flit_make.py index b3b65f41..6b08a0a8 100644 --- a/scripts/flitcli/flit_make.py +++ b/scripts/flitcli/flit_make.py @@ -92,15 +92,14 @@ brief_description = 'Runs the make and adds to the database' -def main(arguments, prog=sys.argv[0]): - 'Main logic here' - parser = argparse.ArgumentParser( - prog=prog, - description=''' - This command runs the full set of tests and adds the results - to the configured database. - ''', - ) +def populate_parser(parser=None): + 'Populate or create an ArgumentParser' + if parser is None: + parser = argparse.ArgumentParser() + parser.description = ''' + This command runs the full set of tests and adds the results + to the configured database. + ''' processors = multiprocessing.cpu_count() parser.add_argument('-j', '--jobs', type=int, default=processors, help=''' @@ -130,6 +129,11 @@ def main(arguments, prog=sys.argv[0]): when creating a new run. This argument is ignored if --append is specified. The default label is ''') + return parser + +def main(arguments, prog=sys.argv[0]): + 'Main logic here' + parser = populate_parser() args = parser.parse_args(arguments) check_call_kwargs = dict() diff --git a/scripts/flitcli/flit_update.py b/scripts/flitcli/flit_update.py index 5519b1a4..30a11981 100644 --- a/scripts/flitcli/flit_update.py +++ b/scripts/flitcli/flit_update.py @@ -93,22 +93,20 @@ brief_description = 'Updates the Makefile based on flit-config.toml' -def parse_args(arguments, prog=sys.argv[0]): - 'Return parsed arugments' - parser = argparse.ArgumentParser( - prog=prog, - description=''' +def populate_parser(parser=None): + 'Populate or create an ArgumentParser' + if parser is None: + parser = argparse.ArgumentParser() + parser.description = ''' Updates the Makefile based on flit-config.toml. The Makefile is autogenerated and should not be modified manually. If there are things you want to replace or add, you can use custom.mk which is included at the end of the Makefile. So, you may add rules, add to variables, or override variables. - ''', - ) + ''' parser.add_argument('-C', '--directory', default='.', help='The directory to initialize') - args = parser.parse_args(arguments) - return args + return parser def flag_name(flag): ''' @@ -373,9 +371,10 @@ def create_makefile(args, makefile='Makefile'): util.process_in_file(os.path.join(conf.data_dir, 'Makefile.in'), makefile, replacements, overwrite=True) -def main(arguments, prog=sys.argv[0]): +def main(arguments): 'Main logic here' - args = parse_args(arguments, prog=prog) + parser = populate_parser() + args = parser.parse_args(arguments) makefile = os.path.join(args.directory, 'Makefile') if os.path.exists(makefile): diff --git a/scripts/flitcli/unfinished/flit_analyze.py b/scripts/flitcli/unfinished/flit_analyze.py index d6bd4e2e..d339a9af 100644 --- a/scripts/flitcli/unfinished/flit_analyze.py +++ b/scripts/flitcli/unfinished/flit_analyze.py @@ -89,15 +89,14 @@ brief_description = 'Runs analysis on a previous flit run' -def main(arguments, prog=sys.argv[0]): - parser = argparse.ArgumentParser( - prog=prog, - description=''' - Runs analysis on a previous flit run. The analysis will be of - the current flit repository and will create a directory called - analysis inside of the flit directory. - ''', - ) +def populate_parser(parser=None): + if parser is None: + parser = argparse.ArgumentParser() + parser.description = ''' + Runs analysis on a previous flit run. The analysis will be of the + current flit repository and will create a directory called analysis + inside of the flit directory. + ''' parser.add_argument('-C', '--directory', default='.', help='Directory containing flit-config.toml') parser.add_argument('-l', '--list', action='store_true', @@ -106,7 +105,11 @@ def main(arguments, prog=sys.argv[0]): help=''' Analyze the given run(s). Defaults to the latest run. - ''',) + ''') + return parser + +def main(arguments, prog=sys.argv[0]): + parser = populate_parser() args = parser.parse_args(arguments) # Subcommand logic here diff --git a/scripts/flitcli/unfinished/flit_check.py b/scripts/flitcli/unfinished/flit_check.py index 5daa80ff..871af597 100644 --- a/scripts/flitcli/unfinished/flit_check.py +++ b/scripts/flitcli/unfinished/flit_check.py @@ -87,26 +87,29 @@ brief_description = 'Verifies the correctness of a config file' -def main(arguments, prog=sys.argv[0]): - parser = argparse.ArgumentParser( - prog=prog, - description=''' - This command only verifies the correctness of the - configurations you have for your flit tests. As part of this - verification, this command checks to see if the remote - connections are capable of being done, such as the connection - to the machines to run the software, the connection to the - database machine, and the connection to the database machine - from the run machine. You may need to provide a few SSH - passwords to do this check. - ''', - ) +def populate_parser(parser=None): + 'Populate or create an ArgumentParser' + if parser is None: + parser = ArgumentParser() + parser.description=''' + This command only verifies the correctness of the configurations you + have for your flit tests. As part of this verification, this command + checks to see if the remote connections are capable of being done, such + as the connection to the machines to run the software, the connection + to the database machine, and the connection to the database machine + from the run machine. You may need to provide a few SSH passwords to + do this check. + ''' parser.add_argument('-C', '--directory', default='.', help='The directory containing flit-config.toml') parser.add_argument('-c', '--config-only', action='store_true', help=''' Only check the config file instead of also the SSH connections and maybe other things.''') + return parser + +def main(arguments): + parser = populate_parser() args = parser.parse_args(arguments) # Subcommand logic here diff --git a/scripts/flitcli/unfinished/flit_run.py b/scripts/flitcli/unfinished/flit_run.py index a6bb18f1..b2f2a1b6 100644 --- a/scripts/flitcli/unfinished/flit_run.py +++ b/scripts/flitcli/unfinished/flit_run.py @@ -87,18 +87,22 @@ brief_description = 'Run flit on the configured remote machine(s)' -def main(arguments, prog=sys.argv[0]): - parser = argparse.ArgumentParser( - prog=prog, - description=''' - Run flit on the configured remote machine(s). Note that you - may need to provide a password for SSH, but that should be - taken care of pretty early on in the process. The results - should be sent to the database computer for later analysis. - ''', - ) +def populate_parser(parser=None): + 'Populate or create an ArgumentParser' + if parser is None: + parser = argparse.ArgumentParser() + parser.description = ''' + Run flit on the configured remote machine(s). Note that you may need + to provide a password for SSH, but that should be taken care of pretty + early on in the process. The results should be sent to the database + computer for later analysis. + ''' parser.add_argument('description', help='A description of the test run (required)') + return parser + +def main(arguments): + parser = populate_parser() args = parser.parse_args(arguments) # Subcommand logic here diff --git a/src/flit.h b/src/flit.h index f4dc1961..2125ffb4 100644 --- a/src/flit.h +++ b/src/flit.h @@ -83,7 +83,7 @@ // This wrapper file simply includes all of the header files for ease of use // and backward-compatability - + #ifndef TOP_FLIT_H #define TOP_FLIT_H diff --git a/src/flit/subprocess.h b/src/flit/subprocess.h index a757e168..38c5382f 100644 --- a/src/flit/subprocess.h +++ b/src/flit/subprocess.h @@ -93,7 +93,7 @@ static Flit_##mainfunc##Factory global_##mainfunc##Factory; \ #include - + namespace flit { struct ProcResult { diff --git a/tests/flit_cli/flit_bisect/data/tests/BisectTest.cpp b/tests/flit_cli/flit_bisect/data/tests/BisectTest.cpp index a6a7cbf0..f273db22 100644 --- a/tests/flit_cli/flit_bisect/data/tests/BisectTest.cpp +++ b/tests/flit_cli/flit_bisect/data/tests/BisectTest.cpp @@ -94,7 +94,7 @@ #include int real_problem_test(int, char**) { - double answer = + double answer = file1_all() + file2_all() + file3_all() + diff --git a/tests/flit_makefile/tst_build_with_compiler_specific_flags.py b/tests/flit_makefile/tst_build_with_compiler_specific_flags.py index 52191ac7..e31fcba0 100644 --- a/tests/flit_makefile/tst_build_with_compiler_specific_flags.py +++ b/tests/flit_makefile/tst_build_with_compiler_specific_flags.py @@ -112,7 +112,7 @@ ... _ = conf.write("fixed_link_flags = '-Wfix-link -Whello'\\n") ... _ = shutil.copy('fake_clang34.py', temp_dir) ... _ = subp.check_output(['make', '--always-make', 'Makefile', -... '-C', temp_dir]) +... '-C', temp_dir]) ... make_out = subp.check_output(['make', 'gt', '-C', temp_dir, ... 'VERBOSE=1']) ... make_out = make_out.decode('utf8').splitlines() diff --git a/tests/flit_makefile/tst_clang34.py b/tests/flit_makefile/tst_clang34.py index 0d18c18a..586c316c 100644 --- a/tests/flit_makefile/tst_clang34.py +++ b/tests/flit_makefile/tst_clang34.py @@ -112,7 +112,7 @@ ... _ = conf.write("type = 'clang'\\n") ... _ = shutil.copy('fake_clang34.py', temp_dir) ... _ = subp.check_output(['make', '--always-make', 'Makefile', -... '-C', temp_dir]) +... '-C', temp_dir]) ... make_out = subp.check_output(['make', 'gt', '-C', temp_dir, ... 'VERBOSE=1']) ... make_out = make_out.decode('utf8').splitlines() diff --git a/tests/test_harness.py b/tests/test_harness.py index b0e508c7..b6a8c221 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -96,7 +96,7 @@ _harness_dir = os.path.dirname(os.path.realpath(__file__)) _flit_dir = os.path.dirname(_harness_dir) -_script_dir = os.path.join(_flit_dir, 'scripts/flitcli') +_script_dir = os.path.join(_flit_dir, 'scripts', 'flitcli') del os From ac86d10c0d284b0cfc172f9a159d811fb92e5d05 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 18 Feb 2020 18:44:52 -0700 Subject: [PATCH 12/21] tests/bash_completion: add autogenerated test This tests that all optional and subcommand choices are completed by the bash-completion script --- .../bash_completion/tst_completion_flit.py | 148 +++------------- .../flit_cli/bash_completion/util/__init__.py | 0 .../bash_completion/util/arginspect.py | 159 ++++++++++++++++++ .../bash_completion/util/completion.py | 113 +++++++++++++ 4 files changed, 298 insertions(+), 122 deletions(-) create mode 100644 tests/flit_cli/bash_completion/util/__init__.py create mode 100644 tests/flit_cli/bash_completion/util/arginspect.py create mode 100644 tests/flit_cli/bash_completion/util/completion.py diff --git a/tests/flit_cli/bash_completion/tst_completion_flit.py b/tests/flit_cli/bash_completion/tst_completion_flit.py index 2b70caa0..018216e9 100755 --- a/tests/flit_cli/bash_completion/tst_completion_flit.py +++ b/tests/flit_cli/bash_completion/tst_completion_flit.py @@ -1,137 +1,41 @@ #!/usr/bin/env python3 import os -import subprocess as subp import sys import unittest -before_path = sys.path[:] -sys.path.append('../..') -import test_harness as th -sys.path = before_path - -def _completion_vars(program, command): - ''' - Returns a dictionary of bash varibles to trigger bash-completion. - - >>> vals = _completion_vars('foo', 'bar --he') - >>> sorted(vals.keys()) - ['COMP_CWORD', 'COMP_LINE', 'COMP_POINT', 'COMP_WORDS'] - >>> [vals[x] for x in sorted(vals.keys())] - [2, 'foo bar --he', 12, 'foo bar --he'] - - >>> vals = _completion_vars('foo', 'bar --help ') - >>> sorted(vals.keys()) - ['COMP_CWORD', 'COMP_LINE', 'COMP_POINT', 'COMP_WORDS'] - >>> [vals[x] for x in sorted(vals.keys())] - [3, 'foo bar --help ', 15, 'foo bar --help '] - ''' - line = program + ' ' + command - words = line.rstrip() - args = command.split() - cword = len(args) - point = len(line) - if line[-1] == ' ': - words += ' ' - cword += 1 - return { - 'COMP_LINE': line, - 'COMP_WORDS': words, - 'COMP_CWORD': cword, - 'COMP_POINT': point, - } +import util.arginspect as arginspect +from util.completion import get_completion -def get_completion(completion_file, program, command): - ''' - Returns the potential completions from a bash-completion - - @param completion_file (str): file containing bash-completion definition - @param program (str): program being invoked and completed (e.g., 'cp') - @param command (str): rest of the command-line args (e.g., '-r ./') - @return (list(str)): All potential completions if the user were to have - entered program + ' ' + command into an interactive terminal and hit - tab twice. - ''' - replacements = _completion_vars(program, command) - replacements['completion_file'] = completion_file - replacements['program'] = program - full_cmdline = ( - r'source {completion_file};' - 'COMP_LINE="{COMP_LINE}"' - ' COMP_WORDS=({COMP_WORDS})' - ' COMP_CWORD={COMP_CWORD}' - ' COMP_POINT={COMP_POINT};' - '$(complete -p {program} |' - ' sed "s/.*-F \\([^ ]*\\) .*/\\1/") &&' - ' for elem in "${{COMPREPLY[@]}}"; do' - ' echo "${{elem}}";' - ' done'.format(**replacements)) - out = subp.check_output(['bash', '-i', '-c', full_cmdline]) - return out.decode('utf-8').splitlines() - -class TestFlitBaseCompletion(unittest.TestCase): - def assertEqualCompletion(self, args, expected): - 'Asserts that the expected completions are obtained' - completion_file = os.path.join(th.config.bash_completion_dir, 'flit') - actual = get_completion(completion_file, 'flit', args) - self.assertEqual(sorted(actual), sorted(expected)) - - def test_no_args(self): - self.assertEqualCompletion('', [ - '-h', '--help', - '-v', '--version', - 'help', - 'bisect', - 'experimental', - 'import', - 'init', - 'make', - 'update', - ]) +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +prev_path = sys.path +sys.path.append(os.path.join(SCRIPT_DIR, '..', '..')) +import test_harness as th +sys.path = prev_path - def test_dash(self): - self.assertEqualCompletion('-', ['-h', '--help', '-v', '--version']) +class TestArgparse_Flit(arginspect.ArgParseTestBase): + FLIT_PROG = 'flit' + FLIT_BASH_COMPLETION = os.path.join( + th._flit_dir, 'scripts', 'bash-completion', 'flit') - def test_dash_help(self): - self.assertEqualCompletion('-h', ['-h']) - self.assertEqualCompletion('-h ', []) - self.assertEqualCompletion('--help', ['--help']) - self.assertEqualCompletion('--help ', []) + def bashcomplete(self, args): + return get_completion(self.FLIT_BASH_COMPLETION, self.FLIT_PROG, args) - def test_dash_version(self): - self.assertEqualCompletion('-v', ['-v']) - self.assertEqualCompletion('-v ', []) - self.assertEqualCompletion('--version', ['--version']) - self.assertEqualCompletion('--version ', []) + def get_parser(self): + subcommands = th.flit.load_subcommands(th.config.script_dir) + subcommands.append(th.flit.create_help_subcommand(subcommands)) + return th.flit.populate_parser(subcommands=subcommands) - def test_nonmatching(self): - self.assertEqualCompletion('-a', []) - self.assertEqualCompletion('a', []) - self.assertEqualCompletion('c', []) - self.assertEqualCompletion('bisecj', []) + def test_empty_available_options(self): + self.assertEmptyAvailableOptions(self.get_parser()) - def test_matching_subcommand(self): - self.assertEqualCompletion('h', ['help']) - self.assertEqualCompletion('bi', ['bisect']) - self.assertEqualCompletion('exp', ['experimental']) - self.assertEqualCompletion('i', ['import', 'init']) - self.assertEqualCompletion('m', ['make']) - self.assertEqualCompletion('up', ['update']) + def test_empty_available_options_for_subparsers(self): + self.assertEachSubparserEmptyAvailableOptions(self.get_parser()) -def main(): - 'Calls unittest.main(), only printing if the tests failed' - from io import StringIO - captured = StringIO() - old_stderr = sys.stderr - try: - sys.stderr = captured - result = unittest.main(exit=False) - finally: - sys.stderr = old_stderr - if not result.result.wasSuccessful(): - print(captured.getvalue()) - return 1 - return 0 + def test_no_positional_args(self): + inspector = arginspect.ParserInspector(self.get_parser()) + # test that there are no positional arguments + self.assertEqual(inspector.position_actions, []) if __name__ == '__main__': - sys.exit(main()) + unittest.main() diff --git a/tests/flit_cli/bash_completion/util/__init__.py b/tests/flit_cli/bash_completion/util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/flit_cli/bash_completion/util/arginspect.py b/tests/flit_cli/bash_completion/util/arginspect.py new file mode 100644 index 00000000..89b6f46e --- /dev/null +++ b/tests/flit_cli/bash_completion/util/arginspect.py @@ -0,0 +1,159 @@ +import abc +import argparse +import unittest + +def is_option_action(action): + return bool(action.option_strings) + +def is_position_action(action): + return not is_option_arg(action) + +def is_subparser_action(action): + return isinstance(action, argparse._SubParsersAction) + +class ActionInspector: + ''' + All Actions have: + - option_strings + - dest + - nargs + - const + - default + - type + - choices + - required + - help + - metavar + + argparse._VersionAction adds: + - version + + argparse._SubParsersAction adds: + - prog + - parser_class + ''' + + NAME_MAP = { + argparse._StoreAction : 'store', + argparse._StoreConstAction : 'store_const', + argparse._StoreTrueAction : 'store_true', + argparse._StoreFalseAction : 'store_false', + argparse._AppendAction : 'append', + argparse._AppendConstAction : 'append_const', + argparse._CountAction : 'count', + argparse._HelpAction : 'help', + argparse._VersionAction : 'version', + argparse._SubParsersAction : 'parsers', + argparse._ExtendAction : 'extend', + } + + def __init__(self, action): + if action.__class__ in self.NAME_MAP: + self.action_type = self.NAME_MAP[action.__class__] + else: + self.action_type = action.__class__.__name__ + + # passthrough of attributes + self.action = action + self.option_strings = action.option_strings + self.dest = action.dest + self.nargs = action.nargs + self.const = action.const + self.default = action.default + self.type = action.type + self.choices = action.choices + if is_subparser_action(action): + self.choices = {key: ParserInspector(val) + for key, val in action.choices.items()} + self.required = action.required + self.help = action.help + self.metavar = action.metavar + + # optional attributes + for attr in ('version', 'prog', 'parser_class'): + if hasattr(action, attr): + setattr(self, attr, getattr(action, attr)) + else: + setattr(self, attr, None) + +class ParserInspector: + 'Introspection on an argparse.ArgumentParser' + + def __init__(self, parser): + self.parser = parser + self.option_actions = [ + ActionInspector(a) for a in self.parser._optionals._group_actions] + self.position_actions = [ + ActionInspector(a) for a in self.parser._positionals._group_actions + if not is_subparser_action(a)] + self.subparser_actions = [] + if self.parser._subparsers: + self.subparser_actions = [ + ActionInspector(a) for a in self.parser._subparsers._group_actions + if is_subparser_action(a)] + self.actions = self.option_actions + self.position_actions + \ + self.subparser_actions + + self.option_strings = sum( + [a.option_strings for a in self.option_actions], start=[]) + self.subparser_choices = sum( + [list(a.choices) for a in self.subparser_actions], start=[]) + +class ArgParseTestBase(unittest.TestCase, metaclass=abc.ABCMeta): + + @abc.abstractmethod + def bashcomplete(self, args): + ''' + Return a list of completions from the current string of arguments. + + This is an abstract method that must be implemented by derivative + classes. + ''' + pass + + def assertEqualCompletion(self, args, expected_completions, msg=None): + 'Asserts that the expected completions are obtained' + actual = self.bashcomplete(args) + self.assertEqual(set(expected_completions), set(actual), msg=msg) + + def assertCompletionContains(self, args, expected_subset, msg=None): + 'Asserts that the expected completions are found in the actual' + actual = self.bashcomplete(args) + self.assertLessEqual(set(expected_subset), set(actual), msg=msg) + + def assertEmptyAvailableOptions(self, parser, cli=''): + ''' + Asserts that all options and subparser choices are present in the bash + completion for the given parser and the current cli. + + Note: The cli is only given to specify which subparser we are currently + doing. The parser given should match the subparser associated with the + cli string. + ''' + inspector = ParserInspector(parser) + + expected_completions = inspector.option_strings + if not inspector.position_actions: + expected_completions.extend(inspector.subparser_choices) + + self.assertCompletionContains( + cli, expected_completions, + msg='args: {}'.format(repr(cli))) + + def assertEachSubparserEmptyAvailableOptions(self, parser, cli=''): + ''' + Asserts that all options and subparser choices are present for the + current parser and for all subparsers recursively. This method calls + assertEmptyAvailableOptions() for the given parser and for all + subparsers recursively. + ''' + inspector = ParserInspector(parser) + + # test this parser level + self.assertEmptyAvailableOptions(parser, cli) + + # test all subparsers recursively + for choice in inspector.subparser_choices: + subinspector = inspector.subparser_actions[0].choices[choice] + self.assertEachSubparserEmptyAvailableOptions( + subinspector.parser, cli='{} {} '.format(cli, choice)) diff --git a/tests/flit_cli/bash_completion/util/completion.py b/tests/flit_cli/bash_completion/util/completion.py new file mode 100644 index 00000000..d69f04fd --- /dev/null +++ b/tests/flit_cli/bash_completion/util/completion.py @@ -0,0 +1,113 @@ +import subprocess as subp + +class Completion(): + def prepare(self, program, command): + self.program = program + self.COMP_LINE = program + ' ' + command + self.COMP_WORDS = self.COMP_LINE.rstrip() + + args=command.split() + self.COMP_CWORD=len(args) + self.COMP_POINT=len(self.COMP_LINE) + + if (self.COMP_LINE[-1] == ' '): + self.COMP_WORDS += " " + self.COMP_CWORD += 1 + + def run(self, completion_file, program, command): + self.prepare(program, command) + full_cmdline = \ + r'source {compfile};' \ + 'COMP_LINE="{COMP_LINE}"' \ + ' COMP_WORDS=({COMP_WORDS})' \ + ' COMP_CWORD={COMP_CWORD}' \ + ' COMP_POINT={COMP_POINT};' \ + '$(complete -p {program} |' \ + ' sed "s/.*-F \\([^ ]*\\) .*/\\1/") &&' \ + ' for elem in "${{COMPREPLY[@]}}"; do' \ + ' echo "${{elem}}";' \ + ' done'.format( + compfile=completion_file, + COMP_LINE=self.COMP_LINE, + COMP_WORDS=self.COMP_WORDS, + COMP_POINT=self.COMP_POINT, + program=self.program, + COMP_CWORD=self.COMP_CWORD, + ) + #print(full_cmdline) + proc = subp.Popen(['bash', '-i', '-c', full_cmdline], stdout=subp.PIPE) + out, _ = proc.communicate() + return out.decode('utf-8').splitlines() + +def _completion_vars(program, command): + ''' + Returns a dictionary of bash varibles to trigger bash-completion. + + >>> vals = _completion_vars('foo', 'bar --he') + >>> sorted(vals.keys()) + ['COMP_CWORD', 'COMP_LINE', 'COMP_POINT', 'COMP_WORDS'] + >>> [vals[x] for x in sorted(vals.keys())] + [2, 'foo bar --he', 12, 'foo bar --he'] + + >>> vals = _completion_vars('foo', 'bar --help ') + >>> sorted(vals.keys()) + ['COMP_CWORD', 'COMP_LINE', 'COMP_POINT', 'COMP_WORDS'] + >>> [vals[x] for x in sorted(vals.keys())] + [3, 'foo bar --help ', 15, 'foo bar --help '] + ''' + line = program + ' ' + command + words = line.rstrip() + args = command.split() + cword = len(args) + point = len(line) + if line[-1] == ' ': + words += ' ' + cword += 1 + return { + 'COMP_LINE': line, + 'COMP_WORDS': words, + 'COMP_CWORD': cword, + 'COMP_POINT': point, + } + +def get_completion(completion_file, program, command): + ''' + Returns the potential completions from a bash-completion + + @param completion_file (str): file containing bash-completion definition + @param program (str): program being invoked and completed (e.g., 'cp') + @param command (str): rest of the command-line args (e.g., '-r ./') + @return (list(str)): All potential completions if the user were to have + entered program + ' ' + command into an interactive terminal and hit + tab twice. + ''' + replacements = _completion_vars(program, command) + replacements['completion_file'] = completion_file + replacements['program'] = program + full_cmdline = ( + r'source {completion_file};' + 'COMP_LINE="{COMP_LINE}"' + ' COMP_WORDS=({COMP_WORDS})' + ' COMP_CWORD={COMP_CWORD}' + ' COMP_POINT={COMP_POINT};' + '$(complete -p {program} |' + ' sed "s/.*-F \\([^ ]*\\) .*/\\1/") &&' + ' for elem in "${{COMPREPLY[@]}}"; do' + ' echo "${{elem}}";' + ' done'.format(**replacements)) + out = subp.check_output(['bash', '-i', '-c', full_cmdline]) + return out.decode('utf-8').splitlines() + +def main(): + 'Compares the two implementations' + cfile = '../completions/bar' + prog = 'bar' + args = 'import ' + complete_1 = Completion().run(cfile, prog, args) + complete_2 = get_completion(cfile, prog, args) + assert complete_1 == complete_2 + from pprint import pprint + pprint(complete_1) + +if __name__ == '__main__': + main() From acd7146f4fc664963f5efd91043034c0fb9154f6 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 18 Feb 2020 19:39:19 -0700 Subject: [PATCH 13/21] tests/bash-completion: Fix a few problems --- scripts/bash-completion/flit | 4 ++-- scripts/flitcli/flit.py | 4 +++- scripts/flitcli/flit_experimental.py | 1 + .../bash_completion/tst_completion_flit.py | 17 ++++++++++++++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/bash-completion/flit b/scripts/bash-completion/flit index 34c90158..5d002196 100644 --- a/scripts/bash-completion/flit +++ b/scripts/bash-completion/flit @@ -23,7 +23,7 @@ _flit__sqlite_files() _flit_help() { local cur available_subcommands - available_subcommands="bisect init make update import" + available_subcommands="-h --help bisect experimental init make update import" cur="${COMP_WORDS[COMP_CWORD]}" COMPREPLY=( $(compgen -W "${available_subcommands}" -- ${cur}) ) } @@ -229,7 +229,7 @@ _flit_experimental() COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" subcommand="${COMP_WORDS[2]}" - available_subcommands="--help --version help ninja" + available_subcommands="-h --help -v --version help ninja" # subcommand completion if [ ${COMP_CWORD} -eq 2 ]; then diff --git a/scripts/flitcli/flit.py b/scripts/flitcli/flit.py index d5b7d075..64ba9520 100755 --- a/scripts/flitcli/flit.py +++ b/scripts/flitcli/flit.py @@ -201,6 +201,7 @@ def create_help_subcommand(subcommands): help_description = 'display help for a specific subcommand' def help_populate_parser(parser=None): + 'populate_parser() for the help subcommand' if parser is None: parser = argparse.ArgumentParser() parser.description = help_description @@ -209,12 +210,13 @@ def help_populate_parser(parser=None): dest='help_subcommand', metavar='subcommand', required=False) - for subcommand in subcommands: + for subcommand in subcommand_map.values(): subparsers.add_parser(subcommand.name, help=subcommand.brief_description) return parser def help_main(arguments): + 'main() for the help subcommand' parser = help_populate_parser(argparse.ArgumentParser()) args = parser.parse_args(arguments) if args.help_subcommand: diff --git a/scripts/flitcli/flit_experimental.py b/scripts/flitcli/flit_experimental.py index 513351c2..79f4b41e 100644 --- a/scripts/flitcli/flit_experimental.py +++ b/scripts/flitcli/flit_experimental.py @@ -108,6 +108,7 @@ def populate_parser(parser=None, subcommands=None): def main(arguments): 'Main logic here' subcommands = flit.load_subcommands(experimental_dir) + subcommands.append(flit.create_help_subcommand(subcommands)) parser = populate_parser(subcommands=subcommands) return flit._main_impl( arguments, diff --git a/tests/flit_cli/bash_completion/tst_completion_flit.py b/tests/flit_cli/bash_completion/tst_completion_flit.py index 018216e9..db3cd685 100755 --- a/tests/flit_cli/bash_completion/tst_completion_flit.py +++ b/tests/flit_cli/bash_completion/tst_completion_flit.py @@ -37,5 +37,20 @@ def test_no_positional_args(self): # test that there are no positional arguments self.assertEqual(inspector.position_actions, []) +def main(): + 'Calls unittest.main(), only printing if the tests failed' + from io import StringIO + captured = StringIO() + old_stderr = sys.stderr + try: + sys.stderr = captured + result = unittest.main(exit=False) + finally: + sys.stderr = old_stderr + if not result.result.wasSuccessful(): + print(captured.getvalue()) + return 1 + return 0 + if __name__ == '__main__': - unittest.main() + sys.exit(main()) From 8636d450533b99c56a1c91997191f2d0009a3b3b Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 18 Feb 2020 20:55:20 -0700 Subject: [PATCH 14/21] Add unittest_main() to test_harness.py --- .../bash_completion/tst_completion_flit.py | 17 +---------------- tests/test_harness.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/flit_cli/bash_completion/tst_completion_flit.py b/tests/flit_cli/bash_completion/tst_completion_flit.py index db3cd685..211f887f 100755 --- a/tests/flit_cli/bash_completion/tst_completion_flit.py +++ b/tests/flit_cli/bash_completion/tst_completion_flit.py @@ -37,20 +37,5 @@ def test_no_positional_args(self): # test that there are no positional arguments self.assertEqual(inspector.position_actions, []) -def main(): - 'Calls unittest.main(), only printing if the tests failed' - from io import StringIO - captured = StringIO() - old_stderr = sys.stderr - try: - sys.stderr = captured - result = unittest.main(exit=False) - finally: - sys.stderr = old_stderr - if not result.result.wasSuccessful(): - print(captured.getvalue()) - return 1 - return 0 - if __name__ == '__main__': - sys.exit(main()) + sys.exit(th.unittest_main()) diff --git a/tests/test_harness.py b/tests/test_harness.py index b6a8c221..8ed2a644 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -93,6 +93,7 @@ from contextlib import contextmanager import os +import sys _harness_dir = os.path.dirname(os.path.realpath(__file__)) _flit_dir = os.path.dirname(_harness_dir) @@ -176,6 +177,22 @@ def touch(filename): from pathlib import Path Path(filename).touch() +def unittest_main(): + 'Calls unittest.main(), only printing if the tests failed' + from io import StringIO + import unittest + captured = StringIO() + old_stderr = sys.stderr + try: + sys.stderr = captured + result = unittest.main(exit=False) + finally: + sys.stderr = old_stderr + if not result.result.wasSuccessful(): + print(captured.getvalue()) + return 1 + return 0 + flit = _path_import(_script_dir, 'flit') config = _path_import(_script_dir, 'flitconfig') util = _path_import(_script_dir, 'flitutil') From dc42f2166619682222caf8da3ca9953c16904844 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 18 Feb 2020 20:55:39 -0700 Subject: [PATCH 15/21] Makefile: add --no-print-directory to recursive calls --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f6dccd6c..2a421aa2 100644 --- a/Makefile +++ b/Makefile @@ -58,11 +58,11 @@ help: .PHONY: check check: - $(MAKE) check --directory tests + $(MAKE) --no-print-directory check --directory tests .PHONY: clean clean: - $(MAKE) clean --directory tests + $(MAKE) --no-print-directory clean --directory tests .PHONY: install install: From a731d976b87dce5f8f33c63aff6241f14129dd41 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 18 Feb 2020 21:18:12 -0700 Subject: [PATCH 16/21] flit.py: make parsing shallow by default Go back to passing the prog argument to main of subcommands so that the help documentation works properly. This change was made to 1. Fix behavior with flit bisect that flit_bisect.py handles, but is not necessarily handled properly by the ArgumentParser made from flit bisect. 2. Reduce parsing time, since it is unnecessary to parse twice 3. Have help usage be correct --- scripts/flitcli/experimental/flit_ninja.py | 3 +- scripts/flitcli/flit.py | 35 +++++++++++-------- scripts/flitcli/flit_bisect.py | 14 ++++---- scripts/flitcli/flit_experimental.py | 12 ++++--- scripts/flitcli/flit_import.py | 3 +- scripts/flitcli/flit_init.py | 3 +- scripts/flitcli/flit_make.py | 3 +- scripts/flitcli/flit_update.py | 3 +- scripts/flitcli/unfinished/flit_analyze.py | 3 +- scripts/flitcli/unfinished/flit_check.py | 3 +- scripts/flitcli/unfinished/flit_run.py | 3 +- .../bash_completion/tst_completion_flit.py | 2 +- 12 files changed, 53 insertions(+), 34 deletions(-) diff --git a/scripts/flitcli/experimental/flit_ninja.py b/scripts/flitcli/experimental/flit_ninja.py index 4312efba..0a5fc755 100755 --- a/scripts/flitcli/experimental/flit_ninja.py +++ b/scripts/flitcli/experimental/flit_ninja.py @@ -664,9 +664,10 @@ def write(self): n.newline() n.build('run', 'phony', [x + comparison_suffix for x in results_files]) -def main(arguments): +def main(arguments, prog=None): 'Main logic here' parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) arguments = [x for x in arguments if x not in ('-q', '--quiet')] diff --git a/scripts/flitcli/flit.py b/scripts/flitcli/flit.py index 64ba9520..8919764f 100755 --- a/scripts/flitcli/flit.py +++ b/scripts/flitcli/flit.py @@ -138,14 +138,16 @@ def load_subcommands(directory): return subcommands -def populate_parser(parser=None, subcommands=None): +def populate_parser(parser=None, subcommands=None, recursive=False): ''' Populate and return the ArgumentParser. If not given, a new one is made. All arguments are optional. - @param subcommands (list(Subcommand)): list of subcommands (default=None) @param parser (argparse.ArgumentParser): parser to populate or None to generate a new one. (default=None) + @param subcommands (list(Subcommand)): list of subcommands (default=None) + @param recursive (bool): True means to add subcommand parsing beyond + the top-level ''' if parser is None: parser = argparse.ArgumentParser() @@ -167,12 +169,14 @@ def populate_parser(parser=None, subcommands=None): required=True) for subcommand in subcommands: subparser = subparsers.add_parser( - subcommand.name, help=subcommand.brief_description) - subcommand.populate_parser(subparser) + subcommand.name, help=subcommand.brief_description, + add_help=recursive) + if recursive: + subcommand.populate_parser(subparser) return parser def main(arguments, module_dir=conf.script_dir, outstream=None, - errstream=None): + errstream=None, prog=None): ''' Main logic here. @@ -181,7 +185,7 @@ def main(arguments, module_dir=conf.script_dir, outstream=None, would go to the console and put it into a StringStream or maybe a file. ''' if outstream is None and errstream is None: - return _main_impl(arguments, module_dir) + return _main_impl(arguments, module_dir, prog=prog) oldout = sys.stdout olderr = sys.stderr try: @@ -189,7 +193,7 @@ def main(arguments, module_dir=conf.script_dir, outstream=None, sys.stdout = outstream if errstream is not None: sys.stderr = errstream - return _main_impl(arguments, module_dir) + return _main_impl(arguments, module_dir, prog=prog) finally: sys.stdout = oldout sys.stderr = olderr @@ -215,31 +219,34 @@ def help_populate_parser(parser=None): help=subcommand.brief_description) return parser - def help_main(arguments): + help_parser = help_populate_parser() + + def help_main(arguments, prog=sys.argv[0]): 'main() for the help subcommand' - parser = help_populate_parser(argparse.ArgumentParser()) - args = parser.parse_args(arguments) + help_parser.prog = prog + args = help_parser.parse_args(arguments) if args.help_subcommand: sub = subcommand_map[args.help_subcommand].populate_parser() sub.print_help() else: - parser.print_help() + help_parser.print_help() return 0 return Subcommand('help', help_description, help_main, help_populate_parser) -def _main_impl(arguments, module_dir): +def _main_impl(arguments, module_dir, prog=None): 'Implementation of main' subcommands = load_subcommands(module_dir) subcommands.append(create_help_subcommand(subcommands)) parser = populate_parser(subcommands=subcommands) - args = parser.parse_args(arguments) + if prog: parser.prog = prog + args, remaining = parser.parse_known_args(arguments) subcommand_map = {sub.name: sub for sub in subcommands} subcommand = subcommand_map[args.subcommand] - return subcommand.main(arguments[1:]) + return subcommand.main(remaining, prog=parser.prog + ' ' + args.subcommand) if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/scripts/flitcli/flit_bisect.py b/scripts/flitcli/flit_bisect.py index 2550bcd4..c5440df0 100644 --- a/scripts/flitcli/flit_bisect.py +++ b/scripts/flitcli/flit_bisect.py @@ -1238,11 +1238,12 @@ def populate_parser(parser=None): ''') return parser -def parse_args(arguments): +def parse_args(arguments, prog=None): ''' Builds a parser, parses the arguments, and returns the parsed arguments. ''' parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) # Split the compilation into separate components @@ -1663,7 +1664,7 @@ def compile_trouble(directory, compiler, optl, switches, compiler_type, if delete: shutil.rmtree(trouble_path) -def run_bisect(arguments): +def run_bisect(arguments, prog=None): ''' The actual function for running the bisect command-line tool. @@ -1680,7 +1681,7 @@ def run_bisect(arguments): return value for sources and symbols are both None. If the search fails in the symbols part, then only the symbols return value is None. ''' - args = parse_args(arguments) + args = parse_args(arguments, prog=prog) if args.compiler_type == 'auto': projconf = util.load_projconf(args.directory) @@ -1983,7 +1984,7 @@ def auto_bisect_worker(arg_queue, result_queue): result_queue.put((row, -1, None, None, None, 1)) raise -def parallel_auto_bisect(arguments, prog=sys.argv[0]): +def parallel_auto_bisect(arguments, prog=None): ''' Runs bisect in parallel under the auto mode. This is only applicable if the --auto-sqlite-run option has been specified in the arguments. @@ -2026,7 +2027,8 @@ def parallel_auto_bisect(arguments, prog=sys.argv[0]): # prepend a compilation and test case so that if the user provided # some, then an error will occur. args = parse_args( - ['--precision', 'double', 'compilation', 'testcase'] + arguments) + ['--precision', 'double', 'compilation', 'testcase'] + arguments, + prog=prog) sqlitefile = args.auto_sqlite_run projconf = util.load_projconf(args.directory) @@ -2159,7 +2161,7 @@ def parallel_auto_bisect(arguments, prog=sys.argv[0]): return return_tot -def main(arguments, prog=sys.argv[0]): +def main(arguments, prog=None): ''' A wrapper around the bisect program. This checks for the --auto-sqlite-run stuff and runs the run_bisect multiple times if so. diff --git a/scripts/flitcli/flit_experimental.py b/scripts/flitcli/flit_experimental.py index 79f4b41e..f5c9ec15 100644 --- a/scripts/flitcli/flit_experimental.py +++ b/scripts/flitcli/flit_experimental.py @@ -105,15 +105,17 @@ def populate_parser(parser=None, subcommands=None): ''' return parser -def main(arguments): +def main(arguments, prog=None): 'Main logic here' subcommands = flit.load_subcommands(experimental_dir) subcommands.append(flit.create_help_subcommand(subcommands)) parser = populate_parser(subcommands=subcommands) - return flit._main_impl( - arguments, - module_dir=os.path.join(conf.script_dir, 'experimental'), - description=description) + if prog: parser.prog = prog + args, remaining = parser.parse_known_args(arguments) + + subcommand_map = {sub.name: sub for sub in subcommands} + subcommand = subcommand_map[args.subcommand] + return subcommand.main(remaining, prog=parser.prog + ' ' + args.subcommand) if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/scripts/flitcli/flit_import.py b/scripts/flitcli/flit_import.py index 49677e4f..d60181e7 100644 --- a/scripts/flitcli/flit_import.py +++ b/scripts/flitcli/flit_import.py @@ -168,9 +168,10 @@ def get_dbfile_from_toml(tomlfile): 'Only sqlite database supported' return projconf['database']['filepath'] -def main(arguments, prog=sys.argv[0]): +def main(arguments, prog=None): 'Main logic here' parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) if args.dbfile is None: diff --git a/scripts/flitcli/flit_init.py b/scripts/flitcli/flit_init.py index 47aa6740..c42d88db 100644 --- a/scripts/flitcli/flit_init.py +++ b/scripts/flitcli/flit_init.py @@ -112,9 +112,10 @@ def populate_parser(parser=None): help='Copy over litmus tests too') return parser -def main(arguments): +def main(arguments, prog=None): 'Main logic here' parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) os.makedirs(args.directory, exist_ok=True) diff --git a/scripts/flitcli/flit_make.py b/scripts/flitcli/flit_make.py index 6b08a0a8..5cbcf94f 100644 --- a/scripts/flitcli/flit_make.py +++ b/scripts/flitcli/flit_make.py @@ -131,9 +131,10 @@ def populate_parser(parser=None): ''') return parser -def main(arguments, prog=sys.argv[0]): +def main(arguments, prog=None): 'Main logic here' parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) check_call_kwargs = dict() diff --git a/scripts/flitcli/flit_update.py b/scripts/flitcli/flit_update.py index 30a11981..ef94fa91 100644 --- a/scripts/flitcli/flit_update.py +++ b/scripts/flitcli/flit_update.py @@ -371,9 +371,10 @@ def create_makefile(args, makefile='Makefile'): util.process_in_file(os.path.join(conf.data_dir, 'Makefile.in'), makefile, replacements, overwrite=True) -def main(arguments): +def main(arguments, prog=None): 'Main logic here' parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) makefile = os.path.join(args.directory, 'Makefile') diff --git a/scripts/flitcli/unfinished/flit_analyze.py b/scripts/flitcli/unfinished/flit_analyze.py index d339a9af..7200f690 100644 --- a/scripts/flitcli/unfinished/flit_analyze.py +++ b/scripts/flitcli/unfinished/flit_analyze.py @@ -108,8 +108,9 @@ def populate_parser(parser=None): ''') return parser -def main(arguments, prog=sys.argv[0]): +def main(arguments, prog=None): parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) # Subcommand logic here diff --git a/scripts/flitcli/unfinished/flit_check.py b/scripts/flitcli/unfinished/flit_check.py index 871af597..a4c84dfc 100644 --- a/scripts/flitcli/unfinished/flit_check.py +++ b/scripts/flitcli/unfinished/flit_check.py @@ -108,8 +108,9 @@ def populate_parser(parser=None): connections and maybe other things.''') return parser -def main(arguments): +def main(arguments, prog=None): parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) # Subcommand logic here diff --git a/scripts/flitcli/unfinished/flit_run.py b/scripts/flitcli/unfinished/flit_run.py index b2f2a1b6..0c69768d 100644 --- a/scripts/flitcli/unfinished/flit_run.py +++ b/scripts/flitcli/unfinished/flit_run.py @@ -101,8 +101,9 @@ def populate_parser(parser=None): help='A description of the test run (required)') return parser -def main(arguments): +def main(arguments, prog=None): parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) # Subcommand logic here diff --git a/tests/flit_cli/bash_completion/tst_completion_flit.py b/tests/flit_cli/bash_completion/tst_completion_flit.py index 211f887f..2c0e2197 100755 --- a/tests/flit_cli/bash_completion/tst_completion_flit.py +++ b/tests/flit_cli/bash_completion/tst_completion_flit.py @@ -24,7 +24,7 @@ def bashcomplete(self, args): def get_parser(self): subcommands = th.flit.load_subcommands(th.config.script_dir) subcommands.append(th.flit.create_help_subcommand(subcommands)) - return th.flit.populate_parser(subcommands=subcommands) + return th.flit.populate_parser(subcommands=subcommands, recursive=True) def test_empty_available_options(self): self.assertEmptyAvailableOptions(self.get_parser()) From c6e561cc7e42bdf0a44a7f30cc9efd7c9fb1493a Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Tue, 18 Feb 2020 21:33:31 -0700 Subject: [PATCH 17/21] Fix version test --- scripts/flitcli/flit.py | 2 +- tests/Makefile | 6 +++--- tests/flit_cli/Makefile | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/flitcli/flit.py b/scripts/flitcli/flit.py index 8919764f..80921fa9 100755 --- a/scripts/flitcli/flit.py +++ b/scripts/flitcli/flit.py @@ -160,7 +160,7 @@ def populate_parser(parser=None, subcommands=None, recursive=False): ''' parser.add_argument('-v', '--version', action='version', version='flit version ' + conf.version, - help='print the version and exit') + help='Print version and exit') if subcommands: subparsers = parser.add_subparsers( title='Subcommands', diff --git a/tests/Makefile b/tests/Makefile index 6766752d..0613414f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -38,11 +38,11 @@ run_tester_py: $(HARNESS_PY) python3 -m doctest $(HARNESS_PY) build__%: - @$(MAKE) build --directory $* + @$(MAKE) --no-print-directory build --directory $* check__%: - @$(MAKE) check --directory $* + @$(MAKE) --no-print-directory check --directory $* clean__%: - @$(MAKE) clean --directory $* + @$(MAKE) --no-print-directory clean --directory $* diff --git a/tests/flit_cli/Makefile b/tests/flit_cli/Makefile index 65f3455d..3d4132d0 100644 --- a/tests/flit_cli/Makefile +++ b/tests/flit_cli/Makefile @@ -23,11 +23,11 @@ help: .PHONY: build__% check__% clean__% build__%: - @$(MAKE) build --directory $* + @$(MAKE) --no-print-directory build --directory $* check__%: - @$(MAKE) check --directory $* + @$(MAKE) --no-print-directory check --directory $* clean__%: - @$(MAKE) clean --directory $* + @$(MAKE) --no-print-directory clean --directory $* From cbab3b50ad1a734b2722215ba6bdaedb562dc440 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Wed, 19 Feb 2020 04:56:34 +0000 Subject: [PATCH 18/21] Fixes for python 3.4 --- scripts/flitcli/flit.py | 6 ++--- .../bash_completion/util/arginspect.py | 25 ++++++++----------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/scripts/flitcli/flit.py b/scripts/flitcli/flit.py index 80921fa9..17ef124c 100755 --- a/scripts/flitcli/flit.py +++ b/scripts/flitcli/flit.py @@ -165,8 +165,7 @@ def populate_parser(parser=None, subcommands=None, recursive=False): subparsers = parser.add_subparsers( title='Subcommands', dest='subcommand', - metavar='subcommand', - required=True) + metavar='subcommand') for subcommand in subcommands: subparser = subparsers.add_parser( subcommand.name, help=subcommand.brief_description, @@ -212,8 +211,7 @@ def help_populate_parser(parser=None): subparsers = parser.add_subparsers( title='Subcommands', dest='help_subcommand', - metavar='subcommand', - required=False) + metavar='subcommand') for subcommand in subcommand_map.values(): subparsers.add_parser(subcommand.name, help=subcommand.brief_description) diff --git a/tests/flit_cli/bash_completion/util/arginspect.py b/tests/flit_cli/bash_completion/util/arginspect.py index 89b6f46e..b5fd3a22 100644 --- a/tests/flit_cli/bash_completion/util/arginspect.py +++ b/tests/flit_cli/bash_completion/util/arginspect.py @@ -11,6 +11,12 @@ def is_position_action(action): def is_subparser_action(action): return isinstance(action, argparse._SubParsersAction) +_p = argparse.ArgumentParser() +_action_map = _p._registries['action'] +_name_map = {v: k for k, v in _action_map.items()} +del _p +del _action_map + class ActionInspector: ''' All Actions have: @@ -33,21 +39,10 @@ class ActionInspector: - parser_class ''' - NAME_MAP = { - argparse._StoreAction : 'store', - argparse._StoreConstAction : 'store_const', - argparse._StoreTrueAction : 'store_true', - argparse._StoreFalseAction : 'store_false', - argparse._AppendAction : 'append', - argparse._AppendConstAction : 'append_const', - argparse._CountAction : 'count', - argparse._HelpAction : 'help', - argparse._VersionAction : 'version', - argparse._SubParsersAction : 'parsers', - argparse._ExtendAction : 'extend', - } + NAME_MAP = _name_map def __init__(self, action): + if action.__class__ in self.NAME_MAP: self.action_type = self.NAME_MAP[action.__class__] else: @@ -95,9 +90,9 @@ def __init__(self, parser): self.subparser_actions self.option_strings = sum( - [a.option_strings for a in self.option_actions], start=[]) + [a.option_strings for a in self.option_actions], []) self.subparser_choices = sum( - [list(a.choices) for a in self.subparser_actions], start=[]) + [list(a.choices) for a in self.subparser_actions], []) class ArgParseTestBase(unittest.TestCase, metaclass=abc.ABCMeta): From e001e3c99a495d33974426fa1f82258ace222523 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Wed, 19 Feb 2020 13:08:38 -0700 Subject: [PATCH 19/21] docs/install: add bash-completion description --- documentation/installation.md | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/documentation/installation.md b/documentation/installation.md index 75303cb7..d26c1724 100644 --- a/documentation/installation.md +++ b/documentation/installation.md @@ -12,6 +12,7 @@ Instruction Contents: * [Compilers](#compilers) * [Optional Dependencies](#optional-dependencies) * [FLiT Setup](#flit-setup) +* [Bash Completion](#bash-completion) * [Database Setup](#database-setup) * [Uninstallation](#uninstallation) @@ -218,6 +219,48 @@ ln -s ~/FLiT/scripts/flitcli/flit.py ~/bin/flit See [FLiT Command-Line](flit-command-line.md) for more information on how to use the command-line tool. +## Bash Completion + +FLiT comes with a bash-completion script to be used with bash. In the +repository, the script can be found at `scripts/bash-completion/flit`. If you +install FLiT using `make install`, it gets installed to +`$PREFIX/share/bash-completion/completions/flit`. Some systems will be able to +use the flit bash completion from this location directly (if `$PREFIX="/usr"`). +Other systems store these scripts in `/etc/bash_completion.d/`. + +If your system stores bash-completion scripts in `/etc/bash_completion.d/`, you +can either copy the script, or create a symbolic link (preferred). + +```bash +sudo ln -s /usr/share/bash-completion/completions/flit /etc/bash_completion.d/flit +``` + +If you do not have sudo permissions or do not want to install bash-completion +for flit system-wide, then you can implement it locally for your user account. +Newer bash-completion installations allow users to have a script in their home +directory called `$HOME/.bash_completion`. We recommend you have a directory +for storing bash-completion scripts. You can put the following in your +`$HOME/.bash_completion` file + +```bash +if [ -d ~/.local/share/bash-completion/completions ]; then + for file in ~/.local/share/bash-completion/completions/*; do + if [ -f "$file" ]; then + source "$file" + fi + done +fi +``` + +Then you can simply copy or symbolically link bash-completion scripts into +`~/.local/share/bash-completion/completions`. If you are using FLiT from the +repository, you can accomplish that with + +```bash +mkdir -p ~/.local/share/bash-completion/completions +ln -s /FLiT/scripts/bash-completion/flit ~/.local/share/bash-completion/completions/flit +``` + ## Database Setup There should be nothing to setup since `sqlite3` should already be present. From c0411ff795c8668e38c09ffb9c88104259605ea2 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Wed, 19 Feb 2020 13:10:08 -0700 Subject: [PATCH 20/21] docs/adding-to-cli: update based on changes - Updated the README.md in scripts/flitcli/ - Added a section in pointing to the to know how to add custom subcommands --- documentation/flit-command-line.md | 15 ++++++++++++ scripts/flitcli/README.md | 37 ++++++++++++++++++------------ 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/documentation/flit-command-line.md b/documentation/flit-command-line.md index 16cf6d27..a84c5f04 100644 --- a/documentation/flit-command-line.md +++ b/documentation/flit-command-line.md @@ -29,6 +29,10 @@ Possible subcommands: * [flit bisect](#flit-bisect): Assign variability blame to files and functions * [flit experimental](#flit-experimental): Access to experimental features +Possibly of interest: + +* [Adding More Subcommands](#adding-more-subcommands) + ## flit help This can display the help documentation for a specific subcommand. This is @@ -217,6 +221,17 @@ You may also read the documentation on [experimental features](experimental-features.md). +## Adding More Subcommands + +The FLiT command-line structure is extremely modular. If you create a file +with the correct naming scheme, put it in the correct directory, and implement +three things, it will automatically be picked up and become available as a +subcommand from the `flit` command-line tool. + +See the flit command-line [README.md](../scripts/flitcli/README.md) for more +information. + + [Prev](litmus-tests.md) | [Table of Contents](README.md) diff --git a/scripts/flitcli/README.md b/scripts/flitcli/README.md index 2143cf92..fc95063b 100644 --- a/scripts/flitcli/README.md +++ b/scripts/flitcli/README.md @@ -8,6 +8,9 @@ things: - `flitconfig.py`: Contains paths to FLiT files needed by the command-line tool - `flit_*.py`: Each subcommand is isolated into its own executable python script. This allows for modularity and easy extensibility. +- `flitelf.py`: helper script for reading ELF binary files +- `flitutil.py`: helper functions used in many subcommands +- `experimental/flit_*.py`: experimental subcommands ## FLiT Subcommands @@ -24,14 +27,14 @@ For this documentation, let us use the example subcommand of squelch. keep this to one or two sentences. More detailed explanations can be given by the help documentation of the subcommand. 3. Implement a main method with the following declaration: - `def main(arguments, prog=sys.argv[0])`. The `prog` argument is intended to - be in the help documentation for that subcommand as the executable that was + `def main(arguments, prog=None)`. The `prog` argument is intended to be in + the help documentation for that subcommand as the executable that was called. It can be passed directly into the `prog` argument for the `argparse.ArgumentParser` class. The `arguments` arg only contain the arguments and not the program name (similar to `sys.argv[1:]`). -4. Implement command-line parsing of the `-h` and `--help` flags in the - `arguments` parameter to `main`. This should show the documentation for - your subcommand and exit. +4. Implement a method called `populate_parser(parser=None)` where it populates + a given `argparse.ArgumentParser` instance (if `None` is given, you create + one in the function) and returns the parser. ## Example Subcommand @@ -45,20 +48,24 @@ import sys brief_description = 'Quiets any communication with the server' -def main(arguments, prog=sys.argv[0]): - parser = argparse.ArgumentParser( - prog=prog, - description=''' - The squelch command rejects any commands to communicate from - the server. This is not an actual command or feature of flit, - this subcommand is only for illustrative purposes on how to - generate a subcommand. - ''', - ) +def populate_parser(parser=None): + if not parser: + parser = ArgumentParser() + parser.description = ''' + The squelch command rejects any commands to communicate from + the server. This is not an actual command or feature of flit, + this subcommand is only for illustrative purposes on how to + generate a subcommand. + ''' parser.add_argument('-s', '--server', default='127.0.0.1', help='The server IP address') parser.add_argument('-v', '--verbose', action='store_true', help='Show verbose messages') + return parser + +def main(arguments, prog=None): + parser = populate_parser() + if prog: parser.prog = prog args = parser.parse_args(arguments) # Subcommand logic here From fde3afadd57b6b44364f86225f6e8bad79f40932 Mon Sep 17 00:00:00 2001 From: Michael Bentley Date: Mon, 16 Mar 2020 11:52:31 -0600 Subject: [PATCH 21/21] tests/bash_completion: add doc to arginspect.py --- .../bash_completion/util/arginspect.py | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/flit_cli/bash_completion/util/arginspect.py b/tests/flit_cli/bash_completion/util/arginspect.py index b5fd3a22..ddfb7bf4 100644 --- a/tests/flit_cli/bash_completion/util/arginspect.py +++ b/tests/flit_cli/bash_completion/util/arginspect.py @@ -1,14 +1,67 @@ import abc import argparse import unittest +import sys def is_option_action(action): + ''' + If the ArgumentParser action (which is ArgumentParser's way to store + possible arguments) is an option argument (meaning it starts with a "-"), + then return True. + + >>> p = argparse.ArgumentParser() + >>> is_option_action(p.add_argument('--number')) + True + + >>> is_option_action(p.add_argument('another_number')) + False + + >>> subparsers = p.add_subparsers() + >>> _ = subparsers.add_parser('subcommand') + >>> is_option_action(subparsers) + False + ''' return bool(action.option_strings) def is_position_action(action): - return not is_option_arg(action) + ''' + If the ArgumentParser action (which is ArgumentParser's way to store + possible arguments) is a positional argument, then return True. + + >>> p = argparse.ArgumentParser() + >>> is_position_action(p.add_argument('--number')) + False + + >>> is_position_action(p.add_argument('another_number')) + True + + >>> subparsers = p.add_subparsers() + >>> _ = subparsers.add_parser('subcommand') + >>> is_position_action(subparsers) + True + ''' + return not is_option_action(action) def is_subparser_action(action): + ''' + If the ArgumentParser action (which is ArgumentParser's way to store + possible arguments) is a subcommand with its own argument parsing, then + return True. + + Note: all subparser actions are also position actions. + + >>> p = argparse.ArgumentParser() + >>> is_subparser_action(p.add_argument('--number')) + False + + >>> is_subparser_action(p.add_argument('another_number')) + False + + >>> subparsers = p.add_subparsers() + >>> _ = subparsers.add_parser('subcommand') + >>> is_subparser_action(subparsers) + True + ''' return isinstance(action, argparse._SubParsersAction) _p = argparse.ArgumentParser() @@ -47,6 +100,8 @@ def __init__(self, action): self.action_type = self.NAME_MAP[action.__class__] else: self.action_type = action.__class__.__name__ + print('Warning: unrecognized action class: {}'.format(self.action_type), + file=sys.stderr) # passthrough of attributes self.action = action