From 115ddcc9dbb95e90492d511fca981f0963ad2622 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 14:05:43 -0700 Subject: [PATCH 01/12] renamed 'environment' test --- .../spack/test/{environment.py => environment_modifications.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/spack/spack/test/{environment.py => environment_modifications.py} (100%) diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment_modifications.py similarity index 100% rename from lib/spack/spack/test/environment.py rename to lib/spack/spack/test/environment_modifications.py From ff7ba9ed8d5a1d77982b2ae1d7f8f910fb880ec0 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 15:22:55 -0700 Subject: [PATCH 02/12] rename 'spack env' command to 'spack build-env' --- lib/spack/spack/cmd/{env.py => build_env.py} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename lib/spack/spack/cmd/{env.py => build_env.py} (91%) diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/build_env.py similarity index 91% rename from lib/spack/spack/cmd/env.py rename to lib/spack/spack/cmd/build_env.py index 29b00e12f26e0c..d3202a0afe3863 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/build_env.py @@ -28,7 +28,7 @@ import os import llnl.util.tty as tty -import spack.build_environment as build_env +import spack.build_environment as build_environment import spack.cmd import spack.cmd.common.arguments as arguments @@ -44,9 +44,9 @@ def setup_parser(subparser): help="specs of package environment to emulate") -def env(parser, args): +def build_env(parser, args): if not args.spec: - tty.die("spack env requires a spec.") + tty.die("spack build-env requires a spec.") # Specs may have spaces in them, so if they do, require that the # caller put a '--' between the spec and the command to be @@ -63,10 +63,10 @@ def env(parser, args): specs = spack.cmd.parse_specs(spec, concretize=True) if len(specs) > 1: - tty.die("spack env only takes one spec.") + tty.die("spack build-env only takes one spec.") spec = specs[0] - build_env.setup_package(spec.package, args.dirty) + build_environment.setup_package(spec.package, args.dirty) if not cmd: # If no command act like the "env" command and print out env vars. From 2365cc8c7a67f9599db75b42802be2407ca3f38e Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 16:12:38 -0700 Subject: [PATCH 03/12] allow writing full spec (including build deps) to dict --- lib/spack/spack/spec.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5050981e2f5bbf..b7dc34959de44f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1444,7 +1444,7 @@ def full_hash(self, length=None): return self._full_hash[:length] - def to_node_dict(self, hash_function=None): + def to_node_dict(self, hash_function=None, all_deps=False): d = syaml_dict() if self.versions: @@ -1476,7 +1476,11 @@ def to_node_dict(self, hash_function=None): # TODO: restore build dependencies here once we have less picky # TODO: concretization. - deps = self.dependencies_dict(deptype=('link', 'run')) + if all_deps: + deptypes = ('link', 'run', 'build') + else: + deptypes = ('link', 'run') + deps = self.dependencies_dict(deptype=deptypes) if deps: if hash_function is None: hash_function = lambda s: s.dag_hash() @@ -1490,10 +1494,14 @@ def to_node_dict(self, hash_function=None): return syaml_dict([(self.name, d)]) - def to_dict(self): + def to_dict(self, all_deps=False): + if all_deps: + deptypes = ('link', 'run', 'build') + else: + deptypes = ('link', 'run') node_list = [] - for s in self.traverse(order='pre', deptype=('link', 'run')): - node = s.to_node_dict() + for s in self.traverse(order='pre', deptype=deptypes): + node = s.to_node_dict(all_deps=all_deps) node[s.name]['hash'] = s.dag_hash() node_list.append(node) From 81e06da1faf80939b4e8ddb43e9e255aaf7c8ac1 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 16:15:57 -0700 Subject: [PATCH 04/12] add convenience function for reading spec dependencies from node dict --- lib/spack/spack/spec.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index b7dc34959de44f..17c4a2e1b0e046 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1570,6 +1570,15 @@ def from_node_dict(node): return spec + @staticmethod + def dependencies_from_node_dict(node): + name = next(iter(node)) + node = node[name] + if 'dependencies' not in node: + return + for t in Spec.read_yaml_dep_specs(node['dependencies']): + yield t + @staticmethod def read_yaml_dep_specs(dependency_dict): """Read the DependencySpec portion of a YAML-formatted Spec. From fcd2217b982c453d136a172c029d5d9847ec235e Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 16:25:09 -0700 Subject: [PATCH 05/12] _concrete should never be cleared when copying Specs --- lib/spack/spack/spec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 17c4a2e1b0e046..aae9d7272ba3b5 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2700,17 +2700,17 @@ def _dup(self, other, deps=True, cleardeps=True, caches=None): deptypes = deps self._dup_deps(other, deptypes, caches) + self._concrete = other._concrete + if caches: self._hash = other._hash self._cmp_key_cache = other._cmp_key_cache self._normal = other._normal - self._concrete = other._concrete self._full_hash = other._full_hash else: self._hash = None self._cmp_key_cache = None self._normal = False - self._concrete = False self._full_hash = None return changed From 46dfedaa3aaad3362211672cbe35c540b2a46208 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 16:34:11 -0700 Subject: [PATCH 06/12] save/restore concrete & patches when exporting/importing Specs to/from node dictionaries --- lib/spack/spack/spec.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index aae9d7272ba3b5..4d980cc8e9b640 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1474,6 +1474,13 @@ def to_node_dict(self, hash_function=None, all_deps=False): 'module': bool(self.external_module) } + d['concrete'] = self._concrete + + if 'patches' in self.variants: + variant = self.variants['patches'] + if hasattr(variant, '_patches_in_order_of_appearance'): + d['patches'] = variant._patches_in_order_of_appearance + # TODO: restore build dependencies here once we have less picky # TODO: concretization. if all_deps: @@ -1565,6 +1572,19 @@ def from_node_dict(node): spec.external_path = None spec.external_module = None + if 'concrete' in node: + spec._concrete = node['concrete'] + + if 'patches' in node: + patches = node['patches'] + if len(patches) > 0: + mvar = spec.variants.setdefault( + 'patches', MultiValuedVariant('patches', ()) + ) + mvar.value = patches + # FIXME: Monkey patches mvar to store patches order + mvar._patches_in_order_of_appearance = patches + # Don't read dependencies here; from_node_dict() is used by # from_yaml() to read the root *and* each dependency spec. From 5e8b027ea988f481027e8a68a465d7da276c8482 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 17:03:42 -0700 Subject: [PATCH 07/12] refactor common arguments --- lib/spack/spack/cmd/common/arguments.py | 15 ++++++ lib/spack/spack/cmd/install.py | 71 ++++++++++++++++--------- lib/spack/spack/cmd/module.py | 33 +++++++----- lib/spack/spack/cmd/spec.py | 15 +++--- lib/spack/spack/cmd/uninstall.py | 41 +++++++------- lib/spack/spack/spec.py | 6 +++ 6 files changed, 115 insertions(+), 66 deletions(-) diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index ea5bc1c25da84e..7f8c2855b9e2ae 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -101,6 +101,11 @@ def _specs(self, **kwargs): '-r', '--dependencies', action='store_true', dest='recurse_dependencies', help='recursively traverse spec dependencies') +_arguments['recurse_dependents'] = Args( + '-R', '--dependents', action='store_true', dest='dependents', + help='also uninstall any packages that depend on the ones given ' + 'via command line') + _arguments['clean'] = Args( '--clean', action='store_false', @@ -131,6 +136,16 @@ def _specs(self, **kwargs): '-t', '--tags', action='append', help='filter a package query by tags') +_arguments['jobs'] = Args( + '-j', '--jobs', action='store', type=int, dest="jobs", + help="explicitly set number of make jobs, default is #cpus.") + +_arguments['install_status'] = Args( + '-I', '--install-status', action='store_true', default=False, + help='show install status of packages. packages can be: ' + 'installed [+], missing and needed by an installed package [-], ' + 'or not installed (no annotation)') + _arguments['no_checksum'] = Args( '-n', '--no-checksum', action='store_true', default=False, help="do not use checksums to verify downloadeded files (unsafe)") diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 6404ca81a375e7..c3f926225aa13d 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -44,18 +44,8 @@ level = "short" -def setup_parser(subparser): - subparser.add_argument( - '--only', - default='package,dependencies', - dest='things_to_install', - choices=['package', 'dependencies'], - help="""select the mode of installation. -the default is to install the package along with all its dependencies. -alternatively one can decide to install only the package or only -the dependencies""" - ) - arguments.add_common_arguments(subparser, ['jobs']) +def add_common_arguments(subparser): + arguments.add_common_arguments(subparser, ['jobs', 'install_status']) subparser.add_argument( '--overwrite', action='store_true', help="reinstall an existing spec, even if it has dependents") @@ -84,14 +74,52 @@ def setup_parser(subparser): subparser.add_argument( '--fake', action='store_true', help="fake install for debug purposes.") + + cd_group = subparser.add_mutually_exclusive_group() + arguments.add_common_arguments(cd_group, ['clean', 'dirty']) + + +def update_kwargs_from_args(args, kwargs): + """Parse cli arguments and construct a dictionary + that will be passed to Package.do_install API""" + + kwargs.update({ + 'keep_prefix': args.keep_prefix, + 'keep_stage': args.keep_stage, + 'restage': not args.dont_restage, + 'install_source': args.install_source, + 'make_jobs': args.jobs, + 'verbose': args.verbose, + 'fake': args.fake, + 'dirty': args.dirty, + 'use_cache': args.use_cache + }) + if hasattr(args, 'setup'): + setups = set() + for arglist_s in args.setup: + for arg in [x.strip() for x in arglist_s.split(',')]: + setups.add(arg) + kwargs['setup'] = setups + tty.msg('Setup={0}'.format(kwargs['setup'])) + + +def setup_parser(subparser): + add_common_arguments(subparser) + subparser.add_argument( + '--only', + default='package,dependencies', + dest='things_to_install', + choices=['package', 'dependencies'], + help="""select the mode of installation. +the default is to install the package along with all its dependencies. +alternatively one can decide to install only the package or only +the dependencies""" + ) subparser.add_argument( '-f', '--file', action='append', default=[], dest='specfiles', metavar='SPEC_YAML_FILE', help="install from file. Read specs to install from .yaml files") - cd_group = subparser.add_mutually_exclusive_group() - arguments.add_common_arguments(cd_group, ['clean', 'dirty']) - subparser.add_argument( 'package', nargs=argparse.REMAINDER, @@ -178,17 +206,10 @@ def install(parser, args, **kwargs): # Parse cli arguments and construct a dictionary # that will be passed to Package.do_install API + update_kwargs_from_args(args, kwargs) kwargs.update({ - 'keep_prefix': args.keep_prefix, - 'keep_stage': args.keep_stage, - 'restage': not args.dont_restage, - 'install_source': args.install_source, - 'install_deps': 'dependencies' in args.things_to_install, - 'make_jobs': args.jobs, - 'verbose': args.verbose, - 'fake': args.fake, - 'dirty': args.dirty, - 'use_cache': args.use_cache + 'install_dependencies': ('dependencies' in args.things_to_install), + 'install_package': ('package' in args.things_to_install) }) if args.run_tests: diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 11bbaf7759301f..35a8d8b100f236 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -57,6 +57,24 @@ def decorator(callback): return decorator +def add_loads_arguments(subparser): + subparser.add_argument( + '--input-only', action='store_false', dest='shell', + help='generate input for module command (instead of a shell script)' + ) + subparser.add_argument( + '-p', '--prefix', dest='prefix', default='', + help='prepend to module names when issuing module load commands' + ) + subparser.add_argument( + '-x', '--exclude', dest='exclude', action='append', default=[], + help="exclude package from output; may be specified multiple times" + ) + arguments.add_common_arguments( + subparser, ['module_type', 'recurse_dependencies'] + ) + + def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='subparser_name') @@ -91,20 +109,9 @@ def setup_parser(subparser): 'loads', help='prompt the list of modules associated with a constraint' ) - loads_parser.add_argument( - '--input-only', action='store_false', dest='shell', - help='generate input for module command (instead of a shell script)' - ) - loads_parser.add_argument( - '-p', '--prefix', dest='prefix', default='', - help='prepend to module names when issuing module load commands' - ) - loads_parser.add_argument( - '-x', '--exclude', dest='exclude', action='append', default=[], - help="exclude package from output; may be specified multiple times" - ) + add_loads_arguments(loads_parser) arguments.add_common_arguments( - loads_parser, ['constraint', 'module_type', 'recurse_dependencies'] + loads_parser, ['constraint'] ) diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index 0bc1c7874e4a44..70374a6c5d90df 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -34,8 +34,9 @@ level = "short" -def setup_parser(subparser): - arguments.add_common_arguments(subparser, ['long', 'very_long']) +def add_common_arguments(subparser): + arguments.add_common_arguments( + subparser, ['long', 'very_long', 'install_status']) subparser.add_argument( '-y', '--yaml', action='store_true', default=False, help='print concrete spec as YAML') @@ -46,11 +47,7 @@ def setup_parser(subparser): subparser.add_argument( '-N', '--namespaces', action='store_true', default=False, help='show fully qualified package names') - subparser.add_argument( - '-I', '--install-status', action='store_true', default=False, - help='show install status of packages. packages can be: ' - 'installed [+], missing and needed by an installed package [-], ' - 'or not installed (no annotation)') + subparser.add_argument( '-t', '--types', action='store_true', default=False, help='show dependency types') @@ -58,6 +55,10 @@ def setup_parser(subparser): 'specs', nargs=argparse.REMAINDER, help="specs of packages") +def setup_parser(subparser): + add_common_arguments(subparser) + + def spec(parser, args): name_fmt = '$.' if args.namespaces else '$_' kwargs = {'cover': args.cover, diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index ec05a23137b08a..cd2df93195ec8e 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -28,6 +28,7 @@ import spack.cmd import spack.package +import spack.cmd.common.arguments as arguments import spack.repo import spack.store @@ -50,11 +51,16 @@ } -def setup_parser(subparser): +def add_common_arguments(subparser): subparser.add_argument( '-f', '--force', action='store_true', dest='force', help="remove regardless of whether other packages depend on this one") + arguments.add_common_arguments( + subparser, ['recurse_dependents', 'yes_to_all']) + +def setup_parser(subparser): + add_common_arguments(subparser) subparser.add_argument( '-a', '--all', action='store_true', dest='all', help="USE CAREFULLY. remove ALL installed packages that match each " @@ -63,15 +69,6 @@ def setup_parser(subparser): "supplied all installed software will be uninstalled. this " "is both useful and dangerous, like rm -r") - subparser.add_argument( - '-R', '--dependents', action='store_true', dest='dependents', - help='also uninstall any packages that depend on the ones given ' - 'via command line') - - subparser.add_argument( - '-y', '--yes-to-all', action='store_true', dest='yes_to_all', - help='assume "yes" is the answer to every confirmation requested') - subparser.add_argument( 'packages', nargs=argparse.REMAINDER, @@ -167,16 +164,12 @@ def num_installed_deps(pkg): item.do_uninstall(force=force) -def get_uninstall_list(args): - specs = [any] - if args.packages: - specs = spack.cmd.parse_specs(args.packages) - +def get_uninstall_list(args, specs): # Gets the list of installed specs that match the ones give via cli # takes care of '-a' is given in the cli uninstall_list = find_matching_specs(specs, args.all, args.force) - # Takes care of '-d' + # Takes care of '-R' dependent_list = installed_dependents(uninstall_list) # Process dependent_list and update uninstall_list @@ -200,12 +193,9 @@ def get_uninstall_list(args): return uninstall_list -def uninstall(parser, args): - if not args.packages and not args.all: - tty.die('uninstall requires at least one package argument.', - ' Use `spack uninstall --all` to uninstall ALL packages.') +def uninstall_specs(args, specs): - uninstall_list = get_uninstall_list(args) + uninstall_list = get_uninstall_list(args, specs) if not uninstall_list: tty.warn('There are no package to uninstall.') @@ -220,3 +210,12 @@ def uninstall(parser, args): # Uninstall everything on the list do_uninstall(uninstall_list, args.force) + + +def uninstall(parser, args): + if not args.packages and not args.all: + tty.die('uninstall requires at least one package argument.', + ' Use `spack uninstall --all` to uninstall ALL packages.') + + uninstall_specs( + args, spack.cmd.parse_specs(args.packages) if args.packages else [any]) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 4d980cc8e9b640..23295b074a253e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -3215,6 +3215,7 @@ def tree(self, **kwargs): prefix = kwargs.pop('prefix', None) show_types = kwargs.pop('show_types', False) deptypes = kwargs.pop('deptypes', ('build', 'link')) + recurse_dependencies = kwargs.pop('recurse_dependencies', True) check_kwargs(kwargs, self.tree) out = "" @@ -3254,6 +3255,11 @@ def tree(self, **kwargs): if d > 0: out += "^" out += node.format(fmt, color=color) + "\n" + + # Check if we wanted just the first line + if not recurse_dependencies: + break + return out def __repr__(self): From c23d73af897d6c6e6027b12515ad08a2c703a47b Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 17:38:28 -0700 Subject: [PATCH 08/12] preserve command_line as the scope of highest precedence --- lib/spack/spack/config.py | 15 ++++++++++++++- lib/spack/spack/test/conftest.py | 9 ++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 5157e918aa530f..5e60ac6e68ed90 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -265,13 +265,26 @@ def __init__(self, *scopes): def push_scope(self, scope): """Add a higher precedence scope to the Configuration.""" + cmd_line_scope = None + if self.scopes: + highest_precedence_scope = list(self.scopes.values())[-1] + if highest_precedence_scope.name == 'command_line': + # If the command-line scope is present, it should always + # be the scope of highest precedence + cmd_line_scope = self.pop_scope() + self.scopes[scope.name] = scope + if cmd_line_scope: + self.scopes['command_line'] = cmd_line_scope def pop_scope(self): """Remove the highest precedence scope and return it.""" name, scope = self.scopes.popitem(last=True) return scope + def remove_scope(self, scope_name): + return self.scopes.pop(scope_name) + @property def file_scopes(self): """List of scopes with an associated file (non-internal scopes).""" @@ -462,7 +475,7 @@ def override(path, value): yield config - scope = config.pop_scope() + scope = config.remove_scope(overrides.name) assert scope is overrides diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index a003df40c896fa..2d461e587c34fe 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -244,9 +244,12 @@ def config(configuration_dir): real_configuration = spack.config.config - spack.config.config = spack.config.Configuration( - *[spack.config.ConfigScope(name, str(configuration_dir.join(name))) - for name in ['site', 'system', 'user']]) + test_scopes = [ + spack.config.ConfigScope(name, str(configuration_dir.join(name))) + for name in ['site', 'system', 'user']] + test_scopes.append(spack.config.InternalConfigScope('command_line')) + + spack.config.config = spack.config.Configuration(*test_scopes) yield spack.config.config From 654885651850a4b41d7b3f1388f8a07fd6e86417 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 17:47:11 -0700 Subject: [PATCH 09/12] rename 'env' command test to 'build_env' to reflect new name --- lib/spack/spack/test/cmd/{env.py => build_env.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename lib/spack/spack/test/cmd/{env.py => build_env.py} (98%) diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/build_env.py similarity index 98% rename from lib/spack/spack/test/cmd/env.py rename to lib/spack/spack/test/cmd/build_env.py index da489c8c820ce1..69ccbb732a925c 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/build_env.py @@ -26,7 +26,7 @@ from spack.main import SpackCommand, SpackCommandError -info = SpackCommand('env') +info = SpackCommand('build-env') @pytest.mark.parametrize('pkg', [ From 7499b72ea3e0c8f902119179f69cb358d19d1958 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 17:53:58 -0700 Subject: [PATCH 10/12] add spack env command, along with env.yaml schema and tests --- .gitignore | 1 + lib/spack/spack/cmd/env.py | 813 ++++++++++++++++++++++++++++++++ lib/spack/spack/schema/env.py | 62 +++ lib/spack/spack/test/cmd/env.py | 168 +++++++ 4 files changed, 1044 insertions(+) create mode 100644 lib/spack/spack/cmd/env.py create mode 100644 lib/spack/spack/schema/env.py create mode 100644 lib/spack/spack/test/cmd/env.py diff --git a/.gitignore b/.gitignore index bb7b14a82c4428..a12ef9b02b9b8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /db /var/spack/stage /var/spack/cache +/var/spack/environments /var/spack/repos/*/index.yaml /var/spack/repos/*/lock *.pyc diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py new file mode 100644 index 00000000000000..d48397f46e53cf --- /dev/null +++ b/lib/spack/spack/cmd/env.py @@ -0,0 +1,813 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty +import spack +import llnl.util.filesystem as fs +import spack.modules +import spack.util.spack_json as sjson +import spack.util.spack_yaml as syaml +import spack.schema.env +import spack.config +import spack.cmd.spec +import spack.cmd.install +import spack.cmd.uninstall +import spack.cmd.module +import spack.cmd.common.arguments as arguments +from spack.config import ConfigScope +from spack.spec import Spec, CompilerSpec, FlagMap +from spack.repo import Repo +from spack.version import VersionList +from contextlib import contextmanager + +import argparse +try: + from itertools import izip_longest as zip_longest +except ImportError: + from itertools import zip_longest +import os +import sys +import shutil + +description = "group a subset of packages" +section = "environment" +level = "long" + +_db_dirname = fs.join_path(spack.paths.var_path, 'environments') + + +def get_env_root(name): + """Given an environment name, determines its root directory""" + return fs.join_path(_db_dirname, name) + + +def get_dotenv_dir(env_root): + """@return Directory in an environment that is owned by Spack""" + return fs.join_path(env_root, '.env') + + +def get_write_paths(env_root): + """Determines the names of temporary and permanent directories to + write machine-generated environment info.""" + tmp_new = fs.join_path(env_root, '.env.new') + dest = get_dotenv_dir(env_root) + tmp_old = fs.join_path(env_root, '.env.old') + return tmp_new, dest, tmp_old + + +class Environment(object): + def clear(self): + self.user_specs = list() + self.concretized_order = list() + self.specs_by_hash = dict() + + def __init__(self, name): + self.name = name + self.clear() + + # Default config + self.yaml = { + 'configs': [''], + 'specs': [] + } + + @property + def path(self): + return get_env_root(self.name) + + def repo_path(self): + return fs.join_path(get_dotenv_dir(self.path), 'repo') + + def add(self, user_spec, report_existing=True): + """Add a single user_spec (non-concretized) to the Environment""" + query_spec = Spec(user_spec) + existing = set(x for x in self.user_specs + if Spec(x).name == query_spec.name) + if existing: + if report_existing: + tty.die("Package {0} was already added to {1}" + .format(query_spec.name, self.name)) + else: + tty.msg("Package {0} was already added to {1}" + .format(query_spec.name, self.name)) + else: + tty.msg('Adding %s to environment %s' % (user_spec, self.name)) + self.user_specs.append(user_spec) + + def remove(self, query_spec): + """Remove specs from an environment that match a query_spec""" + query_spec = Spec(query_spec) + match_index = -1 + for i, spec in enumerate(self.user_specs): + if Spec(spec).name == query_spec.name: + match_index = i + break + + if match_index < 0: + tty.die("Not found: {0}".format(query_spec)) + + del self.user_specs[match_index] + if match_index < len(self.concretized_order): + spec_hash = self.concretized_order[match_index] + del self.concretized_order[match_index] + del self.specs_by_hash[spec_hash] + + def concretize(self, force=False): + """Concretize user_specs in an Environment, creating (fully + concretized) specs. + + force: bool + If set, re-concretize ALL specs, even those that were + already concretized. + """ + + if force: + # Clear previously concretized specs + self.specs_by_hash = dict() + self.concretized_order = list() + + num_concretized = len(self.concretized_order) + new_specs = list() + for user_spec in self.user_specs[num_concretized:]: + tty.msg('Concretizing %s' % user_spec) + + spec = spack.cmd.parse_specs(user_spec)[0] + spec.concretize() + new_specs.append(spec) + dag_hash = spec.dag_hash() + self.specs_by_hash[dag_hash] = spec + self.concretized_order.append(spec.dag_hash()) + + # Display concretized spec to the user + sys.stdout.write(spec.tree( + recurse_dependencies=True, install_status=True, + hashlen=7, hashes=True)) + + return new_specs + + def install(self, install_args=None): + """Do a `spack install` on all the (concretized) + specs in an Environment.""" + + # Make sure log directory exists + logs = fs.join_path(self.path, 'logs') + try: + os.makedirs(logs) + except OSError: + if not os.path.isdir(logs): + raise + + for concretized_hash in self.concretized_order: + spec = self.specs_by_hash[concretized_hash] + + # Parse cli arguments and construct a dictionary + # that will be passed to Package.do_install API + kwargs = dict() + if install_args: + spack.cmd.install.update_kwargs_from_args(install_args, kwargs) + with pushd(self.path): + spec.package.do_install(**kwargs) + + # Link the resulting log file into logs dir + logname = '%s-%s.log' % (spec.name, spec.dag_hash(7)) + logpath = fs.join_path(logs, logname) + try: + os.remove(logpath) + except OSError: + pass + os.symlink(spec.package.build_log_path, logpath) + + def uninstall(self, args): + """Uninstall all the specs in an Environment.""" + specs = self._get_environment_specs(recurse_dependencies=True) + args.all = False + spack.cmd.uninstall.uninstall_specs(args, specs) + + def list(self, stream, **kwargs): + """List the specs in an environment.""" + for user_spec, concretized_hash in zip_longest( + self.user_specs, self.concretized_order): + + stream.write('========= {0}\n'.format(user_spec)) + + if concretized_hash: + concretized_spec = self.specs_by_hash[concretized_hash] + stream.write(concretized_spec.tree(**kwargs)) + + def upgrade_dependency(self, dep_name, dry_run=False): + # TODO: if you have + # w -> x -> y + # and + # v -> x -> y + # then it would be desirable to ensure that w and v refer to the + # same x after upgrading y. This is not currently guaranteed. + new_order = list() + new_deps = list() + for i, spec_hash in enumerate(self.concretized_order): + spec = self.specs_by_hash[spec_hash] + if dep_name in spec: + if dry_run: + tty.msg("Would upgrade {0} for {1}" + .format(spec[dep_name].format(), spec.format())) + else: + new_spec = upgrade_dependency_version(spec, dep_name) + new_order.append(new_spec.dag_hash()) + self.specs_by_hash[new_spec.dag_hash()] = new_spec + new_deps.append(new_spec[dep_name]) + else: + new_order.append(spec_hash) + + if not dry_run: + self.concretized_order = new_order + return new_deps[0] if new_deps else None + + def reset_os_and_compiler(self, compiler=None): + new_order = list() + new_specs_by_hash = {} + for spec_hash in self.concretized_order: + spec = self.specs_by_hash[spec_hash] + new_spec = reset_os_and_compiler(spec, compiler) + new_order.append(new_spec.dag_hash()) + new_specs_by_hash[new_spec.dag_hash()] = new_spec + self.concretized_order = new_order + self.specs_by_hash = new_specs_by_hash + + def _get_environment_specs(self, recurse_dependencies=True): + """Returns the specs of all the packages in an environment. + If these specs appear under different user_specs, only one copy + is added to the list returned.""" + package_to_spec = {} + spec_list = list() + + for spec_hash in self.concretized_order: + spec = self.specs_by_hash[spec_hash] + + specs = spec.traverse(deptype=('link', 'run')) \ + if recurse_dependencies else (spec,) + for dep in specs: + if dep.name in package_to_spec: + tty.warn("{0} takes priority over {1}" + .format(package_to_spec[dep.name].format(), + dep.format())) + else: + package_to_spec[dep.name] = dep + spec_list.append(dep) + + return spec_list + + def to_dict(self): + """Used in serializing to JSON""" + concretized_order = list(self.concretized_order) + concrete_specs = dict() + for spec in self.specs_by_hash.values(): + for s in spec.traverse(): + if s.dag_hash() not in concrete_specs: + concrete_specs[s.dag_hash()] = ( + s.to_node_dict(all_deps=True)) + format = { + 'user_specs': self.user_specs, + 'concretized_order': concretized_order, + 'concrete_specs': concrete_specs, + } + return format + + @staticmethod + def from_dict(name, d): + """Used in deserializing from JSON""" + env = Environment(name) + env.user_specs = list(d['user_specs']) + env.concretized_order = list(d['concretized_order']) + specs_dict = d['concrete_specs'] + + hash_to_node_dict = specs_dict + root_hashes = set(env.concretized_order) + + specs_by_hash = {} + for dag_hash, node_dict in hash_to_node_dict.items(): + specs_by_hash[dag_hash] = Spec.from_node_dict(node_dict) + + for dag_hash, node_dict in hash_to_node_dict.items(): + for dep_name, dep_hash, deptypes in ( + Spec.dependencies_from_node_dict(node_dict)): + specs_by_hash[dag_hash]._add_dependency( + specs_by_hash[dep_hash], deptypes) + + env.specs_by_hash = dict( + (x, y) for x, y in specs_by_hash.items() if x in root_hashes) + + return env + + +def reset_os_and_compiler(spec, compiler=None): + spec = spec.copy() + for x in spec.traverse(): + x.compiler = None + x.architecture = None + x.compiler_flags = FlagMap(x) + x._concrete = False + x._hash = None + if compiler: + spec.compiler = CompilerSpec(compiler) + spec.concretize() + return spec + + +def upgrade_dependency_version(spec, dep_name): + spec = spec.copy() + for x in spec.traverse(): + x._concrete = False + x._hash = None + spec[dep_name].versions = VersionList(':') + spec.concretize() + return spec + + +def check_consistent_env(env_root): + tmp_new, dest, tmp_old = get_write_paths(env_root) + if os.path.exists(tmp_new) or os.path.exists(tmp_old): + tty.die("Partial write state, run 'spack env repair'") + + +def write(environment, new_repo=None): + """Writes an in-memory environment back to its location on disk, + in an atomic manner.""" + + tmp_new, dest, tmp_old = get_write_paths(get_env_root(environment.name)) + + # Write the machine-generated stuff + fs.mkdirp(tmp_new) + # create one file for the environment object + with open(fs.join_path(tmp_new, 'environment.json'), 'w') as F: + sjson.dump(environment.to_dict(), stream=F) + + dest_repo_dir = fs.join_path(tmp_new, 'repo') + if new_repo: + shutil.copytree(new_repo.root, dest_repo_dir) + elif os.path.exists(environment.repo_path()): + shutil.copytree(environment.repo_path(), dest_repo_dir) + + # Swap in new directory atomically + if os.path.exists(dest): + shutil.move(dest, tmp_old) + shutil.move(tmp_new, dest) + if os.path.exists(tmp_old): + shutil.rmtree(tmp_old) + + +def repair(environment_name): + """Recovers from crash during critical section of write(). + Possibilities: + + tmp_new, dest + tmp_new, tmp_old + tmp_old, dest + """ + tmp_new, dest, tmp_old = get_write_paths(get_env_root(environment_name)) + if os.path.exists(tmp_old): + if not os.path.exists(dest): + shutil.move(tmp_new, dest) + else: + shutil.rmtree(tmp_old) + tty.info("Previous update completed") + elif os.path.exists(tmp_new): + tty.info("Previous update did not complete") + else: + tty.info("Previous update may have completed") + + if os.path.exists(tmp_new): + shutil.rmtree(tmp_new) + + +def read(environment_name): + # Check that env is in a consistent state on disk + env_root = get_env_root(environment_name) + + # Read env.yaml file + env_yaml = spack.config._read_config_file( + fs.join_path(env_root, 'env.yaml'), + spack.schema.env.schema) + + dotenv_dir = get_dotenv_dir(env_root) + with open(fs.join_path(dotenv_dir, 'environment.json'), 'r') as F: + environment_dict = sjson.load(F) + environment = Environment.from_dict(environment_name, environment_dict) + if env_yaml: + environment.yaml = env_yaml['env'] + + return environment + + +# =============== Modifies Environment + +def environment_create(args): + if os.path.exists(get_env_root(args.environment)): + raise tty.die("Environment already exists: " + args.environment) + + _environment_create(args.environment) + + +def _environment_create(name, init_config=None): + environment = Environment(name) + + user_specs = list() + config_sections = {} + if init_config: + for key, val in init_config.items(): + if key == 'user_specs': + user_specs.extend(val) + else: + config_sections[key] = val + + for user_spec in user_specs: + environment.add(user_spec) + + write(environment) + + # When creating the environment, the user may specify configuration + # to place in the environment initially. Spack does not interfere + # with this configuration after initialization so it is handled here + if len(config_sections) > 0: + config_basedir = fs.join_path(environment.path, 'config') + os.mkdir(config_basedir) + for key, val in config_sections.items(): + yaml_section = syaml.dump({key: val}, default_flow_style=False) + yaml_file = '{0}.yaml'.format(key) + yaml_path = fs.join_path(config_basedir, yaml_file) + with open(yaml_path, 'w') as F: + F.write(yaml_section) + + +def environment_add(args): + check_consistent_env(get_env_root(args.environment)) + environment = read(args.environment) + parsed_specs = spack.cmd.parse_specs(args.package) + + if args.all: + # Don't allow command-line specs with --all + if len(parsed_specs) > 0: + tty.die('Cannot specify --all and specs too on the command line') + + yaml_specs = environment.yaml['specs'] + if len(yaml_specs) == 0: + tty.msg('No specs to add from env.yaml') + + # Add list of specs from env.yaml file + for user_spec, _ in yaml_specs.items(): # OrderedDict + environment.add(str(user_spec), report_existing=False) + else: + for spec in parsed_specs: + environment.add(str(spec)) + + write(environment) + + +def environment_remove(args): + check_consistent_env(get_env_root(args.environment)) + environment = read(args.environment) + if args.all: + environment.clear() + else: + for spec in spack.cmd.parse_specs(args.package): + environment.remove(spec.format()) + write(environment) + + +def environment_spec(args): + environment = read(args.environment) + prepare_repository(environment, use_repo=args.use_repo) + prepare_config_scope(environment) + spack.cmd.spec.spec(None, args) + + +def environment_concretize(args): + check_consistent_env(get_env_root(args.environment)) + environment = read(args.environment) + _environment_concretize( + environment, use_repo=args.use_repo, force=args.force) + + +def _environment_concretize(environment, use_repo=False, force=False): + """Function body separated out to aid in testing.""" + + # Change global search paths + repo = prepare_repository(environment, use_repo=use_repo) + prepare_config_scope(environment) + + new_specs = environment.concretize(force=force) + + for spec in new_specs: + for dep in spec.traverse(): + dump_to_environment_repo(dep, repo) + + # Moves /.env.new to /.env + write(environment, repo) + +# =============== Does not Modify Environment + + +def environment_install(args): + check_consistent_env(get_env_root(args.environment)) + environment = read(args.environment) + prepare_repository(environment, use_repo=args.use_repo) + environment.install(args) + + +def environment_uninstall(args): + check_consistent_env(get_env_root(args.environment)) + environment = read(args.environment) + prepare_repository(environment) + environment.uninstall(args) + +# ======================================= + + +def dump_to_environment_repo(spec, repo): + dest_pkg_dir = repo.dirname_for_package_name(spec.name) + if not os.path.exists(dest_pkg_dir): + spack.repo.path.dump_provenance(spec, dest_pkg_dir) + + +def prepare_repository(environment, remove=None, use_repo=False): + """Adds environment's repository to the global search path of repos""" + import tempfile + repo_stage = tempfile.mkdtemp() + new_repo_dir = fs.join_path(repo_stage, 'repo') + if os.path.exists(environment.repo_path()): + shutil.copytree(environment.repo_path(), new_repo_dir) + else: + spack.repo.create_repo(new_repo_dir, environment.name) + if remove: + remove_dirs = [] + repo = Repo(new_repo_dir) + for pkg_name in remove: + remove_dirs.append(repo.dirname_for_package_name(pkg_name)) + for d in remove_dirs: + shutil.rmtree(d) + repo = Repo(new_repo_dir) + if use_repo: + spack.repo.put_first(repo) + return repo + + +def prepare_config_scope(environment): + """Adds environment's scope to the global search path + of configuration scopes""" + + # Load up configs + for config_spec in environment.yaml['configs']: + config_name = os.path.split(config_spec)[1] + if config_name == '': + # Use default config for the environment; doesn't have to exist + config_dir = fs.join_path(environment.path, 'config') + if not os.path.isdir(config_dir): + continue + config_name = environment.name + else: + # Use external user-provided config + config_dir = os.path.normpath(os.path.join( + environment.path, config_spec.format(**os.environ))) + if not os.path.isdir(config_dir): + tty.die('Spack config %s (%s) not found' % + (config_name, config_dir)) + + tty.msg('Using Spack config %s scope at %s' % + (config_name, config_dir)) + spack.config.config.push_scope(ConfigScope(config_name, config_dir)) + + +def environment_relocate(args): + environment = read(args.environment) + prepare_repository(environment, use_repo=args.use_repo) + environment.reset_os_and_compiler(compiler=args.compiler) + write(environment) + + +def environment_list(args): + # TODO? option to list packages w/ multiple instances? + environment = read(args.environment) + import sys + environment.list( + sys.stdout, recurse_dependencies=args.recurse_dependencies, + hashes=args.long or args.very_long, + hashlen=None if args.very_long else 7, + install_status=args.install_status) + + +def environment_stage(args): + environment = read(args.environment) + prepare_repository(environment, use_repo=args.use_repo) + for spec in environment.specs_by_hash.values(): + for dep in spec.traverse(): + dep.package.do_stage() + + +def environment_location(args): + environment = read(args.environment) + print(environment.path) + + +@contextmanager +def redirect_stdout(ofname): + """Redirects STDOUT to (by default) a file within the environment; + or else a user-specified filename.""" + with open(ofname, 'w') as f: + original = sys.stdout + sys.stdout = f + yield + sys.stdout = original + + +@contextmanager +def pushd(dir): + original = os.getcwd() + os.chdir(dir) + yield + os.chdir(original) + + +def environment_loads(args): + # Set the module types that have been selected + module_types = args.module_type + if module_types is None: + # If no selection has been made select all of them + module_types = ['tcl'] + + module_types = list(set(module_types)) + + environment = read(args.environment) + recurse_dependencies = args.recurse_dependencies + args.recurse_dependencies = False + ofname = fs.join_path(environment.path, 'loads') + with redirect_stdout(ofname): + specs = environment._get_environment_specs( + recurse_dependencies=recurse_dependencies) + spack.cmd.module.loads(module_types, specs, args) + + print('To load this environment, type:') + print(' source %s' % ofname) + + +def environment_upgrade_dependency(args): + environment = read(args.environment) + repo = prepare_repository( + environment, use_repo=args.use_repo, remove=[args.dep_name]) + new_dep = environment.upgrade_dependency(args.dep_name, args.dry_run) + if not args.dry_run and new_dep: + dump_to_environment_repo(new_dep, repo) + write(environment, repo) + + +def add_use_repo_argument(cmd_parser): + cmd_parser.add_argument( + '--use-env-repo', action='store_true', dest='use_repo', + help='Use package definitions stored in the environment' + ) + + +def setup_parser(subparser): + subparser.add_argument( + 'environment', + help="The environment you are working with" + ) + + sp = subparser.add_subparsers( + metavar='SUBCOMMAND', dest='environment_command') + + create_parser = sp.add_parser('create', help='Make an environment') + create_parser.add_argument( + '--init-file', dest='init_file', + help='File with user specs to add and configuration yaml to use' + ) + + add_parser = sp.add_parser('add', help='Add a spec to an environment') + add_parser.add_argument( + '-a', '--all', action='store_true', dest='all', + help="Add all specs listed in env.yaml") + add_parser.add_argument( + 'package', + nargs=argparse.REMAINDER, + help="Spec of the package to add" + ) + + remove_parser = sp.add_parser( + 'remove', help='Remove a spec from this environment') + remove_parser.add_argument( + '-a', '--all', action='store_true', dest='all', + help="Remove all specs from (clear) the environment") + remove_parser.add_argument( + 'package', + nargs=argparse.REMAINDER, + help="Spec of the package to remove" + ) + + spec_parser = sp.add_parser( + 'spec', help='Concretize sample spec') + spack.cmd.spec.add_common_arguments(spec_parser) + add_use_repo_argument(spec_parser) + + concretize_parser = sp.add_parser( + 'concretize', help='Concretize user specs') + concretize_parser.add_argument( + '-f', '--force', action='store_true', + help="Re-concretize even if already concretized.") + add_use_repo_argument(concretize_parser) + + relocate_parser = sp.add_parser( + 'relocate', + help='Reconcretize environment with new OS and/or compiler') + relocate_parser.add_argument( + '--compiler', + help="Compiler spec to use" + ) + add_use_repo_argument(relocate_parser) + + list_parser = sp.add_parser('list', help='List specs in an environment') + arguments.add_common_arguments( + list_parser, + ['recurse_dependencies', 'long', 'very_long', 'install_status']) + + loads_parser = sp.add_parser( + 'loads', + help='List modules for an installed environment ' + '(see spack module loads)') + spack.cmd.module.add_loads_arguments(loads_parser) + + sp.add_parser( + 'location', + help='Print the root directory of the environment') + + upgrade_parser = sp.add_parser( + 'upgrade', + help='''Upgrade a dependency package in an environment to the latest +version''') + upgrade_parser.add_argument( + 'dep_name', help='Dependency package to upgrade') + upgrade_parser.add_argument( + '--dry-run', action='store_true', dest='dry_run', + help="Just show the updates that would take place") + add_use_repo_argument(upgrade_parser) + + stage_parser = sp.add_parser( + 'stage', + help='Download all source files for all packages in an environment') + add_use_repo_argument(stage_parser) + + config_update_parser = sp.add_parser( + 'update-config', + help='Add config yaml file to environment') + config_update_parser.add_argument( + 'config_files', + nargs=argparse.REMAINDER, + help="Configuration files to add" + ) + + install_parser = sp.add_parser( + 'install', + help='Install all concretized specs in an environment') + spack.cmd.install.add_common_arguments(install_parser) + add_use_repo_argument(install_parser) + + uninstall_parser = sp.add_parser( + 'uninstall', + help='Uninstall all concretized specs in an environment') + spack.cmd.uninstall.add_common_arguments(uninstall_parser) + + +def env(parser, args, **kwargs): + action = { + 'create': environment_create, + 'add': environment_add, + 'spec': environment_spec, + 'concretize': environment_concretize, + 'list': environment_list, + 'loads': environment_loads, + 'location': environment_location, + 'remove': environment_remove, + 'relocate': environment_relocate, + 'upgrade': environment_upgrade_dependency, + 'stage': environment_stage, + 'install': environment_install, + 'uninstall': environment_uninstall + } + action[args.environment_command](args) diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py new file mode 100644 index 00000000000000..00d95ccf78c558 --- /dev/null +++ b/lib/spack/spack/schema/env.py @@ -0,0 +1,62 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Schema for env.yaml configuration file. + +.. literalinclude:: ../spack/schema/env.py + :lines: 32- +""" + + +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack Environments user configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'env': { + 'type': 'object', + 'default': {}, + 'properties': { + 'configs': { + 'type': 'array', + 'default': [], + 'items': {'type': 'string'} + }, + 'specs': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { # user spec + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + } + } + } + } + } + } +} diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py new file mode 100644 index 00000000000000..326cea2ebcbd14 --- /dev/null +++ b/lib/spack/spack/test/cmd/env.py @@ -0,0 +1,168 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import unittest +import tempfile +import shutil +import pytest +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +import spack.cmd.env +import spack.modules +import spack.util.spack_yaml as syaml +from spack.cmd.env import (Environment, prepare_repository, + _environment_concretize, prepare_config_scope, + _environment_create) +from spack.version import Version + + +class TestEnvironment(unittest.TestCase): + def setUp(self): + self.env_dir = spack.cmd.env._db_dirname + spack.cmd.env._db_dirname = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(spack.cmd.env._db_dirname) + spack.cmd.env._db_dirname = self.env_dir + + def test_add(self): + c = Environment('test') + c.add('mpileaks') + assert 'mpileaks' in c.user_specs + + @pytest.mark.usefixtures('config', 'mutable_mock_packages') + def test_concretize(self): + c = Environment('test') + c.add('mpileaks') + c.concretize() + env_specs = c._get_environment_specs() + assert any(x.name == 'mpileaks' for x in env_specs) + + @pytest.mark.usefixtures('config', 'mutable_mock_packages', + 'install_mockery', 'mock_fetch') + def test_env_install(self): + c = Environment('test') + c.add('cmake-client') + c.concretize() + c.install() + env_specs = c._get_environment_specs() + spec = next(x for x in env_specs if x.name == 'cmake-client') + assert spec.package.installed + + @pytest.mark.usefixtures('config', 'mutable_mock_packages') + def test_remove_after_concretize(self): + c = Environment('test') + c.add('mpileaks') + c.concretize() + c.add('python') + c.concretize() + c.remove('mpileaks') + env_specs = c._get_environment_specs() + assert not any(x.name == 'mpileaks' for x in env_specs) + + @pytest.mark.usefixtures('config', 'mutable_mock_packages') + def test_reset_compiler(self): + c = Environment('test') + c.add('mpileaks') + c.concretize() + + first_spec = c.specs_by_hash[c.concretized_order[0]] + available = set(['gcc', 'clang']) + available.remove(first_spec.compiler.name) + new_compiler = next(iter(available)) + c.reset_os_and_compiler(compiler=new_compiler) + + new_spec = c.specs_by_hash[c.concretized_order[0]] + assert new_spec.compiler != first_spec.compiler + + @pytest.mark.usefixtures('config', 'mutable_mock_packages') + def test_environment_list(self): + c = Environment('test') + c.add('mpileaks') + c.concretize() + c.add('python') + mock_stream = StringIO() + c.list(mock_stream) + list_content = mock_stream.getvalue() + assert 'mpileaks' in list_content + assert 'python' in list_content + mpileaks_spec = c.specs_by_hash[c.concretized_order[0]] + assert mpileaks_spec.format() in list_content + + @pytest.mark.usefixtures('config', 'mutable_mock_packages') + def test_upgrade_dependency(self): + c = Environment('test') + c.add('mpileaks ^callpath@0.9') + c.concretize() + + c.upgrade_dependency('callpath') + env_specs = c._get_environment_specs() + callpath_dependents = list(x for x in env_specs if 'callpath' in x) + assert callpath_dependents + for spec in callpath_dependents: + assert spec['callpath'].version == Version('1.0') + + @pytest.mark.usefixtures('config', 'mutable_mock_packages') + def test_init_config(self): + test_config = """user_specs: + - mpileaks +packages: + mpileaks: + version: [2.2] +""" + spack.package_prefs.PackagePrefs._packages_config_cache = None + spack.package_prefs.PackagePrefs._spec_cache = {} + + _environment_create('test', syaml.load(StringIO(test_config))) + c = spack.cmd.env.read('test') + prepare_config_scope(c) + c.concretize() + assert any(x.satisfies('mpileaks@2.2') + for x in c._get_environment_specs()) + + @pytest.mark.usefixtures('config', 'mutable_mock_packages') + def test_to_dict(self): + c = Environment('test') + c.add('mpileaks') + c.concretize() + context_dict = c.to_dict() + c_copy = Environment.from_dict('test_copy', context_dict) + assert c.specs_by_hash == c_copy.specs_by_hash + + @pytest.mark.usefixtures('config', 'mutable_mock_packages') + def test_prepare_repo(self): + c = Environment('testx') + c.add('mpileaks') + _environment_concretize(c) + repo = None + try: + repo = prepare_repository(c) + package = repo.get(spack.spec.Spec('mpileaks')) + assert package.namespace.split('.')[-1] == 'testx' + finally: + if repo: + shutil.rmtree(repo.root) From 4ca464da015efdb47cd07596adb9752f0f216ca4 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Fri, 18 May 2018 17:55:14 -0700 Subject: [PATCH 11/12] add environments documentation --- lib/spack/docs/environments.rst | 317 ++++++++++++++++++++++++++++++++ lib/spack/docs/index.rst | 1 + 2 files changed, 318 insertions(+) create mode 100644 lib/spack/docs/environments.rst diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst new file mode 100644 index 00000000000000..5ae543072b9d70 --- /dev/null +++ b/lib/spack/docs/environments.rst @@ -0,0 +1,317 @@ +.. _environments: + +Environments +============ + +An environment is used to group together a set of specs for the +purpose of building, rebuilding and deploying in a coherent fashion. +Environments provide a number of advantages over the the a la carte +approach of building and loading individual Spack modules: + +#. Environments eliminate the non-determinism encountered when + installing packages and then loading what was just created. + Consider the following sequence: + + .. code-block:: console + + $ spack install mypackage + $ spack load mypackage + + Sometime later, somebody else installs an earlier version of ``mypackage``: + + .. code-block:: console + + $ spack install mypackage@0.9 + + Now the ``spack load mypackage`` command will fail because more + than one version of ``mypackage`` is installed. Thus, ``spack + load`` is non-deterministic, and not a reliable way to load + specific sets of modules one wishes to build and load. + Environments solve this problem by creating a localized context + that is immune to what other versions of packages might be + installed in the global Spack prefix. +#. Environments separate the steps of (a) choosing what to + install, (b) concretizing, and (c) installing. This allows + Environments to remain stable and repeatable, even if Spack packages + are upgraded: specs are only re-concretized when the user + explicitly asks for it. It should even be possible to reliably + transport environments between different computers running + different versions of Spack! +#. Since re-concretization only occurs at controlled times, the + Environment equivalent of many Spack commands are far faster. For + example, ``spack env list -Ilr`` vs. ``spack spec``. +#. Environments allow several specs to be built at once; a more robust + solution than ad-hoc scripts making multiple calls to ``spack + install``. +#. An Environment that is built as a whole can be loaded as a whole. + Spack can generate a script of ``module load`` commands that load + the appropraite environment. And that script will work, without + running Spack, as long as the underlying modules remain installed. + A fast-running ``spack env load`` command would also be + possible. + +Other packaging systems also provide environments that are similar in +some ways to Spack environments; for example, `Conda environments +`_ or +`Python Virtual Environments +`_. Spack environments +some distinctive features: + +#. A spec installed "in" an environment is no different form the same + spec installed anywhere else in Spack. Environments are assembled + simply by collecting together a set of specs. Environments are + loaded by generating a set of ``module load`` commands. (In the + future, environments can also be assembled into a single tree using + techniques from ``spack view``.) +#. Spack Environments may contain more than one version of the same + package; but only a single module for a package may be loaded. + Modules that occur in earlier specs listed in an environment take + precedence over modules that occur later. + +Using Environments +------------------ + +Here we follow a typical use case of creating, concretizing, +installing and loading an environment. + +.. note:: + + Environments are currently in Beta. We expect the user interface + to change. Future changes might also break existing environment + files, particularly ``environment.json``. + + +Creating an Environment +^^^^^^^^^^^^^^^^^^^^^^^ + +An environment is created by: + +.. code-block:: console + + $ spack env myenv create + +Spack then creates the following files: + +* The directory ``var/spack/environments/myenv``. This is the public + part of the environment. + + .. note:: + + All environments are stored in the ``var/spack/environments`` folder. + + The following files may be added to this directory by the user or + Spack: + + * ``env.yaml``: Addition environment specification and configuration + * ``loads``: A script of ``module load`` commands, generated by + ``spack env myenv loads``, which the user may source to use the + built environment. + * ``logs/``: Symbolic links to build logs for specs that are part of + the environment. + * Anything else the user wishes to put in this directory! + +* The directory ``var/spack/environments/myenv/.env``. This directory + is "owned" by Spack, and may be rewritten from time to time. The + user should not put anything in it. It contains: + + * ``environment.json``: Concretized specs for the environment. + * ``repo/``: A repo consisting of the Spack packages used in this + environment. This allows the environment to build the same, in + theory, even on different verions of Spack with different + packages! + +Adding User Specs +^^^^^^^^^^^^^^^^^ + +Once an environment is created, it is time to add some user specs. A +*user spec* is simply a spec typed by the user; for example, what one +would type as part of ``spack install``. An environment can consist +of as many user specs as one likes, and the specs can contain +conflicting versions of packages. For example: + +.. code-block:: console + + $ spack env myenv add mpileaks + $ spack env myenv add python + +Concretizing +^^^^^^^^^^^^ + +Once some user specs have been added to an environment, they can be +concretized. The following command will concretize all user specs +that have been added and not yet concretized: + +.. code-block:: console + + $ spack env myenv concretize + +This command will re-concretize all specs: + +.. code-block:: console + + $ spack env myenv concretize + +Re-concretizing can be useful if packages have changed and one wishes +them to take effect in an environment; however it can lead to +additional re-building of packages. + +Listing +^^^^^^^ + +The ``spack env concretize`` command reports on its results. +This report can be repeated at any time with: + +.. code-block:: console + + $ spack env myenv list -Ilr + +This version of the command reports on hashes and install status of +all the specs in an environment. See ``spack env myenv list -h`` for +customization options. + +Installing +^^^^^^^^^^ + +Once an environment has been concretized, it can be installed. One +can think of this as running ``spack install`` on every concretized +spec in the environment: + +.. code-block:: console + + $ spack env myenv install + +As it installs, ``spack env install`` creates symbolic links in +the ``logs/`` directory in the environment, allowing for easy +inspection of build logs related to that environment. + +Loading +^^^^^^^ + +Once an environment has been installed, the following creates a load script for it: + +.. code-block:: console + + $ spack env myenv loads -r + +This creates a file called ``loads`` in the environment directory. +Sourcing that file in Bash will make the environment available to the +user; and can be included in ``.bashrc`` files, etc. The ``loads`` +file may also be copied out of the environment, renamed, etc. + +Environment Configs +------------------- + +A variety of Spack behaviors are changed through Spack configs. Many +Spack users use a ``~/.spack/packages.yaml`` file to customize or +otherwise direct the behavior of the concretizer; for example, to +choose a specific version of a package, or to use an external package. + +Spack environments can use a config as well; the user just has to +create a ``config/`` sub-directory in the environment, and then add +configuration files to it (eg: ``packages.yaml``). For example, the +following ``config/packages.yaml`` file will direct the concretizer to +use ``python@3.5.2`` in an environment: + +.. code-block:: yaml + + packages: + python: + version: [3.5.2] + +An environment's config will be loaded last; and will have precedence +over all other configs loaded by Spack; for example, the config files +in ``~/.spack``. + +env.yaml +-------- + +Further customization is also possible through a user-created +``env.yaml`` file in the environment's directory. This currently +allows for two further features: + +#. Loading (stacking) more than one config; or loading them from + locations outside the environment. +#. Recording a list of user specs to be part of the environment. + +An example ``env.yaml`` file: + +.. code-block:: yaml + + env: + configs: + - '{HOME}/spscopes/centos7' # Lowest precedence + - '{HOME}/spscopes/gissversions' + - '{HOME}/spscopes/twoway' + - '{HOME}/spscopes/develop' + - config # Highest precedence + specs: + ncview: # Highest precedence + netcdf: + nco: # Lowest precedence + py-sphinx: + +.. note:: + + #. If ``env.yaml`` exists, then Spack will no longer automatically + load from the default environment ``config/`` directory. This is a + bug. To enable the default behavior, use the following in + ``env.yaml``: + + .. code-block:: yaml + + env: + configs: + - config + + #. The effect of configs on concretization can be tested using the + ``spack env spec`` command. + + #. As shown above, the syntax ``{X}`` in config paths will + substitute the system environent variable named ``X``. This + allows configs to be referenced either relative to the + environment directory, or somewhere else on the filesystem. + + +Loading Specs from env.yaml +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If a list of user specs is included in ``env.yaml``, it can be added +to the environment as follows: + +.. code-block:: console + + $ spack env myenv add --all + +This is equivalent to a series of ``spack env myenv add`` calls. +Similarly, all user specs may be removed from the environment with: + +.. code-block:: console + + $ spack env myenv remove --all + +In this way, the list of user specs associated with an environment can +be kept in the environment; rather than in a separate script +elsewhere. + + +Initializing an Environment from a Template +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When creating an environment with ``spack env create`` a user can +provide a template file with the ``--init-file`` option which will +add a set of pre-specified specs and initialize the environment's +configuration. The file has the following format: + +.. code-block:: yaml + + user_specs: + - python + - mpileaks + packages: + ... + compilers: + ... + +Each of the configuration sections (after ``user_specs``) will be +extracted into a corresponding configuration file in the environment; +Spack will not make further edits to the environment configuration. diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index ea80e2c21e2bf7..2097bade7ec912 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -64,6 +64,7 @@ or refer to the full manual below. module_file_support repositories binary_caches + environments command_index package_list From bea5eab933af56b7d5b22936c97a047c7e288473 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Sat, 26 May 2018 23:08:00 -0400 Subject: [PATCH 12/12] Garbage collection for Spack Environments # Conflicts: # lib/spack/spack/cmd/uninstall.py --- lib/spack/spack/cmd/env.py | 24 ++++++++++++++++++++ lib/spack/spack/cmd/uninstall.py | 39 ++++++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index d48397f46e53cf..13f9e135a3ddf6 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -57,6 +57,30 @@ _db_dirname = fs.join_path(spack.paths.var_path, 'environments') +def env_names(): + """Yields names of the environments in this Spack""" + envs = [] + for fname in os.listdir(_db_dirname): + if os.path.isfile( + fs.join_path(_db_dirname, fname, '.env', 'environment.json')): + yield fname + + +def all_hashes(): + """Returns a set of all the hashes used by all environments""" + seen = set() + for env_name in spack.cmd.env.env_names(): + env = spack.cmd.env.read(env_name) + for top_hash in env.concretized_order: + top_spec = env.specs_by_hash[top_hash] + + for d, dep_spec in top_spec.traverse_edges( + order='pre', cover='nodes', depth=True, deptypes=('build', 'link', 'run')): + node = dep_spec.spec + seen.add(node.dag_hash()) + return seen + + def get_env_root(name): """Given an environment name, determines its root directory""" return fs.join_path(_db_dirname, name) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index cd2df93195ec8e..a85007cb37bb93 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -25,9 +25,13 @@ from __future__ import print_function import argparse +import os import spack.cmd import spack.package +import spack.cmd.env +import spack.store +import spack.repository import spack.cmd.common.arguments as arguments import spack.repo import spack.store @@ -69,6 +73,11 @@ def setup_parser(subparser): "supplied all installed software will be uninstalled. this " "is both useful and dangerous, like rm -r") + subparser.add_argument( + '-g', '--garbage', action='store_true', dest='garbage', + help="Uninstalls packages not currently concretized in " + "a Spack Environment.") + subparser.add_argument( 'packages', nargs=argparse.REMAINDER, @@ -193,9 +202,30 @@ def get_uninstall_list(args, specs): return uninstall_list -def uninstall_specs(args, specs): +def all_hashes(): + """Returns a set of all the hashes installed on Spack""" + ret = set() + for spec in spack.store.db.query(): + ret.add(spec.dag_hash()) + return ret + - uninstall_list = get_uninstall_list(args, specs) +def uninstall_specs(args, specs): + uninstall_list = [] + if args.garbage: + all = all_hashes() + env = spack.cmd.env.all_hashes() + garbage = all - env + if len(garbage) == 0 and not args.packages and not args.all: + tty.msg('All packages are used by at least one environment, ' + 'no garbage to uninstall.') + return + + garbage_specs = spack.cmd.parse_specs( + ['/' + x for x in garbage]) + uninstall_list.extend(get_uninstall_list(args, garbage_specs)) + + uninstall_list.extend(get_uninstall_list(args, specs)) if not uninstall_list: tty.warn('There are no package to uninstall.') @@ -213,9 +243,10 @@ def uninstall_specs(args, specs): def uninstall(parser, args): - if not args.packages and not args.all: + if not args.packages and not args.all and not args.garbage: tty.die('uninstall requires at least one package argument.', ' Use `spack uninstall --all` to uninstall ALL packages.') uninstall_specs( - args, spack.cmd.parse_specs(args.packages) if args.packages else [any]) + args, spack.cmd.parse_specs(args.packages) + if args.packages or args.garbage else [any])