Support for environment.yml/lockfiles

README.md

@@ -20,7 +20,7 @@ Assuming you have a [conda-lock](https://github.com/conda-incubator/conda-lock)
 named `~/data_science.lock`:
 ```
 # create the specs and copy source files to ./dist/SPECS and ./dist/SOURCES
-python -m conda_rpms.generate --name data_science --output dist ~/data_science.lock
+python -m conda_rpms.generate ~/data_science.lock data_science.yml --output dist --modulefile_template modulefile.tmpl 
 # build the RPMs to ./dist/RPMS
 rpmbuild -bb --define "_topdir $(pwd)/dist"  dist/SPECS/*.spec
 ```
@@ -41,7 +41,7 @@ Environment RPM
 
 RPM name format: ``<RPM namespace>-env-<env name>``
 
-A environment RPM represents a resolved conda environment.
+An environment RPM represents a resolved conda environment.
 It depends on all Package RPMs that should be installed in order to produce a working environment. The environment RPM knows its target installation prefix, and uses conda functionality at install time to link the Package RPMs to the desired installation prefix.
 
 

setup.cfg

@@ -12,9 +12,10 @@ install_requires=
     jinja2
     ensureconda
     conda_package_handling
+    pyyaml
 
 [options.extras_require]
 test = pytest; mock; pytest-cov
 
 [options.packages.find]
-where = src
+where = src

src/conda_rpms/conda_compat.py

@@ -3,13 +3,13 @@
 from dataclasses import asdict, dataclass, fields
 from pathlib import Path
 from typing import Iterable, List, Optional
+import yaml
 
 from conda.api import PackageCacheData
 from conda_package_handling.api import create as create_conda_tarball
 
 from .logger import log
 
-
 class SpecTarget:
     _ident = "__none__"
     _id_string = "UndefinedSpec"
@@ -47,8 +47,10 @@ def from_url(cls, url: str) -> CondaPackage:
 @dataclass
 class CondaEnvironment(SpecTarget):
     _ident = "env"
-    _id_string = "{name}-{version}"
+    _id_string = "{name}-{label}-{version}"
     name: str
+    label: str
+    tag: str
     version: str
     summary: Optional[str]
 
@@ -76,10 +78,20 @@ def read_conda_lockfile(lockfile: Iterable[str]) -> List[str]:
     return urls
 
 
+def read_conda_environment_yml(env_yml: Iterable[str]) -> CondaEnvironment:
+    """Returns a CondaEnvironemnt instance using metadata from the 
+    environment.yml file.
+
+    """
+    env = yaml.safe_load(env_yml)
+    return CondaEnvironment(*env['name'].split('-'), env['meta']['released'],
+                            env['meta']['version'], None)
+
+
 def make_package_tarball(pkg: CondaPackage) -> Path:
-    """Return the path to a package's associated tar.bz2 tarball.
+    """Returns the path to a package's associated tar.bz2 tarball.
 
-    If the conda pacakge is in the .conda format, this function
+    If the conda package is in the .conda format, this function
     will also convert to the .tar.bz2 format and place it alongside
     the .conda package.
 

src/conda_rpms/generate.py

@@ -9,7 +9,7 @@
 import shutil
 from pathlib import Path
 import sys
-from typing import Iterable, List, NamedTuple, Optional, Union
+from typing import List
 
 import jinja2
 import yaml
@@ -20,6 +20,7 @@
     CondaEnvironment,
     make_package_tarball,
     read_conda_lockfile,
+    read_conda_environment_yml
 )
 
 # configure the installer to use the same version
@@ -33,6 +34,11 @@ class PackageSpec:
     spec: Path  # the path to a written RPM spec file
     tarball: Path  # the path the associated tar.bz2 tarball of files
 
+@dataclass
+class ModulefileConfig:
+    prefix: Path = None # the path of where the modulefile will be written to
+    modulefile: str = None # the contents of the modulefile that will be created
+    default: str = None # the contents of the .version default file
 
 _module_dir = Path(__file__).resolve().parent
 install_script_path = _module_dir / "dist/install.py"
@@ -46,10 +52,10 @@ class PackageSpec:
 
 
 def render_package_spec(pkg_dir: Path, rpm_namespace: str, install_prefix: Path) -> str:
-    """Render an package RPM spec.
+    """Render a package RPM spec.
 
     This relies on the conda package cache having already
-    downloaded and extracted the specific pacakge and
+    downloaded and extracted the specific package and
     stored it in the local cache.
 
     Returns the rendered RPM spec as a string.
@@ -67,10 +73,6 @@ def render_package_spec(pkg_dir: Path, rpm_namespace: str, install_prefix: Path)
     else:
         meta = {}
 
-    meta_about = meta.setdefault("about", {})
-    meta_about.setdefault("license", pkginfo.get("license"))
-    meta_about.setdefault("summary", "The {} package".format(pkginfo["name"]))
-
     return pkg_spec_tmpl.render(
         pkginfo=pkginfo,
         meta=meta,
@@ -106,12 +108,14 @@ def render_environment_spec(
     packages: List[CondaPackage],
     install_prefix: Path,
     rpm_namespace: str,
+    modulefile_config: ModulefileConfig,
 ) -> str:
     return env_spec_tmpl.render(
         env=asdict(environment),
         pkgs=[p.id for p in packages],
         install_prefix=str(install_prefix),
         rpm_prefix=rpm_namespace,
+        module=modulefile_config
     )
 
 
@@ -132,6 +136,7 @@ def write_environment_rpm_spec(
     outdir: Path,
     install_prefix: Path,
     rpm_namespace: str,
+    modulefile_config: ModulefileConfig
 ) -> Path:
     """Write an environment RPM spec to disk.
 
@@ -142,7 +147,8 @@ def write_environment_rpm_spec(
 
     Writes an rpm .spec file to `outdir/SPECS`.
     """
-    spec = render_environment_spec(environment, packages, install_prefix, rpm_namespace)
+    spec = render_environment_spec(environment, packages, install_prefix,
+                                   rpm_namespace, modulefile_config)
     spec_filename = f"{rpm_namespace}-env-{environment.id}.spec"
     return write_spec(spec, outdir / "SPECS" / spec_filename)
 
@@ -173,12 +179,42 @@ def create_installer(
     return PackageSpec(pkg, spec, tarball)
 
 
+def create_rpm_spec(pkg, outdir, install_prefix, rpm_namespace):
+    spec = write_package_rpm_spec(pkg, outdir, install_prefix, rpm_namespace)
+    tarball = make_package_tarball(pkg)
+    return PackageSpec(pkg.id, spec, tarball)
+
+
+def get_modulefile_config(
+        template, prefix, default_version_file, env, install_prefix
+) -> ModulefileConfig:
+    modulefile_config = ModulefileConfig(prefix=prefix)
+
+    # Render the modulefile template 
+    if template:
+        template = Path(template)
+        module_loader = jinja2.FileSystemLoader(template.parent)
+        module_env = jinja2.Environment(loader=module_loader)
+        module_template = module_env.get_template(template.name)
+        modulefile_config.modulefile = module_template.render(env=env, install_prefix=install_prefix)
+
+        # Load the modulefile default version file
+        if default_version_file:
+            with open(default_version_file) as fh:
+                modulefile_config.default = fh.read()
+
+    return modulefile_config
+
+
 def environment_to_rpms(
-    name: str,
+    environment_yml: Path,
     lockfile: Path,
     outdir: Path,
     install_prefix: Path = Path("/opt/conda-dist"),
     rpm_namespace: str = "CondaDist",
+    modulefile_template: Path = None,
+    modulefile_prefix: Path = None,
+    modulefile_default: Path = None
 ):
     """Take an explicit conda lock file and turn it into a set of RPMs."""
     config = outdir, install_prefix, rpm_namespace
@@ -188,13 +224,14 @@ def environment_to_rpms(
 
     sourcedir = outdir / "SOURCES"
     specdir = outdir / "SPECS"
-    for dir in (outdir, sourcedir, specdir):
-        dir.mkdir(exist_ok=True)
+    for directory in (outdir, sourcedir, specdir):
+        directory.mkdir(exist_ok=True)
 
     log.info(f"Generating RPM environment from {lockfile}.")
     log.info(f"{len(urls)} packages to be rendered.")
 
-    log.info(f"Checking conda cache for pacakges.")
+    # Note that this requires all packages to be available in the cache.
+    log.info(f"Checking conda cache for packages.")
     pkgs = [CondaPackage.from_url(url) for url in urls]
 
     log.info("Rendering all package specs.")
@@ -209,14 +246,20 @@ def environment_to_rpms(
         log.info("Coping %s to %s", spec.tarball, sourcedir / spec.tarball.name)
         shutil.copy(spec.tarball, sourcedir / spec.tarball.name)
 
-    environment = CondaEnvironment(args.name, "1.0", None)
-    write_environment_rpm_spec(environment, pkgs, outdir, install_prefix, rpm_namespace)
-
-
-def create_rpm_spec(pkg, outdir, install_prefix, rpm_namespace):
-    spec = write_package_rpm_spec(pkg, outdir, install_prefix, rpm_namespace)
-    tarball = make_package_tarball(pkg)
-    return PackageSpec(pkg.id, spec, tarball)
+    log.info("Rendering env spec.")
+    with open(environment_yml, "r") as f:
+        environment = read_conda_environment_yml(f)
+    modulefile_config  = get_modulefile_config(modulefile_template,
+                                               modulefile_prefix,
+                                               modulefile_default,
+                                               environment,
+                                               install_prefix)
+    write_environment_rpm_spec(environment,
+                               pkgs,
+                               outdir,
+                               install_prefix,
+                               rpm_namespace,
+                               modulefile_config)
 
 
 if __name__ == "__main__":
@@ -225,10 +268,10 @@ def create_rpm_spec(pkg, outdir, install_prefix, rpm_namespace):
 
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "file", help="The conda lockfile to produce an RPM package for."
+        "environment_yml", help="The conda environment.yml to produce an RPM package for."
     )
     parser.add_argument(
-        "--name", "-n", required=True, help="The name of the environment to be created."
+        "lockfile", help="The conda lockfile to produce an RPM package for."
     )
     parser.add_argument(
         "--output",
@@ -255,12 +298,33 @@ def create_rpm_spec(pkg, outdir, install_prefix, rpm_namespace):
             Shared packages will go to {install-prefix}/pkgs/{package}"""
         ),
     )
+    parser.add_argument(
+        "--modulefile_template",
+        default=None,
+        help="The template for the modulefile that will be created."
+    )
+    parser.add_argument(
+        "--modulefile_prefix",
+        default="/opt/conda-dist/modulefiles",
+        help="The destination prefix where modulefiles will be installed."
+    )
+    parser.add_argument(
+        "--modulefile_default",
+        default=None,
+        help=dedent(
+            r"""The .version file that sets the environment being created as
+            the default environment."""
+        ),
+    )
     args = parser.parse_args()
 
     environment_to_rpms(
-        args.name,
-        Path(args.file),
+        Path(args.environment_yml),
+        Path(args.lockfile),
         Path(args.output),
         Path(args.install_prefix),
         args.rpm_namespace,
+        args.modulefile_template,
+        args.modulefile_prefix,
+        args.modulefile_default,
     )

src/conda_rpms/templates/env.spec.template

@@ -1,12 +1,12 @@
-{% set env_dir = '{}/environments/{}/{}'.format(install_prefix, env.name, env.version) %}
+{% set env_dir = '{}/environments/{}/{}'.format(install_prefix, env.name, env.tag) %}
 
-Name:           {{ rpm_prefix }}-env-{{ env.name }}
+Name:           {{ rpm_prefix }}-env-{{ env.name }}-{{ env.label }}
 Version:        {{ env.version }}
 Release:        0
 Summary:        {{ env.summary }}
 
 License:        BSD 3
-BuildRoot:      %{_tmppath}/env-{{ env.name }}-{{env.version}}
+BuildRoot:      %{_tmppath}/env-{{ env.name }}-{{ env.label }}-{{env.version}}
 
 Requires: {{ rpm_prefix}}-installer
 {% for pkg in pkgs -%}
@@ -15,9 +15,10 @@ Requires: {{ rpm_prefix }}-pkg-{{ pkg }}
 
 %description
 
-This is the {{ env.name }} {{rpm_prefix}} environment, described as:
+This is the {{ env.name }}-{{ env.label }} {{ rpm_prefix }} environment which
+currently points to {{ env.name }}/{{ env.tag }} via a symbolic link.
 
-Unlike normal RPMs, {{rpm_prefix}} environment RPMs create a conda environment at
+Unlike normal RPMs, {{ rpm_prefix }} environment RPMs create a conda environment at
 install time (i.e. they are not pre-built). This allows the environments to share
 assets that otherwise wouldn't be sharable, thus making environments themselves very
 lightweight.
@@ -28,6 +29,18 @@ rm -rf $RPM_BUILD_ROOT/
 
 %install
 mkdir -p $RPM_BUILD_ROOT{{ env_dir }}
+ln -s {{ env_dir }} $RPM_BUILD_ROOT{{ install_prefix }}/environments/{{ env.name }}/{{ env.label }}
+{% if module.prefix %}
+mkdir -p $RPM_BUILD_ROOT{{ module.prefix }}
+cat <<'EOF1' > $RPM_BUILD_ROOT{{ module.prefix }}/{{ env.name }}-{{ env.label }}
+{{ module.modulefile }}
+EOF1
+{% if module.default %}
+cat <<'EOF2' > $RPM_BUILD_ROOT{{ module.prefix }}/.version
+{{ module.default }}
+EOF2
+{% endif %}
+{% endif %}
 
 
 # Run *after* the RPM is installed or upgraded. (https://wiki.mageia.org/en/Packagers_RPM_tutorial#Pre-_and_Post-installation_scripts)
@@ -65,3 +78,10 @@ rm -rf $RPM_BUILD_ROOT
 %files
 # All files in this directory are owned by this RPM.
 %dir {{ env_dir }}
+{{ install_prefix }}/environments/{{ env.name }}/{{ env.label }}
+{% if module.prefix %}
+{{ module.prefix }}/{{ env.name }}-{{ env.label }}
+{% if module.default %}
+{{ module.prefix }}/.version
+{% endif %}
+{% endif %}