Skip to content

Commit

Permalink
Merge pull request #44 from pangeo-forge/local-dev
Browse files Browse the repository at this point in the history
Allow running against local directories
  • Loading branch information
yuvipanda authored Nov 9, 2022
2 parents 1aa6585 + 0394459 commit 4c76cb5
Show file tree
Hide file tree
Showing 5 changed files with 67 additions and 38 deletions.
9 changes: 2 additions & 7 deletions pangeo_forge_runner/commands/bake.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
"""
Command to run a pangeo-forge recipe
"""
import tempfile
from datetime import datetime
from pathlib import Path

Expand Down Expand Up @@ -111,12 +110,8 @@ def start(self):
extra={"status": "setup"},
)

# Create a temporary directory where we fetch the feedstock repo and perform all operations
# FIXME: Support running this on an already existing repository, so users can run it
# as they develop their feedstock
with tempfile.TemporaryDirectory() as d:
self.fetch(d)
feedstock = Feedstock(Path(d) / self.feedstock_subdir)
with self.fetch() as checkout_dir:
feedstock = Feedstock(Path(checkout_dir) / self.feedstock_subdir)

self.log.info("Parsing recipes...", extra={"status": "running"})
with redirect_stderr(self.log, {"status": "running"}), redirect_stdout(
Expand Down
68 changes: 42 additions & 26 deletions pangeo_forge_runner/commands/base.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import logging
import os
import sys
import tempfile
from collections.abc import Generator
from contextlib import contextmanager

from pythonjsonlogger import jsonlogger
from repo2docker import contentproviders
Expand Down Expand Up @@ -122,38 +126,50 @@ class BaseCommand(Application):
""",
)

def fetch(self, target_path):
@contextmanager
def fetch(self) -> Generator[str, None, None]:
"""
Fetch repo from url at ref, and check it out to checkout_path
Fetch repo from configured url at ref, and return a directory where it can be accessed.
Uses repo2docker to detect what kinda url is going to be checked out,
and fetches it into checkout_path. No image building or anything is
performed.
checkout_path should be empty.
If self.repo is a local directory that exists, it is just returned - no extra
processing is done. If not, repo2docker is used to detect what kind of URL
is to be checked out (git, zenodo, mercurial, etc) and that is checked out into
a temporary directory, the path of which is returned. When the contextmanager
exits, the temporary directory is cleaned up.
"""
picked_content_provider = None
for ContentProvider in self.content_providers:
cp = ContentProvider()
spec = cp.detect(self.repo, ref=self.ref)
if spec is not None:
picked_content_provider = cp
self.log.info(
"Picked {cp} content "
"provider.\n".format(cp=cp.__class__.__name__),
extra={"status": "fetching"},
if os.path.exists(self.repo):
# We are trying to submit off a local checkout, so we don't need to do much
checkout_dir = self.repo

yield checkout_dir

# No cleanup necessary
return
with tempfile.TemporaryDirectory() as checkout_dir:
picked_content_provider = None
for ContentProvider in self.content_providers:
cp = ContentProvider()
spec = cp.detect(self.repo, ref=self.ref)
if spec is not None:
picked_content_provider = cp
self.log.info(
"Picked {cp} content "
"provider.\n".format(cp=cp.__class__.__name__),
extra={"status": "fetching"},
)
break

if picked_content_provider is None:
raise ValueError(
f"Could not fetch {self.repo}, no matching contentprovider found"
)
break

if picked_content_provider is None:
raise ValueError(
f"Could not fetch {self.repo}, no matching contentprovider found"
)
for log_line in picked_content_provider.fetch(
spec, checkout_dir, yield_output=True
):
self.log.info(log_line, extra=dict(status="fetching"))

for log_line in picked_content_provider.fetch(
spec, target_path, yield_output=True
):
self.log.info(log_line, extra=dict(status="fetching"))
yield checkout_dir

def json_excepthook(self, etype, evalue, traceback):
"""
Expand Down
6 changes: 2 additions & 4 deletions pangeo_forge_runner/commands/expand_meta.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import json
import tempfile
from pathlib import Path

from .. import Feedstock
Expand All @@ -19,9 +18,8 @@ class ExpandMeta(BaseCommand):
flags = common_flags

def start(self):
with tempfile.TemporaryDirectory() as d:
self.fetch(d)
feedstock = Feedstock(Path(d) / self.feedstock_subdir)
with self.fetch() as checkout_dir:
feedstock = Feedstock(Path(checkout_dir) / self.feedstock_subdir)
with redirect_stderr(self.log, {"status": "running"}), redirect_stdout(
self.log, {"status": "running"}
):
Expand Down
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
author_email="[email protected]",
version="0.7.0",
packages=find_packages(),
python_requires=">=3.9",
install_requires=[
"jupyter-repo2docker",
"ruamel.yaml",
Expand Down
21 changes: 20 additions & 1 deletion tests/test_fetch.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import os
import tempfile

import pytest
from repo2docker import contentproviders

Expand All @@ -16,4 +19,20 @@ def test_bad_cp():

bc.repo = "https://example.com"
with pytest.raises(ValueError):
bc.fetch("/tmp")
with bc.fetch() as _:
pass


def test_local():
"""
Test that local directories are just treated as nops
"""
with tempfile.TemporaryDirectory() as d:
bc = BaseCommand()
bc.repo = d

with bc.fetch() as checkout_dir:
assert checkout_dir == d

# Make sure the directory isn't deleted either by the end!
assert os.path.exists(d)

0 comments on commit 4c76cb5

Please sign in to comment.