diff --git a/.github/workflows/test-action.yaml b/.github/workflows/test-action.yaml new file mode 100644 index 0000000..d5ff740 --- /dev/null +++ b/.github/workflows/test-action.yaml @@ -0,0 +1,55 @@ +name: Test action + +on: + push: + branches: ["main"] + pull_request: + branches: [ "*" ] + +jobs: + test: + name: (recipes@${{ matrix.recipes-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + recipes-version: ["0.10.3"] # , "main"] + steps: + - uses: actions/checkout@v4 + - name: "Clone test feedstock" + # Fetches test feedstock (containing example test recipes) from pangeo-forge-recipes + run: | + git clone --depth 1 --branch ${{ matrix.recipes-version }} https://github.com/pangeo-forge/pangeo-forge-recipes.git + - name: "Add requirements.txt" + # The clone step above gives us recipe modules and meta.yaml, but does not contain a requirements file, + # so we add that here, providing the action with the correct version of pangeo-forge-recipes to install + # in the container at action runtime. + run: | + echo "pangeo-forge-recipes==${{ matrix.recipes-version }}" > pangeo-forge-recipes/examples/feedstock/requirements.txt + - name: "Overwrite meta.yaml" + # The example feedstock contains multiple recipes, but we just want to test gpcp + run: | + grep -E 'recipes|gpcp' pangeo-forge-recipes/examples/feedstock/meta.yaml > temp.yaml \ + && mv temp.yaml pangeo-forge-recipes/examples/feedstock/meta.yaml + - name: "Deploy recipes" + uses: ./ + with: + # select_recipe_by_label: true + pangeo_forge_runner_config: > + { + "BaseCommand": { + "feedstock_subdir": "pangeo-forge-recipes/examples/feedstock" + }, + "Bake": { + "prune": true, + "bakery_class": "pangeo_forge_runner.bakery.local.LocalDirectBakery" + }, + "TargetStorage": { + "fsspec_class": "fsspec.implementations.local.LocalFileSystem", + "root_path": "./target" + }, + "InputCacheStorage": { + "fsspec_class": "fsspec.implementations.local.LocalFileSystem", + "root_path": "./cache" + } + } diff --git a/Dockerfile b/Dockerfile index 5a4dbb1..45b151b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,12 @@ -FROM quay.io/pangeo/forge:5e51a29 +FROM ubuntu:22.04 -RUN conda run -n notebook pip install git+https://github.com/pangeo-forge/pangeo-forge-runner@main +RUN apt-get update && apt-get install -y python3 python3-pip + +# install pangeo-forge-runner. to make this faster, let's drop the pangeo-forge-recipes +# dependency from -runner (because recipes is always installed at runtime anyway) +RUN python3 -m pip install pangeo-forge-runner COPY action/deploy_recipe.py /deploy_recipe.py COPY entrypoint.sh /entrypoint.sh -ENV CONDA_ENV=notebook - ENTRYPOINT [ "sh", "/entrypoint.sh" ] diff --git a/action/deploy_recipe.py b/action/deploy_recipe.py index cb15aa5..679352a 100644 --- a/action/deploy_recipe.py +++ b/action/deploy_recipe.py @@ -27,15 +27,19 @@ def deploy_recipe_cmd(cmd: list[str]): """Wrapper for `call_subprocess_run` with extra stdout parsing when deploying recipes.""" stdout = call_subprocess_run(cmd) lastline = json.loads(stdout.splitlines()[-1]) - job_id = lastline["job_id"] - job_name = lastline["job_name"] - print(f"Job submitted with {job_id = } and {job_name = }") + if all([k in lastline for k in ("job_id", "job_name")]): + job_id = lastline["job_id"] + job_name = lastline["job_name"] + print(f"Job submitted with {job_id = } and {job_name = }") + else: + print( + "Keys 'job_id' and/or 'job_name' missing from deploy recipe process stdout, " + "but deploy command did not fail. Perhaps the configured bakery type does not " + "provide this logging information?" + ) def main(): - # set in Dockerfile - conda_env = os.environ["CONDA_ENV"] - # injected by github actions repository = os.environ["GITHUB_REPOSITORY"] # will this fail for external prs? api_url = os.environ["GITHUB_API_URL"] @@ -50,7 +54,6 @@ def main(): select_recipe_by_label = os.environ["INPUT_SELECT_RECIPE_BY_LABEL"] # log variables to stdout - print(f"{conda_env = }") print(f"{head_ref = }") print(f"{sha = }") print(f"{config = }") @@ -79,10 +82,10 @@ def main(): # if calling `pangeo-forge-runner` directly, `--feedstock-subdir` can be passed as a CLI arg. # in the action context, users do not compose their own `pangeo-forge-runner` CLI calls, so if # they want to use a non-default value for feedstock-subdir, it must be passed via the long-form - # name in the config JSON (i.e, `{"BaseCommand": "feedstock-subdir": ...}}`). + # name in the config JSON (i.e, `{"BaseCommand": "feedstock_subdir": ...}}`). feedstock_subdir = ( - config["BaseCommand"]["feedstock-subdir"] - if "BaseCommand" in config and "feedstock-subdir" in config["BaseCommand"] + config["BaseCommand"]["feedstock_subdir"] + if "BaseCommand" in config and "feedstock_subdir" in config["BaseCommand"] else "feedstock" ) # because we've run the actions/checkout step before reaching this point, our current @@ -90,7 +93,7 @@ def main(): # contents directly on the filesystem here, without requesting it from github. if "requirements.txt" in os.listdir(feedstock_subdir): call_subprocess_run( - f"mamba run -n {conda_env} pip install -Ur {feedstock_subdir}/requirements.txt".split() + f"python3 -m pip install -Ur {feedstock_subdir}/requirements.txt".split() ) with tempfile.NamedTemporaryFile("w", suffix=".json") as f: diff --git a/entrypoint.sh b/entrypoint.sh index d9ef33e..346378b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,3 +1,3 @@ #!/bin/sh -l -conda run --no-capture-output -n notebook python3 /deploy_recipe.py +python3 /deploy_recipe.py diff --git a/tests/test_main.py b/tests/test_main.py index 04da8d2..effc35d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -27,7 +27,6 @@ def select_recipe_by_label(request): @pytest.fixture def env(select_recipe_by_label, head_ref): return { - "CONDA_ENV": "notebook", "GITHUB_REPOSITORY": "my/repo", "GITHUB_API_URL": "https://api.github.com", # fixturing of `head_ref` reflects that on `push` @@ -56,8 +55,8 @@ class MockCompletedProcess: returncode: int -@pytest.fixture -def subprocess_run_side_effect(): +@pytest.fixture(params=[True, False], ids=["has_job_id", "no_job_id"]) +def subprocess_run_side_effect(request): def _get_mock_completed_proc(cmd: list[str], *args, **kwargs): # `subprocess.run` is called a few ways, so use a side effect function # to vary the output depending on what arguments it was called with. @@ -69,8 +68,11 @@ def _get_mock_completed_proc(cmd: list[str], *args, **kwargs): returncode=returncode, ) elif "bake" in " ".join(cmd): + # not all bakery types have a job_id, represent that here + has_job_id = request.param + stdout = b'{"job_id": "foo", "job_name": "bar"}' if has_job_id else b'{}' return MockCompletedProcess( - stdout=b'{"job_id": "foo", "job_name": "bar"}', + stdout=stdout, stderr=b"", returncode=0, ) @@ -134,7 +136,7 @@ def test_main( if pip_install_raises: config: dict = json.loads(env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"]) - config.update({"BaseCommand": {"feedstock-subdir": "broken-requirements-feedstock"}}) + config.update({"BaseCommand": {"feedstock_subdir": "broken-requirements-feedstock"}}) env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"] = json.dumps(config) with patch.dict(os.environ, env): @@ -156,8 +158,7 @@ def test_main( else "feedstock" ) expected_cmd = ( - f"mamba run -n {env['CONDA_ENV']} " - f"pip install -Ur {feedstock_subdir}/requirements.txt" + f"python3 -m pip install -Ur {feedstock_subdir}/requirements.txt" ).split() subprocess_run.assert_any_call(expected_cmd, capture_output=True) else: