Add basic integration tests (#164)

* Add integration tests

* lint [integration]

* fix path and flag [integration]

* Fix pytest flag [integration]

* fix test [integration]

* Update tests/integration/test_integration.py

Co-authored-by: Hood Chatham <[email protected]>

* Update tests/integration/test_integration.py

Co-authored-by: Hood Chatham <[email protected]>

* Make decorator [integration]

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix test

* [integration]

---------

Co-authored-by: Hood Chatham <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/main.yml

@@ -12,6 +12,7 @@ concurrency:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    needs: [check-integration-test-trigger]
     env:
       DISPLAY: :99
     strategy:
@@ -51,7 +52,20 @@ jobs:
       - name: Run tests
         shell: bash -l {0}
         run: |
-          pytest -v \
+          pytest -v tests \
+            --cov=micropip \
+            --durations=10 \
+            --dist-dir=./dist/ \
+            --maxfail=15 \
+            --runner=${{ matrix.test-config.runner }} \
+            --rt ${{ matrix.test-config.runtime }}
+
+      - name: Run integration tests
+        if: needs.check-integration-test-trigger.outputs.run-integration-test
+        shell: bash -l {0}
+        run: |
+          pytest -v tests/integration \
+            --integration \
             --cov=micropip \
             --durations=10 \
             --dist-dir=./dist/ \
@@ -64,6 +78,32 @@ jobs:
         with:
           fail_ci_if_error: false
 
+  check-integration-test-trigger:
+    name: test-integration-test-trigger
+    runs-on: ubuntu-latest
+    outputs:
+      run-integration-test: ${{ steps.check-integration-test-trigger.outputs.trigger }}
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - id: check-integration-test-trigger
+        name: Check integration test trigger
+        shell: bash
+        run: |
+          set -e -x
+
+          COMMIT_MSG=$(git log --no-merges -1 --oneline)
+
+          # The integration tests will be triggered on push or on pull_request when the commit
+          # message contains "[integration]" or if it is pushed to main branch.
+          if [[ "$GITHUB_EVENT_NAME" == push && "$GITHUB_REF" == refs/heads/main ||
+                "$COMMIT_MSG" =~ \[integration\] ]]; then
+              echo "trigger=true" >> "$GITHUB_OUTPUT"
+          fi
+
   deploy:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 default_language_version:
-  python: "3.11"
+  python: "3.12"
 
 exclude: (^micropip/externals|^tests/vendored|^tests/test_data)
 repos:

tests/conftest.py

@@ -23,6 +23,13 @@ def pytest_addoption(parser):
         help="Run tests that query remote package indexes.",
     )
 
+    parser.addoption(
+        "--integration",
+        action="store_true",
+        default=None,
+        help="Run integration tests.",
+    )
+
 
 EMSCRIPTEN_VER = "3.1.14"
 PLATFORM = f"emscripten_{EMSCRIPTEN_VER.replace('.', '_')}_wasm32"

tests/integration/test_integration.py

@@ -0,0 +1,87 @@
+# Integration tests for micropip
+# These test often requires querying to the real packages existing in PyPI,
+# to test the micropip's ability to install packages from real world package indexes.
+
+# To prevent sending many requests to remote servers, these tests are disabled by default.
+# To run these tests locally, invoke pytest with the `--integration` flag.
+
+import pytest
+from pytest_pyodide import run_in_pyodide
+
+
+def integration_test_only(func):
+    def wrapper(selenium_standalone_micropip, pytestconfig):
+        if not pytestconfig.getoption("--integration"):
+            pytest.skip("Integration tests are skipped. Use --integration to run them.")
+        func(selenium_standalone_micropip, pytestconfig)
+
+    return wrapper
+
+
+@integration_test_only
+def test_integration_install_basic(selenium_standalone_micropip, pytestconfig):
+    @run_in_pyodide
+    async def _run(selenium):
+        import micropip
+
+        await micropip.install("snowballstemmer")
+
+        import snowballstemmer
+
+        snowballstemmer.stemmer("english")
+
+    _run(selenium_standalone_micropip)
+
+
+@integration_test_only
+def test_integration_list_basic(selenium_standalone_micropip, pytestconfig):
+    @run_in_pyodide
+    async def _run(selenium):
+        import micropip
+
+        await micropip.install("snowballstemmer")
+
+        packages = micropip.list()
+        assert "snowballstemmer" in packages
+
+    _run(selenium_standalone_micropip)
+
+
+@integration_test_only
+def test_integration_uninstall_basic(selenium_standalone_micropip, pytestconfig):
+    @run_in_pyodide
+    async def _run(selenium):
+        import micropip
+
+        await micropip.install("snowballstemmer")
+
+        import snowballstemmer
+
+        snowballstemmer.stemmer("english")
+
+        micropip.uninstall("snowballstemmer")
+
+        packages = micropip.list()
+        assert "snowballstemmer" not in packages
+
+    _run(selenium_standalone_micropip)
+
+
+@integration_test_only
+def test_integration_freeze_basic(selenium_standalone_micropip, pytestconfig):
+    @run_in_pyodide
+    async def _run(selenium):
+        import json
+
+        import micropip
+
+        await micropip.install("snowballstemmer")
+
+        import snowballstemmer
+
+        snowballstemmer.stemmer("english")
+
+        lockfile = micropip.freeze()
+        assert "snowballstemmer" in json.loads(lockfile)["packages"]
+
+    _run(selenium_standalone_micropip)

tests/test_remote_indexes.py → tests/integration/test_remote_indexes.py

@@ -1,7 +1,7 @@
 # This file contains tests that actually query remote package indexes,
 # to ensure that micropip works with real-world package indexes.
 # Since running these tests will send many requests to remote servers,
-# these tests are disabled by default.
+# these tests are disabled by default in CI and local testing.
 #
 # To run these tests, add `--run-remote-index-tests` flag, or
 # these tests can also be run in Github Actions manually.

tests/test_install.py

@@ -6,28 +6,6 @@
 import micropip
 
 
-def test_install_simple(selenium_standalone_micropip):
-    selenium = selenium_standalone_micropip
-    assert (
-        selenium.run_js(
-            """
-            return await pyodide.runPythonAsync(`
-                import os
-                import micropip
-                from pyodide.ffi import to_js
-                # Package 'pyodide-micropip-test' has dependency on 'snowballstemmer'
-                # It is used to test markers support
-                await micropip.install('pyodide-micropip-test')
-                import snowballstemmer
-                stemmer = snowballstemmer.stemmer('english')
-                to_js(stemmer.stemWords('go going goes gone'.split()))
-            `);
-            """
-        )
-        == ["go", "go", "goe", "gone"]
-    )
-
-
 def test_install_custom_url(selenium_standalone_micropip, wheel_catalog):
     selenium = selenium_standalone_micropip
     snowball_wheel = wheel_catalog.get("snowballstemmer")