chore(py): cleanup python CI (datahub-project#11324)

G-Research · Sep 11, 2024 · dfa9775 · dfa9775
1 parent 3755731
commit dfa9775
Show file tree

Hide file tree

Showing 9 changed files with 27 additions and 71 deletions.
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -43,16 +43,16 @@ jobs:
             extra_pip_requirements: "apache-airflow~=2.4.3"
             extra_pip_extras: plugin-v2,test-airflow24
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
+            extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
+            extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
+            extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
             extra_pip_extras: plugin-v2
           - python-version: "3.11"
-            extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt'
+            extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
@@ -73,7 +73,7 @@ jobs:
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
         with:

diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml
@@ -55,7 +55,7 @@ jobs:
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:dagster-plugin:lint :metadata-ingestion-modules:dagster-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
         with:

diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml
@@ -57,7 +57,7 @@ jobs:
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:gx-plugin:lint :metadata-ingestion-modules:gx-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }}
         with:

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -83,7 +83,7 @@ jobs:
       - name: Debug info
         if: always()
         run: |
-          source metadata-ingestion/venv/bin/activate && pip freeze
+          source metadata-ingestion/venv/bin/activate && uv pip freeze
           set -x
           df -hl
           docker image ls

diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml
@@ -43,7 +43,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
@@ -55,7 +55,7 @@ jobs:
         run: ./gradlew :metadata-ingestion-modules:prefect-plugin:lint :metadata-ingestion-modules:prefect-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.10'}}
         with:

diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle
@@ -26,20 +26,16 @@ task environmentSetup(type: Exec) {
   outputs.file(sentinel_file)
   commandLine 'bash', '-c', 
     "${python_executable} -m venv ${venv_name} && set -x && " +
-    "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
+    "${venv_name}/bin/python -m pip install --upgrade uv && " +
     "touch ${sentinel_file}"
 }
 
 task installPackage(type: Exec, dependsOn: [environmentSetup, ':metadata-ingestion:codegen']) {
   def sentinel_file = "${venv_name}/.build_install_package_sentinel"
   inputs.file file('setup.py')
   outputs.file(sentinel_file)
-  // Workaround for https://github.com/yaml/pyyaml/issues/601.
-  // See https://github.com/yaml/pyyaml/issues/601#issuecomment-1638509577.
-  // and https://github.com/datahub-project/datahub/pull/8435.
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pip install 'Cython<3.0' 'PyYAML<6' --no-build-isolation && " +
     "${pip_install_command} -e .[ignore${extra_pip_extras}] ${extra_pip_requirements} &&" +
     "touch ${sentinel_file}"
 }

diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle
@@ -23,7 +23,7 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
   outputs.file(sentinel_file)
   commandLine 'bash', '-c',
       "${python_executable} -m venv ${venv_name} && " +
-      "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
+      "${venv_name}/bin/python -m pip install --upgrade uv && " +
       "touch ${sentinel_file}"
 }
 
@@ -77,30 +77,16 @@ task installDevTest(type: Exec, dependsOn: [installDev]) {
       "touch ${sentinel_file}"
 }
 
-def testFile = hasProperty('testFile') ? testFile : 'unknown'
-task testSingle(dependsOn: [installDevTest]) {
-  doLast {
-  if (testFile != 'unknown')  {
-  exec {
-    commandLine 'bash', '-x', '-c',
-     "source ${venv_name}/bin/activate && pytest ${testFile}"
-    }
-  } else {
-   throw new GradleException("No file provided. Use -PtestFile=<test_file>")
-    }
-  }
-}
-
 task testQuick(type: Exec, dependsOn: installDevTest) {
   // We can't enforce the coverage requirements if we run a subset of the tests.
   inputs.files(project.fileTree(dir: "src/", include: "**/*.py"))
   inputs.files(project.fileTree(dir: "tests/"))
   outputs.dir("${venv_name}")
-  commandLine 'bash', '-x', '-c',
-    "source ${venv_name}/bin/activate && pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+  commandLine 'bash', '-c',
+    "source ${venv_name}/bin/activate && set -x && " +
+    "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
-
 task buildWheel(type: Exec, dependsOn: [environmentSetup]) {
   commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " +
   'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'

diff --git a/metadata-ingestion-modules/gx-plugin/build.gradle b/metadata-ingestion-modules/gx-plugin/build.gradle
@@ -23,7 +23,7 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
   outputs.file(sentinel_file)
   commandLine 'bash', '-c',
       "${python_executable} -m venv ${venv_name} && " +
-      "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
+      "${venv_name}/bin/python -m pip install --upgrade uv && " +
       "touch ${sentinel_file}"
 }
 
@@ -77,30 +77,16 @@ task installDevTest(type: Exec, dependsOn: [installDev]) {
       "touch ${sentinel_file}"
 }
 
-def testFile = hasProperty('testFile') ? testFile : 'unknown'
-task testSingle(dependsOn: [installDevTest]) {
-  doLast {
-  if (testFile != 'unknown')  {
-  exec {
-    commandLine 'bash', '-x', '-c',
-     "source ${venv_name}/bin/activate && pytest ${testFile}"
-    }
-  } else {
-   throw new GradleException("No file provided. Use -PtestFile=<test_file>")
-    }
-  }
-}
-
 task testQuick(type: Exec, dependsOn: installDevTest) {
   // We can't enforce the coverage requirements if we run a subset of the tests.
   inputs.files(project.fileTree(dir: "src/", include: "**/*.py"))
   inputs.files(project.fileTree(dir: "tests/"))
   outputs.dir("${venv_name}")
-  commandLine 'bash', '-x', '-c',
-    "source ${venv_name}/bin/activate && pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+  commandLine 'bash', '-c',
+    "source ${venv_name}/bin/activate && set -x && " +
+    "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
-
 task buildWheel(type: Exec, dependsOn: [environmentSetup]) {
   commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " +
   'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'

diff --git a/metadata-ingestion-modules/prefect-plugin/build.gradle b/metadata-ingestion-modules/prefect-plugin/build.gradle
@@ -23,8 +23,8 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
   outputs.file(sentinel_file)
   commandLine 'bash', '-c',
       "${python_executable} -m venv ${venv_name} && " +
-          "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
-          "touch ${sentinel_file}"
+      "${venv_name}/bin/python -m pip install --upgrade uv && " +
+      "touch ${sentinel_file}"
 }
 
 task installPackage(type: Exec, dependsOn: [environmentSetup, ':metadata-ingestion:codegen']) {
@@ -75,33 +75,21 @@ task installDevTest(type: Exec, dependsOn: [installDev]) {
     "${pip_install_command} -e .[dev,integration-tests] && touch ${sentinel_file}"
 }
 
-def testFile = hasProperty('testFile') ? testFile : 'unknown'
-task testSingle(dependsOn: [installDevTest]) {
-  doLast {
-  if (testFile != 'unknown')  {
-  exec {
-    commandLine 'bash', '-x', '-c',
-     "source ${venv_name}/bin/activate && pytest ${testFile}"
-    }
-  } else {
-   throw new GradleException("No file provided. Use -PtestFile=<test_file>")
-    }
-  }
-}
-
 task testQuick(type: Exec, dependsOn: installDevTest) {
   // We can't enforce the coverage requirements if we run a subset of the tests.
   inputs.files(project.fileTree(dir: "src/", include: "**/*.py"))
   inputs.files(project.fileTree(dir: "tests/"))
   outputs.dir("${venv_name}")
-  commandLine 'bash', '-x', '-c',
-    "source ${venv_name}/bin/activate && pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s"
+  commandLine 'bash', '-c',
+    "source ${venv_name}/bin/activate && set -x && " +
+    "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s"
 }
 
 
 task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
-  commandLine 'bash', '-x', '-c',
-    "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
+  commandLine 'bash', '-c',
+    "source ${venv_name}/bin/activate && set -x && " +
+    "pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
 }