From dfa97758a134dce4254847cc4e5323c5b477158d Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 11 Sep 2024 10:49:16 -0700 Subject: [PATCH] chore(py): cleanup python CI (#11324) --- .github/workflows/airflow-plugin.yml | 10 +++---- .github/workflows/dagster-plugin.yml | 2 +- .github/workflows/gx-plugin.yml | 2 +- .github/workflows/metadata-ingestion.yml | 2 +- .github/workflows/prefect-plugin.yml | 4 +-- .../airflow-plugin/build.gradle | 6 +--- .../dagster-plugin/build.gradle | 22 +++------------ .../gx-plugin/build.gradle | 22 +++------------ .../prefect-plugin/build.gradle | 28 ++++++------------- 9 files changed, 27 insertions(+), 71 deletions(-) diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml index 912dd985f66c0..e75bafdac9628 100644 --- a/.github/workflows/airflow-plugin.yml +++ b/.github/workflows/airflow-plugin.yml @@ -43,16 +43,16 @@ jobs: extra_pip_requirements: "apache-airflow~=2.4.3" extra_pip_extras: plugin-v2,test-airflow24 - python-version: "3.10" - extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt' + extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt" extra_pip_extras: plugin-v2 - python-version: "3.10" - extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt' + extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt" extra_pip_extras: plugin-v2 - python-version: "3.10" - extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt' + extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt" extra_pip_extras: plugin-v2 - python-version: "3.11" - extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt' + extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt" extra_pip_extras: plugin-v2 fail-fast: false steps: @@ -73,7 +73,7 @@ jobs: run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build - name: pip freeze show list installed if: always() - run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze + run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v3 if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }} with: diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml index f0b9038b610d2..37b6c93ec841a 100644 --- a/.github/workflows/dagster-plugin.yml +++ b/.github/workflows/dagster-plugin.yml @@ -55,7 +55,7 @@ jobs: run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:dagster-plugin:lint :metadata-ingestion-modules:dagster-plugin:testQuick - name: pip freeze show list installed if: always() - run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && pip freeze + run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v3 if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }} with: diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml index 06a10883e70f5..99121f81099f2 100644 --- a/.github/workflows/gx-plugin.yml +++ b/.github/workflows/gx-plugin.yml @@ -57,7 +57,7 @@ jobs: run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:gx-plugin:lint :metadata-ingestion-modules:gx-plugin:testQuick - name: pip freeze show list installed if: always() - run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && pip freeze + run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v3 if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }} with: diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index cfb3693d89381..c718febca398a 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -83,7 +83,7 @@ jobs: - name: Debug info if: always() run: | - source metadata-ingestion/venv/bin/activate && pip freeze + source metadata-ingestion/venv/bin/activate && uv pip freeze set -x df -hl docker image ls diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml index 09af0ad3f354a..b0af00f92b772 100644 --- a/.github/workflows/prefect-plugin.yml +++ b/.github/workflows/prefect-plugin.yml @@ -43,7 +43,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: @@ -55,7 +55,7 @@ jobs: run: ./gradlew :metadata-ingestion-modules:prefect-plugin:lint :metadata-ingestion-modules:prefect-plugin:testQuick - name: pip freeze show list installed if: always() - run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && pip freeze + run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v3 if: ${{ always() && matrix.python-version == '3.10'}} with: diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index e874f70db02a3..9506609a10044 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -26,7 +26,7 @@ task environmentSetup(type: Exec) { outputs.file(sentinel_file) commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && set -x && " + - "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " + + "${venv_name}/bin/python -m pip install --upgrade uv && " + "touch ${sentinel_file}" } @@ -34,12 +34,8 @@ task installPackage(type: Exec, dependsOn: [environmentSetup, ':metadata-ingesti def sentinel_file = "${venv_name}/.build_install_package_sentinel" inputs.file file('setup.py') outputs.file(sentinel_file) - // Workaround for https://github.com/yaml/pyyaml/issues/601. - // See https://github.com/yaml/pyyaml/issues/601#issuecomment-1638509577. - // and https://github.com/datahub-project/datahub/pull/8435. commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pip install 'Cython<3.0' 'PyYAML<6' --no-build-isolation && " + "${pip_install_command} -e .[ignore${extra_pip_extras}] ${extra_pip_requirements} &&" + "touch ${sentinel_file}" } diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle index 74ca7cedea3a5..0f11af9ca83d7 100644 --- a/metadata-ingestion-modules/dagster-plugin/build.gradle +++ b/metadata-ingestion-modules/dagster-plugin/build.gradle @@ -23,7 +23,7 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { outputs.file(sentinel_file) commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && " + - "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " + + "${venv_name}/bin/python -m pip install --upgrade uv && " + "touch ${sentinel_file}" } @@ -77,30 +77,16 @@ task installDevTest(type: Exec, dependsOn: [installDev]) { "touch ${sentinel_file}" } -def testFile = hasProperty('testFile') ? testFile : 'unknown' -task testSingle(dependsOn: [installDevTest]) { - doLast { - if (testFile != 'unknown') { - exec { - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest ${testFile}" - } - } else { - throw new GradleException("No file provided. Use -PtestFile=") - } - } -} - task testQuick(type: Exec, dependsOn: installDevTest) { // We can't enforce the coverage requirements if we run a subset of the tests. inputs.files(project.fileTree(dir: "src/", include: "**/*.py")) inputs.files(project.fileTree(dir: "tests/")) outputs.dir("${venv_name}") - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && set -x && " + + "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" } - task buildWheel(type: Exec, dependsOn: [environmentSetup]) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh' diff --git a/metadata-ingestion-modules/gx-plugin/build.gradle b/metadata-ingestion-modules/gx-plugin/build.gradle index f1adbc6676e5b..2288ae6bd8396 100644 --- a/metadata-ingestion-modules/gx-plugin/build.gradle +++ b/metadata-ingestion-modules/gx-plugin/build.gradle @@ -23,7 +23,7 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { outputs.file(sentinel_file) commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && " + - "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " + + "${venv_name}/bin/python -m pip install --upgrade uv && " + "touch ${sentinel_file}" } @@ -77,30 +77,16 @@ task installDevTest(type: Exec, dependsOn: [installDev]) { "touch ${sentinel_file}" } -def testFile = hasProperty('testFile') ? testFile : 'unknown' -task testSingle(dependsOn: [installDevTest]) { - doLast { - if (testFile != 'unknown') { - exec { - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest ${testFile}" - } - } else { - throw new GradleException("No file provided. Use -PtestFile=") - } - } -} - task testQuick(type: Exec, dependsOn: installDevTest) { // We can't enforce the coverage requirements if we run a subset of the tests. inputs.files(project.fileTree(dir: "src/", include: "**/*.py")) inputs.files(project.fileTree(dir: "tests/")) outputs.dir("${venv_name}") - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && set -x && " + + "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" } - task buildWheel(type: Exec, dependsOn: [environmentSetup]) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh' diff --git a/metadata-ingestion-modules/prefect-plugin/build.gradle b/metadata-ingestion-modules/prefect-plugin/build.gradle index b078b8d8de3b3..bc091c6493364 100644 --- a/metadata-ingestion-modules/prefect-plugin/build.gradle +++ b/metadata-ingestion-modules/prefect-plugin/build.gradle @@ -23,8 +23,8 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { outputs.file(sentinel_file) commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && " + - "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " + - "touch ${sentinel_file}" + "${venv_name}/bin/python -m pip install --upgrade uv && " + + "touch ${sentinel_file}" } task installPackage(type: Exec, dependsOn: [environmentSetup, ':metadata-ingestion:codegen']) { @@ -75,33 +75,21 @@ task installDevTest(type: Exec, dependsOn: [installDev]) { "${pip_install_command} -e .[dev,integration-tests] && touch ${sentinel_file}" } -def testFile = hasProperty('testFile') ? testFile : 'unknown' -task testSingle(dependsOn: [installDevTest]) { - doLast { - if (testFile != 'unknown') { - exec { - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest ${testFile}" - } - } else { - throw new GradleException("No file provided. Use -PtestFile=") - } - } -} - task testQuick(type: Exec, dependsOn: installDevTest) { // We can't enforce the coverage requirements if we run a subset of the tests. inputs.files(project.fileTree(dir: "src/", include: "**/*.py")) inputs.files(project.fileTree(dir: "tests/")) outputs.dir("${venv_name}") - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s" + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && set -x && " + + "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s" } task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml" + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && set -x && " + + "pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml" }