Merge a recent version of main into the phase 0 staging branch

.github/CODEOWNERS

@@ -1,16 +1,18 @@
 * @snowflakedb/snowpark-python-api-reviewers
+/src/snowflake/snowpark/ @snowflakedb/snowpark-python-api-reviewers @snowflakedb/snowpark-ir
 /src/snowflake/snowpark/modin/ @snowflakedb/snowpark-pandas-api
+/src/snowflake/snowpark/_internal/ast @snowflakedb/snowpark-ir
 /tests/ast/ @snowflakedb/snowpark-ir
 /tests/unit/ast/  @snowflakedb/snowpark-ir
 /tests/integ/modin/ @snowflakedb/snowpark-pandas-api
 /tests/unit/modin/ @snowflakedb/snowpark-pandas-api
 /docs/modin_api_coverage/ @snowflakedb/snowpark-pandas-api
 /docs/source/modin/ @snowflakedb/snowpark-pandas-api
 /tests/notebooks/ @snowflakedb/snowpark-pandas-api
-/.github/ @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers
-/scripts/ @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers
-setup.py @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers
-tox.ini @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers
+/.github/ @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers @snowflakedb/snowpark-ir
+/scripts/ @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers @snowflakedb/snowpark-ir
+setup.py @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers @snowflakedb/snowpark-ir
+tox.ini @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers @snowflakedb/snowpark-ir
 README.md @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers
 CONTRIBUTING.md @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers
 tests/README.md @snowflakedb/snowpark-pandas-api @snowflakedb/snowpark-python-api-reviewers

.github/pull_request_template.md

@@ -6,7 +6,7 @@ Please answer these questions before creating your pull request. Thanks!
 
    <!---
    In this section, please add a Snowflake Jira issue number.
-   
+
    Note that if a corresponding GitHub issue exists, you should still include
    the Snowflake Jira issue number. For example, for GitHub issue
    https://github.com/snowflakedb/snowpark-python/issues/1400, you should
@@ -24,6 +24,7 @@ Please answer these questions before creating your pull request. Thanks!
    - [ ] I am adding new credentials
    - [ ] I am adding a new dependency
    - [ ] If this is a new feature/behavior, I'm adding the Local Testing parity changes.
+   - [ ] I acknowledge that I have ensured my changes to be thread-safe. Follow the link for more information: [Thread-safe Developer Guidelines](https://docs.google.com/document/d/162d_i4zZ2AfcGRXojj0jByt8EUq-DrSHPPnTa4QvwbA/edit#bookmark=id.e82u4nekq80k)
 
 3. Please describe how your code solves the related issue.
 

.github/scripts/install_protoc.sh

@@ -14,7 +14,7 @@ SCRIPT_NAME="$(basename "$0")"
 
 echo "${SCRIPT_NAME} is running... "
 
-PROTOC_VERSION=28.3
+PROTOC_VERSION=3.20.1
 PROTOC_OS_ARCH=""
 PROTOC_ZIP=""
 
@@ -32,7 +32,7 @@ getOSNameAndArch(){
         PROTOC_OS_ARCH="linux-x86_64"
         ;;
       darwin)
-        PROTOC_OS_ARCH="osx-universal_binary"
+        PROTOC_OS_ARCH="osx-x86_64"
         ;;
       mingw64* | msys* | cygwin*)
         PROTOC_OS_ARCH="win64"
@@ -71,4 +71,8 @@ install() {
 
 install
 
+# mypy-protobuf is used to generated typed Python code from protobuf
+pip install mypy-protobuf
+echo "mypy-protobuf version: $(protoc-gen-mypy --version)"
+
 echo "${SCRIPT_NAME} done."

.github/workflows/daily_modin_precommit.yml

@@ -192,6 +192,71 @@ jobs:
             .tox/.coverage
             .tox/coverage.xml
 
+  test-pandas-patch-versions:
+    name: Test Snowpark pandas with pandas ${{ matrix.pandas-version }}
+    needs: build
+    runs-on: ${{ matrix.os.image_name }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - image_name: ubuntu-latest-64-cores
+            download_name: linux
+        pandas-version: ["2.2.1", "2.2.2"]
+        python-version: ["3.9"]
+        cloud-provider: [aws]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Decrypt parameters.py
+        shell: bash
+        run: .github/scripts/decrypt_parameters.sh
+        env:
+          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
+          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
+      - name: Install protoc
+        shell: bash
+        run: .github/scripts/install_protoc.sh
+      - name: Download wheel(s)
+        uses: actions/download-artifact@v4
+        with:
+          name: wheel
+          path: dist
+      - name: Show wheels downloaded
+        run: ls -lh dist
+        shell: bash
+      - name: Upgrade setuptools, pip and wheel
+        run: python -m pip install -U setuptools pip wheel
+      - name: Install tox
+        run: python -m pip install tox
+      - if: ${{ contains('macos', matrix.os.download_name) }}
+        name: Run Snowpark pandas API doctests
+        run: python -m tox -e "modin_pandas_version-py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci"
+        env:
+          MODIN_PANDAS_PATCH_VERSION: ${{ matrix.pandas-version }}
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
+          # For example, see https://github.com/snowflakedb/snowpark-python/pull/681
+        shell: bash
+      - name: Run Snowpark pandas API tests (excluding doctests)
+        run: python -m tox -e "modin_pandas_version-py${PYTHON_VERSION/\./}-snowparkpandasdailynotdoctest-modin-ci"
+        env:
+          MODIN_PANDAS_PATCH_VERSION: ${{ matrix.pandas-version }}
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+        shell: bash
+
   test-disable-sql-simplifier: # Will be removed after sql simplifier is stable and no option to opt out.
     name: Test Disable SQL Simplifier modin-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
     needs: build
@@ -264,8 +329,8 @@ jobs:
             .tox/.coverage
             .tox/coverage.xml
 
-  test-enable-cte-optimization:
-    name: Test Enable CTE Optimization modin-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+  test-disable-cte-optimization:
+    name: Test Disable CTE Optimization modin-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
     needs: build
     runs-on: ${{ matrix.os.image_name }}
     strategy:
@@ -312,15 +377,15 @@ jobs:
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
-          PYTEST_ADDOPTS: --color=yes --tb=short --enable_cte_optimization
+          PYTEST_ADDOPTS: --color=yes --tb=short --disable_cte_optimization
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
       - name: Run Snowpark pandas API tests (excluding doctests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasdailynotdoctest-modin-ci"
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
-          PYTEST_ADDOPTS: --color=yes --tb=short --enable_cte_optimization --skip_sql_count_check --ignore=tests/integ/modin/test_sql_counter.py
+          PYTEST_ADDOPTS: --color=yes --tb=short --disable_cte_optimization --skip_sql_count_check --ignore=tests/integ/modin/test_sql_counter.py
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
       - name: Combine coverages
@@ -331,7 +396,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           include-hidden-files: true
-          name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-enable-cte-optimization
+          name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-disable-cte-optimization
           path: |
             .tox/.coverage
             .tox/coverage.xml
@@ -406,22 +471,16 @@ jobs:
           python-version: '3.9'
       - name: Upgrade setuptools and pip
         run: python -m pip install -U setuptools pip
+      - name: Install tox
+        run: python -m pip install tox
       - name: Install protoc
         shell: bash
         run: .github/scripts/install_protoc.sh
-      - name: Install Snowpark
-        run: python -m pip install ".[modin-development]"
-      - name: Install Sphinx
-        run: python -m pip install sphinx
       - name: Build document
-        working-directory: docs
-        # treat warning as failure but complete the entire process
-        run: |
-          make clean
-          make html SPHINXOPTS="-W --keep-going"
+        run: python -m tox -e docs
       - name: Upload html files
         uses: actions/upload-artifact@v4
         with:
           include-hidden-files: true
           name: docs
-          path: docs/build/html
+          path: .tox/docs_out