Merge branch 'main' into 2023-11-date_range-fix

.circleci/config.yml

@@ -18,6 +18,29 @@ jobs:
           PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
           LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD
           ci/run_tests.sh
+  linux-musl:
+    docker:
+      - image: quay.io/pypa/musllinux_1_1_aarch64
+    resource_class: arm.large
+    steps:
+      # Install pkgs first to have git in the image
+      # (needed for checkout)
+      - run: |
+          apk update
+          apk add git
+          apk add musl-locales
+      - checkout
+      - run: |
+          /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
+          . ~/virtualenvs/pandas-dev/bin/activate
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
+          python -m pip list --no-cache-dir
+      - run: |
+          . ~/virtualenvs/pandas-dev/bin/activate
+          export PANDAS_CI=1
+          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
   build-aarch64:
     parameters:
       cibw-build:
@@ -89,6 +112,13 @@ workflows:
         equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
       - test-arm
+  test-musl:
+    # Don't run trigger this one when scheduled pipeline runs
+    when:
+      not:
+        equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
+    jobs:
+      - linux-musl
   build-wheels:
     jobs:
       - build-aarch64:
@@ -97,4 +127,11 @@ workflows:
               only: /^v.*/
           matrix:
             parameters:
-              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64", "cp312-manylinux_aarch64"]
+              cibw-build: ["cp39-manylinux_aarch64",
+                           "cp310-manylinux_aarch64",
+                           "cp311-manylinux_aarch64",
+                           "cp312-manylinux_aarch64",
+                           "cp39-musllinux_aarch64",
+                           "cp310-musllinux_aarch64",
+                           "cp311-musllinux_aarch64",
+                           "cp312-musllinux_aarch64",]

.circleci/setup_env.sh

@@ -55,6 +55,6 @@ if pip show pandas 1>/dev/null; then
 fi
 
 echo "Install pandas"
-python -m pip install --no-build-isolation -ve .
+python -m pip install --no-build-isolation -ve . --config-settings=setup-args="--werror"
 
 echo "done"

.github/actions/build_pandas/action.yml

@@ -4,6 +4,12 @@ inputs:
   editable:
     description: Whether to build pandas in editable mode (default true)
     default: true
+  meson_args:
+    description: Extra flags to pass to meson
+    required: false
+  cflags_adds:
+    description: Items to append to the CFLAGS variable
+    required: false
 runs:
   using: composite
   steps:
@@ -24,9 +30,12 @@ runs:
 
     - name: Build Pandas
       run: |
+        export CFLAGS="$CFLAGS ${{ inputs.cflags_adds }}"
         if [[ ${{ inputs.editable }} == "true" ]]; then
-          pip install -e . --no-build-isolation -v --no-deps
+          pip install -e . --no-build-isolation -v --no-deps ${{ inputs.meson_args }} \
+            --config-settings=setup-args="--werror"
         else
-          pip install . --no-build-isolation -v --no-deps
+          pip install . --no-build-isolation -v --no-deps ${{ inputs.meson_args }} \
+            --config-settings=setup-args="--werror"
         fi
       shell: bash -el {0}

.github/actions/run-tests/action.yml

@@ -1,9 +1,16 @@
 name: Run tests and report results
+inputs:
+  preload:
+    description: Preload arguments for sanitizer
+    required: false
+  asan_options:
+    description: Arguments for Address Sanitizer (ASAN)
+    required: false
 runs:
   using: composite
   steps:
     - name: Test
-      run: ci/run_tests.sh
+      run: ${{ inputs.asan_options }} ${{ inputs.preload }} ci/run_tests.sh
       shell: bash -el {0}
 
     - name: Publish test results

.github/actions/setup-conda/action.yml

@@ -11,6 +11,6 @@ runs:
       with:
         environment-file: ${{ inputs.environment-file }}
         environment-name: test
-        condarc-file: ci/condarc.yml
+        condarc-file: ci/.condarc
         cache-environment: true
         cache-downloads: true

.github/workflows/broken-linkcheck.yml

@@ -9,6 +9,7 @@ on:
       - "doc/make.py"
 jobs:
   linkcheck:
+    if: false
     runs-on: ubuntu-latest
     defaults:
       run:

.github/workflows/code-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
 
 env:
   ENV_FILE: environment.yml
@@ -170,7 +170,7 @@ jobs:
 
       - name: Setup Python
         id: setup_python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
           cache: 'pip'

.github/workflows/codeql.yml

@@ -28,8 +28,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: github/codeql-action/init@v2
+      - uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
-      - uses: github/codeql-action/autobuild@v2
-      - uses: github/codeql-action/analyze@v2
+      - uses: github/codeql-action/autobuild@v3
+      - uses: github/codeql-action/analyze@v3

.github/workflows/deprecation-tracking-bot.yml

@@ -19,7 +19,7 @@ jobs:
       issues: write
     runs-on: ubuntu-22.04
     env:
-      DEPRECATION_TRACKER_ISSUE: 50578
+      DEPRECATION_TRACKER_ISSUE: 56596
     steps:
     - uses: actions/github-script@v7
       id: update-deprecation-issue

.github/workflows/docbuild-and-upload.yml

@@ -4,13 +4,13 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
 
 env:
   ENV_FILE: environment.yml
@@ -46,6 +46,9 @@ jobs:
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 
+    - name: Test website
+      run: python -m pytest web/
+
     - name: Build website
       run: python web/pandas_web.py web/pandas --target-path=web/build
 
@@ -82,7 +85,7 @@ jobs:
       run: mv doc/build/html web/build/docs
 
     - name: Save website as an artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: website
         path: web/build

.github/workflows/package-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
     types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
@@ -40,7 +40,7 @@ jobs:
 
       - name: Setup Python
         id: setup_python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 

.github/workflows/stale-pr.yml

@@ -14,7 +14,7 @@ jobs:
     if: github.repository_owner == 'pandas-dev'
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity. Please [update](https://pandas.pydata.org/pandas-docs/stable/development/contributing.html#updating-your-pull-request) and respond to this comment if you're still interested in working on this."

.github/workflows/unit-tests.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
     paths-ignore:
       - "doc/**"
       - "web/**"
@@ -26,7 +26,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml]
+        env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
         include:
@@ -69,21 +69,41 @@ jobs:
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
+          - name: "Copy-on-Write 3.12"
+            env_file: actions-312.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_copy_on_write: "1"
           - name: "Copy-on-Write 3.11 (warnings)"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "warn"
+          - name: "Copy-on-Write 3.10 (warnings)"
+            env_file: actions-310.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_copy_on_write: "warn"
+          - name: "Copy-on-Write 3.9 (warnings)"
+            env_file: actions-39.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_copy_on_write: "warn"
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "--max-worker-restart 0"
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
-            test_args: "-W error::DeprecationWarning -W error::FutureWarning"
+            test_args: "-W error::FutureWarning"
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
+          - name: "ASAN / UBSAN"
+            env_file: actions-311-sanitizers.yaml
+            pattern: "not slow and not network and not single_cpu and not skip_ubsan"
+            asan_options: "ASAN_OPTIONS=detect_leaks=0"
+            preload: LD_PRELOAD=$(gcc -print-file-name=libasan.so)
+            meson_args: --config-settings=setup-args="-Db_sanitize=address,undefined"
+            cflags_adds: -fno-sanitize-recover=all
+            pytest_workers: -1  # disable pytest-xdist as it swallows stderr from ASAN
       fail-fast: false
     name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}
     env:
@@ -93,8 +113,9 @@ jobs:
       PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
       PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
-      PYTEST_WORKERS: 'auto'
+      PYTEST_WORKERS: ${{ matrix.pytest_workers || 'auto' }}
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
+      NPY_PROMOTION_STATE: ${{ matrix.env_file == 'actions-311-numpydev.yaml' && 'weak' || 'legacy' }}
       # Clipboard tests
       QT_QPA_PLATFORM: offscreen
     concurrency:
@@ -162,16 +183,25 @@ jobs:
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
+      with:
+        meson_args: ${{ matrix.meson_args }}
+        cflags_adds: ${{ matrix.cflags_adds }}
 
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
       if: ${{ matrix.name != 'Pypy' }}
+      with:
+        preload: ${{ matrix.preload }}
+        asan_options: ${{ matrix.asan_options }}
       env:
         # Set pattern to not single_cpu if not already set
         PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}
 
     - name: Test (single_cpu)
       uses: ./.github/actions/run-tests
+      with:
+        preload: ${{ matrix.preload }}
+        asan_options: ${{ matrix.asan_options }}
       env:
         PATTERN: 'single_cpu'
         PYTEST_WORKERS: 0
@@ -182,7 +212,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, windows-latest]
-        env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml]
+        env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
     name: ${{ format('{0} {1}', matrix.os, matrix.env_file) }}
@@ -242,7 +272,7 @@ jobs:
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
           python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
-          python -m pip install --no-cache-dir --no-build-isolation -e .
+          python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
           python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
@@ -280,7 +310,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
           python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
-          python -m pip install --no-cache-dir --no-build-isolation -e .
+          python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
 
       - name: Run Tests
@@ -313,7 +343,7 @@ jobs:
     #    To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
     #    to the corresponding posix/windows-macos/sdist etc. workflows.
     # Feel free to modify this comment as necessary.
-    #if: false # Uncomment this to freeze the workflow, comment it to unfreeze
+    if: false # Uncomment this to freeze the workflow, comment it to unfreeze
     defaults:
       run:
         shell: bash -eou pipefail {0}
@@ -342,7 +372,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python Dev Version
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.12-dev'
 
@@ -353,7 +383,7 @@ jobs:
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
           python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov
-          python -m pip install -ve . --no-build-isolation --no-index --no-deps
+          python -m pip install -ve . --no-build-isolation --no-index --no-deps --config-settings=setup-args="--werror"
           python -m pip list
 
       - name: Run Tests