Merge branch 'main' into opt_excel_nrows

Author: ahawryluk

.github/actions/build_pandas/action.yml

@@ -12,6 +12,9 @@ runs:
 
     - name: Build Pandas
       run: |
-        python setup.py build_ext -j 2
+        python setup.py build_ext -j $N_JOBS
         python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
       shell: bash -el {0}
+      env:
+        # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873
+        N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }}

.github/workflows/code-checks.yml

@@ -74,12 +74,16 @@ jobs:
 
     - name: Install pyright
       # note: keep version in sync with .pre-commit-config.yaml
-      run: npm install -g pyright@1.1.230
+      run: npm install -g pyright@1.1.245
 
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
 
+    - name: Check for no warnings when building single-page docs
+      run: ci/code_checks.sh single-docs
+      if: ${{ steps.build.outcome == 'success' }}
+
     - name: Run checks on imported code
       run: ci/code_checks.sh code
       if: ${{ steps.build.outcome == 'success' }}

.github/workflows/docbuild-and-upload.yml

@@ -37,6 +37,7 @@ jobs:
       run: |
         source activate pandas-dev
         python web/pandas_web.py web/pandas --target-path=web/build
+
     - name: Build documentation
       run: |
         source activate pandas-dev

.github/workflows/posix.yml

@@ -28,7 +28,7 @@ jobs:
         pattern: ["not single_cpu", "single_cpu"]
         # Don't test pyarrow v2/3: Causes timeouts in read_csv engine
         # even if tests are skipped/xfailed
-        pyarrow_version: ["5", "7"]
+        pyarrow_version: ["5", "6", "7"]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -61,7 +61,16 @@ jobs:
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_testing_mode: "deprecate"
-            test_args: "-W error"
+            test_args: "-W error::DeprecationWarning:numpy"
+        exclude:
+          - env_file: actions-39.yaml
+            pyarrow_version: "6"
+          - env_file: actions-39.yaml
+            pyarrow_version: "7"
+          - env_file: actions-310.yaml
+            pyarrow_version: "6"
+          - env_file: actions-310.yaml
+            pyarrow_version: "7"
       fail-fast: false
     name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }}
     env:
@@ -171,10 +180,9 @@ jobs:
       run: ci/run_tests.sh
       # TODO: Don't continue on error for PyPy
       continue-on-error: ${{ env.IS_PYPY == 'true' }}
-      if: always()
 
     - name: Build Version
-      run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+      run: conda list
 
     - name: Publish test results
       uses: actions/upload-artifact@v3

.github/workflows/windows.yml

@@ -0,0 +1,75 @@
+name: Windows
+
+on:
+  push:
+    branches:
+      - main
+      - 1.4.x
+  pull_request:
+    branches:
+      - main
+      - 1.4.x
+    paths-ignore:
+      - "doc/**"
+
+env:
+  PANDAS_CI: 1
+  PYTEST_TARGET: pandas
+  PYTEST_WORKERS: auto
+  PATTERN: "not slow and not db and not network and not single_cpu"
+
+
+jobs:
+  pytest:
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
+      fail-fast: false
+    concurrency:
+      # https://github.heygears.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-windows
+      cancel-in-progress: true
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Install Dependencies
+      uses: conda-incubator/setup-miniconda@v2.1.1
+      with:
+        mamba-version: "*"
+        channels: conda-forge
+        activate-environment: pandas-dev
+        channel-priority: strict
+        environment-file: ci/deps/${{ matrix.env_file }}
+        use-only-tar-bz2: true
+
+    - name: Build Pandas
+      uses: ./.github/actions/build_pandas
+
+    - name: Test
+      run: ci/run_tests.sh
+
+    - name: Build Version
+      run: conda list
+
+    - name: Publish test results
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test results
+        path: test-data.xml
+      if: failure()
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v2
+      with:
+        flags: unittests
+        name: codecov-pandas
+        fail_ci_if_error: false

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         types_or: [python, rst, markdown]
         files: ^(pandas|doc)/
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.2.0
     hooks:
     -   id: debug-statements
     -   id: end-of-file-fixer
@@ -56,7 +56,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.1
+    rev: v2.32.0
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -71,7 +71,7 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.2
+    rev: v0.4.1
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa
@@ -89,7 +89,7 @@ repos:
         types: [python]
         stages: [manual]
         # note: keep version in sync with .github/workflows/code-checks.yml
-        additional_dependencies: ['pyright@1.1.230']
+        additional_dependencies: ['pyright@1.1.245']
 -   repo: local
     hooks:
     -   id: flake8-rst
@@ -176,6 +176,13 @@ repos:
         files: ^pandas/core/
         exclude: ^pandas/core/api\.py$
         types: [python]
+    -   id: use-io-common-urlopen
+        name: Use pandas.io.common.urlopen instead of urllib.request.urlopen
+        language: python
+        entry: python scripts/use_io_common_urlopen.py
+        files: ^pandas/
+        exclude: ^pandas/tests/
+        types: [python]
     -   id: no-bool-in-core-generic
         name: Use bool_t instead of bool in pandas/core/generic.py
         entry: python scripts/no_bool_in_generic.py

LICENSES/KLIB_LICENSE

@@ -0,0 +1,23 @@
+The MIT License
+
+Copyright (c) 2008-     Attractive Chaos <attractor@live.co.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -1,4 +1,5 @@
 include RELEASE.md
+include versioneer.py
 
 graft doc
 prune doc/build
@@ -54,9 +55,6 @@ global-exclude *.pxi
 # exclude the whole directory to avoid running related tests in sdist
 prune pandas/tests/io/parser/data
 
-include versioneer.py
-include pandas/_version.py
-include pandas/io/formats/templates/*.tpl
-
+# Selectively re-add *.cxx files that were excluded above
 graft pandas/_libs/src
 graft pandas/_libs/tslibs/src

asv_bench/benchmarks/tslibs/tz_convert.py

@@ -11,10 +11,14 @@
 
 try:
     old_sig = False
-    from pandas._libs.tslibs.tzconversion import tz_convert_from_utc
+    from pandas._libs.tslibs import tz_convert_from_utc
 except ImportError:
-    old_sig = True
-    from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc
+    try:
+        old_sig = False
+        from pandas._libs.tslibs.tzconversion import tz_convert_from_utc
+    except ImportError:
+        old_sig = True
+        from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc
 
 
 class TimeTZConvert:

azure-pipelines.yml

@@ -27,11 +27,6 @@ jobs:
     name: macOS
     vmImage: macOS-10.15
 
-- template: ci/azure/windows.yml
-  parameters:
-    name: Windows
-    vmImage: windows-2019
-
 - job: py38_32bit
   pool:
     vmImage: ubuntu-18.04
@@ -48,6 +43,7 @@ jobs:
         pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
+        export PANDAS_CI=1 && \
         pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
       displayName: 'Run 32-bit manylinux2014 Docker Build / Tests'
 

ci/azure/windows.yml

@@ -12,8 +12,9 @@
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
 #   $ ./ci/code_checks.sh typing        # run static type analysis
+#   $ ./ci/code_checks.sh single-docs   # check single-page docs build warning-free
 
-[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" ]] || \
+[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" || "$1" == "single-docs" ]] || \
     { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|typing]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
@@ -78,8 +79,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (EX04, GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, PR03, PR04, PR05, PR06, PR08, PR09, PR10, RT01, RT04, RT05, SA02, SA03, SS01, SS02, SS03, SS04, SS05)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT04,RT05,SA02,SA03,SS01,SS02,SS03,SS04,SS05
+    MSG='Validate docstrings (EX04, GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, PR03, PR04, PR05, PR06, PR08, PR09, PR10, RT01, RT04, RT05, SA02, SA03, SA04, SS01, SS02, SS03, SS04, SS05)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT04,RT05,SA02,SA03,SA04,SS01,SS02,SS03,SS04,SS05
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi
@@ -102,4 +103,11 @@ if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
     fi
 fi
 
+### SINGLE-PAGE DOCS ###
+if [[ -z "$CHECK" || "$CHECK" == "single-docs" ]]; then
+    python doc/make.py --warnings-are-errors --single pandas.Series.value_counts
+    python doc/make.py --warnings-are-errors --single pandas.Series.str.split
+    python doc/make.py clean
+fi
+
 exit $RET

ci/code_checks.sh

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -27,6 +26,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml