From 7e22245ab5c26638a37e0ddfe6713a9e4c100c2a Mon Sep 17 00:00:00 2001
From: Richard Shadrach <rhshadrach@gmail.com>
Date: Wed, 24 Aug 2022 20:32:50 +0200
Subject: [PATCH 1/2] REGR: Better warning in pivot_table when dropping
nuisance columns
---
pandas/core/reshape/pivot.py | 14 +++++++-
pandas/tests/reshape/test_pivot.py | 8 ++---
pandas/tests/util/test_rewrite_warning.py | 39 ++++++++++++++++++++
pandas/util/_exceptions.py | 44 +++++++++++++++++++++++
4 files changed, 100 insertions(+), 5 deletions(-)
create mode 100644 pandas/tests/util/test_rewrite_warning.py
diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py
index 37e78c7dbf7a2..810a428098df2 100644
--- a/pandas/core/reshape/pivot.py
+++ b/pandas/core/reshape/pivot.py
@@ -21,6 +21,7 @@
Appender,
Substitution,
)
+from pandas.util._exceptions import rewrite_warning
from pandas.core.dtypes.cast import maybe_downcast_to_dtype
from pandas.core.dtypes.common import (
@@ -164,7 +165,18 @@ def __internal_pivot_table(
values = list(values)
grouped = data.groupby(keys, observed=observed, sort=sort)
- agged = grouped.agg(aggfunc)
+ msg = (
+ "pivot_table dropped a column because it failed to aggregate. This behavior "
+ "is deprecated and will raise in a future version of pandas. Select only the "
+ "columns that can be aggregated."
+ )
+ with rewrite_warning(
+ target_message="The default value of numeric_only",
+ target_category=FutureWarning,
+ new_message=msg,
+ ):
+ agged = grouped.agg(aggfunc)
+
if dropna and isinstance(agged, ABCDataFrame) and len(agged.columns):
agged = agged.dropna(how="all")
diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py
index 14ea670fa6cf9..f9119ea43160b 100644
--- a/pandas/tests/reshape/test_pivot.py
+++ b/pandas/tests/reshape/test_pivot.py
@@ -147,7 +147,7 @@ def test_pivot_table_nocols(self):
df = DataFrame(
{"rows": ["a", "b", "c"], "cols": ["x", "y", "z"], "values": [1, 2, 3]}
)
- msg = "The default value of numeric_only"
+ msg = "pivot_table dropped a column because it failed to aggregate"
with tm.assert_produces_warning(FutureWarning, match=msg):
rs = df.pivot_table(columns="cols", aggfunc=np.sum)
xp = df.pivot_table(index="cols", aggfunc=np.sum).T
@@ -911,7 +911,7 @@ def test_no_col(self, data):
# to help with a buglet
data.columns = [k * 2 for k in data.columns]
- msg = "The default value of numeric_only"
+ msg = "pivot_table dropped a column because it failed to aggregate"
with tm.assert_produces_warning(FutureWarning, match=msg):
table = data.pivot_table(index=["AA", "BB"], margins=True, aggfunc=np.mean)
for value_col in table.columns:
@@ -975,7 +975,7 @@ def test_margin_with_only_columns_defined(
}
)
- msg = "The default value of numeric_only"
+ msg = "pivot_table dropped a column because it failed to aggregate"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = df.pivot_table(columns=columns, margins=True, aggfunc=aggfunc)
expected = DataFrame(values, index=Index(["D", "E"]), columns=expected_columns)
@@ -2004,7 +2004,7 @@ def test_pivot_string_func_vs_func(self, f, f_numpy, data):
# GH #18713
# for consistency purposes
- msg = "The default value of numeric_only"
+ msg = "pivot_table dropped a column because it failed to aggregate"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = pivot_table(data, index="A", columns="B", aggfunc=f)
expected = pivot_table(data, index="A", columns="B", aggfunc=f_numpy)
diff --git a/pandas/tests/util/test_rewrite_warning.py b/pandas/tests/util/test_rewrite_warning.py
new file mode 100644
index 0000000000000..f847a06d8ea8d
--- /dev/null
+++ b/pandas/tests/util/test_rewrite_warning.py
@@ -0,0 +1,39 @@
+import warnings
+
+import pytest
+
+from pandas.util._exceptions import rewrite_warning
+
+import pandas._testing as tm
+
+
+@pytest.mark.parametrize(
+ "target_category, target_message, hit",
+ [
+ (FutureWarning, "Target message", True),
+ (FutureWarning, "Target", True),
+ (FutureWarning, "get mess", True),
+ (FutureWarning, "Missed message", False),
+ (DeprecationWarning, "Target message", False),
+ ],
+)
+@pytest.mark.parametrize(
+ "new_category",
+ [
+ None,
+ DeprecationWarning,
+ ],
+)
+def test_rewrite_warning(target_category, target_message, hit, new_category):
+ new_message = "Rewritten message"
+ if hit:
+ expected_category = new_category if new_category else target_category
+ expected_message = new_message
+ else:
+ expected_category = FutureWarning
+ expected_message = "Target message"
+ with tm.assert_produces_warning(expected_category, match=expected_message):
+ with rewrite_warning(
+ target_message, target_category, new_message, new_category
+ ):
+ warnings.warn(message="Target message", category=FutureWarning)
diff --git a/pandas/util/_exceptions.py b/pandas/util/_exceptions.py
index f3a640feb46fc..6cbd710b31b3e 100644
--- a/pandas/util/_exceptions.py
+++ b/pandas/util/_exceptions.py
@@ -3,7 +3,9 @@
import contextlib
import inspect
import os
+import re
from typing import Generator
+import warnings
@contextlib.contextmanager
@@ -47,3 +49,45 @@ def find_stack_level() -> int:
else:
break
return n
+
+
+@contextlib.contextmanager
+def rewrite_warning(
+ target_message: str,
+ target_category: type[Warning],
+ new_message: str,
+ new_category: type[Warning] | None = None,
+) -> None:
+ """
+ Rewrite the message of a warning.
+
+ Parameters
+ ----------
+ target_message : str
+ Warning message to match.
+ target_category : Warning
+ Warning type to match.
+ new_message : str
+ New warning message to emit.
+ new_category : Warning or None, default None
+ New warning type to emit. When None, will be the same as target_category.
+ """
+ if new_category is None:
+ new_category = target_category
+ with warnings.catch_warnings(record=True) as record:
+ yield
+ if len(record) > 0:
+ match = re.compile(target_message)
+ for warning in record:
+ if warning.category is target_category and re.search(
+ match, str(warning.message)
+ ):
+ category, message = new_category, new_message
+ else:
+ category, message = warning.category, warning.message
+ warnings.warn_explicit(
+ message=message,
+ category=category,
+ filename=warning.filename,
+ lineno=warning.lineno,
+ )
From 02a2227a9c49fafb26db5f6484bb1f2ac89195fb Mon Sep 17 00:00:00 2001
From: Richard Shadrach <rhshadrach@gmail.com>
Date: Thu, 10 Nov 2022 11:30:27 -0500
Subject: [PATCH 2/2] type-hint fixups
---
pandas/util/_exceptions.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pandas/util/_exceptions.py b/pandas/util/_exceptions.py
index 6cbd710b31b3e..1eefd06a133fb 100644
--- a/pandas/util/_exceptions.py
+++ b/pandas/util/_exceptions.py
@@ -57,7 +57,7 @@ def rewrite_warning(
target_category: type[Warning],
new_message: str,
new_category: type[Warning] | None = None,
-) -> None:
+) -> Generator[None, None, None]:
"""
Rewrite the message of a warning.
@@ -82,7 +82,8 @@ def rewrite_warning(
if warning.category is target_category and re.search(
match, str(warning.message)
):
- category, message = new_category, new_message
+ category = new_category
+ message: Warning | str = new_message
else:
category, message = warning.category, warning.message
warnings.warn_explicit(