API: User-control of result keys in GroupBy.apply

doc/source/user_guide/groupby.rst

@@ -1017,16 +1017,14 @@ for both ``aggregate`` and ``transform`` in many standard use cases. However,
 
 The dimension of the returned result can also change:
 
-.. ipython::
-
-    In [8]: grouped = df.groupby('A')['C']
+.. ipython:: python
 
-    In [10]: def f(group):
-       ....:     return pd.DataFrame({'original': group,
-       ....:                          'demeaned': group - group.mean()})
-       ....:
+    grouped = df.groupby('A', group_keys=False)['C']
 
-    In [11]: grouped.apply(f)
+    def f(group):
+        return pd.DataFrame({'original': group,
+                             'demeaned': group - group.mean()})
+    grouped.apply(f)
 
 ``apply`` on a Series can operate on a returned value from the applied function,
 that is itself a series, and possibly upcast the result to a DataFrame:

doc/source/whatsnew/v0.25.0.rst

@@ -342,10 +342,15 @@ Now every group is evaluated only a single time.
 
 *New behavior*:
 
-.. ipython:: python
-
-    df.groupby("a").apply(func)
+.. code-block:: python
 
+   In [3]: df.groupby('a').apply(func)
+   x
+   y
+   Out[3]:
+      a  b
+   0  x  1
+   1  y  2
 
 Concatenating sparse values
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^

doc/source/whatsnew/v1.2.0.rst

@@ -142,6 +142,59 @@ See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for mor
 
 Deprecations
 ~~~~~~~~~~~~
+
+:meth:`~DataFrame.groupby` no longer ignores ``group_keys`` for transform-like ``apply``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:meth:`~DataFrame.groupby` will no longer ignore the ``group_keys`` argument for functions passed to ``apply`` that return like-indexed outputs (:issue:`34809`, :issue:`31612`, :issue:`14927`, :issue:`13056`).
+Previous versions of pandas would add the group keys only when the result from the applied function had a different index to the input.
+
+*Previous behavior*:
+
+.. code-block:: python
+
+   >>> # pandas 1.0.4
+   >>> df = pd.DataFrame({"A": [1, 2, 2], "B": [1, 2, 3]})
+   >>> df
+      A  B
+   0  1  1
+   1  2  2
+   2  2  3
+   >>> df.groupby("A").apply(lambda x: x.rename(np.exp))  # Different index
+               A  B
+   A
+   1 1.000000  1  1
+   2 2.718282  2  2
+     7.389056  2  3
+
+   >>> df.groupby("A").apply(lambda x: x)  # Same index
+      A  B
+   0  1  1
+   1  2  2
+   2  2  3
+
+In this future this behavior will change to always respect ``group_keys``, which defaults to True.
+
+*New behavior*:
+
+.. ipython:: python
+
+   df = pd.DataFrame({"A": [1, 2, 2], "B": [1, 2, 3]})
+   df.groupby("A", group_keys=True).apply(lambda x: x)
+   df.groupby("A", group_keys=True).apply(lambda x: x.rename(np.exp))
+
+A warning will be issued if the result would change from pandas 1.0.4
+
+.. ipython:: python
+   :okwarning:
+
+   df.groupby("A").apply(lambda x: x)
+
+
+Other Deprecations
+^^^^^^^^^^^^^^^^^^
+
+-
 - Deprecated parameter ``inplace`` in :meth:`MultiIndex.set_codes` and :meth:`MultiIndex.set_levels` (:issue:`35626`)
 - Deprecated parameter ``dtype`` in :~meth:`Index.copy` on method all index classes. Use the :meth:`Index.astype` method instead for changing dtype(:issue:`35853`)
 -

pandas/_libs/reduction.pyx

@@ -373,7 +373,16 @@ def apply_frame_axis0(object frame, object f, object names,
                     mutated = True
             except AttributeError:
                 # `piece` might not have an index, could be e.g. an int
-                pass
+                # By definition, we are not a transform, so set mutated
+                # to True
+                mutated = True
+            if not mutated:
+                # Also check if the columns are mutated
+                try:
+                    if not piece.columns.equals(chunk.columns):
+                        mutated = True
+                except AttributeError:
+                    mutated = True
 
             if not is_scalar(piece):
                 # Need to copy data to avoid appending references

pandas/core/frame.py

@@ -6516,6 +6516,26 @@ def update(
 a   13.0   13.0
 b   12.3  123.0
 NaN 12.3   33.0
+
+When using ``.apply()``, use ``group_keys`` to include or exclude the group keys.
+
+>>> df = pd.DataFrame({'Animal': ['Falcon', 'Falcon',
+...                               'Parrot', 'Parrot'],
+...                    'Max Speed': [380., 370., 24., 26.]})
+>>> df.groupby("Animal", group_keys=False).apply(lambda x: x)
+   Animal  Max Speed
+0  Falcon      380.0
+1  Falcon      370.0
+2  Parrot       24.0
+3  Parrot       26.0
+
+>>> df.groupby("Animal", group_keys=True).apply(lambda x: x)
+          Animal  Max Speed
+Animal
+Falcon 0  Falcon      380.0
+       1  Falcon      370.0
+Parrot 2  Parrot       24.0
+       3  Parrot       26.0
 """
     )
     @Appender(_shared_docs["groupby"] % _shared_doc_kwargs)
@@ -6526,7 +6546,7 @@ def groupby(
         level=None,
         as_index: bool = True,
         sort: bool = True,
-        group_keys: bool = True,
+        group_keys: bool = no_default,
         squeeze: bool = no_default,
         observed: bool = False,
         dropna: bool = True,

pandas/core/generic.py

@@ -7411,6 +7411,87 @@ def clip(
 
         return result
 
+    _shared_docs[
+        "groupby"
+    ] = """
+        Group %(klass)s using a mapper or by a Series of columns.
+
+        A groupby operation involves some combination of splitting the
+        object, applying a function, and combining the results. This can be
+        used to group large amounts of data and compute operations on these
+        groups.
+
+        Parameters
+        ----------
+        by : mapping, function, label, or list of labels
+            Used to determine the groups for the groupby.
+            If ``by`` is a function, it's called on each value of the object's
+            index. If a dict or Series is passed, the Series or dict VALUES
+            will be used to determine the groups (the Series' values are first
+            aligned; see ``.align()`` method). If an ndarray is passed, the
+            values are used as-is determine the groups. A label or list of
+            labels may be passed to group by the columns in ``self``. Notice
+            that a tuple is interpreted as a (single) key.
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+            Split along rows (0) or columns (1).
+        level : int, level name, or sequence of such, default None
+            If the axis is a MultiIndex (hierarchical), group by a particular
+            level or levels.
+        as_index : bool, default True
+            For aggregated output, return object with group labels as the
+            index. Only relevant for DataFrame input. as_index=False is
+            effectively "SQL-style" grouped output.
+        sort : bool, default True
+            Sort group keys. Get better performance by turning this off.
+            Note this does not influence the order of observations within each
+            group. Groupby preserves the order of rows within each group.
+        group_keys : bool, optional
+            When calling apply, add group keys to index to identify pieces.
+            By default group keys are not included when the result's index
+            (and column) labels match the inputs, and are included otherwise.
+
+            .. versionchanged:: 1.2.0
+
+               Warns that `group_keys` will no longer be ignored when the
+               result from ``apply`` is a like-indexed Series or DataFrame.
+               Specify ``group_keys`` explicitly to include the group keys or
+               not.
+
+        squeeze : bool, default False
+            Reduce the dimensionality of the return type if possible,
+            otherwise return a consistent type.
+
+            .. deprecated:: 1.1.0
+
+        observed : bool, default False
+            This only applies if any of the groupers are Categoricals.
+            If True: only show observed values for categorical groupers.
+            If False: show all values for categorical groupers.
+
+            .. versionadded:: 0.23.0
+        dropna : bool, default True
+            If True, and if group keys contain NA values, NA values together
+            with row/column will be dropped.
+            If False, NA values will also be treated as the key in groups
+
+            .. versionadded:: 1.1.0
+
+        Returns
+        -------
+        %(klass)sGroupBy
+            Returns a groupby object that contains information about the groups.
+
+        See Also
+        --------
+        resample : Convenience method for frequency conversion and resampling
+            of time series.
+
+        Notes
+        -----
+        See the `user guide
+        <https://pandas.pydata.org/pandas-docs/stable/groupby.html>`_ for more.
+        """
+
     def asfreq(
         self: FrameOrSeries,
         freq,
@@ -7678,6 +7759,7 @@ def resample(
         level=None,
         origin: Union[str, TimestampConvertibleTypes] = "start_day",
         offset: Optional[TimedeltaConvertibleTypes] = None,
+        group_keys: bool_t = lib.no_default,
     ) -> "Resampler":
         """
         Resample time-series data.
@@ -7747,6 +7829,12 @@ def resample(
 
             .. versionadded:: 1.1.0
 
+        group_keys : bool, default True
+            Whether to include the group keys in the result index when performing
+            a ``.groupby().apply()`` to the resampled object.
+
+            .. versionadded:: 1.2.0
+
         Returns
         -------
         Resampler object
@@ -8063,6 +8151,7 @@ def resample(
             level=level,
             origin=origin,
             offset=offset,
+            group_keys=group_keys,
         )
 
     def first(self: FrameOrSeries, offset) -> FrameOrSeries:

pandas/core/groupby/generic.py

@@ -422,7 +422,11 @@ def _wrap_transformed_output(
         return result
 
     def _wrap_applied_output(
-        self, keys: Index, values: Optional[List[Any]], not_indexed_same: bool = False
+        self,
+        keys: Index,
+        values: Optional[List[Any]],
+        not_indexed_same: bool = False,
+        override_group_keys: bool = False,
     ) -> FrameOrSeriesUnion:
         """
         Wrap the output of SeriesGroupBy.apply into the expected result.
@@ -466,7 +470,12 @@ def _get_index() -> Index:
             result.name = self._selection_name
             return result
         elif isinstance(values[0], (Series, DataFrame)):
-            return self._concat_objects(keys, values, not_indexed_same=not_indexed_same)
+            return self._concat_objects(
+                keys,
+                values,
+                not_indexed_same=not_indexed_same,
+                override_group_keys=override_group_keys,
+            )
         else:
             # GH #6265 #24880
             result = self.obj._constructor(
@@ -1190,7 +1199,13 @@ def _aggregate_item_by_item(self, func, *args, **kwargs) -> DataFrame:
 
         return self.obj._constructor(result, columns=result_columns)
 
-    def _wrap_applied_output(self, keys, values, not_indexed_same=False):
+    def _wrap_applied_output(
+        self,
+        keys,
+        values,
+        not_indexed_same: bool = False,
+        override_group_keys: bool = False,
+    ) -> FrameOrSeriesUnion:
         if len(keys) == 0:
             return self.obj._constructor(index=keys)
 
@@ -1201,18 +1216,23 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
             # GH9684 - All values are None, return an empty frame.
             return self.obj._constructor()
         elif isinstance(first_not_none, DataFrame):
-            return self._concat_objects(keys, values, not_indexed_same=not_indexed_same)
+            return self._concat_objects(
+                keys,
+                values,
+                not_indexed_same=not_indexed_same,
+                override_group_keys=override_group_keys,
+            )
         else:
             key_index = self.grouper.result_index if self.as_index else None
 
-            if isinstance(first_not_none, Series):
-
+            if isinstance(first_not_none, (Series, DataFrame)):
                 # this is to silence a DeprecationWarning
                 # TODO: Remove when default dtype of empty Series is object
                 kwargs = first_not_none._construct_axes_dict()
-                backup = create_series_with_explicit_dtype(
-                    dtype_if_empty=object, **kwargs
-                )
+                backup: FrameOrSeriesUnion
+                if isinstance(first_not_none, Series):
+                    kwargs["dtype_if_empty"] = object
+                backup = create_series_with_explicit_dtype(**kwargs)
 
                 values = [x if (x is not None) else backup for x in values]
 
@@ -1239,7 +1259,10 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
                             # OR we don't have a multi-index and we have only a
                             # single values
                             return self._concat_objects(
-                                keys, values, not_indexed_same=not_indexed_same
+                                keys,
+                                values,
+                                not_indexed_same=not_indexed_same,
+                                override_group_keys=override_group_keys,
                             )
 
                         # still a series
@@ -1251,7 +1274,12 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
 
                     if not all_indexed_same:
                         # GH 8467
-                        return self._concat_objects(keys, values, not_indexed_same=True)
+                        return self._concat_objects(
+                            keys,
+                            values,
+                            not_indexed_same=True,
+                            override_group_keys=override_group_keys,
+                        )
 
                     # GH6124 if the list of Series have a consistent name,
                     # then propagate that name to the result.
@@ -1621,13 +1649,13 @@ def _gotitem(self, key, ndim: int, subset=None):
                 subset = self.obj[key]
             return SeriesGroupBy(
                 subset,
-                level=self.level,
-                grouper=self.grouper,
                 selection=key,
-                sort=self.sort,
+                grouper=self.grouper,
+                observed=self.observed,
                 group_keys=self.group_keys,
+                level=self.level,
+                sort=self.sort,
                 squeeze=self.squeeze,
-                observed=self.observed,
                 dropna=self.dropna,
             )