diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py index 30c2d10245650..30c65ce42ba50 100644 --- a/python/pyspark/sql/pandas/serializers.py +++ b/python/pyspark/sql/pandas/serializers.py @@ -226,8 +226,7 @@ def create_array(s, t): s = _check_series_convert_timestamps_internal(s, self._timezone) elif t is not None and pa.types.is_map(t): s = _convert_dict_to_map_items(s) - elif is_categorical_dtype(s.dtype): - # Note: This can be removed once minimum pyarrow version is >= 0.16.1 + elif t is None and is_categorical_dtype(s.dtype): s = s.astype(s.dtypes.categories.dtype) try: array = pa.Array.from_pandas(s, mask=mask, type=t, safe=self._safecheck)