API: Change default for Index.union sort

doc/source/whatsnew/v0.24.1.rst

@@ -15,10 +15,34 @@ Whats New in 0.24.1 (February XX, 2019)
 These are the changes in pandas 0.24.1. See :ref:`release` for a full changelog
 including other versions of pandas.
 
+.. _whatsnew_0241.api:
+
+API Changes
+~~~~~~~~~~~
+
+Changing the ``sort`` parameter for :meth:`Index.Union`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The default ``sort`` value for :meth:`Index.union` has changed from ``True`` to ``None``.
+The default *behavior* remains the same: The result is sorted, unless
+
+1. ``self`` and ``other`` are identical
+2. ``self`` or ``other`` is empty
+3. ``self`` or ``other`` contain values that can not be compared (a ``RuntimeWarning`` is raised).
+
+This allows ``sort=True`` to now mean "always sort" A ``TypeError`` is raised if the values cannot be compared.
+
+Changed the behavior of :meth:`Index.intersection` with ``sort=True``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When ``sort=True`` is provided to :meth:`Index.intersection`, the values are always sorted. In 0.24.0,
+the values would not be sorted when ``self`` and ``other`` were identical. Pass ``sort=False`` to not
+sort the values.
+
 .. _whatsnew_0241.regressions:
 
 Fixed Regressions
-^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~
 
 - Bug in :meth:`DataFrame.itertuples` with ``records`` orient raising an ``AttributeError`` when the ``DataFrame`` contained more than 255 columns (:issue:`24939`)
 - Bug in :meth:`DataFrame.itertuples` orient converting integer column names to strings prepended with an underscore (:issue:`24940`)
@@ -27,7 +51,7 @@ Fixed Regressions
 .. _whatsnew_0241.enhancements:
 
 Enhancements
-^^^^^^^^^^^^
+~~~~~~~~~~~~
 
 
 .. _whatsnew_0241.bug_fixes:

pandas/_libs/lib.pyx

@@ -233,11 +233,14 @@ def fast_unique_multiple(list arrays, sort: bool=True):
             if val not in table:
                 table[val] = stub
                 uniques.append(val)
-    if sort:
+    if sort is None:
         try:
             uniques.sort()
         except Exception:
+            # TODO: RuntimeWarning?
             pass
+    elif sort:
+        uniques.sort()
 
     return uniques
 

pandas/core/indexes/base.py

@@ -2245,18 +2245,34 @@ def _get_reconciled_name_object(self, other):
             return self._shallow_copy(name=name)
         return self
 
-    def union(self, other, sort=True):
+    def union(self, other, sort=None):
         """
         Form the union of two Index objects.
 
         Parameters
         ----------
         other : Index or array-like
-        sort : bool, default True
-            Sort the resulting index if possible
+        sort : bool or None, default None
+            Whether to sort the resulting Index.
+
+            * None : Sort the result, except when
+
+              1. `self` and `other` are equal.
+              2. `self` or `other` has length 0.
+              3. Some values in `self` or `other` cannot be compared.
+                 A RuntimeWarning is issued in this case.
+
+            * True : sort the result. A TypeError is raised when the
+              values cannot be compared.
+            * False : do not sort the result.
 
             .. versionadded:: 0.24.0
 
+            .. versionchanged:: 0.24.1
+
+               Changed the default `sort` to None, matching the
+               behavior of pandas 0.23.4 and earlier.
+
         Returns
         -------
         union : Index
@@ -2273,10 +2289,16 @@ def union(self, other, sort=True):
         other = ensure_index(other)
 
         if len(other) == 0 or self.equals(other):
-            return self._get_reconciled_name_object(other)
+            result = self._get_reconciled_name_object(other)
+            if sort:
+                result = result.sort_values()
+            return result
 
         if len(self) == 0:
-            return other._get_reconciled_name_object(self)
+            result = other._get_reconciled_name_object(self)
+            if sort:
+                result = result.sort_values()
+            return result
 
         # TODO: is_dtype_union_equal is a hack around
         # 1. buggy set ops with duplicates (GH #13432)
@@ -2319,13 +2341,16 @@ def union(self, other, sort=True):
             else:
                 result = lvals
 
-            if sort:
+            if sort is None:
                 try:
                     result = sorting.safe_sort(result)
                 except TypeError as e:
                     warnings.warn("{}, sort order is undefined for "
                                   "incomparable objects".format(e),
                                   RuntimeWarning, stacklevel=3)
+            elif sort:
+                # raise if not sortable.
+                result = sorting.safe_sort(result)
 
         # for subclasses
         return self._wrap_setop_result(other, result)
@@ -2342,8 +2367,12 @@ def intersection(self, other, sort=False):
         Parameters
         ----------
         other : Index or array-like
-        sort : bool, default False
-            Sort the resulting index if possible
+        sort : bool or None, default False
+            Whether to sort the resulting index.
+
+            * False : do not sort the result.
+            * True : sort the result. A TypeError is raised when the
+              values cannot be compared.
 
             .. versionadded:: 0.24.0
 
@@ -2367,7 +2396,10 @@ def intersection(self, other, sort=False):
         other = ensure_index(other)
 
         if self.equals(other):
-            return self._get_reconciled_name_object(other)
+            result = self._get_reconciled_name_object(other)
+            if sort:
+                result = result.sort_values()
+            return result
 
         if not is_dtype_equal(self.dtype, other.dtype):
             this = self.astype('O')

pandas/core/indexes/multi.py

@@ -2879,18 +2879,34 @@ def equal_levels(self, other):
                 return False
         return True
 
-    def union(self, other, sort=True):
+    def union(self, other, sort=None):
         """
         Form the union of two MultiIndex objects
 
         Parameters
         ----------
         other : MultiIndex or array / Index of tuples
-        sort : bool, default True
-            Sort the resulting MultiIndex if possible
+        sort : bool or None, default None
+            Whether to sort the resulting Index.
+
+            * None : Sort the result, except when
+
+              1. `self` and `other` are equal.
+              2. `self` has length 0.
+              3. Some values in `self` or `other` cannot be compared.
+                 A RuntimeWarning is issued in this case.
+
+            * True : sort the result. A TypeError is raised when the
+              values cannot be compared.
+            * False : do not sort the result.
 
             .. versionadded:: 0.24.0
 
+            .. versionchanged:: 0.24.1
+
+               Changed the default `sort` to None, matching the
+               behavior of pandas 0.23.4 and earlier.
+
         Returns
         -------
         Index
@@ -2901,8 +2917,12 @@ def union(self, other, sort=True):
         other, result_names = self._convert_can_do_setop(other)
 
         if len(other) == 0 or self.equals(other):
+            if sort:
+                return self.sort_values()
             return self
 
+        # TODO: Index.union returns other when `len(self)` is 0.
+
         uniq_tuples = lib.fast_unique_multiple([self._ndarray_values,
                                                 other._ndarray_values],
                                                sort=sort)
@@ -2917,7 +2937,7 @@ def intersection(self, other, sort=False):
         Parameters
         ----------
         other : MultiIndex or array / Index of tuples
-        sort : bool, default True
+        sort : bool, default False
             Sort the resulting MultiIndex if possible
 
             .. versionadded:: 0.24.0
@@ -2934,6 +2954,8 @@ def intersection(self, other, sort=False):
         other, result_names = self._convert_can_do_setop(other)
 
         if self.equals(other):
+            if sort:
+                return self.sort_values()
             return self
 
         self_tuples = self._ndarray_values

pandas/tests/indexes/multi/test_set_ops.py

@@ -203,10 +203,16 @@ def test_union(idx, sort):
 
     # corner case, pass self or empty thing:
     the_union = idx.union(idx, sort=sort)
-    assert the_union is idx
+    if sort:
+        tm.assert_index_equal(the_union, idx.sort_values())
+    else:
+        assert the_union is idx
 
     the_union = idx.union(idx[:0], sort=sort)
-    assert the_union is idx
+    if sort:
+        tm.assert_index_equal(the_union, idx.sort_values())
+    else:
+        assert the_union is idx
 
     # won't work in python 3
     # tuples = _index.values
@@ -238,7 +244,10 @@ def test_intersection(idx, sort):
 
     # corner case, pass self
     the_int = idx.intersection(idx, sort=sort)
-    assert the_int is idx
+    if sort:
+        tm.assert_index_equal(the_int, idx.sort_values())
+    else:
+        assert the_int is idx
 
     # empty intersection: disjoint
     empty = idx[:2].intersection(idx[2:], sort=sort)
@@ -249,3 +258,47 @@ def test_intersection(idx, sort):
     # tuples = _index.values
     # result = _index & tuples
     # assert result.equals(tuples)
+
+
+def test_intersect_equal_sort():
+    idx = pd.MultiIndex.from_product([[1, 0], ['a', 'b']])
+    sorted_ = pd.MultiIndex.from_product([[0, 1], ['a', 'b']])
+    tm.assert_index_equal(idx.intersection(idx, sort=False), idx)
+    tm.assert_index_equal(idx.intersection(idx, sort=True), sorted_)
+
+
+@pytest.mark.parametrize('slice_', [slice(None), slice(0)])
+def test_union_sort_other_empty(slice_):
+    # https://github.com/pandas-dev/pandas/issues/24959
+    idx = pd.MultiIndex.from_product([[1, 0], ['a', 'b']])
+
+    # default, sort=None
+    other = idx[slice_]
+    tm.assert_index_equal(idx.union(other), idx)
+    # MultiIndex does not special case empty.union(idx)
+    # tm.assert_index_equal(other.union(idx), idx)
+
+    # sort=False
+    tm.assert_index_equal(idx.union(other, sort=False), idx)
+
+    # sort=True
+    result = idx.union(other, sort=True)
+    expected = pd.MultiIndex.from_product([[0, 1], ['a', 'b']])
+    tm.assert_index_equal(result, expected)
+
+
+def test_union_sort_other_incomparable():
+    # https://github.com/pandas-dev/pandas/issues/24959
+    idx = pd.MultiIndex.from_product([[1, pd.Timestamp('2000')], ['a', 'b']])
+
+    # default, sort=None
+    result = idx.union(idx[:1])
+    tm.assert_index_equal(result, idx)
+
+    # sort=False
+    result = idx.union(idx[:1], sort=False)
+    tm.assert_index_equal(result, idx)
+
+    # sort=True
+    with pytest.raises(TypeError, match='Cannot compare'):
+        idx.union(idx[:1], sort=True)