diff --git a/ibis/backends/dask/executor.py b/ibis/backends/dask/executor.py
index 1077b79e980b6..660d75a8c1105 100644
--- a/ibis/backends/dask/executor.py
+++ b/ibis/backends/dask/executor.py
@@ -364,8 +364,14 @@ def visit(cls, op: ops.Sort, parent, keys):
         # 2. sort the dataframe using those columns
         # 3. drop the sort key columns
         ascending = [key.ascending for key in op.keys]
-        na_pos_dict = {True: "first", False: "last"}
-        na_position = na_pos_dict[all(key.nulls_first for key in op.keys)]
+        nulls_first = [key.nulls_first for key in op.keys]
+
+        if all(nulls_first):
+            na_position = "first"
+        elif not any(nulls_first):
+            na_position = "last"
+        else:
+            raise ValueError("dask does not support different columns ordering")
 
         newcols = {gen_name("sort_key"): col for col in keys}
         names = list(newcols.keys())
diff --git a/ibis/backends/pandas/executor.py b/ibis/backends/pandas/executor.py
index b25f6f5e7066d..0a81024340746 100644
--- a/ibis/backends/pandas/executor.py
+++ b/ibis/backends/pandas/executor.py
@@ -593,8 +593,14 @@ def visit(cls, op: ops.Sort, parent, keys):
         # 2. sort the dataframe using those columns
         # 3. drop the sort key columns
         ascending = [key.ascending for key in op.keys]
-        na_pos_dict = {True: "first", False: "last"}
-        na_position = na_pos_dict[all(key.nulls_first for key in op.keys)]
+        nulls_first = [key.nulls_first for key in op.keys]
+
+        if all(nulls_first):
+            na_position = "first"
+        elif not any(nulls_first):
+            na_position = "last"
+        else:
+            raise ValueError("pandas does not support different columns ordering")
 
         newcols = {gen_name("sort_key"): col for col in keys}
         names = list(newcols.keys())