test(pyspark): add tests to read split CSV outputs

ibis/backends/pyspark/tests/test_export.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import pandas.testing as tm
+import pytest
+
+pytest.importorskip("pyspark")
+
+from pyspark.sql import SparkSession  # noqa: E402
+
+from ibis.backends.pyspark.datatypes import PySparkSchema  # noqa: E402
+
+
+@pytest.fixture
+def awards_players(con):
+    return con.table("awards_players")
+
+
+def test_table_to_csv(tmp_path, awards_players):
+    outcsv = tmp_path / "out.csv"
+
+    # avoid pandas NaNonense
+    columns = ["playerID", "awardID", "yearID", "lgID"]
+    awards_players = awards_players.select(columns)
+
+    awards_players.to_csv(outcsv)
+
+    spark = SparkSession.builder.getOrCreate()
+    df = spark.read.csv(
+        str(outcsv), schema=PySparkSchema.from_ibis(awards_players.schema())
+    ).toPandas()
+
+    tm.assert_frame_equal(
+        awards_players.to_pandas().sort_values(by=columns).reset_index(drop=True),
+        df.sort_values(by=columns).reset_index(drop=True),
+    )

ibis/backends/pyspark/tests/test_window.py

@@ -5,7 +5,7 @@
 
 import ibis
 
-pyspark = pytest.importorskip("pyspark")
+pytest.importorskip("pyspark")
 
 import pyspark.sql.functions as F  # noqa: E402
 from pyspark.sql.window import Window  # noqa: E402