Skip to content

Commit

Permalink
add pyspark test
Browse files Browse the repository at this point in the history
  • Loading branch information
ion-elgreco committed Mar 20, 2024
1 parent 0e56002 commit 14c30b1
Showing 1 changed file with 34 additions and 0 deletions.
34 changes: 34 additions & 0 deletions python/tests/pyspark_integration/test_writer_readable.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,3 +96,37 @@ def test_issue_1591_roundtrip_special_characters(tmp_path: pathlib.Path):

loaded = DeltaTable(spark_path).to_pandas()
assert loaded.shape == data.shape


@pytest.mark.pyspark
@pytest.mark.integration
def test_read_checkpointed_table(tmp_path: pathlib.Path):
data = pa.table(
{
"int": pa.array([1]),
}
)
write_deltalake(tmp_path, data)

dt = DeltaTable(tmp_path)
dt.create_checkpoint()

assert_spark_read_equal(data, str(tmp_path))


@pytest.mark.pyspark
@pytest.mark.integration
def test_read_checkpointed_features_table(tmp_path: pathlib.Path):
from datetime import datetime

data = pa.table(
{
"timestamp": pa.array([datetime(2010, 1, 1)]),
}
)
write_deltalake(tmp_path, data)

dt = DeltaTable(tmp_path)
dt.create_checkpoint()

assert_spark_read_equal(data, str(tmp_path))

0 comments on commit 14c30b1

Please sign in to comment.