diff --git a/ibis/backends/pyspark/tests/conftest.py b/ibis/backends/pyspark/tests/conftest.py index 7ffce3ac297e..d4c068a03228 100644 --- a/ibis/backends/pyspark/tests/conftest.py +++ b/ibis/backends/pyspark/tests/conftest.py @@ -168,6 +168,16 @@ def connect(*, tmpdir, worker_id, **kw): .config("spark.sql.streaming.schemaInference", True) ) + config = ( + config.config( + "spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", + ) + .config("spark.sql.catalog.local", "org.apache.iceberg.spark.SparkCatalog") + .config("spark.sql.catalog.local.type", "hadoop") + .config("spark.sql.catalog.local.warehouse", "icehouse") + ) + try: from delta.pip_utils import configure_spark_with_delta_pip except ImportError: diff --git a/poetry-overrides.nix b/poetry-overrides.nix index e2e4d39a8cde..ee5829919c46 100644 --- a/poetry-overrides.nix +++ b/poetry-overrides.nix @@ -1 +1,16 @@ -_final: _prev: { } +final: prev: { + pyspark = prev.pyspark.overridePythonAttrs (attrs: + let + icebergJarUrl = "https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.5.2/iceberg-spark-runtime-3.5_2.12-1.5.2.jar"; + icebergJar = final.pkgs.fetchurl { + name = "iceberg-spark-runtime-3.5_2.12-1.5.2.jar"; + url = icebergJarUrl; + sha256 = "12v1704h0bq3qr2fci0mckg9171lyr8v6983wpa83k06v1w4pv1a"; + }; + in + { + postInstall = attrs.postInstall or "" + '' + cp ${icebergJar} $out/${final.python.sitePackages}/pyspark/jars/${icebergJar.name} + ''; + }); +}