Merge branch 'main' of github.com:apache/iceberg-python into fd-warnings

apache · Oct 16, 2023 · af818f2 · af818f2
2 parents 1a06e8f + 8f8b622
commit af818f2
Show file tree

Hide file tree

Showing 37 changed files with 1,217 additions and 550 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -28,6 +28,16 @@ github:
     - apache
     - hacktoberfest
     - pyiceberg
+  enabled_merge_buttons:
+    merge: false
+    squash: true
+    rebase: true
+  protected_branches:
+    main:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+
+      required_linear_history: true
   features:
     wiki: true
     issues: true

diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml
@@ -25,7 +25,7 @@ on:
       version:
         description: 'Version'
         type: string
-        default: 'master'
+        default: 'main'
 
 
 jobs:
@@ -50,18 +50,16 @@ jobs:
 
       - name: Set version
         run: python -m poetry version "${{ inputs.version }}"
-        working-directory: ./python
-        if: "${{ github.event.inputs.version != 'master' }}"
+        if: "${{ github.event.inputs.version != 'main' }}"
 
       # Publish the source distribution with the version that's in
       # the repository, otherwise the tests will fail
       - name: Compile source distribution
         run: python3 -m poetry build --format=sdist
         if: startsWith(matrix.os, 'ubuntu')
-        working-directory: ./python
 
       - name: Build wheels
-        uses: pypa/[email protected].0
+        uses: pypa/[email protected].2
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

diff --git a/dev/Dockerfile b/dev/Dockerfile
@@ -38,7 +38,7 @@ WORKDIR ${SPARK_HOME}
 
 ENV SPARK_VERSION=3.4.1
 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.4_2.12
-ENV ICEBERG_VERSION=1.3.1
+ENV ICEBERG_VERSION=1.4.0
 ENV AWS_SDK_VERSION=2.20.18
 ENV PYICEBERG_VERSION=0.4.0
 
@@ -50,13 +50,9 @@ RUN curl --retry 3 -s -C - https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}
 RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar -Lo iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
  && mv iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar /opt/spark/jars
 
-# Download Java AWS SDK
-RUN curl -s https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar -Lo bundle-${AWS_SDK_VERSION}.jar \
- && mv bundle-${AWS_SDK_VERSION}.jar /opt/spark/jars
-
-# Download URL connection client required for S3FileIO
-RUN curl -s https://repo1.maven.org/maven2/software/amazon/awssdk/url-connection-client/${AWS_SDK_VERSION}/url-connection-client-${AWS_SDK_VERSION}.jar -Lo url-connection-client-${AWS_SDK_VERSION}.jar \
- && mv url-connection-client-${AWS_SDK_VERSION}.jar /opt/spark/jars
+# Download AWS bundle
+RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
+ && mv iceberg-aws-bundle-${ICEBERG_VERSION}.jar /opt/spark/jars
 
 COPY spark-defaults.conf /opt/spark/conf
 ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}"

diff --git a/dev/provision.py b/dev/provision.py
@@ -279,3 +279,25 @@
         (CAST('2023-03-12' AS date), CAST('2023-03-12 12:22:00' AS timestamp), 12, 'l');
     """
     )
+
+# There is an issue with CREATE OR REPLACE
+# https://github.com/apache/iceberg/issues/8756
+spark.sql(
+    """
+DROP TABLE IF EXISTS default.test_table_version
+"""
+)
+
+spark.sql(
+    """
+CREATE TABLE default.test_table_version (
+    dt     date,
+    number integer,
+    letter string
+)
+USING iceberg
+TBLPROPERTIES (
+    'format-version'='1'
+);
+"""
+)
diff --git a/dev/spark-defaults.conf b/dev/spark-defaults.conf
@@ -20,7 +20,7 @@ spark.sql.catalog.demo                 org.apache.iceberg.spark.SparkCatalog
 spark.sql.catalog.demo.type            rest
 spark.sql.catalog.demo.uri             http://rest:8181
 spark.sql.catalog.demo.io-impl         org.apache.iceberg.aws.s3.S3FileIO
-spark.sql.catalog.demo.warehouse       s3a://warehouse/wh/
+spark.sql.catalog.demo.warehouse       s3://warehouse/wh/
 spark.sql.catalog.demo.s3.endpoint     http://minio:9000
 spark.sql.defaultCatalog               demo
 spark.eventLog.enabled                 true

diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md
@@ -195,9 +195,9 @@ Renaming a field in an Iceberg table is simple:
 
 ```python
 with table.update_schema() as update:
-    update.rename("retries", "num_retries")
+    update.rename_column("retries", "num_retries")
     # This will rename `confirmed_by` to `exchange`
-    update.rename("properties.confirmed_by", "exchange")
+    update.rename_column("properties.confirmed_by", "exchange")
 ```
 
 ### Move column

diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md
@@ -43,7 +43,7 @@ If you want to install the library on the host, you can simply run `pip3 install
 To set up IDEA with Poetry ([also on Loom](https://www.loom.com/share/6d36464d45f244729d91003e7f671fd2)):
 
 - Open up the Python project in IntelliJ
-- Make sure that you're on latest master (that includes Poetry)
+- Make sure that you're on latest main (that includes Poetry)
 - Go to File -> Project Structure (⌘;)
 - Go to Platform Settings -> SDKs
 - Click the + sign -> Add Python SDK

diff --git a/mkdocs/docs/how-to-release.md b/mkdocs/docs/how-to-release.md
@@ -54,10 +54,10 @@ Both the source distribution (`sdist`) and the binary distributions (`wheels`) n
 
 Before committing the files to the Apache SVN artifact distribution SVN hashes need to be generated, and those need to be signed with gpg to make sure that they are authentic.
 
-Go to [Github Actions and run the `Python release` action](https://github.com/apache/iceberg/actions/workflows/python-release.yml). **Set the version to master, since we cannot modify the source**. Download the zip, and sign the files:
+Go to [Github Actions and run the `Python release` action](https://github.com/apache/iceberg/actions/workflows/python-release.yml). **Set the version to main, since we cannot modify the source**. Download the zip, and sign the files:
 
 ```bash
-cd release-master/
+cd release-main/
 
 for name in $(ls pyiceberg-*.whl pyiceberg-*.tar.gz)
 do

diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml
@@ -17,8 +17,8 @@
 ---
 site_name: PyIceberg
 site_url: https://py.iceberg.apache.org/
-repo_url: "https://github.com/apache/iceberg/tree/master/python"
-repo_name: "apache/iceberg/python"
+repo_url: "https://github.com/apache/iceberg-python"
+repo_name: "apache/iceberg-python"
 
 plugins:
   - gen-files:

diff --git a/mkdocs/requirements.txt b/mkdocs/requirements.txt
@@ -16,10 +16,10 @@
 # under the License.
 
 mkdocs==1.5.3
-griffe==0.36.4
+griffe==0.36.5
 jinja2==3.1.2
 mkdocstrings==0.23.0
-mkdocstrings-python==1.7.1
+mkdocstrings-python==1.7.3
 mkdocs-literate-nav==0.6.1
 mkdocs-autorefs==0.5.0
 mkdocs-gen-files==0.5.0