diff --git a/.github/workflows/beam_PreCommit_PythonDocs.yml b/.github/workflows/beam_PreCommit_PythonDocs.yml index 12639421af8b..aa11539b3f9a 100644 --- a/.github/workflows/beam_PreCommit_PythonDocs.yml +++ b/.github/workflows/beam_PreCommit_PythonDocs.yml @@ -82,7 +82,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default - python-version: default + python-version: 3.8 - name: run pythonDocsPreCommit script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index fdbae21336e5..741758bc2133 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -302,10 +302,10 @@ jobs: path: beam-site token: ${{ github.event.inputs.REPO_TOKEN }} ref: release-docs - - name: Install Python 3.9 + - name: Install Python 3.8 uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.8' - name: Install node uses: actions/setup-node@v4 with: diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d1e99f2bd579..ae159ccfaf42 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -89,6 +89,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + ref: "v2.60.0-RC1" + repository: apache/beam - name: Install python uses: actions/setup-python@v5 with: @@ -96,7 +99,7 @@ jobs: - name: Get tag id: get_tag run: | - echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT + echo "TAG=v2.60.0-RC1" >> $GITHUB_OUTPUT - name: Check whether an -RC tag was applied to the commit. id: is_rc run: | @@ -229,8 +232,8 @@ jobs: {"os": "windows-latest", "runner": "windows-latest", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}", arch: "auto" }, {"os": "ubuntu-20.04", "runner": [self-hosted, ubuntu-20.04, main], "python": "${{ needs.check_env_variables.outputs.py-versions-test }}", arch: "aarch64" } ] - # Keep in sync with PY_VERSIONS_FULL env var abvove - if changed, change that as well. - py_version: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"] + # Keep in sync (without asterisks) with PY_VERSIONS_FULL env var above - if changed, change that as well. + py_version: ["cp38-", "cp39-", "cp310-", "cp311-", "cp312-"] steps: - name: Download python source distribution from artifacts # Pinned to v3 because of https://github.com/actions/download-artifact/issues/249 @@ -260,7 +263,7 @@ jobs: if: ${{ contains(matrix.os_python.python, matrix.py_version) }} working-directory: apache-beam-source env: - CIBW_BUILD: ${{ matrix.py_version }} + CIBW_BUILD: ${{ matrix.py_version }}* # TODO: https://github.com/apache/beam/issues/23048 CIBW_SKIP: "*-musllinux_*" CIBW_BEFORE_BUILD: pip install cython==0.29.36 numpy --config-settings=setup-args="-Dallow-noblas=true" && pip install --upgrade setuptools @@ -280,16 +283,16 @@ jobs: - name: Upload wheels as artifacts if: ${{ contains(matrix.os_python.python, matrix.py_version) }} # Pinned to v3 because of https://github.com/actions/upload-artifact?tab=readme-ov-file#breaking-changes - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheelhouse-${{ matrix.os_python.os }}${{ (matrix.os_python.arch == 'aarch64' && '-aarch64') || '' }} + name: wheelhouse-${{ matrix.os_python.os }}-${{ matrix.py_version }}${{ (matrix.os_python.arch == 'aarch64' && '-aarch64') || '' }} path: apache-beam-source/wheelhouse/ - name: Build RC wheels # Only build wheel if it is one of the target versions for this platform, otherwise no-op if: ${{ needs.build_source.outputs.is_rc == 1 && contains(matrix.os_python.python, matrix.py_version) }} working-directory: apache-beam-source-rc env: - CIBW_BUILD: ${{ matrix.py_version }} + CIBW_BUILD: ${{ matrix.py_version }}* # TODO: https://github.com/apache/beam/issues/23048 CIBW_SKIP: "*-musllinux_*" CIBW_BEFORE_BUILD: pip install cython==0.29.36 numpy --config-settings=setup-args="-Dallow-noblas=true" && pip install --upgrade setuptools @@ -308,7 +311,7 @@ jobs: # Pinned to v3 because of https://github.com/actions/download-artifact/issues/249 uses: actions/upload-artifact@v4 with: - name: wheelhouse-rc${{ needs.build_source.outputs.rc_num }}-${{ matrix.os_python.os }}${{ (matrix.arch == 'aarch64' && '-aarch64') || '' }} + name: wheelhouse-rc${{ needs.build_source.outputs.rc_num }}-${{ matrix.py_version }}${{ matrix.os_python.os }}${{ (matrix.os_python.arch == 'aarch64' && '-aarch64') || '' }} path: apache-beam-source-rc/wheelhouse/ upload_wheels_to_gcs: diff --git a/sdks/java/javadoc/build.gradle b/sdks/java/javadoc/build.gradle index c0622b173043..7aacb9fdd176 100644 --- a/sdks/java/javadoc/build.gradle +++ b/sdks/java/javadoc/build.gradle @@ -27,6 +27,56 @@ plugins { id 'org.apache.beam.module' } applyJavaNature(publish: false) description = "Apache Beam :: SDKs :: Java :: Aggregated Javadoc" +// The following dependencies are excluded from javadoc external link due to +// "Error fetching URL" in Java11+ +def excludedFromLink = [ + 'com.azure/azure-sdk-bom', + 'io.cdap.cdap/cdap-api-common', + 'io.cdap.cdap/cdap-common', + 'io.cdap.cdap/cdap-etl-api-spark', + 'io.cdap/hubspot-plugins', + 'io.cdap.plugin/salesforce-plugins', + 'io.cdap.plugin/servicenow-plugins', + 'io.cdap.plugin/zendesk-plugins', + 'com.google.api-ads/google-ads-stubs-v17', + 'com.google.api-client/google-api-client-java6', + 'com.google.apis/google-api-services-cloudresourcemanager', + 'com.google.apis/google-api-services-healthcare', + 'com.google.cloud.dataflow/google-cloud-dataflow-java-proto-library-all', + 'com.google.cloud/libraries-bom', + 'org.apache.hadoop/hadoop-client', + 'org.apache.hadoop/hadoop-minicluster', + 'com.fasterxml.jackson.module/jackson-module-scala_2.11', + 'com.fasterxml.jackson.module/jackson-module-scala_2.12', + 'com.github.jbellis/jamm', + 'org.apache.logging.log4j/log4j-to-slf4j', + 'org.apache.logging.log4j/log4j-slf4j-impl', + 'org.mockito/mockito-inline', + 'org.apache.nemo/nemo-compiler-frontend-beam', + 'io.netty/netty-all', + 'io.netty/netty-tcnative-boringssl-static', + 'io.netty/netty-transport-native-epoll', + 'com.singlestore/singlestore-jdbc-client', + 'com.solacesystems/sol-jcsmp', + 'org.apache.spark/spark-core_2.11', + 'org.apache.spark/spark-streaming_2.11', + 'org.apache.spark/spark-core_2.12', + 'org.apache.spark/spark-network-common_2.12', + 'org.apache.spark/spark-sql_2.12', + 'org.apache.spark/spark-streaming_2.12', + 'org.apache.tephra/tephra-api', + 'org.testcontainers/azure', + 'org.testcontainers/gcloud', + 'org.testcontainers/solace', + 'org.apache.beam/beam-vendor-grpc-1_60_1', + 'org.apache.beam/beam-vendor-guava-32_1_2-jre', + 'org.apache.beam/beam-vendor-calcite-1_28_0', + 'org.codehaus.woodstox/woodstox-core-asl', + 'org.apache.arrow/arrow-memory-core', + 'org.apache.arrow/arrow-memory-netty', + 'org.hamcrest/hamcrest', +] + for (p in rootProject.subprojects) { if (!p.path.equals(project.path) && !p.path.startsWith(':sdks:java:bom')) { evaluationDependsOn(p.path) @@ -55,12 +105,20 @@ task allJavadoc() { task aggregateJavadoc(type: Javadoc) { def createJavadocIOUrlForDependency = { def dependency = dependencies.create(it) - return 'https://static.javadoc.io/' + dependency.getGroup() + '/' + dependency.getName() + '/' + dependency.getVersion() + def identifier = dependency.getGroup() + '/' + dependency.getName() + if (excludedFromLink.contains(identifier)) { + return '' + } + def url = 'https://static.javadoc.io/' + identifier + if (dependency.getVersion() != null) { + url += '/' + dependency.getVersion() + } + return url } def exportedJavadocProjects = getExportedJavadocProjects() source exportedJavadocProjects.collect { project(it).sourceSets.main.allJava } - classpath = files(exportedJavadocProjects.collect { project(it).sourceSets.main.compileClasspath }) + classpath = files(exportedJavadocProjects.collect { project(it).sourceSets.main.runtimeClasspath }) destinationDir = file("${buildDir}/docs/javadoc") failOnError = true @@ -80,7 +138,10 @@ task aggregateJavadoc(type: Javadoc) { title "Apache Beam " + project.version overview 'overview.html' for (dep in project.library.java.values()) { - links createJavadocIOUrlForDependency(dep) + def link = createJavadocIOUrlForDependency(dep) + if (!link.isEmpty()) { + links link + } } } } diff --git a/sdks/python/apache_beam/io/filesystems.py b/sdks/python/apache_beam/io/filesystems.py index a32b85332b60..ae6c393c5c32 100644 --- a/sdks/python/apache_beam/io/filesystems.py +++ b/sdks/python/apache_beam/io/filesystems.py @@ -398,7 +398,7 @@ def report_source_lineage(path, level=None): Args: path: string path to be reported. level: the level of file path. default to - :class:`~apache_beam.io.filesystem.FileSystem.Lineage`.FILE. + :class:`~apache_beam.io.filesystem.FileSystem.LineageLevel`.FILE. """ filesystem = FileSystems.get_filesystem(path) filesystem.report_lineage(path, Lineage.sources(), level=level) @@ -411,7 +411,7 @@ def report_sink_lineage(path, level=None): Args: path: string path to be reported. level: the level of file path. default to - :class:`~apache_beam.io.filesystem.FileSystem.Lineage`.FILE. + :class:`~apache_beam.io.filesystem.FileSystem.LineageLevel`.FILE. """ filesystem = FileSystems.get_filesystem(path) filesystem.report_lineage(path, Lineage.sinks(), level=level)