diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 6aa61ac11ad..c0aea717f82 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -119,6 +119,8 @@ Improvements This will enable client and server hostName check settings to be governed by the same environment variable. If users want separate client/server settings, they can manually override the solr.jetty.ssl.sniHostCheck option in SOLR_OPTS. (Houston Putman) +* SOLR-16970: SOLR_OPTS is now able to override options set by the Solr control scripts, "bin/solr" and "bin/solr.cmd". (Houston Putman) + Optimizations --------------------- @@ -166,6 +168,9 @@ Bug Fixes * SOLR-16044: SlowRequest logging is no longer disabled if SolrCore logger set to ERROR (janhoy, hossman) +* SOLR-16415: asyncId must not have '/'; enforce this. Enhance ZK cleanup to process directories + instead of fail. (David Smiley, Paul McArthur) + * SOLR-16963: The "solr.jetty.ssl.verifyClientHostName" sysProp and "SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION" envVar have been fixed, and the setting once again tells the server to check the originating client hostname against the client certificate when doing mTLS. (Houston Putman, Tomás Fernández Löbbe) diff --git a/solr/bin/solr b/solr/bin/solr index d4b09d82ea3..cc72f621094 100644 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -628,7 +628,7 @@ function jetty_port() { function run_tool() { # shellcheck disable=SC2086 - "$JAVA" ${SOLR_TOOL_OPTS:-} $SOLR_SSL_OPTS $AUTHC_OPTS ${SOLR_ZK_CREDS_AND_ACLS:-} -Dsolr.install.dir="$SOLR_TIP" \ + "$JAVA" $SOLR_SSL_OPTS $AUTHC_OPTS ${SOLR_ZK_CREDS_AND_ACLS:-} ${SOLR_TOOL_OPTS:-} -Dsolr.install.dir="$SOLR_TIP" \ -Dlog4j.configurationFile="$DEFAULT_SERVER_DIR/resources/log4j2-console.xml" \ -classpath "$DEFAULT_SERVER_DIR/solr-webapp/webapp/WEB-INF/lib/*:$DEFAULT_SERVER_DIR/lib/ext/*:$DEFAULT_SERVER_DIR/lib/*" \ org.apache.solr.cli.SolrCLI "$@" @@ -742,7 +742,7 @@ function stop_solr() { if [ -n "$SOLR_PID" ]; then echo -e "Sending stop command to Solr running on port $SOLR_PORT ... waiting up to $SOLR_STOP_WAIT seconds to allow Jetty process $SOLR_PID to stop gracefully." # shellcheck disable=SC2086 - "$JAVA" ${SOLR_TOOL_OPTS:-} $SOLR_SSL_OPTS $AUTHC_OPTS -jar "$DIR/start.jar" "STOP.PORT=$THIS_STOP_PORT" "STOP.KEY=$STOP_KEY" --stop || true + "$JAVA" $SOLR_SSL_OPTS $AUTHC_OPTS ${SOLR_TOOL_OPTS:-} -jar "$DIR/start.jar" "STOP.PORT=$THIS_STOP_PORT" "STOP.KEY=$STOP_KEY" --stop || true (loops=0 while true do @@ -1222,6 +1222,7 @@ fi FG="false" FORCE=false SOLR_OPTS=(${SOLR_OPTS:-}) +SCRIPT_SOLR_OPTS=() PASS_TO_RUN_EXAMPLE=() if [ $# -gt 0 ]; then @@ -1370,6 +1371,7 @@ if [ $# -gt 0 ]; then break # out-of-args, stop looping elif [ "${1:0:2}" == "-D" ]; then # pass thru any opts that begin with -D (java system props) + # These should go to the end of SOLR_OPTS, as they should override everything else SOLR_OPTS+=("$1") PASS_TO_RUN_EXAMPLE+=("$1") shift @@ -1388,20 +1390,20 @@ fi # Solr modules option if [[ -n "${SOLR_MODULES:-}" ]] ; then - SOLR_OPTS+=("-Dsolr.modules=$SOLR_MODULES") + SCRIPT_SOLR_OPTS+=("-Dsolr.modules=$SOLR_MODULES") fi # Default placement plugin if [[ -n "${SOLR_PLACEMENTPLUGIN_DEFAULT:-}" ]] ; then - SOLR_OPTS+=("-Dsolr.placementplugin.default=$SOLR_PLACEMENTPLUGIN_DEFAULT") + SCRIPT_SOLR_OPTS+=("-Dsolr.placementplugin.default=$SOLR_PLACEMENTPLUGIN_DEFAULT") fi # Remote streaming and stream body if [ "${SOLR_ENABLE_REMOTE_STREAMING:-false}" == "true" ]; then - SOLR_OPTS+=("-Dsolr.enableRemoteStreaming=true") + SCRIPT_SOLR_OPTS+=("-Dsolr.enableRemoteStreaming=true") fi if [ "${SOLR_ENABLE_STREAM_BODY:-false}" == "true" ]; then - SOLR_OPTS+=("-Dsolr.enableStreamBody=true") + SCRIPT_SOLR_OPTS+=("-Dsolr.enableStreamBody=true") fi : ${SOLR_SERVER_DIR:=$DEFAULT_SERVER_DIR} @@ -1494,15 +1496,15 @@ if [[ "$SCRIPT_CMD" == "stop" ]]; then fi if [ -n "${SOLR_PORT_ADVERTISE:-}" ]; then - SOLR_OPTS+=("-Dsolr.port.advertise=$SOLR_PORT_ADVERTISE") + SCRIPT_SOLR_OPTS+=("-Dsolr.port.advertise=$SOLR_PORT_ADVERTISE") fi if [ -n "${SOLR_JETTY_HOST:-}" ]; then - SOLR_OPTS+=("-Dsolr.jetty.host=$SOLR_JETTY_HOST") + SCRIPT_SOLR_OPTS+=("-Dsolr.jetty.host=$SOLR_JETTY_HOST") fi if [ -n "${SOLR_ZK_EMBEDDED_HOST:-}" ]; then - SOLR_OPTS+=("-Dsolr.zk.embedded.host=$SOLR_ZK_EMBEDDED_HOST") + SCRIPT_SOLR_OPTS+=("-Dsolr.zk.embedded.host=$SOLR_ZK_EMBEDDED_HOST") fi : "${STOP_PORT:=$((SOLR_PORT - 1000))}" @@ -1736,8 +1738,8 @@ else JAVA_MEM_OPTS=("-Xms$SOLR_HEAP" "-Xmx$SOLR_HEAP") fi -# Pick default for Java thread stack size, and then add to SOLR_OPTS -SOLR_OPTS+=(${SOLR_JAVA_STACK_SIZE:-"-Xss256k"}) +# Pick default for Java thread stack size, and then add to SCRIPT_SOLR_OPTS +SCRIPT_SOLR_OPTS+=(${SOLR_JAVA_STACK_SIZE:-"-Xss256k"}) : "${SOLR_TIMEZONE:=UTC}" @@ -1778,79 +1780,83 @@ function start_solr() { fi if [ -n "${SOLR_WAIT_FOR_ZK:-}" ]; then - SOLR_OPTS+=("-DwaitForZk=$SOLR_WAIT_FOR_ZK") + SCRIPT_SOLR_OPTS+=("-DwaitForZk=$SOLR_WAIT_FOR_ZK") fi if [ -n "${SOLR_DATA_HOME:-}" ]; then - SOLR_OPTS+=("-Dsolr.data.home=$SOLR_DATA_HOME") + SCRIPT_SOLR_OPTS+=("-Dsolr.data.home=$SOLR_DATA_HOME") fi if [ -n "${SOLR_DELETE_UNKNOWN_CORES:-}" ]; then - SOLR_OPTS+=("-Dsolr.deleteUnknownCores=$SOLR_DELETE_UNKNOWN_CORES") + SCRIPT_SOLR_OPTS+=("-Dsolr.deleteUnknownCores=$SOLR_DELETE_UNKNOWN_CORES") fi - # If SSL-related system props are set, add them to SOLR_OPTS + # If SSL-related system props are set, add them to SCRIPT_SOLR_OPTS if [ "$SOLR_SSL_ENABLED" == "true" ]; then # If using SSL and solr.jetty.https.port not set explicitly, use the jetty.port SSL_PORT_PROP="-Dsolr.jetty.https.port=$SOLR_PORT" - SOLR_OPTS+=($SOLR_SSL_OPTS "$SSL_PORT_PROP") + SCRIPT_SOLR_OPTS+=($SOLR_SSL_OPTS "$SSL_PORT_PROP") fi - # If authentication system props are set, add them to SOLR_OPTS + # If authentication system props are set, add them to SCRIPT_SOLR_OPTS if [ -n "$AUTHC_OPTS" ]; then - SOLR_OPTS+=($AUTHC_OPTS) + SCRIPT_SOLR_OPTS+=($AUTHC_OPTS) fi - # If there are internal options set by Solr (users should not use this variable), add them to SOLR_OPTS + # If there are internal options set by Solr (users should not use this variable), add them to SCRIPT_SOLR_OPTS if [ -n "$SOLR_OPTS_INTERNAL" ]; then - SOLR_OPTS+=($SOLR_OPTS_INTERNAL) + SCRIPT_SOLR_OPTS+=($SOLR_OPTS_INTERNAL) fi - # If a heap dump directory is specified, enable it in SOLR_OPTS + # If a heap dump directory is specified, enable it in SCRIPT_SOLR_OPTS if [[ -z "${SOLR_HEAP_DUMP_DIR:-}" ]] && [[ "${SOLR_HEAP_DUMP:-}" == "true" ]]; then SOLR_HEAP_DUMP_DIR="${SOLR_LOGS_DIR}/dumps" fi if [[ -n "${SOLR_HEAP_DUMP_DIR:-}" ]]; then - SOLR_OPTS+=("-XX:+HeapDumpOnOutOfMemoryError") - SOLR_OPTS+=("-XX:HeapDumpPath=$SOLR_HEAP_DUMP_DIR/solr-$(date +%s)-pid$$.hprof") + SCRIPT_SOLR_OPTS+=("-XX:+HeapDumpOnOutOfMemoryError") + SCRIPT_SOLR_OPTS+=("-XX:HeapDumpPath=$SOLR_HEAP_DUMP_DIR/solr-$(date +%s)-pid$$.hprof") fi if $verbose ; then echo -e "\nStarting Solr using the following settings:" - echo -e " JAVA = $JAVA" - echo -e " SOLR_SERVER_DIR = $SOLR_SERVER_DIR" - echo -e " SOLR_HOME = $SOLR_HOME" - echo -e " SOLR_HOST = ${SOLR_HOST:-}" - echo -e " SOLR_PORT = $SOLR_PORT" - echo -e " STOP_PORT = $STOP_PORT" - echo -e " JAVA_MEM_OPTS = ${JAVA_MEM_OPTS[*]}" - echo -e " GC_TUNE = ${GC_TUNE_ARR[*]}" - echo -e " GC_LOG_OPTS = ${GC_LOG_OPTS[*]}" - echo -e " SOLR_TIMEZONE = $SOLR_TIMEZONE" + echo -e " JAVA = $JAVA" + echo -e " SOLR_SERVER_DIR = $SOLR_SERVER_DIR" + echo -e " SOLR_HOME = $SOLR_HOME" + echo -e " SOLR_HOST = ${SOLR_HOST:-}" + echo -e " SOLR_PORT = $SOLR_PORT" + echo -e " STOP_PORT = $STOP_PORT" + echo -e " JAVA_MEM_OPTS = ${JAVA_MEM_OPTS[*]}" + echo -e " GC_TUNE = ${GC_TUNE_ARR[*]}" + echo -e " GC_LOG_OPTS = ${GC_LOG_OPTS[*]}" + echo -e " SOLR_TIMEZONE = $SOLR_TIMEZONE" if [ "$SOLR_MODE" == "solrcloud" ]; then - echo -e " CLOUD_MODE_OPTS = ${CLOUD_MODE_OPTS[*]}" + echo -e " CLOUD_MODE_OPTS = ${CLOUD_MODE_OPTS[*]}" fi if [ -n "${SOLR_OPTS:-}" ]; then - echo -e " SOLR_OPTS = ${SOLR_OPTS[*]}" + echo -e " SOLR_OPTS (USER) = ${SOLR_OPTS[*]}" + fi + + if [ -n "${SCRIPT_SOLR_OPTS:-}" ]; then + echo -e " SOLR_OPTS (SCRIPT) = ${SCRIPT_SOLR_OPTS[*]}" fi if [ -n "${SOLR_ADDL_ARGS:-}" ]; then - echo -e " SOLR_ADDL_ARGS = $SOLR_ADDL_ARGS" + echo -e " SOLR_ADDL_ARGS = $SOLR_ADDL_ARGS" fi if [ "${ENABLE_REMOTE_JMX_OPTS:-false}" == "true" ]; then - echo -e " RMI_PORT = ${RMI_PORT:-}" - echo -e " REMOTE_JMX_OPTS = ${REMOTE_JMX_OPTS[*]}" + echo -e " RMI_PORT = ${RMI_PORT:-}" + echo -e " REMOTE_JMX_OPTS = ${REMOTE_JMX_OPTS[*]}" fi if [ -n "${SOLR_LOG_LEVEL:-}" ]; then - echo -e " SOLR_LOG_LEVEL = $SOLR_LOG_LEVEL" + echo -e " SOLR_LOG_LEVEL = $SOLR_LOG_LEVEL" fi if [ -n "${SOLR_DATA_HOME:-}" ]; then - echo -e " SOLR_DATA_HOME = $SOLR_DATA_HOME" + echo -e " SOLR_DATA_HOME = $SOLR_DATA_HOME" fi echo fi @@ -1865,14 +1871,14 @@ function start_solr() { # Workaround for JIT crash, see https://issues.apache.org/jira/browse/SOLR-16463 if [[ "$JAVA_VER_NUM" -ge "17" ]] ; then - SOLR_OPTS+=("-XX:CompileCommand=exclude,com.github.benmanes.caffeine.cache.BoundedLocalCache::put") + SCRIPT_SOLR_OPTS+=("-XX:CompileCommand=exclude,com.github.benmanes.caffeine.cache.BoundedLocalCache::put") echo "Java $JAVA_VER_NUM detected. Enabled workaround for SOLR-16463" fi # Vector optimizations are only supported for Java 20 and 21 for now. # This will need to change as Lucene is upgraded and newer Java versions are released if [[ "$JAVA_VER_NUM" -ge "20" ]] && [[ "$JAVA_VER_NUM" -le "21" ]] ; then - SOLR_OPTS+=("--add-modules" "jdk.incubator.vector") + SCRIPT_SOLR_OPTS+=("--add-modules" "jdk.incubator.vector") echo "Java $JAVA_VER_NUM detected. Incubating Panama Vector APIs have been enabled" fi @@ -1886,7 +1892,7 @@ function start_solr() { # OOME is thrown. Program operation after OOME is unpredictable. "-XX:+CrashOnOutOfMemoryError" "-XX:ErrorFile=${SOLR_LOGS_DIR}/jvm_crash_%p.log" \ "-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.install.dir=$SOLR_TIP" "-Dsolr.install.symDir=$SOLR_TIP_SYM" \ - "-Dsolr.default.confdir=$DEFAULT_CONFDIR" "${LOG4J_CONFIG[@]}" "${SOLR_OPTS[@]}" "${SECURITY_MANAGER_OPTS[@]}" "${SOLR_ADMIN_UI}") + "-Dsolr.default.confdir=$DEFAULT_CONFDIR" "${LOG4J_CONFIG[@]}" "${SCRIPT_SOLR_OPTS[@]}" "${SECURITY_MANAGER_OPTS[@]}" "${SOLR_ADMIN_UI}" "${SOLR_OPTS[@]}") mk_writable_dir "$SOLR_LOGS_DIR" "Logs" if [[ -n "${SOLR_HEAP_DUMP_DIR:-}" ]]; then diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index c58410aeebf..d10ac598057 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -805,22 +805,24 @@ IF NOT "%SOLR_HOST%"=="" ( set SOLR_HOST_ARG= ) +set SCRIPT_SOLR_OPTS= + REM Solr modules option IF DEFINED SOLR_MODULES ( - set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.modules=%SOLR_MODULES%" + set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.modules=%SOLR_MODULES%" ) REM Default placement plugin IF DEFINED SOLR_PLACEMENTPLUGIN_DEFAULT ( - set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.placementplugin.default=%SOLR_PLACEMENTPLUGIN_DEFAULT%" + set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.placementplugin.default=%SOLR_PLACEMENTPLUGIN_DEFAULT%" ) REM Remote streaming and stream body IF "%SOLR_ENABLE_REMOTE_STREAMING%"=="true" ( - set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.enableRemoteStreaming=true" + set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.enableRemoteStreaming=true" ) IF "%SOLR_ENABLE_STREAM_BODY%"=="true" ( - set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.enableStreamBody=true" + set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.enableStreamBody=true" ) IF "%SOLR_SERVER_DIR%"=="" set "SOLR_SERVER_DIR=%DEFAULT_SERVER_DIR%" @@ -923,7 +925,7 @@ IF "%SCRIPT_CMD%"=="stop" ( set found_it=1 @echo Stopping Solr process %%N running on port %SOLR_PORT% IF "%STOP_PORT%"=="" set /A STOP_PORT=%SOLR_PORT% - 1000 - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" %SOLR_JETTY_CONFIG% STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop + "%JAVA%" %SOLR_SSL_OPTS% %SOLR_TOOL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" %SOLR_JETTY_CONFIG% STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop del "%SOLR_TIP%"\bin\solr-%SOLR_PORT%.port REM wait for the process to terminate CALL :wait_for_process_exit %%N !SOLR_STOP_WAIT! @@ -948,15 +950,15 @@ IF "%SOLR_PORT%"=="" set SOLR_PORT=8983 IF "%STOP_PORT%"=="" set /A STOP_PORT=%SOLR_PORT% - 1000 IF DEFINED SOLR_PORT_ADVERTISE ( - set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.port.advertise=%SOLR_PORT_ADVERTISE%" + set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.port.advertise=%SOLR_PORT_ADVERTISE%" ) IF DEFINED SOLR_JETTY_HOST ( - set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.jetty.host=%SOLR_JETTY_HOST%" + set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.jetty.host=%SOLR_JETTY_HOST%" ) IF DEFINED SOLR_ZK_EMBEDDED_HOST ( - set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.zk.embedded.host=%SOLR_ZK_EMBEDDED_HOST%" + set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.zk.embedded.host=%SOLR_ZK_EMBEDDED_HOST%" ) IF "%SCRIPT_CMD%"=="start" ( @@ -1097,7 +1099,7 @@ IF "%SOLR_ADMIN_UI_DISABLED%"=="true" ( IF NOT "%SOLR_HEAP%"=="" set SOLR_JAVA_MEM=-Xms%SOLR_HEAP% -Xmx%SOLR_HEAP% IF "%SOLR_JAVA_MEM%"=="" set SOLR_JAVA_MEM=-Xms512m -Xmx512m IF "%SOLR_JAVA_STACK_SIZE%"=="" set SOLR_JAVA_STACK_SIZE=-Xss256k -set SOLR_OPTS=%SOLR_JAVA_STACK_SIZE% %SOLR_OPTS% +set SCRIPT_SOLR_OPTS=%SOLR_JAVA_STACK_SIZE% %SCRIPT_SOLR_OPTS% IF "%SOLR_TIMEZONE%"=="" set SOLR_TIMEZONE=UTC IF "%GC_TUNE%"=="" ( @@ -1112,14 +1114,14 @@ IF "%GC_TUNE%"=="" ( REM Workaround for JIT crash, see https://issues.apache.org/jira/browse/SOLR-16463 if !JAVA_MAJOR_VERSION! GEQ 17 ( - set SOLR_OPTS=%SOLR_OPTS% -XX:CompileCommand=exclude,com.github.benmanes.caffeine.cache.BoundedLocalCache::put + set SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -XX:CompileCommand=exclude,com.github.benmanes.caffeine.cache.BoundedLocalCache::put echo Java %JAVA_MAJOR_VERSION% detected. Enabled workaround for SOLR-16463 ) REM Vector optimizations are only supported for Java 20 and 21 for now. REM This will need to change as Lucene is upgraded and newer Java versions are released if !JAVA_MAJOR_VERSION! GEQ 20 if !JAVA_MAJOR_VERSION! LEQ 21 ( - set SOLR_OPTS=%SOLR_OPTS% --add-modules jdk.incubator.vector + set SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% --add-modules jdk.incubator.vector echo Java %JAVA_MAJOR_VERSION% detected. Incubating Panama Vector APIs have been enabled ) @@ -1149,44 +1151,48 @@ if !JAVA_MAJOR_VERSION! GEQ 9 if NOT "%JAVA_VENDOR%" == "OpenJ9" ( IF "%verbose%"=="1" ( @echo Starting Solr using the following settings: - CALL :safe_echo " JAVA = %JAVA%" - CALL :safe_echo " SOLR_SERVER_DIR = %SOLR_SERVER_DIR%" - CALL :safe_echo " SOLR_HOME = %SOLR_HOME%" - @echo SOLR_HOST = %SOLR_HOST% - @echo SOLR_PORT = %SOLR_PORT% - @echo STOP_PORT = %STOP_PORT% - @echo SOLR_JAVA_MEM = %SOLR_JAVA_MEM% - @echo GC_TUNE = !GC_TUNE! - @echo GC_LOG_OPTS = %GC_LOG_OPTS% - @echo SOLR_TIMEZONE = %SOLR_TIMEZONE% + CALL :safe_echo " JAVA = %JAVA%" + CALL :safe_echo " SOLR_SERVER_DIR = %SOLR_SERVER_DIR%" + CALL :safe_echo " SOLR_HOME = %SOLR_HOME%" + @echo SOLR_HOST = %SOLR_HOST% + @echo SOLR_PORT = %SOLR_PORT% + @echo STOP_PORT = %STOP_PORT% + @echo SOLR_JAVA_MEM = %SOLR_JAVA_MEM% + @echo GC_TUNE = !GC_TUNE! + @echo GC_LOG_OPTS = %GC_LOG_OPTS% + @echo SOLR_TIMEZONE = %SOLR_TIMEZONE% IF "%SOLR_MODE%"=="solrcloud" ( - @echo CLOUD_MODE_OPTS = %CLOUD_MODE_OPTS% + @echo CLOUD_MODE_OPTS = %CLOUD_MODE_OPTS% ) IF NOT "%SOLR_OPTS%"=="" ( - @echo SOLR_OPTS = %SOLR_OPTS% + @echo SOLR_OPTS (USER) = %SOLR_OPTS% + ) + + IF NOT "%SCRIPT_SOLR_OPTS%"=="" ( + @echo SOLR_OPTS (SCRIPT) = %SCRIPT_SOLR_OPTS% ) IF NOT "%SOLR_ADDL_ARGS%"=="" ( - CALL :safe_echo " SOLR_ADDL_ARGS = %SOLR_ADDL_ARGS%" + CALL :safe_echo " SOLR_ADDL_ARGS = %SOLR_ADDL_ARGS%" ) IF NOT "%SOLR_JETTY_ADDL_CONFIG%"=="" ( - CALL :safe_echo " SOLR_JETTY_ADDL_CONFIG = %SOLR_JETTY_ADDL_CONFIG%" + CALL :safe_echo " SOLR_JETTY_ADDL_CONFIG = %SOLR_JETTY_ADDL_CONFIG%" ) IF "%ENABLE_REMOTE_JMX_OPTS%"=="true" ( - @echo RMI_PORT = !RMI_PORT! - @echo REMOTE_JMX_OPTS = %REMOTE_JMX_OPTS% + @echo RMI_PORT = !RMI_PORT! + @echo REMOTE_JMX_OPTS = %REMOTE_JMX_OPTS% ) IF NOT "%SOLR_LOG_LEVEL%"=="" ( - @echo SOLR_LOG_LEVEL = !SOLR_LOG_LEVEL! + @echo SOLR_LOG_LEVEL = !SOLR_LOG_LEVEL! ) IF NOT "%SOLR_DATA_HOME%"=="" ( - @echo SOLR_DATA_HOME = !SOLR_DATA_HOME! + @echo SOLR_DATA_HOME = !SOLR_DATA_HOME! ) @echo. @@ -1207,7 +1213,7 @@ IF NOT "!IP_ACL_OPTS!"=="" set "START_OPTS=%START_OPTS% !IP_ACL_OPTS!" IF NOT "%REMOTE_JMX_OPTS%"=="" set "START_OPTS=%START_OPTS% %REMOTE_JMX_OPTS%" IF NOT "%SOLR_ADDL_ARGS%"=="" set "START_OPTS=%START_OPTS% %SOLR_ADDL_ARGS%" IF NOT "%SOLR_HOST_ARG%"=="" set "START_OPTS=%START_OPTS% %SOLR_HOST_ARG%" -IF NOT "%SOLR_OPTS%"=="" set "START_OPTS=%START_OPTS% %SOLR_OPTS%" +IF NOT "%SCRIPT_SOLR_OPTS%"=="" set "START_OPTS=%START_OPTS% %SCRIPT_SOLR_OPTS%" IF NOT "%SOLR_OPTS_INTERNAL%"=="" set "START_OPTS=%START_OPTS% %SOLR_OPTS_INTERNAL%" IF NOT "!SECURITY_MANAGER_OPTS!"=="" set "START_OPTS=%START_OPTS% !SECURITY_MANAGER_OPTS!" IF "%SOLR_SSL_ENABLED%"=="true" ( @@ -1223,6 +1229,9 @@ set "START_OPTS=%START_OPTS% -Dsolr.log.dir=%SOLR_LOGS_DIR_QUOTED% -Djava.util.l IF NOT "%SOLR_DATA_HOME%"=="" set "START_OPTS=%START_OPTS% -Dsolr.data.home=%SOLR_DATA_HOME_QUOTED%" IF NOT DEFINED LOG4J_CONFIG set "LOG4J_CONFIG=%SOLR_SERVER_DIR%\resources\log4j2.xml" +REM This should be the last thing added to START_OPTS, so that users can override as much as possible +IF NOT "%SOLR_OPTS%"=="" set "START_OPTS=%START_OPTS% %SOLR_OPTS%" + cd /d "%SOLR_SERVER_DIR%" IF NOT EXIST "%SOLR_LOGS_DIR%" ( @@ -1272,7 +1281,7 @@ IF "%FG%"=="1" ( set SOLR_START_WAIT=30 ) REM now wait to see Solr come online ... - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" -Dsolr.default.confdir="%DEFAULT_CONFDIR%"^ + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dsolr.default.confdir="%DEFAULT_CONFDIR%"^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI status -maxWaitSecs !SOLR_START_WAIT! -solrUrl !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%SOLR_PORT%/solr @@ -1287,7 +1296,7 @@ goto done :run_example REM Run the requested example -"%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ +"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -Dsolr.install.symDir="%SOLR_TIP%" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ @@ -1311,7 +1320,7 @@ for /f "usebackq" %%i in (`dir /b "%SOLR_TIP%\bin" ^| findstr /i "^solr-.*\.port set has_info=1 echo Found Solr process %%k running on port !SOME_SOLR_PORT! REM Passing in %2 (-h or -help) directly is captured by a custom help path for usage output - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI status -solrUrl !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:!SOME_SOLR_PORT!/solr %2 @@ -1326,7 +1335,7 @@ set has_info= goto done :run_solrcli -"%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ +"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI %* @@ -1475,7 +1484,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="The -d option must be set for upconfig." goto zk_short_usage ) - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -confname !CONFIGSET_NAME! -confdir !CONFIGSET_DIR! -zkHost !ZK_HOST! %ZK_VERBOSE%^ @@ -1489,7 +1498,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="The -d option must be set for downconfig." goto zk_short_usage ) - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -confname !CONFIGSET_NAME! -confdir !CONFIGSET_DIR! -zkHost !ZK_HOST! %ZK_VERBOSE% @@ -1508,7 +1517,7 @@ IF "!ZK_OP!"=="upconfig" ( goto zk_short_usage ) ) - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -src !ZK_SRC! -dst !ZK_DST! -recurse !ZK_RECURSE! %ZK_VERBOSE% @@ -1521,7 +1530,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG=" must be specified for 'mv' command" goto zk_short_usage ) - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -src !ZK_SRC! -dst !ZK_DST! %ZK_VERBOSE% @@ -1530,7 +1539,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="Zookeeper path to remove must be specified when using the 'rm' command" goto zk_short_usage ) - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -path !ZK_SRC! -recurse !ZK_RECURSE! %ZK_VERBOSE% @@ -1539,7 +1548,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="Zookeeper path to remove must be specified when using the 'ls' command" goto zk_short_usage ) - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -path !ZK_SRC! -recurse !ZK_RECURSE! %ZK_VERBOSE% @@ -1548,7 +1557,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="Zookeeper path to create must be specified when using the 'mkroot' command" goto zk_short_usage ) - "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%SOLR_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -path !ZK_SRC! %ZK_VERBOSE% @@ -1609,7 +1618,7 @@ if "!AUTH_PORT!"=="" ( ) ) ) -"%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ +"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI auth %AUTH_PARAMS% -solrIncludeFile "%SOLR_INCLUDE%" -authConfDir "%SOLR_HOME%" ^ diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java index 323c378f71d..fb2347c3e82 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java @@ -16,6 +16,7 @@ */ package org.apache.solr.cloud; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -27,12 +28,17 @@ import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A distributed map. This supports basic map functions e.g. get, put, contains for interaction with * zk which don't have to be ordered i.e. DistributedQueue. */ public class DistributedMap { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected final String dir; protected SolrZkClient zookeeper; @@ -54,7 +60,14 @@ public DistributedMap(SolrZkClient zookeeper, String dir) { this.zookeeper = zookeeper; } + private void assertKeyFormat(String trackingId) { + if (trackingId == null || trackingId.length() == 0 || trackingId.contains("/")) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported key format: " + trackingId); + } + } + public void put(String trackingId, byte[] data) throws KeeperException, InterruptedException { + assertKeyFormat(trackingId); zookeeper.makePath( dir + "/" + PREFIX + trackingId, data, CreateMode.PERSISTENT, null, false, true); } @@ -62,10 +75,11 @@ public void put(String trackingId, byte[] data) throws KeeperException, Interrup /** * Puts an element in the map only if there isn't one with the same trackingId already * - * @return True if the the element was added. False if it wasn't (because the key already exists) + * @return True if the element was added. False if it wasn't (because the key already exists) */ public boolean putIfAbsent(String trackingId, byte[] data) throws KeeperException, InterruptedException { + assertKeyFormat(trackingId); try { zookeeper.makePath( dir + "/" + PREFIX + trackingId, data, CreateMode.PERSISTENT, null, true, true); @@ -94,10 +108,15 @@ public int size() throws KeeperException, InterruptedException { * not deleted exception an exception occurred while deleting */ public boolean remove(String trackingId) throws KeeperException, InterruptedException { + final var path = dir + "/" + PREFIX + trackingId; try { - zookeeper.delete(dir + "/" + PREFIX + trackingId, -1, true); + zookeeper.delete(path, -1, true); } catch (KeeperException.NoNodeException e) { return false; + } catch (KeeperException.NotEmptyException hack) { + // because dirty data before we enforced the rules on put() (trackingId shouldn't have slash) + log.warn("Cleaning malformed key ID starting with {}", path); + zookeeper.clean(path); } return true; } diff --git a/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java b/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java index ab495fffea8..c4c8923b218 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java +++ b/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java @@ -91,13 +91,11 @@ protected boolean lessThan(Long a, Long b) { for (String child : children) { Long id = childToModificationZxid.get(child); if (id != null && id <= topElementMzxId) { - try { - zookeeper.delete(dir + "/" + child, -1, true); - if (onOverflowObserver != null) - onOverflowObserver.onChildDelete(child.substring(PREFIX.length())); - } catch (KeeperException.NoNodeException ignored) { - // this could happen if multiple threads try to clean the same map - } + String trackingId = child.substring(PREFIX.length()); + boolean removed = remove(trackingId); + if (removed && onOverflowObserver != null) { + onOverflowObserver.onChildDelete(trackingId); + } // else, probably multiple threads cleaning the queue simultaneously } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestDistributedMap.java b/solr/core/src/test/org/apache/solr/cloud/TestDistributedMap.java index 0b3325e96e5..aa5915f007b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestDistributedMap.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestDistributedMap.java @@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.io.file.PathUtils; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -218,6 +219,36 @@ public void testClear() throws KeeperException, InterruptedException { } } + public void testMalformed() throws InterruptedException, KeeperException { + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkHost()) + .withTimeout(10000, TimeUnit.MILLISECONDS) + .build()) { + String path = getAndMakeInitialPath(zkClient); + DistributedMap map = createMap(zkClient, path); + expectThrows(SolrException.class, () -> map.put("has/slash", new byte[0])); + } + } + + public void testRemoveMalformed() throws InterruptedException, KeeperException { + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkHost()) + .withTimeout(10000, TimeUnit.MILLISECONDS) + .build()) { + String path = getAndMakeInitialPath(zkClient); + // Add a "legacy" / malformed key + final var key = "slash/test/0"; + zkClient.makePath(path + "/" + DistributedMap.PREFIX + key, new byte[0], true); + + DistributedMap map = createMap(zkClient, path); + assertEquals(1, map.size()); + map.remove("slash"); + assertEquals(0, map.size()); + } + } + protected DistributedMap createMap(SolrZkClient zkClient, String path) { return new DistributedMap(zkClient, path); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSizeLimitedDistributedMap.java b/solr/core/src/test/org/apache/solr/cloud/TestSizeLimitedDistributedMap.java index 493b7bf062b..a534e2e2a30 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSizeLimitedDistributedMap.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSizeLimitedDistributedMap.java @@ -25,9 +25,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.zookeeper.KeeperException; public class TestSizeLimitedDistributedMap extends TestDistributedMap { @@ -136,4 +138,32 @@ public void testConcurrentCleanup() throws Exception { protected DistributedMap createMap(SolrZkClient zkClient, String path) { return new SizeLimitedDistributedMap(zkClient, path, Overseer.NUM_RESPONSES_TO_STORE, null); } + + public void testCleanupForKeysWithSlashes() throws InterruptedException, KeeperException { + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkHost()) + .withTimeout(10000, TimeUnit.MILLISECONDS) + .build()) { + int maxEntries = 10; + String path = getAndMakeInitialPath(zkClient); + + // Add a "legacy" / malformed key + zkClient.makePath(path + "/" + DistributedMap.PREFIX + "slash/test/0", new byte[0], true); + + AtomicInteger overFlowCounter = new AtomicInteger(); + DistributedMap map = + new SizeLimitedDistributedMap( + zkClient, path, maxEntries, (element) -> overFlowCounter.incrementAndGet()); + + // Now add regular keys until we reach the size limit of the map. + // Once we hit the limit, the oldest item (the one we added above with slashes) is deleted, + // but that fails. + for (int i = 1; i <= maxEntries; ++i) { + map.put(String.valueOf(i), new byte[0]); + } + assertTrue(map.size() <= maxEntries); + assertEquals(1, overFlowCounter.get()); + } + } } diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index 310aa45049c..aa366eaafd6 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -256,6 +256,7 @@ task integrationTests(type: BatsTask) { environment SOLR_TIP: distDir.toString() environment SOLR_HOME: solrHome environment SOLR_LOGS_DIR: "$solrHome/logs" + environment TEST_OUTPUT_DIR: integrationTestOutput environment TEST_FAILURE_DIR: solrTestFailuresDir environment BATS_LIB_PREFIX: "$nodeProjectDir/node_modules" } diff --git a/solr/packaging/test/test_ssl.bats b/solr/packaging/test/test_ssl.bats index a4c397ae57a..bfde07b9570 100644 --- a/solr/packaging/test/test_ssl.bats +++ b/solr/packaging/test/test_ssl.bats @@ -57,6 +57,7 @@ teardown() { run solr create -c test -s 2 assert_output --partial "Created collection 'test'" +<<<<<<< HEAD run curl --http2 --cacert "$ssl_dir/solr-ssl.pem" -H "Host: test.solr.apache.org" 'https://127.0.0.1:8983/solr/test/select?q=*:*' assert_output --partial '"numFound":0' @@ -97,9 +98,55 @@ teardown() { assert_output --partial "Created collection 'test'" run curl --http2 --cacert "$ssl_dir/solr-ssl.pem" -k 'https://localhost:8983/solr/test/select?q=*:*' +======= + run curl --http2 --cacert "$ssl_dir/solr-ssl.pem" 'https://127.0.0.1:8983/solr/test/select?q=*:*' +>>>>>>> apache/main assert_output --partial '"numFound":0' } +@test "use different hostname when not checking peer-name" { + # Create a keystore + export ssl_dir="${BATS_TEST_TMPDIR}/ssl" + mkdir -p "$ssl_dir" + ( + cd "$ssl_dir" + rm -f solr-ssl.keystore.p12 solr-ssl.pem + # Using a CN that is not localhost, as we will not be checking peer-name + keytool -genkeypair -alias solr-ssl -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore solr-ssl.keystore.p12 -storetype PKCS12 -ext "SAN=DNS:test.solr.apache.org,IP:127.0.0.1" -dname "CN=test.solr.apache.org, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" + openssl pkcs12 -in solr-ssl.keystore.p12 -out solr-ssl.pem -passin pass:secret -passout pass:secret + ) + + # Set ENV_VARs so that Solr uses this keystore + export SOLR_SSL_ENABLED=true + export SOLR_SSL_KEY_STORE=$ssl_dir/solr-ssl.keystore.p12 + export SOLR_SSL_KEY_STORE_PASSWORD=secret + export SOLR_SSL_TRUST_STORE=$ssl_dir/solr-ssl.keystore.p12 + export SOLR_SSL_TRUST_STORE_PASSWORD=secret + export SOLR_SSL_NEED_CLIENT_AUTH=false + export SOLR_SSL_WANT_CLIENT_AUTH=false + export SOLR_SSL_CHECK_PEER_NAME=false + # Remove later when SOLR-16963 is resolved + export SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false + export SOLR_HOST=localhost + + solr start -c + solr assert --started https://localhost:8983/solr --timeout 5000 + + run solr create -c test -s 2 + assert_output --partial "Created collection 'test'" + + run curl --http2 --cacert "$ssl_dir/solr-ssl.pem" -k 'https://localhost:8983/solr/test/select?q=*:*' + assert_output --partial '"numFound":0' + + export SOLR_SSL_CHECK_PEER_NAME=true + # Remove later when SOLR-16963 is resolved + export SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=true + + # This should fail the peername check + run ! solr api -get 'https://localhost:8983/solr/test/select?q=*:*' + assert_output --partial 'Server refused connection' +} + @test "start solr with ssl and auth" { # Create a keystore export ssl_dir="${BATS_TEST_TMPDIR}/ssl" @@ -204,7 +251,7 @@ teardown() { export SOLR_SECURITY_MANAGER_ENABLED=true export SOLR_OPTS="-Djava.io.tmpdir=${test_tmp_dir}" export SOLR_LOG_LEVEL="DEBUG" - export SOLR_TOOL_OPTS="-Djava.io.tmpdir=${test_tmp_dir}" + export SOLR_TOOL_OPTS="-Djava.io.tmpdir=${test_tmp_dir} -Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake" export ssl_dir="${BATS_TEST_TMPDIR}/ssl" export server_ssl_dir="${ssl_dir}/server" @@ -439,8 +486,8 @@ teardown() { export SOLR_SSL_TRUST_STORE= export SOLR_SSL_TRUST_STORE_PASSWORD= - solr assert --started https://localhost:8984/solr --timeout 5000 solr assert --started https://localhost:8983/solr --timeout 5000 + solr assert --started https://localhost:8984/solr --timeout 5000 run solr create -c test -s 2 assert_output --partial "Created collection 'test'" @@ -454,7 +501,7 @@ teardown() { export SOLR_SSL_CLIENT_KEY_STORE= export SOLR_SSL_CLIENT_KEY_STORE_PASSWORD= - # mTLS requires a keyStore, so just using the truststore would fail if mTLS was "NEED"ed, however it is only "WANT"ed, so its ok. + # mTLS requires a keyStore, so just using the truststore would fail if mTLS was "NEED"ed, however it is only "WANT"ed, so its ok run solr api -get 'https://localhost:8983/solr/test/select?q=*:*&rows=0' assert_output --partial '"numFound":0' } diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/collections-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/collections-api.adoc index 59955461b91..452c384bcc0 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/collections-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/collections-api.adoc @@ -36,6 +36,7 @@ Because this API has a large number of commands and options, we've grouped the c Since some collection API calls can be long running tasks (such as SPLITSHARD), you can optionally have the calls run asynchronously. Specifying `async=` enables you to make an asynchronous call, the status of which can be requested using the <> call at any time. +The ID provided can be any string so long as it doesn't have a `/` in it. As of now, REQUESTSTATUS does not automatically clean up the tracking data structures, meaning the status of completed or failed tasks stays stored in ZooKeeper unless cleared manually. DELETESTATUS can be used to clear the stored statuses.