SOLR-17540: Remove Hadoop Auth Module

.github/labeler.yml

@@ -122,11 +122,6 @@ module:gcs-repository:
       - any-glob-to-any-file:
           - solr/modules/gcs-repository/**
 
-module:hadoop-auth:
-  - changed-files:
-      - any-glob-to-any-file:
-          - solr/modules/hadoop-auth/**
-
 module:hdfs:
   - changed-files:
       - any-glob-to-any-file:

gradle/libs.versions.toml

@@ -38,7 +38,6 @@ apache-httpcomponents-httpclient = "4.5.14"
 apache-httpcomponents-httpcore = "4.4.16"
 apache-httpcomponents-httpmime = "4.5.14"
 apache-kafka = "3.7.1"
-apache-kerby = "2.0.3"
 apache-log4j = "2.21.0"
 apache-lucene = "9.11.1"
 apache-opennlp = "1.9.4"
@@ -228,14 +227,10 @@ apache-curator-client = { module = "org.apache.curator:curator-client", version.
 apache-curator-framework = { module = "org.apache.curator:curator-framework", version.ref = "apache-curator" }
 apache-curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref = "apache-curator" }
 apache-curator-test = { module = "org.apache.curator:curator-test", version.ref = "apache-curator" }
-apache-hadoop-annotations = { module = "org.apache.hadoop:hadoop-annotations", version.ref = "apache-hadoop" }
-apache-hadoop-auth = { module = "org.apache.hadoop:hadoop-auth", version.ref = "apache-hadoop" }
 apache-hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.ref = "apache-hadoop" }
 apache-hadoop-client-minicluster = { module = "org.apache.hadoop:hadoop-client-minicluster", version.ref = "apache-hadoop" }
 apache-hadoop-client-runtime = { module = "org.apache.hadoop:hadoop-client-runtime", version.ref = "apache-hadoop" }
-apache-hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "apache-hadoop" }
 apache-hadoop-hdfs = { module = "org.apache.hadoop:hadoop-hdfs", version.ref = "apache-hadoop" }
-apache-hadoop-minikdc = { module = "org.apache.hadoop:hadoop-minikdc", version.ref = "apache-hadoop" }
 apache-hadoop-thirdparty-shadedguava = { module = "org.apache.hadoop.thirdparty:hadoop-shaded-guava", version.ref = "apache-hadoop-thirdparty" }
 apache-httpcomponents-httpclient = { module = "org.apache.httpcomponents:httpclient", version.ref = "apache-httpcomponents-httpclient" }
 apache-httpcomponents-httpcore = { module = "org.apache.httpcomponents:httpcore", version.ref = "apache-httpcomponents-httpcore" }
@@ -244,8 +239,6 @@ apache-kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref
 apache-kafka-kafka213 = { module = "org.apache.kafka:kafka_2.13", version.ref = "apache-kafka" }
 apache-kafka-server-common = { module = "org.apache.kafka:kafka-server-common", version.ref = "apache-kafka" }
 apache-kafka-streams = { module = "org.apache.kafka:kafka-streams", version.ref = "apache-kafka" }
-apache-kerby-core = { module = "org.apache.kerby:kerb-core", version.ref = "apache-kerby" }
-apache-kerby-util = { module = "org.apache.kerby:kerb-util", version.ref = "apache-kerby" }
 apache-log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "apache-log4j" }
 apache-log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "apache-log4j" }
 apache-log4j-jul = { module = "org.apache.logging.log4j:log4j-jul", version.ref = "apache-log4j" }

gradle/testing/randomization/policies/solr-tests.policy

@@ -100,7 +100,7 @@ grant {
   permission java.lang.RuntimePermission "closeClassLoader";
   // needed by HttpSolrClient
   permission java.lang.RuntimePermission "getFileSystemAttributes";
-  // needed by hadoop auth (TODO: there is a cleaner way to handle this)
+  // needed by hadoop hdfs (TODO: there is a cleaner way to handle this)
   permission java.lang.RuntimePermission "loadLibrary.jaas";
   permission java.lang.RuntimePermission "loadLibrary.jaas_unix";
   permission java.lang.RuntimePermission "loadLibrary.jaas_nt";

gradle/validation/dependencies.gradle

@@ -179,9 +179,6 @@ allprojects {
                 handler.add(conf.name, libs.apache.httpcomponents.httpmime, {
                     because 'version alignment for consistency across project'
                 })
-                handler.add(conf.name, libs.apache.kerby.core, {
-                    because 'version alignment for consistency across project'
-                })
                 handler.add(conf.name, libs.apache.zookeeper.zookeeper, {
                     because 'version alignment for consistency across project'
                 })

gradle/validation/rat-sources.gradle

@@ -106,10 +106,6 @@ allprojects {
                     exclude "src/test-files/META-INF/services/*"
                     break
 
-                case ":solr:modules:hadoop-auth":
-                    exclude "src/test-files/**/*.conf"
-                    break
-
                 case ":solr:modules:hdfs":
                     exclude "src/test-files/**/*.aff"
                     exclude "src/test-files/**/*.dic"

settings.gradle

@@ -48,7 +48,6 @@ include "solr:modules:cross-dc"
 include "solr:modules:opentelemetry"
 include "solr:modules:extraction"
 include "solr:modules:gcs-repository"
-include "solr:modules:hadoop-auth"
 include "solr:modules:hdfs"
 include "solr:modules:jwt-auth"
 include "solr:modules:langid"

solr/bin/solr

@@ -318,13 +318,13 @@ fi
 if [ -z "${SOLR_AUTH_TYPE:-}" ] && [ -n "${SOLR_AUTHENTICATION_OPTS:-}" ]; then
   echo "WARNING: SOLR_AUTHENTICATION_OPTS environment variable configured without associated SOLR_AUTH_TYPE variable"
   echo "         Please configure SOLR_AUTH_TYPE environment variable with the authentication type to be used."
-  echo "         Currently supported authentication types are [kerberos, basic]"
+  echo "         Currently supported authentication types are [basic]"
 fi
 
 if [ -n "${SOLR_AUTH_TYPE:-}" ] && [ -n "${SOLR_AUTHENTICATION_CLIENT_BUILDER:-}" ]; then
   echo "WARNING: SOLR_AUTHENTICATION_CLIENT_BUILDER and SOLR_AUTH_TYPE environment variables are configured together."
   echo "         Use SOLR_AUTH_TYPE environment variable to configure authentication type to be used. "
-  echo "         Currently supported authentication types are [kerberos, basic]"
+  echo "         Currently supported authentication types are [basic]"
   echo "         The value of SOLR_AUTHENTICATION_CLIENT_BUILDER environment variable will be ignored"
 fi
 
@@ -333,9 +333,6 @@ if [ -n "${SOLR_AUTH_TYPE:-}" ]; then
     basic)
       SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
       ;;
-    kerberos)
-      SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder"
-      ;;
     *)
       echo "ERROR: Value specified for SOLR_AUTH_TYPE environment variable is invalid."
       exit 1

solr/bin/solr.cmd

@@ -203,15 +203,15 @@ IF NOT DEFINED SOLR_AUTH_TYPE (
   IF DEFINED SOLR_AUTHENTICATION_OPTS (
     echo WARNING: SOLR_AUTHENTICATION_OPTS variable configured without associated SOLR_AUTH_TYPE variable
     echo          Please configure SOLR_AUTH_TYPE variable with the authentication type to be used.
-    echo          Currently supported authentication types are [kerberos, basic]
+    echo          Currently supported authentication types are [basic]
   )
 )
 
 IF DEFINED SOLR_AUTH_TYPE (
   IF DEFINED SOLR_AUTHENTICATION_CLIENT_BUILDER (
     echo WARNING: SOLR_AUTHENTICATION_CLIENT_BUILDER and SOLR_AUTH_TYPE variables are configured together
     echo          Use SOLR_AUTH_TYPE variable to configure authentication type to be used
-    echo          Currently supported authentication types are [kerberos, basic]
+    echo          Currently supported authentication types are [basic]
     echo          The value of SOLR_AUTHENTICATION_CLIENT_BUILDER configuration variable will be ignored
   )
 )
@@ -220,12 +220,8 @@ IF DEFINED SOLR_AUTH_TYPE (
   IF /I "%SOLR_AUTH_TYPE%" == "basic" (
     set SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
   ) ELSE (
-    IF /I "%SOLR_AUTH_TYPE%" == "kerberos" (
-      set SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
-    ) ELSE (
-      echo ERROR: Value specified for SOLR_AUTH_TYPE configuration variable is invalid.
-      goto err
-    )
+    echo ERROR: Value specified for SOLR_AUTH_TYPE configuration variable is invalid.
+    goto err
   )
 )
 

solr/core/src/java/org/apache/solr/cli/AuthTool.java

@@ -30,7 +30,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
-import java.util.Base64;
 import java.util.List;
 import java.util.Locale;
 import java.util.stream.Collectors;
@@ -39,7 +38,6 @@
 import org.apache.commons.cli.Options;
 import org.apache.lucene.util.Constants;
 import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.security.Sha256AuthenticationProvider;
 import org.apache.zookeeper.KeeperException;
@@ -52,7 +50,7 @@ public class AuthTool extends ToolBase {
           .longOpt("type")
           .hasArg()
           .desc(
-              "The authentication mechanism to enable (basicAuth or kerberos). Defaults to 'basicAuth'.")
+              "The authentication mechanism to enable (currently only basicAuth). Defaults to 'basicAuth'.")
           .build();
 
   private static final Option PROMPT_OPTION =
@@ -64,14 +62,6 @@ public class AuthTool extends ToolBase {
               "Prompts the user to provide the credentials. Use either --credentials or --prompt, not both.")
           .build();
 
-  private static final Option CONFIG_OPTION =
-      Option.builder()
-          .longOpt("config")
-          .hasArgs()
-          .desc(
-              "Configuration parameters (Solr startup parameters). Required for Kerberos authentication.")
-          .build();
-
   private static final Option BLOCK_UNKNOWN_OPTION =
       Option.builder()
           .longOpt("block-unknown")
@@ -152,7 +142,6 @@ public Options getOptions() {
     return super.getOptions()
         .addOption(TYPE_OPTION)
         .addOption(PROMPT_OPTION)
-        .addOption(CONFIG_OPTION)
         .addOption(BLOCK_UNKNOWN_OPTION)
         .addOption(SOLR_INCLUDE_FILE_OPTION)
         .addOption(UPDATE_INCLUDE_FILE_OPTION)
@@ -176,118 +165,6 @@ private void ensureArgumentIsValidBooleanIfPresent(CommandLine cli, Option optio
     }
   }
 
-  private void handleKerberos(CommandLine cli) throws Exception {
-    String cmd = cli.getArgs()[0];
-    boolean updateIncludeFileOnly =
-        Boolean.parseBoolean(cli.getOptionValue(UPDATE_INCLUDE_FILE_OPTION, "false"));
-    String securityJson =
-        "{"
-            + "\n  \"authentication\":{"
-            + "\n   \"class\":\"solr.KerberosPlugin\""
-            + "\n  }"
-            + "\n}";
-
-    switch (cmd) {
-      case "enable":
-        String zkHost = null;
-        boolean zkInaccessible = false;
-
-        if (!updateIncludeFileOnly) {
-          try {
-            zkHost = CLIUtils.getZkHost(cli);
-          } catch (Exception ex) {
-            CLIO.out(
-                "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n"
-                    + securityJson
-                    + "Dsolr.httpclient.config=[basicAuthConfFile]\n");
-            zkInaccessible = true;
-          }
-          if (zkHost == null) {
-            if (!zkInaccessible) {
-              CLIO.out(
-                  "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n"
-                      + securityJson
-                      + "\n");
-              zkInaccessible = true;
-            }
-          }
-
-          // check if security is already enabled or not
-          if (!zkInaccessible) {
-            try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) {
-              checkSecurityJsonExists(zkClient);
-            } catch (Exception ex) {
-              CLIO.out(
-                  "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n"
-                      + securityJson
-                      + "\n");
-              zkInaccessible = true;
-            }
-          }
-        }
-
-        if (!updateIncludeFileOnly) {
-          if (!zkInaccessible) {
-            echoIfVerbose("Uploading following security.json: " + securityJson);
-            try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) {
-              zkClient.setData(
-                  "/security.json", securityJson.getBytes(StandardCharsets.UTF_8), true);
-            } catch (Exception ex) {
-              CLIO.out(
-                  "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n"
-                      + securityJson);
-            }
-          }
-        }
-
-        String config = StrUtils.join(Arrays.asList(cli.getOptionValues(CONFIG_OPTION)), ' ');
-        // config is base64 encoded (to get around parsing problems), decode it
-        config = config.replace(" ", "");
-        config =
-            new String(
-                Base64.getDecoder().decode(config.getBytes(StandardCharsets.UTF_8)),
-                StandardCharsets.UTF_8);
-        config = config.replace("\n", "").replace("\r", "");
-
-        String solrIncludeFilename = cli.getOptionValue(SOLR_INCLUDE_FILE_OPTION);
-        File includeFile = new File(solrIncludeFilename);
-        if (!includeFile.exists() || !includeFile.canWrite()) {
-          CLIO.out(
-              "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable.");
-          printAuthEnablingInstructions(config);
-          System.exit(0);
-        }
-
-        // update the solr.in.sh file to contain the necessary authentication lines
-        updateIncludeFileEnableAuth(includeFile.toPath(), null, config);
-        echo(
-            "Successfully enabled Kerberos authentication; please restart any running Solr nodes.");
-        return;
-      case "disable":
-        clearSecurityJson(cli, updateIncludeFileOnly);
-
-        solrIncludeFilename = cli.getOptionValue(SOLR_INCLUDE_FILE_OPTION);
-        includeFile = new File(solrIncludeFilename);
-        if (!includeFile.exists() || !includeFile.canWrite()) {
-          CLIO.out(
-              "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable.");
-          CLIO.out(
-              "Security has been disabled. Please remove any SOLR_AUTH_TYPE or SOLR_AUTHENTICATION_OPTS configuration from solr.in.sh/solr.in.cmd.\n");
-          System.exit(0);
-        }
-
-        // update the solr.in.sh file to comment out the necessary authentication lines
-        updateIncludeFileDisableAuth(includeFile.toPath());
-        return;
-      default:
-        CLIO.out("Valid auth commands are: enable, disable.");
-        SolrCLI.exit(1);
-    }
-
-    CLIO.out("Options not understood.");
-    SolrCLI.exit(1);
-  }
-
   private void handleBasicAuth(CommandLine cli) throws Exception {
     String cmd = cli.getArgs()[0];
     boolean prompt = Boolean.parseBoolean(cli.getOptionValue(PROMPT_OPTION, "false"));
@@ -409,8 +286,7 @@ private void handleBasicAuth(CommandLine cli) throws Exception {
             StandardCharsets.UTF_8);
 
         // update the solr.in.sh file to contain the necessary authentication lines
-        updateIncludeFileEnableAuth(
-            includeFile.toPath(), basicAuthConfFile.getAbsolutePath(), null);
+        updateIncludeFileEnableAuth(includeFile.toPath(), basicAuthConfFile.getAbsolutePath());
         final String successMessage =
             String.format(
                 Locale.ROOT,
@@ -498,40 +374,16 @@ private void printAuthEnablingInstructions(String username, String password) {
     }
   }
 
-  private void printAuthEnablingInstructions(String kerberosConfig) {
-    if (Constants.WINDOWS) {
-      CLIO.out(
-          "\nAdd the following lines to the solr.in.cmd file so that the solr.cmd script can use subsequently.\n");
-      CLIO.out(
-          "set SOLR_AUTH_TYPE=kerberos\n"
-              + "set SOLR_AUTHENTICATION_OPTS=\""
-              + kerberosConfig
-              + "\"\n");
-    } else {
-      CLIO.out(
-          "\nAdd the following lines to the solr.in.sh file so that the ./solr script can use subsequently.\n");
-      CLIO.out(
-          "SOLR_AUTH_TYPE=\"kerberos\"\n"
-              + "SOLR_AUTHENTICATION_OPTS=\""
-              + kerberosConfig
-              + "\"\n");
-    }
-  }
-
   /**
    * This will update the include file (e.g. solr.in.sh / solr.in.cmd) with the authentication
    * parameters.
    *
    * @param includeFile The include file
    * @param basicAuthConfFile If basicAuth, the path of the file containing credentials. If not,
    *     null.
-   * @param kerberosConfig If kerberos, the config string containing startup parameters. If not,
-   *     null.
    */
-  private void updateIncludeFileEnableAuth(
-      Path includeFile, String basicAuthConfFile, String kerberosConfig) throws IOException {
-    assert !(basicAuthConfFile != null
-        && kerberosConfig != null); // only one of the two needs to be populated
+  private void updateIncludeFileEnableAuth(Path includeFile, String basicAuthConfFile)
+      throws IOException {
     List<String> includeFileLines = Files.readAllLines(includeFile, StandardCharsets.UTF_8);
     for (int i = 0; i < includeFileLines.size(); i++) {
       String line = includeFileLines.get(i);
@@ -558,17 +410,6 @@ private void updateIncludeFileEnableAuth(
         includeFileLines.add(
             "SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=" + basicAuthConfFile + "\"");
       }
-    } else { // for kerberos
-      if (Constants.WINDOWS) {
-        includeFileLines.add("REM The following lines added by solr.cmd for enabling BasicAuth");
-        includeFileLines.add("set SOLR_AUTH_TYPE=kerberos");
-        includeFileLines.add(
-            "set SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=basicAuthConfFile\"");
-      } else {
-        includeFileLines.add("# The following lines added by ./solr for enabling BasicAuth");
-        includeFileLines.add("SOLR_AUTH_TYPE=\"kerberos\"");
-        includeFileLines.add("SOLR_AUTHENTICATION_OPTS=\"" + kerberosConfig + "\"");
-      }
     }
 
     String lines = includeFileLines.stream().collect(Collectors.joining(System.lineSeparator()));
@@ -609,15 +450,13 @@ public void runImpl(CommandLine cli) throws Exception {
     ensureArgumentIsValidBooleanIfPresent(cli, UPDATE_INCLUDE_FILE_OPTION);
 
     String type = cli.getOptionValue(TYPE_OPTION, "basicAuth");
+    // switch structure is here to support future auth options like oAuth
     switch (type) {
       case "basicAuth":
         handleBasicAuth(cli);
         break;
-      case "kerberos":
-        handleKerberos(cli);
-        break;
       default:
-        throw new IllegalStateException("Only type=basicAuth or kerberos supported at the moment.");
+        throw new IllegalStateException("Only type=basicAuth supported at the moment.");
     }
   }
 }