From bc098889f71f56734b293d421d31c6b0acecb52f Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Mon, 18 Dec 2023 17:56:17 +0800 Subject: [PATCH 1/8] [ISSUE-1190]feat(server-common): Support custom filter --- .../lakehouse/iceberg/IcebergRESTService.java | 1 + docs/security.md | 50 ++++++++++++++++++- .../gravitino/server/web/JettyServer.java | 17 ++++++- .../server/web/JettyServerConfig.java | 28 +++++++++-- .../gravitino/server/GravitinoServer.java | 1 + 5 files changed, 90 insertions(+), 7 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java index cea2df4440d..bf23f6c0393 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java @@ -61,6 +61,7 @@ protected void configure() { Servlet servlet = new ServletContainer(config); server.addServlet(servlet, "/iceberg/*"); + server.addCustomFilters("/iceberg/*"); server.addFilter(new AuthenticationFilter(), "/iceberg/*"); } diff --git a/docs/security.md b/docs/security.md index 9919c9d6fe8..03892e14c73 100644 --- a/docs/security.md +++ b/docs/security.md @@ -173,7 +173,7 @@ Both Gravitino server and Iceberg REST service can configure HTTPS. | `gravitino.auxService.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.uxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | @@ -258,3 +258,51 @@ If you want to use the command `curl`, you can follow the commands: openssl x509 -inform der -in $JAVA_HOME/localhost.crt -out certificate.pem curl -v -X GET --cacert ./certificate.pem -H "Accept: application/vnd.gravitino.v1+json" -H "Content-Type: application/json" https://localhost:8433/api/version ``` + +## Custom filter + +Gravitino supports custom filters to implement the user specified logic to satisfy different safety needs. + +### Gravitino server's configuration +| Configuration item | Description | Default value | Required | Since version | +|---------------------------------------------|------------------------------------------------------------------|---------------|-----------|---------------| +| `gravitino.server.webserver.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | Yes | 0.4.0 | +The filter should be a standard javax servlet Filter. +Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.server.webserver..param.=` + +For example: +```text +gravitino.server.webserver.customFilters=com.test.filter1 +gravitino.server.webserver.com.test.filter1.param.name1=foo +gravitino.server.webserver.com.test.filter1.param.name2=ba +``` + +### Iceberg REST service's configuration +| Configuration item | Description | Default value | Required | Since version | +|-----------------------------------------------------|--------------------------------------------------------------------|---------------|-----------|---------------| +| `gravitino.auxService.iceberg-rest.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | Yes | 0.4.0 | +The filter should be a standard javax servlet Filter. +Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.auxService.iceberg-rest..param.=` + +For example: +```text +gravitino.auxService.iceberg-rest.customFilters=com.test.filter1 +gravitino.auxService.iceberg-rest.com.test.filter1.param.name1=foo +gravitino.auxService.iceberg-rest.com.test.filter1.param.name2=ba +``` + +### Example + +If you want to use a cross-origin filter for the Gravitino server, you can follow the steps. + +```shell +wget https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-servlets/9.4.51.v20230217/jetty-servlets-9.4.51.v20230217.jar +cp jetty-servlets-9.4.51.v20230217.jar /lib +``` + +You can refer to the [Configurations](gravitino-server-config.md) and append the configurations to the conf/gravitino.conf. + +```text +gravitino.server.webserver.customFilter=org.eclipse.jetty.servlets.CrossOriginFilter +gravitino.server.webserver.org.eclipse.jetty.servlets.CrossOriginFilter.param.allowedOrigins=* +``` \ No newline at end of file diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java index cd6592d2ee8..db9075479c7 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java @@ -19,6 +19,7 @@ import java.security.AccessController; import java.security.PrivilegedAction; import java.util.EnumSet; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; @@ -104,7 +105,7 @@ public synchronized void initialize( StringUtils.isNotBlank(serverConfig.getTrustStorePath()), "If enables the authentication of the client, must set trustStorePath"); Preconditions.checkArgument( - StringUtils.isNotBlank(serverConfig.getTrustStorePasword()), + StringUtils.isNotBlank(serverConfig.getTrustStorePassword()), "If enables the authentication of the client, must set trustStorePassword"); } ServerConnector httpsConnector = @@ -123,7 +124,7 @@ public synchronized void initialize( serverConfig.getSupportedAlgorithms(), serverConfig.isEnableClientAuth(), serverConfig.getTrustStorePath(), - serverConfig.getTrustStorePasword(), + serverConfig.getTrustStorePassword(), serverConfig.getTrustStoreType()); server.addConnector(httpsConnector); } else { @@ -433,4 +434,16 @@ public Thread run() { public ThreadPool getThreadPool() { return server.getThreadPool(); } + + public void addCustomFilters(String pathSpec) { + for (String filterName : serverConfig.getCustomFilters()) { + FilterHolder filterHolder = new FilterHolder(); + filterHolder.setClassName(filterName); + for (Map.Entry entry : + serverConfig.getAllWithPrefix(String.format("%s.param.", filterName)).entrySet()) { + filterHolder.setInitParameter(entry.getKey(), entry.getValue()); + } + servletContextHandler.addFilter(filterHolder, pathSpec, EnumSet.allOf(DispatcherType.class)); + } + } } diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java index 90c68b234e1..1fa5b61a12c 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java @@ -171,6 +171,13 @@ public final class JettyServerConfig { .stringConf() .createWithDefault("JKS"); + public static final ConfigEntry CUSTOM_FILTERS = + new ConfigBuilder("customFilters") + .doc("Comma separated list of filter class names to apply to the APIs") + .version("0.4.0") + .stringConf() + .createWithDefault(""); + private final String host; private final int httpPort; @@ -199,7 +206,9 @@ public final class JettyServerConfig { private final Set enableCipherAlgorithms; private final boolean enableClientAuth; private final String trustStorePath; - private final String trustStorePasword; + private final String trustStorePassword; + + private final Set customFilters; private final String trustStoreType; private final Config internalConfig; @@ -245,8 +254,11 @@ private JettyServerConfig(Map configs) { Sets.newHashSet(internalConfig.get(ENABLE_CIPHER_ALGORITHMS).split(SPLITTER))); this.enableClientAuth = internalConfig.get(ENABLE_CLIENT_AUTH); this.trustStorePath = internalConfig.get(SSL_TRUST_STORE_PATH); - this.trustStorePasword = internalConfig.get(SSL_TRUST_STORE_PASSWORD); + this.trustStorePassword = internalConfig.get(SSL_TRUST_STORE_PASSWORD); this.trustStoreType = internalConfig.get(SSL_TRUST_STORE_TYPE); + this.customFilters = + Collections.unmodifiableSet( + Sets.newHashSet(internalConfig.get(CUSTOM_FILTERS).split(SPLITTER))); } public static JettyServerConfig fromConfig(Config config, String prefix) { @@ -330,14 +342,18 @@ public String getTrustStorePath() { return trustStorePath; } - public String getTrustStorePasword() { - return trustStorePasword; + public String getTrustStorePassword() { + return trustStorePassword; } public String getTrustStoreType() { return trustStoreType; } + public Map getAllWithPrefix(String prefix) { + return internalConfig.getConfigsWithPrefix(prefix); + } + private SSLContext getDefaultSSLContext() { try { return SSLContext.getDefault(); @@ -366,6 +382,10 @@ public Set getSupportedAlgorithms() { return supportedAlgorithms; } + public Set getCustomFilters() { + return customFilters; + } + @VisibleForTesting Set getSupportedCipherSuites() { SSLContext context = diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index 415078a60da..5e0bd678de1 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -87,6 +87,7 @@ protected void configure() { server.addServlet(servlet, "/api/*"); Servlet configServlet = new ConfigServlet(serverConfig); server.addServlet(configServlet, "/configs"); + server.addCustomFilters("/api/*"); server.addFilter(new VersioningFilter(), "/api/*"); server.addFilter(new AuthenticationFilter(), "/api/*"); } From 18faf4d871732382845a53e75b48fc207e8353d8 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Mon, 18 Dec 2023 19:22:50 +0800 Subject: [PATCH 2/8] fix --- docs/security.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/security.md b/docs/security.md index 03892e14c73..3b21848c245 100644 --- a/docs/security.md +++ b/docs/security.md @@ -259,7 +259,7 @@ openssl x509 -inform der -in $JAVA_HOME/localhost.crt -out certificate.pem curl -v -X GET --cacert ./certificate.pem -H "Accept: application/vnd.gravitino.v1+json" -H "Content-Type: application/json" https://localhost:8433/api/version ``` -## Custom filter +## Custom filters Gravitino supports custom filters to implement the user specified logic to satisfy different safety needs. From d5272f50f187763fa24e70ec9be47fba089fed63 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Tue, 19 Dec 2023 15:35:26 +0800 Subject: [PATCH 3/8] fix --- docs/security.md | 4 ++-- .../java/com/datastrato/gravitino/server/web/JettyServer.java | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/security.md b/docs/security.md index 3b21848c245..8b78796e6e0 100644 --- a/docs/security.md +++ b/docs/security.md @@ -297,12 +297,12 @@ If you want to use a cross-origin filter for the Gravitino server, you can follo ```shell wget https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-servlets/9.4.51.v20230217/jetty-servlets-9.4.51.v20230217.jar -cp jetty-servlets-9.4.51.v20230217.jar /lib +cp jetty-servlets-9.4.51.v20230217.jar /libs ``` You can refer to the [Configurations](gravitino-server-config.md) and append the configurations to the conf/gravitino.conf. ```text -gravitino.server.webserver.customFilter=org.eclipse.jetty.servlets.CrossOriginFilter +gravitino.server.webserver.customFilters=org.eclipse.jetty.servlets.CrossOriginFilter gravitino.server.webserver.org.eclipse.jetty.servlets.CrossOriginFilter.param.allowedOrigins=* ``` \ No newline at end of file diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java index db9075479c7..10f89c95c9a 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java @@ -437,6 +437,9 @@ public ThreadPool getThreadPool() { public void addCustomFilters(String pathSpec) { for (String filterName : serverConfig.getCustomFilters()) { + if (StringUtils.isBlank(filterName)) { + continue; + } FilterHolder filterHolder = new FilterHolder(); filterHolder.setClassName(filterName); for (Map.Entry entry : From 55184a69753a8d9dedac9d7e75287a6ad58e9d1d Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Wed, 20 Dec 2023 11:24:05 +0800 Subject: [PATCH 4/8] [MINOR] fix(docs): Modify the required value for the document --- docs/security.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/security.md b/docs/security.md index 8b78796e6e0..03e313ae243 100644 --- a/docs/security.md +++ b/docs/security.md @@ -55,11 +55,11 @@ GravitinoClient client = GravitinoClient.builder(uri) | Configuration item | Description | Default value | Required | Since version | |---------------------------------------------------|-----------------------------------------------------------------------------|-------------------|-----------------------------------------|---------------| -| `gravitino.authenticator` | The authenticator which Gravitino uses, setting as `simple` or `oauth`. | `simple` | Yes | 0.3.0 | -| `gravitino.authenticator.oauth.serviceAudience` | The audience name when Gravitino uses OAuth as the authenticator. | `GravitinoServer` | Yes if use `oauth` as the authenticator | 0.3.0 | -| `gravitino.authenticator.oauth.allowSkewSecs` | The JWT allows skew seconds when Gravitino uses OAuth as the authenticator. | `0` | Yes if use `oauth` as the authenticator | 0.3.0 | +| `gravitino.authenticator` | The authenticator which Gravitino uses, setting as `simple` or `oauth`. | `simple` | No | 0.3.0 | +| `gravitino.authenticator.oauth.serviceAudience` | The audience name when Gravitino uses OAuth as the authenticator. | `GravitinoServer` | No | 0.3.0 | +| `gravitino.authenticator.oauth.allowSkewSecs` | The JWT allows skew seconds when Gravitino uses OAuth as the authenticator. | `0` | No | 0.3.0 | | `gravitino.authenticator.oauth.defaultSignKey` | The signing key of JWT when Gravitino uses OAuth as the authenticator. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | -| `gravitino.authenticator.oauth.signAlgorithmType` | The signature algorithm when Gravitino uses OAuth as the authenticator. | `RS256` | Yes if use `oauth` as the authenticator | 0.3.0 | +| `gravitino.authenticator.oauth.signAlgorithmType` | The signature algorithm when Gravitino uses OAuth as the authenticator. | `RS256` | No | 0.3.0 | | `gravitino.authenticator.oauth.serverUri` | The URI of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | | `gravitino.authenticator.oauth.tokenPath` | The path for token of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | @@ -152,35 +152,35 @@ Both Gravitino server and Iceberg REST service can configure HTTPS. | Configuration item | Description | Default value | Required | Since version | |-----------------------------------------------------|--------------------------------------------------------------------|---------------|---------------------------------------------------|---------------| -| `gravitino.server.webserver.enableHttps` | Enables HTTPS. | `false` | Yes | 0.3.0 | -| `gravitino.server.webserver.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | +| `gravitino.server.webserver.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | No | 0.3.0 | | `gravitino.server.webserver.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.server.webserver.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | | `gravitino.server.webserver.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | none | No | 0.3.0 | -| `gravitino.server.webserver.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.enableClientAuth` | Enables the authentication of the client. | `false` | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.trustStorePath` | Path to the trust store file. | none | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.trustStorePassword` | Password to the trust store. | none | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.server.webserver.trustStoreType` | The type to the trust store. | `JKS` | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.server.webserver.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | +| `gravitino.server.webserver.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | +| `gravitino.server.webserver.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | +| `gravitino.server.webserver.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.server.webserver.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.server.webserver.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | ### Iceberg REST service's configuration | Configuration item | Description | Default value | Required | Since version | |------------------------------------------------------------|--------------------------------------------------------------------|---------------|---------------------------------------------------|---------------| -| `gravitino.auxService.iceberg-rest.enableHttps` | Enables HTTPS. | `false` | Yes | 0.3.0 | -| `gravitino.auxService.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | +| `gravitino.auxService.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | No | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.auxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | | `gravitino.auxService.iceberg-rest.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.auxService.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | | `gravitino.auxService.iceberg-rest.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | -| `gravitino.auxService.iceberg-rest.enableClientAuth` | Enables the authentication of the client. | `false` | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | | `gravitino.auxService.iceberg-rest.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | | `gravitino.auxService.iceberg-rest.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.auxService.iceberg-rest.trustStoreType` | The type to the trust store. | `JKS` | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.auxService.iceberg-rest.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | Refer to the "Additional JSSE Standard Names" section of the [Java security guide](https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#jssenames) for the list of protocols related to tlsProtocol. You can find the list of `tlsProtocol` values for Java 8 in this document. From 239e55d1d5b41a4ac657af18b84fb43194aa05e5 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Wed, 20 Dec 2023 11:32:33 +0800 Subject: [PATCH 5/8] fix --- docs/security.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/security.md b/docs/security.md index 03e313ae243..5adc503ec5f 100644 --- a/docs/security.md +++ b/docs/security.md @@ -264,9 +264,9 @@ curl -v -X GET --cacert ./certificate.pem -H "Accept: application/vnd.gravitino. Gravitino supports custom filters to implement the user specified logic to satisfy different safety needs. ### Gravitino server's configuration -| Configuration item | Description | Default value | Required | Since version | -|---------------------------------------------|------------------------------------------------------------------|---------------|-----------|---------------| -| `gravitino.server.webserver.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | Yes | 0.4.0 | +| Configuration item | Description | Default value | Required | Since version | +|---------------------------------------------|------------------------------------------------------------------|---------------|----------|---------------| +| `gravitino.server.webserver.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | No | 0.4.0 | The filter should be a standard javax servlet Filter. Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.server.webserver..param.=` @@ -278,9 +278,9 @@ gravitino.server.webserver.com.test.filter1.param.name2=ba ``` ### Iceberg REST service's configuration -| Configuration item | Description | Default value | Required | Since version | -|-----------------------------------------------------|--------------------------------------------------------------------|---------------|-----------|---------------| -| `gravitino.auxService.iceberg-rest.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | Yes | 0.4.0 | +| Configuration item | Description | Default value | Required | Since version | +|-----------------------------------------------------|--------------------------------------------------------------------|---------------|----------|---------------| +| `gravitino.auxService.iceberg-rest.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | No | 0.4.0 | The filter should be a standard javax servlet Filter. Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.auxService.iceberg-rest..param.=` From 9a9c107a381eac67a877b8a64fbc4f0f6d3f4dd1 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Wed, 20 Dec 2023 11:32:42 +0800 Subject: [PATCH 6/8] Revert "[MINOR] fix(docs): Modify the required value for the document" This reverts commit 55184a69753a8d9dedac9d7e75287a6ad58e9d1d. --- docs/security.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/security.md b/docs/security.md index 5adc503ec5f..14e492ec950 100644 --- a/docs/security.md +++ b/docs/security.md @@ -55,11 +55,11 @@ GravitinoClient client = GravitinoClient.builder(uri) | Configuration item | Description | Default value | Required | Since version | |---------------------------------------------------|-----------------------------------------------------------------------------|-------------------|-----------------------------------------|---------------| -| `gravitino.authenticator` | The authenticator which Gravitino uses, setting as `simple` or `oauth`. | `simple` | No | 0.3.0 | -| `gravitino.authenticator.oauth.serviceAudience` | The audience name when Gravitino uses OAuth as the authenticator. | `GravitinoServer` | No | 0.3.0 | -| `gravitino.authenticator.oauth.allowSkewSecs` | The JWT allows skew seconds when Gravitino uses OAuth as the authenticator. | `0` | No | 0.3.0 | +| `gravitino.authenticator` | The authenticator which Gravitino uses, setting as `simple` or `oauth`. | `simple` | Yes | 0.3.0 | +| `gravitino.authenticator.oauth.serviceAudience` | The audience name when Gravitino uses OAuth as the authenticator. | `GravitinoServer` | Yes if use `oauth` as the authenticator | 0.3.0 | +| `gravitino.authenticator.oauth.allowSkewSecs` | The JWT allows skew seconds when Gravitino uses OAuth as the authenticator. | `0` | Yes if use `oauth` as the authenticator | 0.3.0 | | `gravitino.authenticator.oauth.defaultSignKey` | The signing key of JWT when Gravitino uses OAuth as the authenticator. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | -| `gravitino.authenticator.oauth.signAlgorithmType` | The signature algorithm when Gravitino uses OAuth as the authenticator. | `RS256` | No | 0.3.0 | +| `gravitino.authenticator.oauth.signAlgorithmType` | The signature algorithm when Gravitino uses OAuth as the authenticator. | `RS256` | Yes if use `oauth` as the authenticator | 0.3.0 | | `gravitino.authenticator.oauth.serverUri` | The URI of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | | `gravitino.authenticator.oauth.tokenPath` | The path for token of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | @@ -152,35 +152,35 @@ Both Gravitino server and Iceberg REST service can configure HTTPS. | Configuration item | Description | Default value | Required | Since version | |-----------------------------------------------------|--------------------------------------------------------------------|---------------|---------------------------------------------------|---------------| -| `gravitino.server.webserver.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | -| `gravitino.server.webserver.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | No | 0.3.0 | +| `gravitino.server.webserver.enableHttps` | Enables HTTPS. | `false` | Yes | 0.3.0 | +| `gravitino.server.webserver.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | Yes if use HTTPS | 0.3.0 | | `gravitino.server.webserver.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.server.webserver.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | +| `gravitino.server.webserver.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | | `gravitino.server.webserver.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | -| `gravitino.server.webserver.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | -| `gravitino.server.webserver.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | -| `gravitino.server.webserver.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.server.webserver.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.server.webserver.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | +| `gravitino.server.webserver.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | none | No | 0.3.0 | +| `gravitino.server.webserver.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.enableClientAuth` | Enables the authentication of the client. | `false` | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.trustStorePath` | Path to the trust store file. | none | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.trustStorePassword` | Password to the trust store. | none | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.server.webserver.trustStoreType` | The type to the trust store. | `JKS` | Yes if use HTTPS and the authentication of client | 0.3.0 | ### Iceberg REST service's configuration | Configuration item | Description | Default value | Required | Since version | |------------------------------------------------------------|--------------------------------------------------------------------|---------------|---------------------------------------------------|---------------| -| `gravitino.auxService.iceberg-rest.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | -| `gravitino.auxService.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | No | 0.3.0 | +| `gravitino.auxService.iceberg-rest.enableHttps` | Enables HTTPS. | `false` | Yes | 0.3.0 | +| `gravitino.auxService.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.auxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | +| `gravitino.auxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.auxService.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | +| `gravitino.auxService.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | -| `gravitino.auxService.iceberg-rest.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | +| `gravitino.auxService.iceberg-rest.enableClientAuth` | Enables the authentication of the client. | `false` | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | | `gravitino.auxService.iceberg-rest.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.auxService.iceberg-rest.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | +| `gravitino.auxService.iceberg-rest.trustStoreType` | The type to the trust store. | `JKS` | Yes if use HTTPS and the authentication of client | 0.3.0 | Refer to the "Additional JSSE Standard Names" section of the [Java security guide](https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#jssenames) for the list of protocols related to tlsProtocol. You can find the list of `tlsProtocol` values for Java 8 in this document. From 831feeeaae14879293f8be662c49e26c7ec80cef Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Wed, 20 Dec 2023 11:34:42 +0800 Subject: [PATCH 7/8] fixed --- docs/security.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/security.md b/docs/security.md index 14e492ec950..6d6d7cbf536 100644 --- a/docs/security.md +++ b/docs/security.md @@ -264,9 +264,11 @@ curl -v -X GET --cacert ./certificate.pem -H "Accept: application/vnd.gravitino. Gravitino supports custom filters to implement the user specified logic to satisfy different safety needs. ### Gravitino server's configuration + | Configuration item | Description | Default value | Required | Since version | |---------------------------------------------|------------------------------------------------------------------|---------------|----------|---------------| | `gravitino.server.webserver.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | No | 0.4.0 | + The filter should be a standard javax servlet Filter. Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.server.webserver..param.=` @@ -278,9 +280,11 @@ gravitino.server.webserver.com.test.filter1.param.name2=ba ``` ### Iceberg REST service's configuration + | Configuration item | Description | Default value | Required | Since version | |-----------------------------------------------------|--------------------------------------------------------------------|---------------|----------|---------------| | `gravitino.auxService.iceberg-rest.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | No | 0.4.0 | + The filter should be a standard javax servlet Filter. Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.auxService.iceberg-rest..param.=` @@ -297,7 +301,7 @@ If you want to use a cross-origin filter for the Gravitino server, you can follo ```shell wget https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-servlets/9.4.51.v20230217/jetty-servlets-9.4.51.v20230217.jar -cp jetty-servlets-9.4.51.v20230217.jar /libs +cp jetty-servlets-9.4.51.v20230217.jar /libs ``` You can refer to the [Configurations](gravitino-server-config.md) and append the configurations to the conf/gravitino.conf. From d72c39d5fc0efc34427141788b5bb08f5f7bdadf Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 21 Dec 2023 19:02:19 +0800 Subject: [PATCH 8/8] fix --- docs/gravitino-server-config.md | 28 +++++----- docs/iceberg-rest-service.md | 5 ++ docs/security.md | 52 ------------------- .../server/web/JettyServerConfig.java | 10 ++-- .../server/web/TestJettyServerConfig.java | 18 +++++++ 5 files changed, 45 insertions(+), 68 deletions(-) diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 8ee97a166cf..ea90870ce19 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -23,18 +23,22 @@ The `gravitino.conf` file lists the configuration items in the following table. ### Gravitino HTTP Server configuration -| Configuration item | Description | Default value | Required | Since version | -|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|----------|---------------| -| `gravitino.server.webserver.host` | The host of Gravitino server. | `0.0.0.0` | No | 0.1.0 | -| `gravitino.server.webserver.httpPort` | The port on which the Gravitino server listens for incoming connections. | `8090` | No | 0.1.0 | -| `gravitino.server.webserver.minThreads` | The minimum number of threads in the thread pool used by Jetty webserver. `minThreads` is 8 if the value is less than 8. | `Math.max(Math.min(Runtime.getRuntime().availableProcessors() * 2, 100), 8)` | No | 0.2.0 | -| `gravitino.server.webserver.maxThreads` | The maximum number of threads in the thread pool used by Jetty webserver. `maxThreads` is 8 if the value is less than 8, and `maxThreads` must be great or equal to `minThreads`. | `Math.max(Runtime.getRuntime().availableProcessors() * 4, 400)` | No | 0.1.0 | -| `gravitino.server.webserver.threadPoolWorkQueueSize` | The size of the queue in the thread pool used by Jetty webserver. | `100` | No | 0.1.0 | -| `gravitino.server.webserver.stopTimeout` | Time in milliseconds to gracefully shutdown the Jetty webserver, for more, please see `org.eclipse.jetty.server.Server#setStopTimeout`. | `30000` | No | 0.2.0 | -| `gravitino.server.webserver.idleTimeout` | The timeout in milliseconds of idle connections. | `30000` | No | 0.2.0 | -| `gravitino.server.webserver.requestHeaderSize` | Maximum size of HTTP requests. | `131072` | No | 0.1.0 | -| `gravitino.server.webserver.responseHeaderSize` | Maximum size of HTTP responses. | `131072` | No | 0.1.0 | -| `gravitino.server.shutdown.timeout` | Time in milliseconds to gracefully shutdown of the Gravitino webserver. | `3000` | No | 0.2.0 | +| Configuration item | Description | Default value | Required | Since version | +|-------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|----------|---------------| +| `gravitino.server.webserver.host` | The host of Gravitino server. | `0.0.0.0` | No | 0.1.0 | +| `gravitino.server.webserver.httpPort` | The port on which the Gravitino server listens for incoming connections. | `8090` | No | 0.1.0 | +| `gravitino.server.webserver.minThreads` | The minimum number of threads in the thread pool used by Jetty webserver. `minThreads` is 8 if the value is less than 8. | `Math.max(Math.min(Runtime.getRuntime().availableProcessors() * 2, 100), 8)` | No | 0.2.0 | +| `gravitino.server.webserver.maxThreads` | The maximum number of threads in the thread pool used by Jetty webserver. `maxThreads` is 8 if the value is less than 8, and `maxThreads` must be great or equal to `minThreads`. | `Math.max(Runtime.getRuntime().availableProcessors() * 4, 400)` | No | 0.1.0 | +| `gravitino.server.webserver.threadPoolWorkQueueSize` | The size of the queue in the thread pool used by Jetty webserver. | `100` | No | 0.1.0 | +| `gravitino.server.webserver.stopTimeout` | Time in milliseconds to gracefully shutdown the Jetty webserver, for more, please see `org.eclipse.jetty.server.Server#setStopTimeout`. | `30000` | No | 0.2.0 | +| `gravitino.server.webserver.idleTimeout` | The timeout in milliseconds of idle connections. | `30000` | No | 0.2.0 | +| `gravitino.server.webserver.requestHeaderSize` | Maximum size of HTTP requests. | `131072` | No | 0.1.0 | +| `gravitino.server.webserver.responseHeaderSize` | Maximum size of HTTP responses. | `131072` | No | 0.1.0 | +| `gravitino.server.shutdown.timeout` | Time in milliseconds to gracefully shutdown of the Gravitino webserver. | `3000` | No | 0.2.0 | +| `gravitino.server.webserver.customFilters` | Comma separated list of filter class names to apply to the APIs. | (none) | No | 0.4.0 | + +The filter in the customFilters should be a standard javax servlet Filter. +Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.server.webserver..param.=` ### Storage configuration diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 86fd757de73..3d13acef210 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -42,6 +42,11 @@ Deploy the Gravitino server to the `GRAVITINO_HOME` directory. You can find the | `gravitino.auxService.iceberg-rest.idleTimeout` | The timeout in ms of idle connections. | `30000` | No | 0.2.0 | | `gravitino.auxService.iceberg-rest.requestHeaderSize` | The maximum size of an HTTP request. | `131072` | No | 0.2.0 | | `gravitino.auxService.iceberg-rest.responseHeaderSize` | The maximum size of an HTTP response. | `131072` | No | 0.2.0 | +| `gravitino.auxService.iceberg-rest.customFilters` | Comma separated list of filter class names to apply to the APIs. | (none) | No | 0.4.0 | + + +The filter in the customFilters should be a standard javax servlet Filter. +Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.auxService.iceberg-rest..param.=` :::caution You must set `gravitino.auxService.iceberg-rest.httpPort` explicitly, like `9001`. diff --git a/docs/security.md b/docs/security.md index 6d6d7cbf536..69df5475dc8 100644 --- a/docs/security.md +++ b/docs/security.md @@ -258,55 +258,3 @@ If you want to use the command `curl`, you can follow the commands: openssl x509 -inform der -in $JAVA_HOME/localhost.crt -out certificate.pem curl -v -X GET --cacert ./certificate.pem -H "Accept: application/vnd.gravitino.v1+json" -H "Content-Type: application/json" https://localhost:8433/api/version ``` - -## Custom filters - -Gravitino supports custom filters to implement the user specified logic to satisfy different safety needs. - -### Gravitino server's configuration - -| Configuration item | Description | Default value | Required | Since version | -|---------------------------------------------|------------------------------------------------------------------|---------------|----------|---------------| -| `gravitino.server.webserver.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | No | 0.4.0 | - -The filter should be a standard javax servlet Filter. -Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.server.webserver..param.=` - -For example: -```text -gravitino.server.webserver.customFilters=com.test.filter1 -gravitino.server.webserver.com.test.filter1.param.name1=foo -gravitino.server.webserver.com.test.filter1.param.name2=ba -``` - -### Iceberg REST service's configuration - -| Configuration item | Description | Default value | Required | Since version | -|-----------------------------------------------------|--------------------------------------------------------------------|---------------|----------|---------------| -| `gravitino.auxService.iceberg-rest.customFilters` | Comma separated list of filter class names to apply to the APIs. | `` | No | 0.4.0 | - -The filter should be a standard javax servlet Filter. -Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.auxService.iceberg-rest..param.=` - -For example: -```text -gravitino.auxService.iceberg-rest.customFilters=com.test.filter1 -gravitino.auxService.iceberg-rest.com.test.filter1.param.name1=foo -gravitino.auxService.iceberg-rest.com.test.filter1.param.name2=ba -``` - -### Example - -If you want to use a cross-origin filter for the Gravitino server, you can follow the steps. - -```shell -wget https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-servlets/9.4.51.v20230217/jetty-servlets-9.4.51.v20230217.jar -cp jetty-servlets-9.4.51.v20230217.jar /libs -``` - -You can refer to the [Configurations](gravitino-server-config.md) and append the configurations to the conf/gravitino.conf. - -```text -gravitino.server.webserver.customFilters=org.eclipse.jetty.servlets.CrossOriginFilter -gravitino.server.webserver.org.eclipse.jetty.servlets.CrossOriginFilter.param.allowedOrigins=* -``` \ No newline at end of file diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java index 1fa5b61a12c..2a06d1704f3 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java @@ -171,12 +171,12 @@ public final class JettyServerConfig { .stringConf() .createWithDefault("JKS"); - public static final ConfigEntry CUSTOM_FILTERS = + public static final ConfigEntry> CUSTOM_FILTERS = new ConfigBuilder("customFilters") .doc("Comma separated list of filter class names to apply to the APIs") .version("0.4.0") .stringConf() - .createWithDefault(""); + .createWithOptional(); private final String host; @@ -257,8 +257,10 @@ private JettyServerConfig(Map configs) { this.trustStorePassword = internalConfig.get(SSL_TRUST_STORE_PASSWORD); this.trustStoreType = internalConfig.get(SSL_TRUST_STORE_TYPE); this.customFilters = - Collections.unmodifiableSet( - Sets.newHashSet(internalConfig.get(CUSTOM_FILTERS).split(SPLITTER))); + internalConfig + .get(CUSTOM_FILTERS) + .map(filters -> Collections.unmodifiableSet(Sets.newHashSet(filters.split(SPLITTER)))) + .orElse(Collections.emptySet()); } public static JettyServerConfig fromConfig(Config config, String prefix) { diff --git a/server-common/src/test/java/com/datastrato/gravitino/server/web/TestJettyServerConfig.java b/server-common/src/test/java/com/datastrato/gravitino/server/web/TestJettyServerConfig.java index 40b6ec8c943..60c15dd7a34 100644 --- a/server-common/src/test/java/com/datastrato/gravitino/server/web/TestJettyServerConfig.java +++ b/server-common/src/test/java/com/datastrato/gravitino/server/web/TestJettyServerConfig.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.server.web; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.config.ConfigBuilder; import com.google.common.collect.Sets; import java.util.Collections; import java.util.Optional; @@ -44,4 +45,21 @@ public void testCipherAlgorithms() { Assertions.assertIterableEquals( Sets.newHashSet(algorithm), jettyServerConfig.getSupportedAlgorithms()); } + + @Test + public void testCustomFilters() { + Config emptyconfig = new Config() {}; + JettyServerConfig jettyServerConfig = JettyServerConfig.fromConfig(emptyconfig, ""); + Assertions.assertTrue(jettyServerConfig.getCustomFilters().isEmpty()); + + Config somethingConfig = new Config() {}; + somethingConfig.set(JettyServerConfig.CUSTOM_FILTERS, Optional.of("1,2")); + somethingConfig.set(new ConfigBuilder("1.1").stringConf(), "test"); + somethingConfig.set(new ConfigBuilder("1.2").stringConf(), "test"); + jettyServerConfig = JettyServerConfig.fromConfig(somethingConfig, ""); + Assertions.assertIterableEquals( + Sets.newHashSet("1", "2"), jettyServerConfig.getCustomFilters()); + Assertions.assertTrue(jettyServerConfig.getAllWithPrefix("2.").isEmpty()); + Assertions.assertEquals(2, jettyServerConfig.getAllWithPrefix("1.").size()); + } }