From ee09bd18f9589a7dce8900b3217252b3a8f04576 Mon Sep 17 00:00:00 2001 From: 1289220708 <1289220708@qq.com> Date: Fri, 1 Mar 2024 21:16:31 +0800 Subject: [PATCH 1/2] fix(server): fix server slow log, support loader import & client IP --- .../hugegraph/api/filter/AccessLogFilter.java | 22 ++++++++-- .../hugegraph/api/filter/PathFilter.java | 40 ++++++++++++------- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java index 7a4a9b97d7..5c26ff7e6c 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.api.filter; +import static org.apache.hugegraph.api.filter.PathFilter.REQUEST_PARAMS_JSON; import static org.apache.hugegraph.api.filter.PathFilter.REQUEST_TIME; import static org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_FAILED_COUNTER; import static org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_RESPONSE_TIME_HISTOGRAM; @@ -24,7 +25,9 @@ import static org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_TOTAL_COUNTER; import java.io.IOException; +import java.net.InetAddress; import java.net.URI; +import java.net.UnknownHostException; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.config.ServerOptions; @@ -38,6 +41,7 @@ import jakarta.ws.rs.container.ContainerResponseContext; import jakarta.ws.rs.container.ContainerResponseFilter; import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.UriInfo; import jakarta.ws.rs.ext.Provider; // TODO: should add test for this class @@ -109,13 +113,25 @@ public void filter(ContainerRequestContext requestContext, // Record slow query if meet needs, watch out the perf if (timeThreshold > 0 && executeTime > timeThreshold && needRecordLog(requestContext)) { - // TODO: set RequestBody null, handle it later & should record "client IP" - LOG.info("[Slow Query] execTime={}ms, body={}, method={}, path={}, query={}", - executeTime, null, method, path, uri.getQuery()); + + LOG.info("[Slow Query] ip={} execTime={}ms, body={}, method={}, path={}, query={}", + getClientIP(requestContext), executeTime, + requestContext.getProperty(REQUEST_PARAMS_JSON), method, path, + uri.getQuery()); } } } + private String getClientIP(ContainerRequestContext requestContext) { + try { + UriInfo uriInfo = requestContext.getUriInfo(); + String host = uriInfo.getRequestUri().getHost(); + return InetAddress.getByName(host).getHostAddress(); + } catch (UnknownHostException e) { + return "unknown"; + } + } + private boolean statusOk(int status) { return status >= 200 && status < 300; } diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java index dda43b3fba..0d3691a109 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java @@ -17,9 +17,16 @@ package org.apache.hugegraph.api.filter; +import static org.apache.hugegraph.api.API.CHARSET; + +import java.io.BufferedInputStream; import java.io.IOException; +import org.apache.commons.io.Charsets; +import org.apache.commons.io.IOUtils; + import jakarta.inject.Singleton; +import jakarta.ws.rs.HttpMethod; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.container.PreMatching; @@ -35,24 +42,29 @@ public class PathFilter implements ContainerRequestFilter { @Override public void filter(ContainerRequestContext context) throws IOException { - context.setProperty(REQUEST_TIME, System.currentTimeMillis()); + long startTime = System.currentTimeMillis(); + + context.setProperty(REQUEST_TIME, startTime); + + recordRequestJson(context); + } - // TODO: temporarily comment it to fix loader bug, handle it later - /*// record the request json + private void recordRequestJson(ContainerRequestContext context) throws IOException { String method = context.getMethod(); - String requestParamsJson = ""; if (method.equals(HttpMethod.POST)) { - requestParamsJson = IOUtils.toString(context.getEntityStream(), - Charsets.toCharset(CHARSET)); - // replace input stream because we have already read it - InputStream in = IOUtils.toInputStream(requestParamsJson, Charsets.toCharset(CHARSET)); - context.setEntityStream(in); + BufferedInputStream bufferedStream = new BufferedInputStream(context.getEntityStream()); + + bufferedStream.mark(Integer.MAX_VALUE); + + context.setProperty(REQUEST_PARAMS_JSON, + IOUtils.toString(bufferedStream, Charsets.toCharset(CHARSET))); + + bufferedStream.reset(); + + context.setEntityStream(bufferedStream); } else if (method.equals(HttpMethod.GET)) { - MultivaluedMap pathParameters = context.getUriInfo() - .getPathParameters(); - requestParamsJson = pathParameters.toString(); + context.setProperty(REQUEST_PARAMS_JSON, + context.getUriInfo().getPathParameters().toString()); } - - context.setProperty(REQUEST_PARAMS_JSON, requestParamsJson);*/ } } From 071943c2806c0ed67b7bbac2933eb054b8155663 Mon Sep 17 00:00:00 2001 From: 1289220708 <1289220708@qq.com> Date: Sat, 15 Jun 2024 18:07:42 +0800 Subject: [PATCH 2/2] fix(server): support put/delete & when 512 length then log cut --- .../apache/hugegraph/api/filter/PathFilter.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java index 0d3691a109..23722dc1c6 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java @@ -39,6 +39,7 @@ public class PathFilter implements ContainerRequestFilter { public static final String REQUEST_TIME = "request_time"; public static final String REQUEST_PARAMS_JSON = "request_params_json"; + public static final int MAX_SLOW_LOG_BODY_LENGTH = 512; @Override public void filter(ContainerRequestContext context) throws IOException { @@ -46,18 +47,22 @@ public void filter(ContainerRequestContext context) throws IOException { context.setProperty(REQUEST_TIME, startTime); - recordRequestJson(context); + collectRequestParams(context); } - private void recordRequestJson(ContainerRequestContext context) throws IOException { + private void collectRequestParams(ContainerRequestContext context) throws IOException { String method = context.getMethod(); - if (method.equals(HttpMethod.POST)) { + if (method.equals(HttpMethod.POST) || method.equals(HttpMethod.PUT) || + method.equals(HttpMethod.DELETE)) { BufferedInputStream bufferedStream = new BufferedInputStream(context.getEntityStream()); bufferedStream.mark(Integer.MAX_VALUE); + String body = IOUtils.toString(bufferedStream, + Charsets.toCharset(CHARSET)); + body = body.length() > MAX_SLOW_LOG_BODY_LENGTH ? + body.substring(0, MAX_SLOW_LOG_BODY_LENGTH) : body; - context.setProperty(REQUEST_PARAMS_JSON, - IOUtils.toString(bufferedStream, Charsets.toCharset(CHARSET))); + context.setProperty(REQUEST_PARAMS_JSON, body); bufferedStream.reset();