From 4ea80bb4fe0e2b0eb4535832f2b9871cc1601f10 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 26 Jul 2024 12:44:49 +0530 Subject: [PATCH 01/10] Add task report stages and counters in response of /sql/statements/queryId API --- .../msq/counters/CounterSnapshotsTree.java | 34 ++++++++++++ .../msq/sql/entity/SqlStatementResult.java | 53 ++++++++++++++++-- .../sql/resources/SqlStatementResource.java | 43 ++++++++++++--- .../sql/entity/SqlStatementResultTest.java | 4 +- .../SqlMSQStatementResourcePostTest.java | 4 +- .../resources/SqlStatementResourceTest.java | 54 +++++++++++++++---- 6 files changed, 170 insertions(+), 22 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java index 8936e104bd69..0f7d990ef931 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java @@ -28,6 +28,7 @@ import org.apache.druid.msq.indexing.report.MSQTaskReportPayload; import java.util.Map; +import java.util.Objects; /** * Tree of {@link CounterSnapshots} (named counter snapshots) organized by stage and worker. @@ -108,4 +109,37 @@ private void putAll(final Map> otherMap) } } } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CounterSnapshotsTree that = (CounterSnapshotsTree) o; + synchronized (snapshotsMap) { + return Objects.equals(snapshotsMap, that.snapshotsMap); + } + } + + @Override + public int hashCode() + { + synchronized (snapshotsMap) { + return Objects.hashCode(snapshotsMap); + } + } + + @Override + public String toString() + { + synchronized (snapshotsMap) { + return "CounterSnapshotsTree{" + + "snapshotsMap=" + snapshotsMap + + '}'; + } + } } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java index de66550a5875..a650f7325c4c 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java @@ -23,6 +23,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.error.ErrorResponse; +import org.apache.druid.msq.counters.CounterSnapshotsTree; +import org.apache.druid.msq.indexing.report.MSQStagesReport; import org.apache.druid.msq.sql.SqlStatementState; import org.joda.time.DateTime; @@ -51,6 +53,24 @@ public class SqlStatementResult @Nullable private final ErrorResponse errorResponse; + @Nullable + private final MSQStagesReport stages; + + @Nullable + private final CounterSnapshotsTree counters; + + public SqlStatementResult( + String queryId, + SqlStatementState state, + DateTime createdAt, + List sqlRowSignature, + Long durationMs, + ResultSetInformation resultSetInformation, + ErrorResponse errorResponse + ) + { + this(queryId, state, createdAt, sqlRowSignature, durationMs, resultSetInformation, errorResponse, null, null); + } @JsonCreator public SqlStatementResult( @@ -67,8 +87,11 @@ public SqlStatementResult( @Nullable @JsonProperty("result") ResultSetInformation resultSetInformation, @Nullable @JsonProperty("errorDetails") - ErrorResponse errorResponse - + ErrorResponse errorResponse, + @Nullable @JsonProperty("stages") + MSQStagesReport stages, + @Nullable @JsonProperty("counters") + CounterSnapshotsTree counters ) { this.queryId = queryId; @@ -78,6 +101,8 @@ public SqlStatementResult( this.durationMs = durationMs; this.resultSetInformation = resultSetInformation; this.errorResponse = errorResponse; + this.stages = stages; + this.counters = counters; } @JsonProperty @@ -130,6 +155,22 @@ public ErrorResponse getErrorResponse() return errorResponse; } + @JsonProperty("stages") + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + public MSQStagesReport getStages() + { + return stages; + } + + @JsonProperty("counters") + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + public CounterSnapshotsTree getCounters() + { + return counters; + } + @Override public boolean equals(Object o) @@ -150,7 +191,7 @@ public boolean equals(Object o) ) && Objects.equals(resultSetInformation, that.resultSetInformation) && Objects.equals( errorResponse == null ? null : errorResponse.getAsMap(), that.errorResponse == null ? null : that.errorResponse.getAsMap() - ); + ) && Objects.equals(stages, that.stages) && Objects.equals(counters, that.counters); } @Override @@ -163,7 +204,9 @@ public int hashCode() sqlRowSignature, durationMs, resultSetInformation, - errorResponse == null ? null : errorResponse.getAsMap() + errorResponse == null ? null : errorResponse.getAsMap(), + stages, + counters ); } @@ -180,6 +223,8 @@ public String toString() ", errorResponse=" + (errorResponse == null ? "{}" : errorResponse.getAsMap().toString()) + + ", stages=" + stages + + ", counters=" + counters + '}'; } } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java index 4beb2a869ef0..2172bc69e913 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java @@ -47,6 +47,7 @@ import org.apache.druid.java.util.common.guava.Yielders; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.msq.counters.CounterSnapshotsTree; import org.apache.druid.msq.exec.ResultsContext; import org.apache.druid.msq.guice.MultiStageQuery; import org.apache.druid.msq.indexing.MSQControllerTask; @@ -55,6 +56,7 @@ import org.apache.druid.msq.indexing.destination.MSQDestination; import org.apache.druid.msq.indexing.destination.MSQSelectDestination; import org.apache.druid.msq.indexing.destination.TaskReportMSQDestination; +import org.apache.druid.msq.indexing.report.MSQStagesReport; import org.apache.druid.msq.indexing.report.MSQTaskReportPayload; import org.apache.druid.msq.kernel.StageDefinition; import org.apache.druid.msq.shuffle.input.DurableStorageInputChannelFactory; @@ -231,7 +233,9 @@ public Response doPost(final SqlQuery sqlQuery, @Context final HttpServletReques @Path("/{id}") @Produces(MediaType.APPLICATION_JSON) public Response doGetStatus( - @PathParam("id") final String queryId, @Context final HttpServletRequest req + @PathParam("id") final String queryId, + @QueryParam("detail") boolean detail, + @Context final HttpServletRequest req ) { try { @@ -242,7 +246,8 @@ public Response doGetStatus( queryId, authenticationResult, true, - Action.READ + Action.READ, + detail ); if (sqlStatementResult.isPresent()) { @@ -369,7 +374,8 @@ public Response deleteQuery(@PathParam("id") final String queryId, @Context fina queryId, authenticationResult, false, - Action.WRITE + Action.WRITE, + false ); if (sqlStatementResult.isPresent()) { switch (sqlStatementResult.get().getState()) { @@ -479,7 +485,7 @@ private Response buildTaskResponse(Sequence sequence, AuthenticationRe } String taskId = String.valueOf(firstRow[0]); - Optional statementResult = getStatementStatus(taskId, authenticationResult, true, Action.READ); + Optional statementResult = getStatementStatus(taskId, authenticationResult, true, Action.READ, false); if (statementResult.isPresent()) { return Response.status(Response.Status.OK).entity(statementResult.get()).build(); @@ -565,7 +571,8 @@ private Optional getStatementStatus( String queryId, AuthenticationResult authenticationResult, boolean withResults, - Action forAction + Action forAction, + boolean detail ) throws DruidException { TaskStatusResponse taskResponse = contactOverlord(overlordClient.taskStatus(queryId), queryId); @@ -592,6 +599,28 @@ private Optional getStatementStatus( jsonMapper ); } else { + MSQStagesReport stages = null; + CounterSnapshotsTree counters = null; + + if (detail) { + MSQTaskReportPayload msqTaskReportPayload; + try { + msqTaskReportPayload = SqlStatementResourceHelper.getPayload(contactOverlord(overlordClient.taskReportAsMap(queryId), queryId)); + } + catch (DruidException e) { + if (e.getErrorCode().equals("notFound")) { + msqTaskReportPayload = null; + } else { + throw e; + } + } + + if (msqTaskReportPayload != null) { + stages = msqTaskReportPayload.getStages(); + counters = msqTaskReportPayload.getCounters(); + } + } + Optional> signature = SqlStatementResourceHelper.getSignature(msqControllerTask); return Optional.of(new SqlStatementResult( queryId, @@ -605,7 +634,9 @@ private Optional getStatementStatus( sqlStatementState, msqControllerTask.getQuerySpec().getDestination() ).orElse(null) : null, - null + null, + stages, + counters )); } } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java index 03c017b7442d..e5c424a9b52c 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java @@ -87,7 +87,9 @@ public void sanityTest() throws JsonProcessingException + " sqlRowSignature=[ColumnNameAndTypes{colName='_time', sqlTypeName='TIMESTAMP', nativeTypeName='LONG'}, ColumnNameAndTypes{colName='alias', sqlTypeName='VARCHAR', nativeTypeName='STRING'}, ColumnNameAndTypes{colName='market', sqlTypeName='VARCHAR', nativeTypeName='STRING'}]," + " durationInMs=100," + " resultSetInformation=ResultSetInformation{numTotalRows=1, totalSizeInBytes=1, resultFormat=object, records=null, dataSource='ds', pages=[PageInformation{id=0, numRows=null, sizeInBytes=1, worker=null, partition=null}]}," - + " errorResponse={error=druidException, errorCode=QueryNotSupported, persona=USER, category=UNCATEGORIZED, errorMessage=QueryNotSupported, context={}}}", + + " errorResponse={error=druidException, errorCode=QueryNotSupported, persona=USER, category=UNCATEGORIZED, errorMessage=QueryNotSupported, context={}}," + + " stages=null," + + " counters=null}", SQL_STATEMENT_RESULT.toString() ); } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java index 1b0483d0b5a3..a9cd5254dd32 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java @@ -689,7 +689,7 @@ public void testInsert() ); Assert.assertEquals(expected, actual); - Response getResponse = resource.doGetStatus(actual.getQueryId(), SqlStatementResourceTest.makeOkRequest()); + Response getResponse = resource.doGetStatus(actual.getQueryId(), false, SqlStatementResourceTest.makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), getResponse.getStatus()); Assert.assertEquals(expected, getResponse.getEntity()); @@ -732,7 +732,7 @@ public void testReplaceAll() ); Assert.assertEquals(expected, actual); - Response getResponse = resource.doGetStatus(actual.getQueryId(), SqlStatementResourceTest.makeOkRequest()); + Response getResponse = resource.doGetStatus(actual.getQueryId(), false, SqlStatementResourceTest.makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), getResponse.getStatus()); Assert.assertEquals(expected, getResponse.getEntity()); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java index a97ee01297fc..fa308b0471bb 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java @@ -690,7 +690,7 @@ public void init() throws Exception @Test public void testMSQSelectAcceptedQuery() { - Response response = resource.doGetStatus(ACCEPTED_SELECT_MSQ_QUERY, makeOkRequest()); + Response response = resource.doGetStatus(ACCEPTED_SELECT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); Assert.assertEquals( new SqlStatementResult( @@ -724,7 +724,7 @@ public void testMSQSelectAcceptedQuery() public void testMSQSelectRunningQuery() { - Response response = resource.doGetStatus(RUNNING_SELECT_MSQ_QUERY, makeOkRequest()); + Response response = resource.doGetStatus(RUNNING_SELECT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); Assert.assertEquals( new SqlStatementResult( @@ -754,10 +754,41 @@ public void testMSQSelectRunningQuery() ); } + @Test + public void testMSQSelectRunningQueryWithDetail() + { + Mockito.when(overlordClient.taskReportAsMap(RUNNING_SELECT_MSQ_QUERY)) + .thenAnswer(inv -> Futures.immediateFuture(TaskReport.buildTaskReports(selectTaskReport.get()))); + Response response = resource.doGetStatus(RUNNING_SELECT_MSQ_QUERY, true, makeOkRequest()); + Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + + SqlStatementResult expectedSqlStatementResult = new SqlStatementResult( + RUNNING_SELECT_MSQ_QUERY, + SqlStatementState.RUNNING, + CREATED_TIME, + COL_NAME_AND_TYPES, + null, + null, + null, + selectTaskReport.get().getPayload().getStages(), + selectTaskReport.get().getPayload().getCounters() + ); + + Assert.assertEquals( + expectedSqlStatementResult, + response.getEntity() + ); + + Assert.assertEquals( + Response.Status.ACCEPTED.getStatusCode(), + resource.deleteQuery(RUNNING_SELECT_MSQ_QUERY, makeOkRequest()).getStatus() + ); + } + @Test public void testFinishedSelectMSQQuery() throws Exception { - Response response = resource.doGetStatus(FINISHED_SELECT_MSQ_QUERY, makeOkRequest()); + Response response = resource.doGetStatus(FINISHED_SELECT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); Assert.assertEquals(objectMapper.writeValueAsString(new SqlStatementResult( FINISHED_SELECT_MSQ_QUERY, @@ -825,7 +856,7 @@ private void assertExpectedResults(String expectedResult, Response resultsRespon public void testFailedMSQQuery() { for (String queryID : ImmutableList.of(ERRORED_SELECT_MSQ_QUERY, ERRORED_INSERT_MSQ_QUERY)) { - assertExceptionMessage(resource.doGetStatus(queryID, makeOkRequest()), FAILURE_MSG, Response.Status.OK); + assertExceptionMessage(resource.doGetStatus(queryID, false, makeOkRequest()), FAILURE_MSG, Response.Status.OK); assertExceptionMessage( resource.doGetResults(queryID, 0L, null, makeOkRequest()), StringUtils.format( @@ -845,7 +876,7 @@ public void testFailedMSQQuery() @Test public void testFinishedInsertMSQQuery() { - Response response = resource.doGetStatus(FINISHED_INSERT_MSQ_QUERY, makeOkRequest()); + Response response = resource.doGetStatus(FINISHED_INSERT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); Assert.assertEquals(new SqlStatementResult( FINISHED_INSERT_MSQ_QUERY, @@ -876,7 +907,7 @@ public void testFinishedInsertMSQQuery() public void testNonMSQTasks() { for (String queryID : ImmutableList.of(RUNNING_NON_MSQ_TASK, FAILED_NON_MSQ_TASK, FINISHED_NON_MSQ_TASK)) { - assertNotFound(resource.doGetStatus(queryID, makeOkRequest()), queryID); + assertNotFound(resource.doGetStatus(queryID, false, makeOkRequest()), queryID); assertNotFound(resource.doGetResults(queryID, 0L, null, makeOkRequest()), queryID); assertNotFound(resource.deleteQuery(queryID, makeOkRequest()), queryID); } @@ -885,7 +916,7 @@ public void testNonMSQTasks() @Test public void testMSQInsertAcceptedQuery() { - Response response = resource.doGetStatus(ACCEPTED_INSERT_MSQ_TASK, makeOkRequest()); + Response response = resource.doGetStatus(ACCEPTED_INSERT_MSQ_TASK, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); Assert.assertEquals( new SqlStatementResult( @@ -918,7 +949,7 @@ public void testMSQInsertAcceptedQuery() @Test public void testMSQInsertRunningQuery() { - Response response = resource.doGetStatus(RUNNING_INSERT_MSQ_QUERY, makeOkRequest()); + Response response = resource.doGetStatus(RUNNING_INSERT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); Assert.assertEquals( new SqlStatementResult( @@ -955,6 +986,7 @@ public void testAPIBehaviourWithSuperUsers() Response.Status.OK.getStatusCode(), resource.doGetStatus( RUNNING_SELECT_MSQ_QUERY, + false, makeExpectedReq(makeAuthResultForUser(SUPERUSER)) ).getStatus() ); @@ -984,6 +1016,7 @@ public void testAPIBehaviourWithDifferentUserAndNoStatePermission() Response.Status.FORBIDDEN.getStatusCode(), resource.doGetStatus( RUNNING_SELECT_MSQ_QUERY, + false, makeExpectedReq(differentUserAuthResult) ).getStatus() ); @@ -1013,6 +1046,7 @@ public void testAPIBehaviourWithDifferentUserAndStateRPermission() Response.Status.OK.getStatusCode(), resource.doGetStatus( RUNNING_SELECT_MSQ_QUERY, + false, makeExpectedReq(differentUserAuthResult) ).getStatus() ); @@ -1042,6 +1076,7 @@ public void testAPIBehaviourWithDifferentUserAndStateWPermission() Response.Status.FORBIDDEN.getStatusCode(), resource.doGetStatus( RUNNING_SELECT_MSQ_QUERY, + false, makeExpectedReq(differentUserAuthResult) ).getStatus() ); @@ -1071,6 +1106,7 @@ public void testAPIBehaviourWithDifferentUserAndStateRWPermission() Response.Status.OK.getStatusCode(), resource.doGetStatus( RUNNING_SELECT_MSQ_QUERY, + false, makeExpectedReq(differentUserAuthResult) ).getStatus() ); @@ -1107,7 +1143,7 @@ public void testTaskIdNotFound() Assert.assertEquals( Response.Status.NOT_FOUND.getStatusCode(), - resource.doGetStatus(taskIdNotFound, makeOkRequest()).getStatus() + resource.doGetStatus(taskIdNotFound, false, makeOkRequest()).getStatus() ); Assert.assertEquals( Response.Status.NOT_FOUND.getStatusCode(), From b611a6ba09e2364a8d13914879c367218ad4bef9 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 26 Jul 2024 18:22:55 +0530 Subject: [PATCH 02/10] Add new fields for failed scenarios + Add warnings field --- .../msq/sql/entity/SqlStatementResult.java | 27 ++++++++++-- .../sql/resources/SqlStatementResource.java | 44 +++++++------------ .../msq/util/SqlStatementResourceHelper.java | 35 ++++++++++++--- .../sql/entity/SqlStatementResultTest.java | 3 +- .../resources/SqlStatementResourceTest.java | 16 +++++-- 5 files changed, 83 insertions(+), 42 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java index a650f7325c4c..6fd74a836043 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.error.ErrorResponse; import org.apache.druid.msq.counters.CounterSnapshotsTree; +import org.apache.druid.msq.indexing.error.MSQErrorReport; import org.apache.druid.msq.indexing.report.MSQStagesReport; import org.apache.druid.msq.sql.SqlStatementState; import org.joda.time.DateTime; @@ -59,6 +60,9 @@ public class SqlStatementResult @Nullable private final CounterSnapshotsTree counters; + @Nullable + private final List warnings; + public SqlStatementResult( String queryId, SqlStatementState state, @@ -69,7 +73,7 @@ public SqlStatementResult( ErrorResponse errorResponse ) { - this(queryId, state, createdAt, sqlRowSignature, durationMs, resultSetInformation, errorResponse, null, null); + this(queryId, state, createdAt, sqlRowSignature, durationMs, resultSetInformation, errorResponse, null, null, null); } @JsonCreator @@ -91,7 +95,9 @@ public SqlStatementResult( @Nullable @JsonProperty("stages") MSQStagesReport stages, @Nullable @JsonProperty("counters") - CounterSnapshotsTree counters + CounterSnapshotsTree counters, + @Nullable @JsonProperty("warnings") + List warnings ) { this.queryId = queryId; @@ -103,6 +109,7 @@ public SqlStatementResult( this.errorResponse = errorResponse; this.stages = stages; this.counters = counters; + this.warnings = warnings; } @JsonProperty @@ -171,6 +178,13 @@ public CounterSnapshotsTree getCounters() return counters; } + @JsonProperty("warnings") + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + public List getWarnings() + { + return warnings; + } @Override public boolean equals(Object o) @@ -191,7 +205,10 @@ public boolean equals(Object o) ) && Objects.equals(resultSetInformation, that.resultSetInformation) && Objects.equals( errorResponse == null ? null : errorResponse.getAsMap(), that.errorResponse == null ? null : that.errorResponse.getAsMap() - ) && Objects.equals(stages, that.stages) && Objects.equals(counters, that.counters); + ) && Objects.equals(stages, that.stages) && Objects.equals(counters, that.counters) && Objects.equals( + warnings, + that.warnings + ); } @Override @@ -206,7 +223,8 @@ public int hashCode() resultSetInformation, errorResponse == null ? null : errorResponse.getAsMap(), stages, - counters + counters, + warnings ); } @@ -225,6 +243,7 @@ public String toString() : errorResponse.getAsMap().toString()) + ", stages=" + stages + ", counters=" + counters + + ", warnings=" + warnings + '}'; } } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java index 2172bc69e913..7575dd00dcb8 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java @@ -47,7 +47,6 @@ import org.apache.druid.java.util.common.guava.Yielders; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.msq.counters.CounterSnapshotsTree; import org.apache.druid.msq.exec.ResultsContext; import org.apache.druid.msq.guice.MultiStageQuery; import org.apache.druid.msq.indexing.MSQControllerTask; @@ -56,7 +55,6 @@ import org.apache.druid.msq.indexing.destination.MSQDestination; import org.apache.druid.msq.indexing.destination.MSQSelectDestination; import org.apache.druid.msq.indexing.destination.TaskReportMSQDestination; -import org.apache.druid.msq.indexing.report.MSQStagesReport; import org.apache.druid.msq.indexing.report.MSQTaskReportPayload; import org.apache.druid.msq.kernel.StageDefinition; import org.apache.druid.msq.shuffle.input.DurableStorageInputChannelFactory; @@ -589,38 +587,27 @@ private Optional getStatementStatus( MSQControllerTask msqControllerTask = getMSQControllerTaskAndCheckPermission(queryId, authenticationResult, forAction); SqlStatementState sqlStatementState = SqlStatementResourceHelper.getSqlStatementState(statusPlus); + MSQTaskReportPayload msqTaskReportPayload = null; + try { + msqTaskReportPayload = SqlStatementResourceHelper.getPayload(contactOverlord(overlordClient.taskReportAsMap(queryId), queryId)); + } + catch (DruidException e) { + if (!e.getErrorCode().equals("notFound")) { + throw e; + } + } + if (SqlStatementState.FAILED == sqlStatementState) { return SqlStatementResourceHelper.getExceptionPayload( queryId, taskResponse, statusPlus, sqlStatementState, - contactOverlord(overlordClient.taskReportAsMap(queryId), queryId), - jsonMapper + msqTaskReportPayload, + jsonMapper, + detail ); } else { - MSQStagesReport stages = null; - CounterSnapshotsTree counters = null; - - if (detail) { - MSQTaskReportPayload msqTaskReportPayload; - try { - msqTaskReportPayload = SqlStatementResourceHelper.getPayload(contactOverlord(overlordClient.taskReportAsMap(queryId), queryId)); - } - catch (DruidException e) { - if (e.getErrorCode().equals("notFound")) { - msqTaskReportPayload = null; - } else { - throw e; - } - } - - if (msqTaskReportPayload != null) { - stages = msqTaskReportPayload.getStages(); - counters = msqTaskReportPayload.getCounters(); - } - } - Optional> signature = SqlStatementResourceHelper.getSignature(msqControllerTask); return Optional.of(new SqlStatementResult( queryId, @@ -635,8 +622,9 @@ private Optional getStatementStatus( msqControllerTask.getQuerySpec().getDestination() ).orElse(null) : null, null, - stages, - counters + detail ? SqlStatementResourceHelper.getQueryStagesReport(msqTaskReportPayload) : null, + detail ? SqlStatementResourceHelper.getQueryCounters(msqTaskReportPayload) : null, + detail ? SqlStatementResourceHelper.getQueryWarningDetails(msqTaskReportPayload) : null )); } } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/SqlStatementResourceHelper.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/SqlStatementResourceHelper.java index 4f07dcb2cc02..db83e5f78a40 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/SqlStatementResourceHelper.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/SqlStatementResourceHelper.java @@ -250,11 +250,12 @@ public static Optional getExceptionPayload( TaskStatusResponse taskResponse, TaskStatusPlus statusPlus, SqlStatementState sqlStatementState, - TaskReport.ReportMap msqPayload, - ObjectMapper jsonMapper + MSQTaskReportPayload msqTaskReportPayload, + ObjectMapper jsonMapper, + boolean detail ) { - final MSQErrorReport exceptionDetails = getQueryExceptionDetails(getPayload(msqPayload)); + final MSQErrorReport exceptionDetails = getQueryExceptionDetails(msqTaskReportPayload); final MSQFault fault = exceptionDetails == null ? null : exceptionDetails.getFault(); if (exceptionDetails == null || fault == null) { return Optional.of(new SqlStatementResult( @@ -267,7 +268,10 @@ public static Optional getExceptionPayload( DruidException.forPersona(DruidException.Persona.DEVELOPER) .ofCategory(DruidException.Category.UNCATEGORIZED) .build("%s", taskResponse.getStatus().getErrorMsg()) - .toErrorResponse() + .toErrorResponse(), + detail ? getQueryStagesReport(msqTaskReportPayload) : null, + detail ? getQueryCounters(msqTaskReportPayload) : null, + detail ? getQueryWarningDetails(msqTaskReportPayload) : null )); } @@ -293,7 +297,10 @@ protected DruidException makeException(DruidException.DruidExceptionBuilder bob) ex.withContext(exceptionContext); return ex; } - }).toErrorResponse() + }).toErrorResponse(), + detail ? getQueryStagesReport(msqTaskReportPayload) : null, + detail ? getQueryCounters(msqTaskReportPayload) : null, + detail ? getQueryWarningDetails(msqTaskReportPayload) : null )); } @@ -374,6 +381,24 @@ private static MSQErrorReport getQueryExceptionDetails(MSQTaskReportPayload payl return payload == null ? null : payload.getStatus().getErrorReport(); } + @Nullable + public static List getQueryWarningDetails(MSQTaskReportPayload payload) + { + return payload == null ? null : new ArrayList<>(payload.getStatus().getWarningReports()); + } + + @Nullable + public static MSQStagesReport getQueryStagesReport(MSQTaskReportPayload payload) + { + return payload == null ? null : payload.getStages(); + } + + @Nullable + public static CounterSnapshotsTree getQueryCounters(MSQTaskReportPayload payload) + { + return payload == null ? null : payload.getCounters(); + } + @Nullable public static MSQTaskReportPayload getPayload(TaskReport.ReportMap reportMap) { diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java index e5c424a9b52c..e03932a23dd3 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java @@ -89,7 +89,8 @@ public void sanityTest() throws JsonProcessingException + " resultSetInformation=ResultSetInformation{numTotalRows=1, totalSizeInBytes=1, resultFormat=object, records=null, dataSource='ds', pages=[PageInformation{id=0, numRows=null, sizeInBytes=1, worker=null, partition=null}]}," + " errorResponse={error=druidException, errorCode=QueryNotSupported, persona=USER, category=UNCATEGORIZED, errorMessage=QueryNotSupported, context={}}," + " stages=null," - + " counters=null}", + + " counters=null," + + " warnings=null}", SQL_STATEMENT_RESULT.toString() ); } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java index fa308b0471bb..e170d8a2c30e 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java @@ -97,6 +97,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -449,7 +450,11 @@ private void setupMocks(OverlordClient indexingServiceClient) ))); - Mockito.when(indexingServiceClient.taskReportAsMap(FINISHED_SELECT_MSQ_QUERY)) + Mockito.when(indexingServiceClient.taskReportAsMap(ArgumentMatchers.eq(FINISHED_SELECT_MSQ_QUERY))) + .thenAnswer(inv -> Futures.immediateFuture(TaskReport.buildTaskReports(selectTaskReport.get()))); + Mockito.when(indexingServiceClient.taskReportAsMap(ArgumentMatchers.eq(ACCEPTED_SELECT_MSQ_QUERY))) + .thenAnswer(inv -> Futures.immediateFuture(TaskReport.buildTaskReports(selectTaskReport.get()))); + Mockito.when(indexingServiceClient.taskReportAsMap(ArgumentMatchers.eq(RUNNING_SELECT_MSQ_QUERY))) .thenAnswer(inv -> Futures.immediateFuture(TaskReport.buildTaskReports(selectTaskReport.get()))); Mockito.when(indexingServiceClient.taskStatus(ArgumentMatchers.eq(ERRORED_SELECT_MSQ_QUERY))) @@ -584,6 +589,10 @@ private void setupMocks(OverlordClient indexingServiceClient) Mockito.when(indexingServiceClient.taskReportAsMap(ArgumentMatchers.eq(FINISHED_INSERT_MSQ_QUERY))) .thenReturn(Futures.immediateFuture(TaskReport.buildTaskReports(MSQ_INSERT_TASK_REPORT))); + Mockito.when(indexingServiceClient.taskReportAsMap(ArgumentMatchers.eq(ACCEPTED_INSERT_MSQ_TASK))) + .thenReturn(Futures.immediateFuture(TaskReport.buildTaskReports(MSQ_INSERT_TASK_REPORT))); + Mockito.when(indexingServiceClient.taskReportAsMap(ArgumentMatchers.eq(RUNNING_INSERT_MSQ_QUERY))) + .thenReturn(Futures.immediateFuture(TaskReport.buildTaskReports(MSQ_INSERT_TASK_REPORT))); Mockito.when(indexingServiceClient.taskPayload(FINISHED_INSERT_MSQ_QUERY)) .thenReturn(Futures.immediateFuture(new TaskPayloadResponse( @@ -757,8 +766,6 @@ public void testMSQSelectRunningQuery() @Test public void testMSQSelectRunningQueryWithDetail() { - Mockito.when(overlordClient.taskReportAsMap(RUNNING_SELECT_MSQ_QUERY)) - .thenAnswer(inv -> Futures.immediateFuture(TaskReport.buildTaskReports(selectTaskReport.get()))); Response response = resource.doGetStatus(RUNNING_SELECT_MSQ_QUERY, true, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); @@ -771,7 +778,8 @@ public void testMSQSelectRunningQueryWithDetail() null, null, selectTaskReport.get().getPayload().getStages(), - selectTaskReport.get().getPayload().getCounters() + selectTaskReport.get().getPayload().getCounters(), + new ArrayList<>(selectTaskReport.get().getPayload().getStatus().getWarningReports()) ); Assert.assertEquals( From 7f864cb96ff44e193e733391f258daeaaf8f19ad Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Sat, 27 Jul 2024 00:55:52 +0530 Subject: [PATCH 03/10] Fix static check: openjdk17 strict compilation --- .../org/apache/druid/msq/counters/CounterSnapshotsTree.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java index 0f7d990ef931..707a9794296a 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java @@ -121,7 +121,9 @@ public boolean equals(Object o) } CounterSnapshotsTree that = (CounterSnapshotsTree) o; synchronized (snapshotsMap) { - return Objects.equals(snapshotsMap, that.snapshotsMap); + synchronized (that.snapshotsMap) { + return Objects.equals(snapshotsMap, that.snapshotsMap); + } } } From afa0c9648dd790042ffa8792eef327656ae9694d Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Sat, 27 Jul 2024 01:43:39 +0530 Subject: [PATCH 04/10] Attempt to fix web console check --- .../msq/sql/resources/SqlStatementResource.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java index 7575dd00dcb8..98a6e207b432 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java @@ -587,9 +587,9 @@ private Optional getStatementStatus( MSQControllerTask msqControllerTask = getMSQControllerTaskAndCheckPermission(queryId, authenticationResult, forAction); SqlStatementState sqlStatementState = SqlStatementResourceHelper.getSqlStatementState(statusPlus); - MSQTaskReportPayload msqTaskReportPayload = null; + Optional msqTaskReportPayload = Optional.empty(); try { - msqTaskReportPayload = SqlStatementResourceHelper.getPayload(contactOverlord(overlordClient.taskReportAsMap(queryId), queryId)); + msqTaskReportPayload = Optional.ofNullable(SqlStatementResourceHelper.getPayload(contactOverlord(overlordClient.taskReportAsMap(queryId), queryId))); } catch (DruidException e) { if (!e.getErrorCode().equals("notFound")) { @@ -603,7 +603,7 @@ private Optional getStatementStatus( taskResponse, statusPlus, sqlStatementState, - msqTaskReportPayload, + msqTaskReportPayload.orElse(null), jsonMapper, detail ); @@ -622,9 +622,9 @@ private Optional getStatementStatus( msqControllerTask.getQuerySpec().getDestination() ).orElse(null) : null, null, - detail ? SqlStatementResourceHelper.getQueryStagesReport(msqTaskReportPayload) : null, - detail ? SqlStatementResourceHelper.getQueryCounters(msqTaskReportPayload) : null, - detail ? SqlStatementResourceHelper.getQueryWarningDetails(msqTaskReportPayload) : null + detail ? SqlStatementResourceHelper.getQueryStagesReport(msqTaskReportPayload.orElse(null)) : null, + detail ? SqlStatementResourceHelper.getQueryCounters(msqTaskReportPayload.orElse(null)) : null, + detail ? SqlStatementResourceHelper.getQueryWarningDetails(msqTaskReportPayload.orElse(null)) : null )); } } From 6a1fc4e327ab9590559ed851a83cf01300c9b781 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Sat, 27 Jul 2024 02:07:06 +0530 Subject: [PATCH 05/10] Attempt 2 to fix web console check --- .../sql/resources/SqlStatementResource.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java index 98a6e207b432..1ee5f1030a40 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlStatementResource.java @@ -113,6 +113,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; import java.util.stream.Collectors; @@ -587,15 +588,19 @@ private Optional getStatementStatus( MSQControllerTask msqControllerTask = getMSQControllerTaskAndCheckPermission(queryId, authenticationResult, forAction); SqlStatementState sqlStatementState = SqlStatementResourceHelper.getSqlStatementState(statusPlus); - Optional msqTaskReportPayload = Optional.empty(); - try { - msqTaskReportPayload = Optional.ofNullable(SqlStatementResourceHelper.getPayload(contactOverlord(overlordClient.taskReportAsMap(queryId), queryId))); - } - catch (DruidException e) { - if (!e.getErrorCode().equals("notFound")) { + Supplier> msqTaskReportPayloadSupplier = () -> { + try { + return Optional.ofNullable(SqlStatementResourceHelper.getPayload( + contactOverlord(overlordClient.taskReportAsMap(queryId), queryId) + )); + } + catch (DruidException e) { + if (e.getErrorCode().equals("notFound")) { + return Optional.empty(); + } throw e; } - } + }; if (SqlStatementState.FAILED == sqlStatementState) { return SqlStatementResourceHelper.getExceptionPayload( @@ -603,7 +608,7 @@ private Optional getStatementStatus( taskResponse, statusPlus, sqlStatementState, - msqTaskReportPayload.orElse(null), + msqTaskReportPayloadSupplier.get().orElse(null), jsonMapper, detail ); @@ -622,9 +627,9 @@ private Optional getStatementStatus( msqControllerTask.getQuerySpec().getDestination() ).orElse(null) : null, null, - detail ? SqlStatementResourceHelper.getQueryStagesReport(msqTaskReportPayload.orElse(null)) : null, - detail ? SqlStatementResourceHelper.getQueryCounters(msqTaskReportPayload.orElse(null)) : null, - detail ? SqlStatementResourceHelper.getQueryWarningDetails(msqTaskReportPayload.orElse(null)) : null + detail ? SqlStatementResourceHelper.getQueryStagesReport(msqTaskReportPayloadSupplier.get().orElse(null)) : null, + detail ? SqlStatementResourceHelper.getQueryCounters(msqTaskReportPayloadSupplier.get().orElse(null)) : null, + detail ? SqlStatementResourceHelper.getQueryWarningDetails(msqTaskReportPayloadSupplier.get().orElse(null)) : null )); } } From 2774447e0f379669d86bc7c55fafc9bb0ca40f32 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 29 Jul 2024 12:07:37 +0530 Subject: [PATCH 06/10] Mark method parameter nullable --- .../druid/msq/util/SqlStatementResourceHelper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/SqlStatementResourceHelper.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/SqlStatementResourceHelper.java index db83e5f78a40..0820342ba727 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/SqlStatementResourceHelper.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/SqlStatementResourceHelper.java @@ -360,7 +360,7 @@ public Object[] next() } @Nullable - public static MSQStagesReport.Stage getFinalStage(MSQTaskReportPayload msqTaskReportPayload) + public static MSQStagesReport.Stage getFinalStage(@Nullable MSQTaskReportPayload msqTaskReportPayload) { if (msqTaskReportPayload == null || msqTaskReportPayload.getStages().getStages() == null) { return null; @@ -376,25 +376,25 @@ public static MSQStagesReport.Stage getFinalStage(MSQTaskReportPayload msqTaskRe } @Nullable - private static MSQErrorReport getQueryExceptionDetails(MSQTaskReportPayload payload) + private static MSQErrorReport getQueryExceptionDetails(@Nullable MSQTaskReportPayload payload) { return payload == null ? null : payload.getStatus().getErrorReport(); } @Nullable - public static List getQueryWarningDetails(MSQTaskReportPayload payload) + public static List getQueryWarningDetails(@Nullable MSQTaskReportPayload payload) { return payload == null ? null : new ArrayList<>(payload.getStatus().getWarningReports()); } @Nullable - public static MSQStagesReport getQueryStagesReport(MSQTaskReportPayload payload) + public static MSQStagesReport getQueryStagesReport(@Nullable MSQTaskReportPayload payload) { return payload == null ? null : payload.getStages(); } @Nullable - public static CounterSnapshotsTree getQueryCounters(MSQTaskReportPayload payload) + public static CounterSnapshotsTree getQueryCounters(@Nullable MSQTaskReportPayload payload) { return payload == null ? null : payload.getCounters(); } From 2320e8483e7e8f40aafa6f62751a897010febd8d Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 29 Jul 2024 13:00:27 +0530 Subject: [PATCH 07/10] Remove equals and hashcode from SqlStatementResult and CounterSnapshotsTree --- .../msq/counters/CounterSnapshotsTree.java | 26 ---------- .../msq/sql/entity/SqlStatementResult.java | 43 ----------------- .../sql/entity/SqlStatementResultTest.java | 8 ---- .../SqlMSQStatementResourcePostTest.java | 37 +++++++++++--- .../resources/SqlStatementResourceTest.java | 48 ++++++++++++++----- 5 files changed, 66 insertions(+), 96 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java index 707a9794296a..dce2fe7ac3ac 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/counters/CounterSnapshotsTree.java @@ -28,7 +28,6 @@ import org.apache.druid.msq.indexing.report.MSQTaskReportPayload; import java.util.Map; -import java.util.Objects; /** * Tree of {@link CounterSnapshots} (named counter snapshots) organized by stage and worker. @@ -110,31 +109,6 @@ private void putAll(final Map> otherMap) } } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - CounterSnapshotsTree that = (CounterSnapshotsTree) o; - synchronized (snapshotsMap) { - synchronized (that.snapshotsMap) { - return Objects.equals(snapshotsMap, that.snapshotsMap); - } - } - } - - @Override - public int hashCode() - { - synchronized (snapshotsMap) { - return Objects.hashCode(snapshotsMap); - } - } - @Override public String toString() { diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java index 6fd74a836043..bd33d76adb1e 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java @@ -31,7 +31,6 @@ import javax.annotation.Nullable; import java.util.List; -import java.util.Objects; public class SqlStatementResult { @@ -186,48 +185,6 @@ public List getWarnings() return warnings; } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SqlStatementResult that = (SqlStatementResult) o; - return Objects.equals(queryId, that.queryId) && state == that.state && Objects.equals( - createdAt, - that.createdAt - ) && Objects.equals(sqlRowSignature, that.sqlRowSignature) && Objects.equals( - durationMs, - that.durationMs - ) && Objects.equals(resultSetInformation, that.resultSetInformation) && Objects.equals( - errorResponse == null ? null : errorResponse.getAsMap(), - that.errorResponse == null ? null : that.errorResponse.getAsMap() - ) && Objects.equals(stages, that.stages) && Objects.equals(counters, that.counters) && Objects.equals( - warnings, - that.warnings - ); - } - - @Override - public int hashCode() - { - return Objects.hash( - queryId, - state, - createdAt, - sqlRowSignature, - durationMs, - resultSetInformation, - errorResponse == null ? null : errorResponse.getAsMap(), - stages, - counters, - warnings - ); - } - @Override public String toString() { diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java index e03932a23dd3..96ef0ac6b1f1 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java @@ -71,14 +71,6 @@ public void sanityTest() throws JsonProcessingException { Assert.assertEquals(JSON_STRING, MAPPER.writeValueAsString(SQL_STATEMENT_RESULT)); - Assert.assertEquals( - SQL_STATEMENT_RESULT, - MAPPER.readValue(MAPPER.writeValueAsString(SQL_STATEMENT_RESULT), SqlStatementResult.class) - ); - Assert.assertEquals( - SQL_STATEMENT_RESULT.hashCode(), - MAPPER.readValue(MAPPER.writeValueAsString(SQL_STATEMENT_RESULT), SqlStatementResult.class).hashCode() - ); Assert.assertEquals( "SqlStatementResult{" + "queryId='q1'," diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java index a9cd5254dd32..cef3e00daa2d 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java @@ -206,7 +206,7 @@ public void emptyInsert() new ResultSetInformation(0L, 0L, null, "foo1", null, null), null ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); } @Test @@ -236,7 +236,7 @@ public void emptyReplace() new ResultSetInformation(0L, 0L, null, "foo1", null, null), null ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); } @Test @@ -282,7 +282,7 @@ protected DruidException makeException(DruidException.DruidExceptionBuilder bob) } }).toErrorResponse() ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); } @Test @@ -687,11 +687,11 @@ public void testInsert() new ResultSetInformation(NullHandling.sqlCompatible() ? 6L : 5L, 0L, null, "foo1", null, null), null ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); Response getResponse = resource.doGetStatus(actual.getQueryId(), false, SqlStatementResourceTest.makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), getResponse.getStatus()); - Assert.assertEquals(expected, getResponse.getEntity()); + assertSqlStatementResult(expected, (SqlStatementResult) getResponse.getEntity()); Response resultsResponse = resource.doGetResults( actual.getQueryId(), @@ -730,11 +730,11 @@ public void testReplaceAll() new ResultSetInformation(NullHandling.sqlCompatible() ? 6L : 5L, 0L, null, "foo1", null, null), null ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); Response getResponse = resource.doGetStatus(actual.getQueryId(), false, SqlStatementResourceTest.makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), getResponse.getStatus()); - Assert.assertEquals(expected, getResponse.getEntity()); + assertSqlStatementResult(expected, (SqlStatementResult) getResponse.getEntity()); Response resultsResponse = resource.doGetResults( actual.getQueryId(), @@ -754,4 +754,27 @@ private static Map defaultAsyncContext() return context; } + private void assertSqlStatementResult(SqlStatementResult expected, SqlStatementResult actual) + { + Assert.assertEquals(expected.getQueryId(), actual.getQueryId()); + Assert.assertEquals(expected.getCreatedAt(), actual.getCreatedAt()); + Assert.assertEquals(expected.getSqlRowSignature(), actual.getSqlRowSignature()); + Assert.assertEquals(expected.getDurationMs(), actual.getDurationMs()); + Assert.assertEquals(expected.getStages(), actual.getStages()); + Assert.assertEquals(expected.getState(), actual.getState()); + Assert.assertEquals(expected.getWarnings(), actual.getWarnings()); + Assert.assertEquals(expected.getResultSetInformation(), actual.getResultSetInformation()); + + if (actual.getCounters() == null || expected.getCounters() == null) { + Assert.assertEquals(expected.getCounters(), actual.getCounters()); + } else { + Assert.assertEquals(expected.getCounters().toString(), actual.getCounters().toString()); + } + + if (actual.getErrorResponse() == null || expected.getErrorResponse() == null) { + Assert.assertEquals(expected.getErrorResponse(), actual.getErrorResponse()); + } else { + Assert.assertEquals(expected.getErrorResponse().getAsMap(), actual.getErrorResponse().getAsMap()); + } + } } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java index e170d8a2c30e..4ea2993050ed 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java @@ -701,7 +701,7 @@ public void testMSQSelectAcceptedQuery() { Response response = resource.doGetStatus(ACCEPTED_SELECT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals( + assertSqlStatementResult( new SqlStatementResult( ACCEPTED_SELECT_MSQ_QUERY, SqlStatementState.ACCEPTED, @@ -711,7 +711,7 @@ public void testMSQSelectAcceptedQuery() null, null ), - response.getEntity() + (SqlStatementResult) response.getEntity() ); assertExceptionMessage( @@ -735,7 +735,7 @@ public void testMSQSelectRunningQuery() Response response = resource.doGetStatus(RUNNING_SELECT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals( + assertSqlStatementResult( new SqlStatementResult( RUNNING_SELECT_MSQ_QUERY, SqlStatementState.RUNNING, @@ -745,7 +745,7 @@ public void testMSQSelectRunningQuery() null, null ), - response.getEntity() + (SqlStatementResult) response.getEntity() ); assertExceptionMessage( @@ -782,9 +782,9 @@ public void testMSQSelectRunningQueryWithDetail() new ArrayList<>(selectTaskReport.get().getPayload().getStatus().getWarningReports()) ); - Assert.assertEquals( + assertSqlStatementResult( expectedSqlStatementResult, - response.getEntity() + (SqlStatementResult) response.getEntity() ); Assert.assertEquals( @@ -886,7 +886,7 @@ public void testFinishedInsertMSQQuery() { Response response = resource.doGetStatus(FINISHED_INSERT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals(new SqlStatementResult( + assertSqlStatementResult(new SqlStatementResult( FINISHED_INSERT_MSQ_QUERY, SqlStatementState.SUCCESS, CREATED_TIME, @@ -894,7 +894,7 @@ public void testFinishedInsertMSQQuery() 100L, new ResultSetInformation(null, null, null, "test", null, null), null - ), response.getEntity()); + ), (SqlStatementResult) response.getEntity()); Assert.assertEquals( Response.Status.OK.getStatusCode(), @@ -926,7 +926,7 @@ public void testMSQInsertAcceptedQuery() { Response response = resource.doGetStatus(ACCEPTED_INSERT_MSQ_TASK, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals( + assertSqlStatementResult( new SqlStatementResult( ACCEPTED_INSERT_MSQ_TASK, SqlStatementState.ACCEPTED, @@ -936,7 +936,7 @@ public void testMSQInsertAcceptedQuery() null, null ), - response.getEntity() + (SqlStatementResult) response.getEntity() ); assertExceptionMessage( @@ -959,7 +959,7 @@ public void testMSQInsertRunningQuery() { Response response = resource.doGetStatus(RUNNING_INSERT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals( + assertSqlStatementResult( new SqlStatementResult( RUNNING_INSERT_MSQ_QUERY, SqlStatementState.RUNNING, @@ -969,7 +969,7 @@ public void testMSQInsertRunningQuery() null, null ), - response.getEntity() + (SqlStatementResult) response.getEntity() ); assertExceptionMessage( @@ -1168,4 +1168,28 @@ public void testIsEnabled() { Assert.assertEquals(Response.Status.OK.getStatusCode(), resource.isEnabled(makeOkRequest()).getStatus()); } + + private void assertSqlStatementResult(SqlStatementResult expected, SqlStatementResult actual) + { + Assert.assertEquals(expected.getQueryId(), actual.getQueryId()); + Assert.assertEquals(expected.getCreatedAt(), actual.getCreatedAt()); + Assert.assertEquals(expected.getSqlRowSignature(), actual.getSqlRowSignature()); + Assert.assertEquals(expected.getDurationMs(), actual.getDurationMs()); + Assert.assertEquals(expected.getStages(), actual.getStages()); + Assert.assertEquals(expected.getState(), actual.getState()); + Assert.assertEquals(expected.getWarnings(), actual.getWarnings()); + Assert.assertEquals(expected.getResultSetInformation(), actual.getResultSetInformation()); + + if (actual.getCounters() == null || expected.getCounters() == null) { + Assert.assertEquals(expected.getCounters(), actual.getCounters()); + } else { + Assert.assertEquals(expected.getCounters().toString(), actual.getCounters().toString()); + } + + if (actual.getErrorResponse() == null || expected.getErrorResponse() == null) { + Assert.assertEquals(expected.getErrorResponse(), actual.getErrorResponse()); + } else { + Assert.assertEquals(expected.getErrorResponse().getAsMap(), actual.getErrorResponse().getAsMap()); + } + } } From daa425ed853d02ab61ff268b0b2ac3d0d6cf5e25 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 29 Jul 2024 13:04:36 +0530 Subject: [PATCH 08/10] Add equals and hashcode back into SqlStatementResult with updated condition --- .../msq/sql/entity/SqlStatementResult.java | 46 ++++++++++++++++++ .../sql/entity/SqlStatementResultTest.java | 8 ++++ .../SqlMSQStatementResourcePostTest.java | 37 +++----------- .../resources/SqlStatementResourceTest.java | 48 +++++-------------- 4 files changed, 73 insertions(+), 66 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java index bd33d76adb1e..508d73703518 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java @@ -31,6 +31,7 @@ import javax.annotation.Nullable; import java.util.List; +import java.util.Objects; public class SqlStatementResult { @@ -185,6 +186,51 @@ public List getWarnings() return warnings; } + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SqlStatementResult that = (SqlStatementResult) o; + return Objects.equals(queryId, that.queryId) && state == that.state && Objects.equals( + createdAt, + that.createdAt + ) && Objects.equals(sqlRowSignature, that.sqlRowSignature) && Objects.equals( + durationMs, + that.durationMs + ) && Objects.equals(resultSetInformation, that.resultSetInformation) && Objects.equals( + errorResponse == null ? null : errorResponse.getAsMap(), + that.errorResponse == null ? null : that.errorResponse.getAsMap() + ) && Objects.equals(stages, that.stages) && Objects.equals( + counters == null ? null : counters.toString(), + that.counters == null ? null : that.counters.toString() + ) && Objects.equals( + warnings, + that.warnings + ); + } + + @Override + public int hashCode() + { + return Objects.hash( + queryId, + state, + createdAt, + sqlRowSignature, + durationMs, + resultSetInformation, + errorResponse == null ? null : errorResponse.getAsMap(), + stages, + counters == null ? null : counters.toString(), + warnings + ); + } + @Override public String toString() { diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java index 96ef0ac6b1f1..e03932a23dd3 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java @@ -71,6 +71,14 @@ public void sanityTest() throws JsonProcessingException { Assert.assertEquals(JSON_STRING, MAPPER.writeValueAsString(SQL_STATEMENT_RESULT)); + Assert.assertEquals( + SQL_STATEMENT_RESULT, + MAPPER.readValue(MAPPER.writeValueAsString(SQL_STATEMENT_RESULT), SqlStatementResult.class) + ); + Assert.assertEquals( + SQL_STATEMENT_RESULT.hashCode(), + MAPPER.readValue(MAPPER.writeValueAsString(SQL_STATEMENT_RESULT), SqlStatementResult.class).hashCode() + ); Assert.assertEquals( "SqlStatementResult{" + "queryId='q1'," diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java index cef3e00daa2d..a9cd5254dd32 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java @@ -206,7 +206,7 @@ public void emptyInsert() new ResultSetInformation(0L, 0L, null, "foo1", null, null), null ); - assertSqlStatementResult(expected, actual); + Assert.assertEquals(expected, actual); } @Test @@ -236,7 +236,7 @@ public void emptyReplace() new ResultSetInformation(0L, 0L, null, "foo1", null, null), null ); - assertSqlStatementResult(expected, actual); + Assert.assertEquals(expected, actual); } @Test @@ -282,7 +282,7 @@ protected DruidException makeException(DruidException.DruidExceptionBuilder bob) } }).toErrorResponse() ); - assertSqlStatementResult(expected, actual); + Assert.assertEquals(expected, actual); } @Test @@ -687,11 +687,11 @@ public void testInsert() new ResultSetInformation(NullHandling.sqlCompatible() ? 6L : 5L, 0L, null, "foo1", null, null), null ); - assertSqlStatementResult(expected, actual); + Assert.assertEquals(expected, actual); Response getResponse = resource.doGetStatus(actual.getQueryId(), false, SqlStatementResourceTest.makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), getResponse.getStatus()); - assertSqlStatementResult(expected, (SqlStatementResult) getResponse.getEntity()); + Assert.assertEquals(expected, getResponse.getEntity()); Response resultsResponse = resource.doGetResults( actual.getQueryId(), @@ -730,11 +730,11 @@ public void testReplaceAll() new ResultSetInformation(NullHandling.sqlCompatible() ? 6L : 5L, 0L, null, "foo1", null, null), null ); - assertSqlStatementResult(expected, actual); + Assert.assertEquals(expected, actual); Response getResponse = resource.doGetStatus(actual.getQueryId(), false, SqlStatementResourceTest.makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), getResponse.getStatus()); - assertSqlStatementResult(expected, (SqlStatementResult) getResponse.getEntity()); + Assert.assertEquals(expected, getResponse.getEntity()); Response resultsResponse = resource.doGetResults( actual.getQueryId(), @@ -754,27 +754,4 @@ private static Map defaultAsyncContext() return context; } - private void assertSqlStatementResult(SqlStatementResult expected, SqlStatementResult actual) - { - Assert.assertEquals(expected.getQueryId(), actual.getQueryId()); - Assert.assertEquals(expected.getCreatedAt(), actual.getCreatedAt()); - Assert.assertEquals(expected.getSqlRowSignature(), actual.getSqlRowSignature()); - Assert.assertEquals(expected.getDurationMs(), actual.getDurationMs()); - Assert.assertEquals(expected.getStages(), actual.getStages()); - Assert.assertEquals(expected.getState(), actual.getState()); - Assert.assertEquals(expected.getWarnings(), actual.getWarnings()); - Assert.assertEquals(expected.getResultSetInformation(), actual.getResultSetInformation()); - - if (actual.getCounters() == null || expected.getCounters() == null) { - Assert.assertEquals(expected.getCounters(), actual.getCounters()); - } else { - Assert.assertEquals(expected.getCounters().toString(), actual.getCounters().toString()); - } - - if (actual.getErrorResponse() == null || expected.getErrorResponse() == null) { - Assert.assertEquals(expected.getErrorResponse(), actual.getErrorResponse()); - } else { - Assert.assertEquals(expected.getErrorResponse().getAsMap(), actual.getErrorResponse().getAsMap()); - } - } } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java index 4ea2993050ed..e170d8a2c30e 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java @@ -701,7 +701,7 @@ public void testMSQSelectAcceptedQuery() { Response response = resource.doGetStatus(ACCEPTED_SELECT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - assertSqlStatementResult( + Assert.assertEquals( new SqlStatementResult( ACCEPTED_SELECT_MSQ_QUERY, SqlStatementState.ACCEPTED, @@ -711,7 +711,7 @@ public void testMSQSelectAcceptedQuery() null, null ), - (SqlStatementResult) response.getEntity() + response.getEntity() ); assertExceptionMessage( @@ -735,7 +735,7 @@ public void testMSQSelectRunningQuery() Response response = resource.doGetStatus(RUNNING_SELECT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - assertSqlStatementResult( + Assert.assertEquals( new SqlStatementResult( RUNNING_SELECT_MSQ_QUERY, SqlStatementState.RUNNING, @@ -745,7 +745,7 @@ public void testMSQSelectRunningQuery() null, null ), - (SqlStatementResult) response.getEntity() + response.getEntity() ); assertExceptionMessage( @@ -782,9 +782,9 @@ public void testMSQSelectRunningQueryWithDetail() new ArrayList<>(selectTaskReport.get().getPayload().getStatus().getWarningReports()) ); - assertSqlStatementResult( + Assert.assertEquals( expectedSqlStatementResult, - (SqlStatementResult) response.getEntity() + response.getEntity() ); Assert.assertEquals( @@ -886,7 +886,7 @@ public void testFinishedInsertMSQQuery() { Response response = resource.doGetStatus(FINISHED_INSERT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - assertSqlStatementResult(new SqlStatementResult( + Assert.assertEquals(new SqlStatementResult( FINISHED_INSERT_MSQ_QUERY, SqlStatementState.SUCCESS, CREATED_TIME, @@ -894,7 +894,7 @@ public void testFinishedInsertMSQQuery() 100L, new ResultSetInformation(null, null, null, "test", null, null), null - ), (SqlStatementResult) response.getEntity()); + ), response.getEntity()); Assert.assertEquals( Response.Status.OK.getStatusCode(), @@ -926,7 +926,7 @@ public void testMSQInsertAcceptedQuery() { Response response = resource.doGetStatus(ACCEPTED_INSERT_MSQ_TASK, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - assertSqlStatementResult( + Assert.assertEquals( new SqlStatementResult( ACCEPTED_INSERT_MSQ_TASK, SqlStatementState.ACCEPTED, @@ -936,7 +936,7 @@ public void testMSQInsertAcceptedQuery() null, null ), - (SqlStatementResult) response.getEntity() + response.getEntity() ); assertExceptionMessage( @@ -959,7 +959,7 @@ public void testMSQInsertRunningQuery() { Response response = resource.doGetStatus(RUNNING_INSERT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - assertSqlStatementResult( + Assert.assertEquals( new SqlStatementResult( RUNNING_INSERT_MSQ_QUERY, SqlStatementState.RUNNING, @@ -969,7 +969,7 @@ public void testMSQInsertRunningQuery() null, null ), - (SqlStatementResult) response.getEntity() + response.getEntity() ); assertExceptionMessage( @@ -1168,28 +1168,4 @@ public void testIsEnabled() { Assert.assertEquals(Response.Status.OK.getStatusCode(), resource.isEnabled(makeOkRequest()).getStatus()); } - - private void assertSqlStatementResult(SqlStatementResult expected, SqlStatementResult actual) - { - Assert.assertEquals(expected.getQueryId(), actual.getQueryId()); - Assert.assertEquals(expected.getCreatedAt(), actual.getCreatedAt()); - Assert.assertEquals(expected.getSqlRowSignature(), actual.getSqlRowSignature()); - Assert.assertEquals(expected.getDurationMs(), actual.getDurationMs()); - Assert.assertEquals(expected.getStages(), actual.getStages()); - Assert.assertEquals(expected.getState(), actual.getState()); - Assert.assertEquals(expected.getWarnings(), actual.getWarnings()); - Assert.assertEquals(expected.getResultSetInformation(), actual.getResultSetInformation()); - - if (actual.getCounters() == null || expected.getCounters() == null) { - Assert.assertEquals(expected.getCounters(), actual.getCounters()); - } else { - Assert.assertEquals(expected.getCounters().toString(), actual.getCounters().toString()); - } - - if (actual.getErrorResponse() == null || expected.getErrorResponse() == null) { - Assert.assertEquals(expected.getErrorResponse(), actual.getErrorResponse()); - } else { - Assert.assertEquals(expected.getErrorResponse().getAsMap(), actual.getErrorResponse().getAsMap()); - } - } } From 174df9a6901d740e8dfcb13c61af5b98014a593a Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Wed, 31 Jul 2024 10:30:16 +0530 Subject: [PATCH 09/10] Revert "Add equals and hashcode back into SqlStatementResult with updated condition" This reverts commit daa425ed853d02ab61ff268b0b2ac3d0d6cf5e25. --- .../msq/sql/entity/SqlStatementResult.java | 46 ------------------ .../sql/entity/SqlStatementResultTest.java | 8 ---- .../SqlMSQStatementResourcePostTest.java | 37 +++++++++++--- .../resources/SqlStatementResourceTest.java | 48 ++++++++++++++----- 4 files changed, 66 insertions(+), 73 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java index 508d73703518..bd33d76adb1e 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/entity/SqlStatementResult.java @@ -31,7 +31,6 @@ import javax.annotation.Nullable; import java.util.List; -import java.util.Objects; public class SqlStatementResult { @@ -186,51 +185,6 @@ public List getWarnings() return warnings; } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SqlStatementResult that = (SqlStatementResult) o; - return Objects.equals(queryId, that.queryId) && state == that.state && Objects.equals( - createdAt, - that.createdAt - ) && Objects.equals(sqlRowSignature, that.sqlRowSignature) && Objects.equals( - durationMs, - that.durationMs - ) && Objects.equals(resultSetInformation, that.resultSetInformation) && Objects.equals( - errorResponse == null ? null : errorResponse.getAsMap(), - that.errorResponse == null ? null : that.errorResponse.getAsMap() - ) && Objects.equals(stages, that.stages) && Objects.equals( - counters == null ? null : counters.toString(), - that.counters == null ? null : that.counters.toString() - ) && Objects.equals( - warnings, - that.warnings - ); - } - - @Override - public int hashCode() - { - return Objects.hash( - queryId, - state, - createdAt, - sqlRowSignature, - durationMs, - resultSetInformation, - errorResponse == null ? null : errorResponse.getAsMap(), - stages, - counters == null ? null : counters.toString(), - warnings - ); - } - @Override public String toString() { diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java index e03932a23dd3..96ef0ac6b1f1 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/entity/SqlStatementResultTest.java @@ -71,14 +71,6 @@ public void sanityTest() throws JsonProcessingException { Assert.assertEquals(JSON_STRING, MAPPER.writeValueAsString(SQL_STATEMENT_RESULT)); - Assert.assertEquals( - SQL_STATEMENT_RESULT, - MAPPER.readValue(MAPPER.writeValueAsString(SQL_STATEMENT_RESULT), SqlStatementResult.class) - ); - Assert.assertEquals( - SQL_STATEMENT_RESULT.hashCode(), - MAPPER.readValue(MAPPER.writeValueAsString(SQL_STATEMENT_RESULT), SqlStatementResult.class).hashCode() - ); Assert.assertEquals( "SqlStatementResult{" + "queryId='q1'," diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java index a9cd5254dd32..cef3e00daa2d 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java @@ -206,7 +206,7 @@ public void emptyInsert() new ResultSetInformation(0L, 0L, null, "foo1", null, null), null ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); } @Test @@ -236,7 +236,7 @@ public void emptyReplace() new ResultSetInformation(0L, 0L, null, "foo1", null, null), null ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); } @Test @@ -282,7 +282,7 @@ protected DruidException makeException(DruidException.DruidExceptionBuilder bob) } }).toErrorResponse() ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); } @Test @@ -687,11 +687,11 @@ public void testInsert() new ResultSetInformation(NullHandling.sqlCompatible() ? 6L : 5L, 0L, null, "foo1", null, null), null ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); Response getResponse = resource.doGetStatus(actual.getQueryId(), false, SqlStatementResourceTest.makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), getResponse.getStatus()); - Assert.assertEquals(expected, getResponse.getEntity()); + assertSqlStatementResult(expected, (SqlStatementResult) getResponse.getEntity()); Response resultsResponse = resource.doGetResults( actual.getQueryId(), @@ -730,11 +730,11 @@ public void testReplaceAll() new ResultSetInformation(NullHandling.sqlCompatible() ? 6L : 5L, 0L, null, "foo1", null, null), null ); - Assert.assertEquals(expected, actual); + assertSqlStatementResult(expected, actual); Response getResponse = resource.doGetStatus(actual.getQueryId(), false, SqlStatementResourceTest.makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), getResponse.getStatus()); - Assert.assertEquals(expected, getResponse.getEntity()); + assertSqlStatementResult(expected, (SqlStatementResult) getResponse.getEntity()); Response resultsResponse = resource.doGetResults( actual.getQueryId(), @@ -754,4 +754,27 @@ private static Map defaultAsyncContext() return context; } + private void assertSqlStatementResult(SqlStatementResult expected, SqlStatementResult actual) + { + Assert.assertEquals(expected.getQueryId(), actual.getQueryId()); + Assert.assertEquals(expected.getCreatedAt(), actual.getCreatedAt()); + Assert.assertEquals(expected.getSqlRowSignature(), actual.getSqlRowSignature()); + Assert.assertEquals(expected.getDurationMs(), actual.getDurationMs()); + Assert.assertEquals(expected.getStages(), actual.getStages()); + Assert.assertEquals(expected.getState(), actual.getState()); + Assert.assertEquals(expected.getWarnings(), actual.getWarnings()); + Assert.assertEquals(expected.getResultSetInformation(), actual.getResultSetInformation()); + + if (actual.getCounters() == null || expected.getCounters() == null) { + Assert.assertEquals(expected.getCounters(), actual.getCounters()); + } else { + Assert.assertEquals(expected.getCounters().toString(), actual.getCounters().toString()); + } + + if (actual.getErrorResponse() == null || expected.getErrorResponse() == null) { + Assert.assertEquals(expected.getErrorResponse(), actual.getErrorResponse()); + } else { + Assert.assertEquals(expected.getErrorResponse().getAsMap(), actual.getErrorResponse().getAsMap()); + } + } } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java index e170d8a2c30e..4ea2993050ed 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/resources/SqlStatementResourceTest.java @@ -701,7 +701,7 @@ public void testMSQSelectAcceptedQuery() { Response response = resource.doGetStatus(ACCEPTED_SELECT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals( + assertSqlStatementResult( new SqlStatementResult( ACCEPTED_SELECT_MSQ_QUERY, SqlStatementState.ACCEPTED, @@ -711,7 +711,7 @@ public void testMSQSelectAcceptedQuery() null, null ), - response.getEntity() + (SqlStatementResult) response.getEntity() ); assertExceptionMessage( @@ -735,7 +735,7 @@ public void testMSQSelectRunningQuery() Response response = resource.doGetStatus(RUNNING_SELECT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals( + assertSqlStatementResult( new SqlStatementResult( RUNNING_SELECT_MSQ_QUERY, SqlStatementState.RUNNING, @@ -745,7 +745,7 @@ public void testMSQSelectRunningQuery() null, null ), - response.getEntity() + (SqlStatementResult) response.getEntity() ); assertExceptionMessage( @@ -782,9 +782,9 @@ public void testMSQSelectRunningQueryWithDetail() new ArrayList<>(selectTaskReport.get().getPayload().getStatus().getWarningReports()) ); - Assert.assertEquals( + assertSqlStatementResult( expectedSqlStatementResult, - response.getEntity() + (SqlStatementResult) response.getEntity() ); Assert.assertEquals( @@ -886,7 +886,7 @@ public void testFinishedInsertMSQQuery() { Response response = resource.doGetStatus(FINISHED_INSERT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals(new SqlStatementResult( + assertSqlStatementResult(new SqlStatementResult( FINISHED_INSERT_MSQ_QUERY, SqlStatementState.SUCCESS, CREATED_TIME, @@ -894,7 +894,7 @@ public void testFinishedInsertMSQQuery() 100L, new ResultSetInformation(null, null, null, "test", null, null), null - ), response.getEntity()); + ), (SqlStatementResult) response.getEntity()); Assert.assertEquals( Response.Status.OK.getStatusCode(), @@ -926,7 +926,7 @@ public void testMSQInsertAcceptedQuery() { Response response = resource.doGetStatus(ACCEPTED_INSERT_MSQ_TASK, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals( + assertSqlStatementResult( new SqlStatementResult( ACCEPTED_INSERT_MSQ_TASK, SqlStatementState.ACCEPTED, @@ -936,7 +936,7 @@ public void testMSQInsertAcceptedQuery() null, null ), - response.getEntity() + (SqlStatementResult) response.getEntity() ); assertExceptionMessage( @@ -959,7 +959,7 @@ public void testMSQInsertRunningQuery() { Response response = resource.doGetStatus(RUNNING_INSERT_MSQ_QUERY, false, makeOkRequest()); Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assert.assertEquals( + assertSqlStatementResult( new SqlStatementResult( RUNNING_INSERT_MSQ_QUERY, SqlStatementState.RUNNING, @@ -969,7 +969,7 @@ public void testMSQInsertRunningQuery() null, null ), - response.getEntity() + (SqlStatementResult) response.getEntity() ); assertExceptionMessage( @@ -1168,4 +1168,28 @@ public void testIsEnabled() { Assert.assertEquals(Response.Status.OK.getStatusCode(), resource.isEnabled(makeOkRequest()).getStatus()); } + + private void assertSqlStatementResult(SqlStatementResult expected, SqlStatementResult actual) + { + Assert.assertEquals(expected.getQueryId(), actual.getQueryId()); + Assert.assertEquals(expected.getCreatedAt(), actual.getCreatedAt()); + Assert.assertEquals(expected.getSqlRowSignature(), actual.getSqlRowSignature()); + Assert.assertEquals(expected.getDurationMs(), actual.getDurationMs()); + Assert.assertEquals(expected.getStages(), actual.getStages()); + Assert.assertEquals(expected.getState(), actual.getState()); + Assert.assertEquals(expected.getWarnings(), actual.getWarnings()); + Assert.assertEquals(expected.getResultSetInformation(), actual.getResultSetInformation()); + + if (actual.getCounters() == null || expected.getCounters() == null) { + Assert.assertEquals(expected.getCounters(), actual.getCounters()); + } else { + Assert.assertEquals(expected.getCounters().toString(), actual.getCounters().toString()); + } + + if (actual.getErrorResponse() == null || expected.getErrorResponse() == null) { + Assert.assertEquals(expected.getErrorResponse(), actual.getErrorResponse()); + } else { + Assert.assertEquals(expected.getErrorResponse().getAsMap(), actual.getErrorResponse().getAsMap()); + } + } } From eda21bcaed0f37e90b92472f9408a6c1128ee703 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Wed, 31 Jul 2024 20:58:48 +0530 Subject: [PATCH 10/10] Update docs --- docs/api-reference/sql-api.md | 432 +++++++++++++++++++++++++++++++++- 1 file changed, 429 insertions(+), 3 deletions(-) diff --git a/docs/api-reference/sql-api.md b/docs/api-reference/sql-api.md index bf58d2364262..e1fb53bc6eba 100644 --- a/docs/api-reference/sql-api.md +++ b/docs/api-reference/sql-api.md @@ -629,10 +629,21 @@ Retrieves information about the query associated with the given query ID. The re - `sizeInBytes`: the size of the page. - `id`: the page number that you can use to reference a specific page when you get query results. +If the optional query parameter `detail` is supplied, then the response also includes the following: +- A `stages` object that summarizes information about the different stages being used for query execution, such as stage number, phase, start time, duration, input and output information, processing methods, and partitioning. +- A `counters` object that provides details on the rows, bytes, and files processed at various stages for each worker across different channels, along with sort progress. +- A `warnings` object that provides details about any warnings. + #### URL `GET` `/druid/v2/sql/statements/{queryId}` +#### Query parameters +* `detail` (optional) + * Type: Boolean + * Default: false + * Fetch additional details about the query, which includes the information about different stages, counters for each stage, and any warnings. + #### Responses @@ -672,7 +683,7 @@ The following example retrieves the status of a query with specified ID `query-9 ```shell -curl "http://ROUTER_IP:ROUTER_PORT/druid/v2/sql/statements/query-9b93f6f7-ab0e-48f5-986a-3520f84f0804" +curl "http://ROUTER_IP:ROUTER_PORT/druid/v2/sql/statements/query-9b93f6f7-ab0e-48f5-986a-3520f84f0804?detail=true" ``` @@ -680,7 +691,7 @@ curl "http://ROUTER_IP:ROUTER_PORT/druid/v2/sql/statements/query-9b93f6f7-ab0e-4 ```HTTP -GET /druid/v2/sql/statements/query-9b93f6f7-ab0e-48f5-986a-3520f84f0804 HTTP/1.1 +GET /druid/v2/sql/statements/query-9b93f6f7-ab0e-48f5-986a-3520f84f0804?detail=true HTTP/1.1 Host: http://ROUTER_IP:ROUTER_PORT ``` @@ -835,7 +846,422 @@ Host: http://ROUTER_IP:ROUTER_PORT "sizeInBytes": 375 } ] - } + }, + "stages": [ + { + "stageNumber": 0, + "definition": { + "id": "query-9b93f6f7-ab0e-48f5-986a-3520f84f0804_0", + "input": [ + { + "type": "table", + "dataSource": "wikipedia", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ], + "filter": { + "type": "equals", + "column": "user", + "matchValueType": "STRING", + "matchValue": "BlueMoon2662" + }, + "filterFields": [ + "user" + ] + } + ], + "processor": { + "type": "scan", + "query": { + "queryType": "scan", + "dataSource": { + "type": "inputNumber", + "inputNumber": 0 + }, + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "virtualColumns": [ + { + "type": "expression", + "name": "v0", + "expression": "'BlueMoon2662'", + "outputType": "STRING" + } + ], + "resultFormat": "compactedList", + "limit": 1001, + "filter": { + "type": "equals", + "column": "user", + "matchValueType": "STRING", + "matchValue": "BlueMoon2662" + }, + "columns": [ + "__time", + "added", + "channel", + "cityName", + "comment", + "commentLength", + "countryIsoCode", + "countryName", + "deleted", + "delta", + "deltaBucket", + "diffUrl", + "flags", + "isAnonymous", + "isMinor", + "isNew", + "isRobot", + "isUnpatrolled", + "metroCode", + "namespace", + "page", + "regionIsoCode", + "regionName", + "v0" + ], + "context": { + "__resultFormat": "array", + "__user": "allowAll", + "enableWindowing": true, + "executionMode": "async", + "finalize": true, + "maxNumTasks": 2, + "maxParseExceptions": 0, + "queryId": "33b53acb-7533-4880-a81b-51c16c489eab", + "scanSignature": "[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"added\",\"type\":\"LONG\"},{\"name\":\"channel\",\"type\":\"STRING\"},{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"comment\",\"type\":\"STRING\"},{\"name\":\"commentLength\",\"type\":\"LONG\"},{\"name\":\"countryIsoCode\",\"type\":\"STRING\"},{\"name\":\"countryName\",\"type\":\"STRING\"},{\"name\":\"deleted\",\"type\":\"LONG\"},{\"name\":\"delta\",\"type\":\"LONG\"},{\"name\":\"deltaBucket\",\"type\":\"LONG\"},{\"name\":\"diffUrl\",\"type\":\"STRING\"},{\"name\":\"flags\",\"type\":\"STRING\"},{\"name\":\"isAnonymous\",\"type\":\"STRING\"},{\"name\":\"isMinor\",\"type\":\"STRING\"},{\"name\":\"isNew\",\"type\":\"STRING\"},{\"name\":\"isRobot\",\"type\":\"STRING\"},{\"name\":\"isUnpatrolled\",\"type\":\"STRING\"},{\"name\":\"metroCode\",\"type\":\"STRING\"},{\"name\":\"namespace\",\"type\":\"STRING\"},{\"name\":\"page\",\"type\":\"STRING\"},{\"name\":\"regionIsoCode\",\"type\":\"STRING\"},{\"name\":\"regionName\",\"type\":\"STRING\"},{\"name\":\"v0\",\"type\":\"STRING\"}]", + "sqlOuterLimit": 1001, + "sqlQueryId": "33b53acb-7533-4880-a81b-51c16c489eab", + "sqlStringifyArrays": false + }, + "columnTypes": [ + "LONG", + "LONG", + "STRING", + "STRING", + "STRING", + "LONG", + "STRING", + "STRING", + "LONG", + "LONG", + "LONG", + "STRING", + "STRING", + "STRING", + "STRING", + "STRING", + "STRING", + "STRING", + "STRING", + "STRING", + "STRING", + "STRING", + "STRING", + "STRING" + ], + "granularity": { + "type": "all" + }, + "legacy": false + } + }, + "signature": [ + { + "name": "__boost", + "type": "LONG" + }, + { + "name": "__time", + "type": "LONG" + }, + { + "name": "added", + "type": "LONG" + }, + { + "name": "channel", + "type": "STRING" + }, + { + "name": "cityName", + "type": "STRING" + }, + { + "name": "comment", + "type": "STRING" + }, + { + "name": "commentLength", + "type": "LONG" + }, + { + "name": "countryIsoCode", + "type": "STRING" + }, + { + "name": "countryName", + "type": "STRING" + }, + { + "name": "deleted", + "type": "LONG" + }, + { + "name": "delta", + "type": "LONG" + }, + { + "name": "deltaBucket", + "type": "LONG" + }, + { + "name": "diffUrl", + "type": "STRING" + }, + { + "name": "flags", + "type": "STRING" + }, + { + "name": "isAnonymous", + "type": "STRING" + }, + { + "name": "isMinor", + "type": "STRING" + }, + { + "name": "isNew", + "type": "STRING" + }, + { + "name": "isRobot", + "type": "STRING" + }, + { + "name": "isUnpatrolled", + "type": "STRING" + }, + { + "name": "metroCode", + "type": "STRING" + }, + { + "name": "namespace", + "type": "STRING" + }, + { + "name": "page", + "type": "STRING" + }, + { + "name": "regionIsoCode", + "type": "STRING" + }, + { + "name": "regionName", + "type": "STRING" + }, + { + "name": "v0", + "type": "STRING" + } + ], + "shuffleSpec": { + "type": "mix" + }, + "maxWorkerCount": 1 + }, + "phase": "FINISHED", + "workerCount": 1, + "partitionCount": 1, + "shuffle": "mix", + "output": "localStorage", + "startTime": "2024-07-31T15:20:21.255Z", + "duration": 103 + }, + { + "stageNumber": 1, + "definition": { + "id": "query-9b93f6f7-ab0e-48f5-986a-3520f84f0804_1", + "input": [ + { + "type": "stage", + "stage": 0 + } + ], + "processor": { + "type": "limit", + "limit": 1001 + }, + "signature": [ + { + "name": "__boost", + "type": "LONG" + }, + { + "name": "__time", + "type": "LONG" + }, + { + "name": "added", + "type": "LONG" + }, + { + "name": "channel", + "type": "STRING" + }, + { + "name": "cityName", + "type": "STRING" + }, + { + "name": "comment", + "type": "STRING" + }, + { + "name": "commentLength", + "type": "LONG" + }, + { + "name": "countryIsoCode", + "type": "STRING" + }, + { + "name": "countryName", + "type": "STRING" + }, + { + "name": "deleted", + "type": "LONG" + }, + { + "name": "delta", + "type": "LONG" + }, + { + "name": "deltaBucket", + "type": "LONG" + }, + { + "name": "diffUrl", + "type": "STRING" + }, + { + "name": "flags", + "type": "STRING" + }, + { + "name": "isAnonymous", + "type": "STRING" + }, + { + "name": "isMinor", + "type": "STRING" + }, + { + "name": "isNew", + "type": "STRING" + }, + { + "name": "isRobot", + "type": "STRING" + }, + { + "name": "isUnpatrolled", + "type": "STRING" + }, + { + "name": "metroCode", + "type": "STRING" + }, + { + "name": "namespace", + "type": "STRING" + }, + { + "name": "page", + "type": "STRING" + }, + { + "name": "regionIsoCode", + "type": "STRING" + }, + { + "name": "regionName", + "type": "STRING" + }, + { + "name": "v0", + "type": "STRING" + } + ], + "shuffleSpec": { + "type": "maxCount", + "clusterBy": { + "columns": [ + { + "columnName": "__boost", + "order": "ASCENDING" + } + ] + }, + "partitions": 1 + }, + "maxWorkerCount": 1 + }, + "phase": "FINISHED", + "workerCount": 1, + "partitionCount": 1, + "shuffle": "globalSort", + "output": "localStorage", + "startTime": "2024-07-31T15:20:21.355Z", + "duration": 10, + "sort": true + } + ], + "counters": { + "0": { + "0": { + "input0": { + "type": "channel", + "rows": [ + 24433 + ], + "bytes": [ + 7393933 + ], + "files": [ + 22 + ], + "totalFiles": [ + 22 + ] + } + } + }, + "1": { + "0": { + "sortProgress": { + "type": "sortProgress", + "totalMergingLevels": -1, + "levelToTotalBatches": {}, + "levelToMergedBatches": {}, + "totalMergersForUltimateLevel": -1, + "triviallyComplete": true, + "progressDigest": 1 + } + } + } + }, + "warnings": [] } ```