From 4824218afbade6dbb1b5fe1d7447bd0b510096fc Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Fri, 20 Nov 2026 11:27:02 +0800 Subject: [PATCH] Fix success status of a query that is removed while its first RPC starts it Removing a query from the Coordinator stops it and releases its result handle, but does not move its state machine on dev/1.3. If cleanUpStaleQueries reclaims a query whose first RPC is still dispatching (the RPC lasted longer than timeout + 60 s) and the dispatch then succeeds, the RPC returned SUCCESS_STATUS without a result set. JDBC executeQuery and Session.executeQueryStatement failed with a NullPointerException on columnNameList; Statement.execute and Session.executeNonQueryStatement reported a statement that succeeded. Coordinator.execution now checks, after the start, whether a statement it registered as a query is still registered. If it is not and the status says success, the status is replaced with QUERY_WAS_KILLED(715) and the message the fetch RPCs already use for a query that is gone. A failure status is returned unchanged, and statements that are not queries are not affected. master reports an error here since #19224, which fails the state machine in stopAndCleanup. That change is not ported; this is a dev/1.3-only fix. --- .../iotdb/db/i18n/DataNodeQueryMessages.java | 4 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 3 + .../thrift/impl/ClientRPCServiceImpl.java | 4 +- .../db/queryengine/plan/Coordinator.java | 23 ++ .../plan/CoordinatorRemovedQueryTest.java | 230 ++++++++++++++++++ 5 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorRemovedQueryTest.java diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 073473b47423..2fc5997715d2 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -71,4 +71,8 @@ private DataNodeQueryMessages() {} "Delete After Loading {} error."; public static final String LOG_FAILED_TO_AUTHORIZE_CQ_ARG_FOR_USER_ARG_ARG_B436A81F = "Failed to authorize CQ {} for user {}: {}"; + public static final String + EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E = + "Query is not found, it may be killed by others, timeout or some other runtime errors," + + " you can see more details in server log."; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 370792a20cc2..45b057fbc44d 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -67,4 +67,7 @@ private DataNodeQueryMessages() {} public static final String LOG_DELETE_AFTER_LOADING_ARG_ERROR_C606A060 = "加载后删除 {} 时出错。"; public static final String LOG_FAILED_TO_AUTHORIZE_CQ_ARG_FOR_USER_ARG_ARG_B436A81F = "CQ {} 的用户 {} 鉴权失败:{}"; + public static final String + EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E = + "未找到查询,可能已被其他用户终止、超时或发生其他运行时错误,更多详情请查看服务器日志。"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 9607e51e7e73..a68a019b3fea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.basic.BasicOpenSessionResp; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; @@ -250,7 +251,8 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); private static final String NO_QUERY_EXECUTION_ERR_MSG = - "Query is not found, it may be killed by others, timeout or some other runtime errors, you can see more details in server log."; + DataNodeQueryMessages + .EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E; @FunctionalInterface public interface SelectResult { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index 00fd3344119c..45d02975487b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.ClientPoolFactory; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient; @@ -32,6 +33,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.query.QueryTimeoutRuntimeException; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.common.DataNodeEndPoints; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; @@ -50,6 +52,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.utils.SetThreadName; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -163,6 +167,19 @@ private ExecutionResult execution( // if it's write request and the result status needs to retry result.status.setNeedRetry(true); } + if (execution.isQuery() + && queryExecutionMap.get(queryId) != execution + && isSuccessful(result.status)) { + // The query was removed while it was being started, for example by cleanUpStaleQueries + // during a dispatch that took too long. Removing a query does not fail its state, so its + // status still says success, but nobody can fetch its result any more. + return new ExecutionResult( + result.queryId, + RpcUtils.getStatus( + TSStatusCode.QUERY_WAS_KILLED, + DataNodeQueryMessages + .EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E)); + } return result; } catch (RuntimeException | Error e) { // Analysis can fail before registration. Only link the query ID to its SQL here; callers @@ -178,6 +195,12 @@ private ExecutionResult execution( } } + private static boolean isSuccessful(TSStatus status) { + return status != null + && (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || status.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()); + } + /** This method is called by the write method. So it does not set the timeout parameter. */ public ExecutionResult executeForTreeModel( Statement statement, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorRemovedQueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorRemovedQueryTest.java new file mode 100644 index 000000000000..62b824d51b21 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorRemovedQueryTest.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; +import org.apache.iotdb.db.queryengine.plan.execution.QueryExecution; +import org.apache.iotdb.db.queryengine.plan.planner.IPlanner; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.read.common.block.TsBlock; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiFunction; +import java.util.function.Consumer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** What the RPC that executes a statement is told when its query is removed while it starts. */ +public class CoordinatorRemovedQueryTest { + + // A query id is never reused, as a terminated query releases itself asynchronously by its id + private static final AtomicLong NEXT_QUERY_ID = new AtomicLong(20_001L); + + private static final long QUERY_TIMEOUT_IN_MS = 2_000L; + // Coordinator.cleanUpStaleQueries spares a query until it exceeds its timeout by this long + private static final long STALE_QUERY_GRACE_PERIOD_IN_MS = 60_000L; + + private static int previousDataNodeId; + private static ExecutorService stateMachineExecutor; + + private final List executedQueryIds = new ArrayList<>(); + + @BeforeClass + public static void setUp() { + // the coordinator builds its query id generator from the data node id of this node + previousDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0); + stateMachineExecutor = Executors.newSingleThreadExecutor(); + } + + @AfterClass + public static void tearDown() { + stateMachineExecutor.shutdownNow(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(previousDataNodeId); + } + + @After + public void releaseExecutedQueries() { + // a query that had to be left alone is still registered + executedQueryIds.forEach(Coordinator.getInstance()::cleanupQueryExecution); + } + + @Test + public void testQueryReclaimedWhileItStartsIsReportedAsNotFound() throws Throwable { + long queryId = nextQueryId(); + + // The start of the query takes so long, for example in the dispatch of its fragment + // instances, that the stale query cleanup reclaims the query. The start then succeeds. + ExecutionResult result = + startStatement( + queryId, + QueryType.READ, + context -> { + context.setStartTime( + System.currentTimeMillis() + - (QUERY_TIMEOUT_IN_MS + STALE_QUERY_GRACE_PERIOD_IN_MS + 1_000L)); + Coordinator.getInstance().cleanUpStaleQueries(); + }); + + assertNull(Coordinator.getInstance().getQueryExecution(queryId)); + // nobody can fetch the result of this query, so the statement must not be reported as + // successful + assertEquals(TSStatusCode.QUERY_WAS_KILLED.getStatusCode(), result.status.getCode()); + assertEquals( + DataNodeQueryMessages + .EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E, + result.status.getMessage()); + } + + @Test + public void testQueryThatStaysRegisteredIsReportedAsSuccessful() throws Throwable { + long queryId = nextQueryId(); + + ExecutionResult result = startStatement(queryId, QueryType.READ, context -> {}); + + assertNotNull(Coordinator.getInstance().getQueryExecution(queryId)); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.status.getCode()); + } + + @Test + public void testStatementThatIsNotAQueryIsReportedAsSuccessful() throws Throwable { + long queryId = nextQueryId(); + + // only queries are registered, so a missing execution means nothing for any other statement + ExecutionResult result = startStatement(queryId, QueryType.WRITE, context -> {}); + + assertNull(Coordinator.getInstance().getQueryExecution(queryId)); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.status.getCode()); + } + + @Test + public void testFailureOfQueryRemovedWhileItStartsIsKept() throws Throwable { + long queryId = nextQueryId(); + TSStatus failure = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, "dispatch failed"); + IQueryExecution execution = mock(IQueryExecution.class); + when(execution.isQuery()).thenReturn(true); + when(execution.getQueryId()).thenReturn("removed_and_failed"); + doAnswer( + invocation -> { + Coordinator.getInstance().cleanupQueryExecution(queryId); + return null; + }) + .when(execution) + .start(); + when(execution.getStatus()) + .thenReturn(new ExecutionResult(new QueryId("removed_and_failed"), failure)); + + ExecutionResult result = execute(queryId, (queryContext, startTime) -> execution); + + assertNull(Coordinator.getInstance().getQueryExecution(queryId)); + // the reason of the failure tells more than "not found" + assertEquals(failure, result.status); + } + + private long nextQueryId() { + long queryId = NEXT_QUERY_ID.getAndIncrement(); + executedQueryIds.add(queryId); + return queryId; + } + + /** + * Executes a statement the way a client RPC does and returns what that RPC is told. The result of + * the statement is already in memory, so no fragment instance is dispatched. {@code + * whileItStarts} runs after the statement has been registered and before its start returns. + */ + private static ExecutionResult startStatement( + long queryId, QueryType queryType, Consumer whileItStarts) throws Throwable { + IAnalysis analysis = mock(IAnalysis.class); + when(analysis.canSkipExecute(any(MPPQueryContext.class))) + .thenAnswer( + invocation -> { + whileItStarts.accept(invocation.getArgument(0)); + return true; + }); + when(analysis.constructResultForMemorySource(any(MPPQueryContext.class))) + .thenReturn(new TsBlock(0)); + IPlanner planner = mock(IPlanner.class); + when(planner.analyze(any(MPPQueryContext.class))).thenReturn(analysis); + return execute( + queryId, + (queryContext, startTime) -> { + queryContext.setQueryType(queryType); + queryContext.setTimeOut(QUERY_TIMEOUT_IN_MS); + queryContext.setStartTime(startTime); + return new QueryExecution(planner, queryContext, stateMachineExecutor); + }); + } + + private static ExecutionResult execute( + long queryId, BiFunction queryExecutionFactory) + throws Throwable { + Method execution = + Coordinator.class.getDeclaredMethod( + "execution", + long.class, + SessionInfo.class, + String.class, + boolean.class, + boolean.class, + BiFunction.class); + execution.setAccessible(true); + try { + return (ExecutionResult) + execution.invoke( + Coordinator.getInstance(), + queryId, + new SessionInfo(0, "test", ZoneId.systemDefault(), "test"), + "SELECT s FROM t1", + false, + false, + queryExecutionFactory); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } +}