From 3a611d818fb371d733d4d5768d88e92a698e4894 Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 19 Nov 2026 15:23:09 +0800 Subject: [PATCH 1/2] [Query] Fix stale query cleanup never reclaiming idle queries Trigger: a client runs a query and then neither fetches the rest of the result nor closes the statement, and no fragment instance of the query is left to fail it. Reproduced on a 1C2D cluster with a query sent to DataNode 1 whose data is on DataNode 2 (20000 rows, timeout 2s): the client reads the first batch and then stays idle. Wrong behaviour: Coordinator.cleanUpStaleQueries runs every 2s to reclaim the queries that exceeded their timeout by more than 60s, and it skipped every query for which IQueryExecution.isActive() returned true. isActive() is documented as "there is an active RPC for this query", but QueryExecution and ConfigExecution returned startTimeOfCurrentRpc == -1, i.e. true when no RPC is in progress. The cleanup therefore skipped exactly the abandoned queries and only reclaimed queries with an RPC in progress. The query above was still registered in the Coordinator 90s later, and the client could still fetch all of its rows. Corrected behaviour: - cleanUpStaleQueries no longer depends on isActive(): every query that exceeded its timeout by more than 60s is reclaimed. The query above is cleaned up with a QueryTimeoutRuntimeException 63s after it started, and a later fetch is told that the query is not found (715). - isActive() returns true iff an RPC of the query is in progress, as documented. QueryExecution.cleanUpCoordinatorContextMapIfNeeded had been written against the inverted value, so its check is negated and its behaviour is unchanged. The cleanup deliberately does not start to skip the queries with an RPC in progress, which is what the isActive() check was meant to do, because the cleanup of such queries is relied on today: - executeBatchStatement (for a query statement) and querySchemaTemplate register an execution that they never release, and the end of their RPC is never recorded. The stale query cleanup is what releases these executions (measured: 65-66s after start with a 5s timeout, before and after this change). Skipping them would keep them forever. - an RPC that is still in progress 60s after the timeout is stuck, e.g. in a dispatch that does not return, and stopping the query is what unblocks it (QueryTerminalCleanupTest). So what was reclaimed before is still reclaimed, and the queries without an RPC in progress are reclaimed in addition. (cherry picked from commit 78bf43b042f34433cfa48fbc47a8707dd04586b2) Backport of apache/iotdb#19330 to dev/1.3: - QueryExecution.cleanUpCoordinatorContextMapIfNeeded keeps the cleanupQueryExecution(queryId, null, t) call of dev/1.3; only its condition is negated, as in the source commit. - CoordinatorTest and QueryExecutionLifecycleTest do not exist on dev/1.3. They are added with the tests of this change only, adapted to the Coordinator.execution and SessionInfo signatures and to the QueryType values of dev/1.3. - Not ported: the assertions on the UDF query scope of an execution and on the QUERY_TIMEOUT status of a reclaimed query. dev/1.3 has no UDF query scope, and its stopAndCleanup() does not move the query state machine, so the status of a reclaimed query stays SUCCESS there. The ported tests assert that the query is removed from the Coordinator. --- .../db/queryengine/plan/Coordinator.java | 10 +- .../plan/execution/IQueryExecution.java | 2 + .../plan/execution/QueryExecution.java | 4 +- .../execution/config/ConfigExecution.java | 2 +- .../execution/ConfigExecutionTest.java | 24 ++ .../db/queryengine/plan/CoordinatorTest.java | 258 ++++++++++++++++++ .../QueryExecutionLifecycleTest.java | 75 +++++ 7 files changed, 369 insertions(+), 6 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecutionLifecycleTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index 00fd3344119c0..0e6e4d6e96bad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -331,14 +331,18 @@ public void cleanupQueryExecution( * We need to reclaim resources from queries that have exceeded their timeout by more than one * minute. This indicates that the associated clients have failed to perform proper resource * cleanup. + * + *

Whether an RPC of the query is in progress does not matter here. The timeout bounds the + * whole lifetime of a query, so a query without an RPC in progress has been abandoned by its + * client, and an RPC that is still in progress this long after the timeout is stuck, e.g. in a + * dispatch that never returns, and is only unblocked by stopping the query. Sparing the queries + * with an RPC in progress would also keep forever every query whose caller never reports the end + * of its RPC through {@link #recordExecutionTime}. */ public void cleanUpStaleQueries() { long currentTime = System.currentTimeMillis(); queryExecutionMap.forEach( (queryId, queryExecution) -> { - if (queryExecution.isActive()) { - return; - } long timeout = queryExecution.getTimeout(); long queryStartTime = queryExecution.getStartExecutionTime(); long executeTime = currentTime - queryStartTime; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java index d41db48afec10..97db9772befdf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java @@ -78,6 +78,8 @@ public interface IQueryExecution { * finishes. On the other hand, if there is no active RPC, it means that the client has finished * fetching results or has not started fetching results yet, and the QueryExecution can be safely * cleaned up. + * + * @return true if an RPC of this query is in progress, false otherwise */ boolean isActive(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java index c99af2a8ffe47..dd8a050ac8b44 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java @@ -432,7 +432,7 @@ private void releaseResource(Throwable t) { * ClientRPCServiceImpl is responsible for performing the cleanup. */ private synchronized void cleanUpCoordinatorContextMapIfNeeded(Throwable t) { - if (isActive()) { + if (!isActive()) { Coordinator.getInstance().cleanupQueryExecution(context.getLocalQueryId(), null, t); } } @@ -446,7 +446,7 @@ private synchronized void cleanUpCoordinatorContextMapIfNeeded(Throwable t) { * cleaned up. */ public synchronized boolean isActive() { - return startTimeOfCurrentRpc == -1; + return startTimeOfCurrentRpc != -1; } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java index 33173a5e59858..b4ca53f27e17c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java @@ -257,7 +257,7 @@ public void updateCurrentRpcStartTime(long startTime) { @Override public boolean isActive() { - return startTimeOfCurrentRpc == -1; + return startTimeOfCurrentRpc != -1; } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/ConfigExecutionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/ConfigExecutionTest.java index 32dcd88f42df2..1fc7d5c389e1d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/ConfigExecutionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/ConfigExecutionTest.java @@ -46,6 +46,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class ConfigExecutionTest { @@ -157,6 +158,29 @@ public void exceptionAfterInvokeGetStatusTest() { } } + @Test + public void isActiveTellsWhetherAnRpcOfTheQueryIsInProgress() { + ExecutorService executor = getExecutor(); + try { + IConfigTask task = + (clientManager) -> immediateFuture(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + ConfigExecution execution = new ConfigExecution(genMPPQueryContext(), executor, task); + // the execution is created by the RPC that executes the statement + assertTrue(execution.isActive()); + // that RPC returns, and the client has not asked for the result yet + execution.recordExecutionTime(1); + assertFalse(execution.isActive()); + // an RPC that fetches the result arrives + execution.updateCurrentRpcStartTime(System.nanoTime()); + assertTrue(execution.isActive()); + // and returns as well + execution.recordExecutionTime(1); + assertFalse(execution.isActive()); + } finally { + executor.shutdownNow(); + } + } + private MPPQueryContext genMPPQueryContext() { MPPQueryContext context = new MPPQueryContext(new QueryId("query1")); context.setQueryType(QueryType.WRITE); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorTest.java new file mode 100644 index 0000000000000..b2dd8c33f5901 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorTest.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; +import org.apache.iotdb.db.queryengine.plan.execution.QueryExecution; +import org.apache.iotdb.db.queryengine.plan.planner.IPlanner; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.read.common.block.TsBlock; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiFunction; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class CoordinatorTest { + + // A query id is never reused, as a terminated query releases itself asynchronously by its id + private static final AtomicLong NEXT_QUERY_ID = new AtomicLong(10_001L); + + private static final long QUERY_TIMEOUT_IN_MS = 2_000L; + // Coordinator.cleanUpStaleQueries spares a query until it exceeds its timeout by this long + private static final long STALE_QUERY_GRACE_PERIOD_IN_MS = 60_000L; + + private static int previousDataNodeId; + private static ExecutorService stateMachineExecutor; + + private final List executedQueryIds = new ArrayList<>(); + + @BeforeClass + public static void setUp() { + // the coordinator builds its query id generator from the data node id of this node + previousDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0); + stateMachineExecutor = Executors.newSingleThreadExecutor(); + } + + @AfterClass + public static void tearDown() { + stateMachineExecutor.shutdownNow(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(previousDataNodeId); + } + + @After + public void releaseExecutedQueries() { + // a query that had to be left alone is still registered + executedQueryIds.forEach(Coordinator.getInstance()::cleanupQueryExecution); + } + + @Test + public void testStaleQueryWithoutRpcInProgressIsCleanedUp() throws Throwable { + QueryExecution execution = executeQuery(QUERY_TIMEOUT_IN_MS); + // the RPC that executed the statement returns, and the client never fetches the result + finishRpc(execution); + setElapsedTime(execution, QUERY_TIMEOUT_IN_MS + STALE_QUERY_GRACE_PERIOD_IN_MS + 1_000L); + + Coordinator.getInstance().cleanUpStaleQueries(); + + assertNull(Coordinator.getInstance().getQueryExecution(queryIdOf(execution))); + } + + @Test + public void testStaleQueryWithRpcInProgressIsCleanedUp() throws Throwable { + // the RPC that executed the statement has not returned yet, or its caller never reports that + // it returned + QueryExecution executing = executeQuery(QUERY_TIMEOUT_IN_MS); + // that RPC returned, and a later RPC is fetching the result + QueryExecution fetching = executeQuery(QUERY_TIMEOUT_IN_MS); + finishRpc(fetching); + fetching.updateCurrentRpcStartTime(System.nanoTime()); + long elapsedTime = QUERY_TIMEOUT_IN_MS + STALE_QUERY_GRACE_PERIOD_IN_MS + 1_000L; + setElapsedTime(executing, elapsedTime); + setElapsedTime(fetching, elapsedTime); + + Coordinator.getInstance().cleanUpStaleQueries(); + + // an RPC still in progress that long after the timeout is stuck, and stopping the query is + // what unblocks it + for (QueryExecution execution : new QueryExecution[] {executing, fetching}) { + assertNull(Coordinator.getInstance().getQueryExecution(queryIdOf(execution))); + } + } + + @Test + public void testQueryWithRpcInProgressWithinGracePeriodIsLeftAlone() throws Throwable { + QueryExecution execution = executeQuery(QUERY_TIMEOUT_IN_MS); + // timed out, but not yet by more than the grace period + setElapsedTime(execution, QUERY_TIMEOUT_IN_MS + STALE_QUERY_GRACE_PERIOD_IN_MS / 2); + + Coordinator.getInstance().cleanUpStaleQueries(); + + assertLeftAlone(execution); + } + + @Test + public void testIdleQueryWithinGracePeriodIsLeftAlone() throws Throwable { + QueryExecution execution = executeQuery(QUERY_TIMEOUT_IN_MS); + finishRpc(execution); + // timed out, but not yet by more than the grace period + setElapsedTime(execution, QUERY_TIMEOUT_IN_MS + STALE_QUERY_GRACE_PERIOD_IN_MS / 2); + + Coordinator.getInstance().cleanUpStaleQueries(); + + assertLeftAlone(execution); + } + + @Test + public void testIdleQueryWithoutPositiveTimeoutIsLeftAlone() throws Throwable { + QueryExecution zeroTimeout = executeQuery(0L); + QueryExecution negativeTimeout = executeQuery(-1L); + finishRpc(zeroTimeout); + finishRpc(negativeTimeout); + setElapsedTime(zeroTimeout, 10 * STALE_QUERY_GRACE_PERIOD_IN_MS); + setElapsedTime(negativeTimeout, 10 * STALE_QUERY_GRACE_PERIOD_IN_MS); + + Coordinator.getInstance().cleanUpStaleQueries(); + + assertLeftAlone(zeroTimeout); + assertLeftAlone(negativeTimeout); + } + + @Test + public void testKilledQueryIsReleasedUnlessAnRpcIsInProgress() throws Throwable { + // the RPC that executed the statement has not returned yet + QueryExecution executing = executeQuery(QUERY_TIMEOUT_IN_MS); + // that RPC returned, and the client has not fetched the result yet + QueryExecution idle = executeQuery(QUERY_TIMEOUT_IN_MS); + finishRpc(idle); + + executing.cancel(); + idle.cancel(); + + // the state machine thread answers getStatus after it has run the cleanup of the killed query + assertEquals(TSStatusCode.QUERY_WAS_KILLED.getStatusCode(), statusCodeOf(executing)); + assertEquals(TSStatusCode.QUERY_WAS_KILLED.getStatusCode(), statusCodeOf(idle)); + // the RPC in progress releases its query when it returns, nothing would release the idle one + assertSame(executing, Coordinator.getInstance().getQueryExecution(queryIdOf(executing))); + assertNull(Coordinator.getInstance().getQueryExecution(queryIdOf(idle))); + } + + /** + * Executes a query the way a client RPC does. When this method returns the query is RUNNING and + * registered in the coordinator, and the RPC that created it is still in progress. The result of + * the query is already in memory, so no fragment instance is dispatched. + */ + private QueryExecution executeQuery(long timeout) throws Throwable { + long queryId = NEXT_QUERY_ID.getAndIncrement(); + executedQueryIds.add(queryId); + IAnalysis analysis = mock(IAnalysis.class); + when(analysis.canSkipExecute(any(MPPQueryContext.class))).thenReturn(true); + when(analysis.constructResultForMemorySource(any(MPPQueryContext.class))) + .thenReturn(new TsBlock(0)); + IPlanner planner = mock(IPlanner.class); + when(planner.analyze(any(MPPQueryContext.class))).thenReturn(analysis); + QueryExecution[] createdExecution = new QueryExecution[1]; + execute( + queryId, + "SELECT s FROM t1", + (queryContext, startTime) -> { + queryContext.setTimeOut(timeout); + queryContext.setStartTime(startTime); + createdExecution[0] = new QueryExecution(planner, queryContext, stateMachineExecutor); + return createdExecution[0]; + }); + return createdExecution[0]; + } + + /** What the RPC in progress does when it returns to the client. */ + private static void finishRpc(QueryExecution execution) { + Coordinator.getInstance().recordExecutionTime(queryIdOf(execution), 1L); + } + + /** Makes the query look as if it had been started {@code elapsedTimeInMs} ago. */ + private static void setElapsedTime(QueryExecution execution, long elapsedTimeInMs) { + execution.getContext().setStartTime(System.currentTimeMillis() - elapsedTimeInMs); + } + + private static long queryIdOf(QueryExecution execution) { + return execution.getContext().getLocalQueryId(); + } + + private static int statusCodeOf(QueryExecution execution) { + return execution.getStatus().status.getCode(); + } + + private static void assertLeftAlone(QueryExecution execution) { + assertSame(execution, Coordinator.getInstance().getQueryExecution(queryIdOf(execution))); + // still running, so its result can still be fetched + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), statusCodeOf(execution)); + } + + private static void execute( + long queryId, + String sql, + BiFunction queryExecutionFactory) + throws Throwable { + Method execution = + Coordinator.class.getDeclaredMethod( + "execution", + long.class, + SessionInfo.class, + String.class, + boolean.class, + boolean.class, + BiFunction.class); + execution.setAccessible(true); + try { + execution.invoke( + Coordinator.getInstance(), + queryId, + new SessionInfo(0, "test", ZoneId.systemDefault(), "test"), + sql, + false, + false, + queryExecutionFactory); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecutionLifecycleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecutionLifecycleTest.java new file mode 100644 index 0000000000000..3226505a4901a --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecutionLifecycleTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution; + +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.planner.IPlanner; + +import org.junit.Test; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class QueryExecutionLifecycleTest { + + @Test + public void isActiveTellsWhetherAnRpcOfTheQueryIsInProgress() throws Exception { + ExecutorService notifications = Executors.newSingleThreadExecutor(); + try { + QueryExecution execution = new QueryExecution(planner(), context(), notifications); + // the execution is created by the RPC that executes the statement + assertTrue(execution.isActive()); + // that RPC returns, and the client has not asked for the result yet + execution.recordExecutionTime(1); + assertFalse(execution.isActive()); + // an RPC that fetches the result arrives + execution.updateCurrentRpcStartTime(System.nanoTime()); + assertTrue(execution.isActive()); + // and returns as well + execution.recordExecutionTime(1); + assertFalse(execution.isActive()); + } finally { + notifications.shutdownNow(); + assertTrue(notifications.awaitTermination(10, TimeUnit.SECONDS)); + } + } + + private static IPlanner planner() { + IPlanner planner = mock(IPlanner.class); + when(planner.analyze(any())).thenReturn(mock(IAnalysis.class)); + return planner; + } + + private static MPPQueryContext context() { + MPPQueryContext context = new MPPQueryContext(new QueryId("is_active")); + context.setQueryType(QueryType.READ); + return context; + } +} From 1749fd9c7bdb6559a973c4fa3a01854fc837b610 Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 19 Nov 2026 20:47:05 +0800 Subject: [PATCH 2/2] [Query] Run the query statements of a batch to their end and release them Trigger 1: a statement that is a query is sent through executeBatchStatement (JDBC Statement.executeBatch), for example select s1 into root.db.dst(s1) from root.db.src (tree model) INSERT INTO dst SELECT * FROM src (table model) Wrong behaviour: the handler only started such a statement and returned its status. A batch has no result set, so nobody fetched the result of the statement, and a query only makes progress while its result is consumed: - with the source data on the DataNode that received the batch, the statement wrote nothing (0 of 2000 rows) or only a part (200 of the 1200 rows of IoTDBInsertQueryIT), and the batch reported success; - with the source data on another DataNode, the rows were written about 0.5s after the batch had returned. In both cases the execution stayed registered in the Coordinator, and listed by SHOW QUERIES, until the stale query cleanup removed it 60s after its timeout. Corrected behaviour: the handler consumes the result of such a statement and then releases its execution. The statement has been run to its end when the batch returns, a failure while it runs is reported in the status of the statement, and nothing stays registered. Trigger 2: a schema template query (querySchemaTemplate, for example Session.showMeasurementsInTemplate). Wrong behaviour: the handler returns the whole result in its response, but it left the execution registered until the stale query cleanup removed it 60s after its timeout. Corrected behaviour: the handler releases the execution when it returns. (cherry picked from commit 9c3af0cfd3bcca347bba24a550a00a6a57372624) Backport of apache/iotdb#19330 to dev/1.3: - ClientRPCServiceImpl: the code around both handlers differs on dev/1.3 (no table model branch, the query id is requested at one place), so the changes were re-applied by hand on the dev/1.3 file: the consumeResult helper, consuming the result and releasing the execution per statement in executeBatchStatement, and releasing the execution in executeTemplateQueryStatement. - IoTDBShowQueriesIT does not exist on dev/1.3. It is added with the two tests of this change and the small fixture they need, and uses the ColumnHeaderConstant of dev/1.3. - IoTDBExecuteBatchIT starts a cluster per test on dev/1.3, so the new test has no cleanup block. - Not ported: IoTDBInsertQueryIT.testInsertQueryInBatch and the table model example above. dev/1.3 has no table model. --- .../iotdb/db/it/IoTDBExecuteBatchIT.java | 27 ++++ .../iotdb/db/it/IoTDBShowQueriesIT.java | 119 ++++++++++++++++++ .../thrift/impl/ClientRPCServiceImpl.java | 36 +++++- 3 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBShowQueriesIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java index f1822fd4bb420..4f266f03bfa72 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java @@ -81,6 +81,33 @@ public void testJDBCExecuteBatch() { } } + @Test + public void testJDBCExecuteBatchWithSelectInto() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.addBatch("insert into root.batch_into.src(timestamp,s1) values(1,1.0)"); + statement.addBatch("insert into root.batch_into.src(timestamp,s1) values(2,2.0)"); + statement.addBatch("select s1 into root.batch_into.dst(s1) from root.batch_into.src"); + statement.executeBatch(); + statement.clearBatch(); + // A batch has no result set to fetch, so the rows must be written when it returns. + ResultSet resultSet = statement.executeQuery("select s1 from root.batch_into.dst"); + int count = 0; + + long[] timestamps = {1, 2}; + double[] values = {1.0, 2.0}; + + while (resultSet.next()) { + assertEquals(timestamps[count], resultSet.getLong(ColumnHeaderConstant.TIME)); + assertEquals(values[count], resultSet.getDouble("root.batch_into.dst.s1"), 0.0); + count++; + } + assertEquals(timestamps.length, count); + } catch (SQLException e) { + fail(e.getMessage()); + } + } + @Test public void testJDBCExecuteBatchForCreateMultiTimeSeriesPlan() { try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBShowQueriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBShowQueriesIT.java new file mode 100644 index 0000000000000..1cf8cb051595f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBShowQueriesIT.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it; + +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.QUERY_ID; +import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.STATEMENT; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBShowQueriesIT { + private static final String DATABASE = "root.show_queries"; + private static final String DEVICE = DATABASE + ".d1"; + private static final String TIMESERIES = DEVICE + ".s1"; + private static final String BATCH_QUERY = "SELECT s1 FROM " + DEVICE; + private static final String SHOW_QUERIES = "SHOW QUERIES"; + private static final String TEMPLATE = "show_queries_template"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + session.executeNonQueryStatement("CREATE TIMESERIES " + TIMESERIES + " INT32"); + session.executeNonQueryStatement("INSERT INTO " + DEVICE + "(time, s1) VALUES (1, 1)"); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void queryInBatchReleaseTest() throws Exception { + try (ISession adminSession = EnvFactory.getEnv().getSessionConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + Set queryIdsBefore = getOtherQueryIds(adminSession); + statement.addBatch(BATCH_QUERY); + statement.executeBatch(); + // A batch only returns the status of its statements, so nobody can fetch or close the query. + assertNoQueryIsLeft(adminSession, queryIdsBefore); + } + } + + @Test + public void schemaTemplateQueryReleaseTest() throws Exception { + try (ISession adminSession = EnvFactory.getEnv().getSessionConnection()) { + adminSession.executeNonQueryStatement("CREATE DEVICE TEMPLATE " + TEMPLATE + " (s1 INT32)"); + try { + Set queryIdsBefore = getOtherQueryIds(adminSession); + // The measurements come back in the response, so nothing is left to fetch or close. + Assert.assertEquals( + Collections.singletonList("s1"), adminSession.showMeasurementsInTemplate(TEMPLATE)); + assertNoQueryIsLeft(adminSession, queryIdsBefore); + } finally { + adminSession.executeNonQueryStatement("DROP DEVICE TEMPLATE " + TEMPLATE); + } + } + } + + /** Ids of the registered queries, except the SHOW QUERIES statement that lists them. */ + private static Set getOtherQueryIds(ISession session) throws Exception { + Set queryIds = new HashSet<>(); + try (SessionDataSet dataSet = session.executeQueryStatement(SHOW_QUERIES)) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + if (!SHOW_QUERIES.equals(iterator.getString(STATEMENT))) { + queryIds.add(iterator.getString(QUERY_ID)); + } + } + } + return queryIds; + } + + private static void assertNoQueryIsLeft(ISession session, Set queryIdsBefore) + throws Exception { + Set leftQueryIds = getOtherQueryIds(session); + leftQueryIds.removeAll(queryIdsBefore); + Assert.assertEquals(Collections.emptySet(), leftQueryIds); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 9607e51e7e73c..9e352550a69c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -1616,6 +1616,19 @@ public TSExecuteStatementResp executeStatement(TSExecuteStatementReq req) { return executeStatementInternal(req, OLD_SELECT_RESULT); } + /** Runs a query to its end and discards its result. */ + private static void consumeResult(IQueryExecution queryExecution) throws IoTDBException { + if (queryExecution == null) { + // only queries are registered, any other statement has already been executed + return; + } + while (queryExecution.hasNextResult()) { + if (!queryExecution.getBatchResult().isPresent()) { + break; + } + } + } + @Override public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) { long t1 = System.nanoTime(); @@ -1632,6 +1645,8 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) { long t2 = System.nanoTime(); String type = null; OperationQuota quota = null; + Long queryId = null; + Throwable t = null; try { Statement s = StatementGenerator.createStatement(statement, clientSession.getZoneId()); if (s == null) { @@ -1652,7 +1667,7 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) { AuditLogger.log(statement, s); } - long queryId = SESSION_MANAGER.requestQueryId(); + queryId = SESSION_MANAGER.requestQueryId(); type = s.getType() == null ? null : s.getType().name(); // create and cache dataset @@ -1685,9 +1700,15 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) { if (result.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() && result.status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { isAllSuccessful = false; + } else if (queryId != null) { + // A batch returns no result set, so nobody fetches the result of a statement that is a + // query. Without a consumer such a statement is not run to its end, which matters for + // the ones that write, such as SELECT INTO. + consumeResult(COORDINATOR.getQueryExecution(queryId)); } results.add(result.status); } catch (Exception e) { + t = e; LOGGER.warn("Error occurred when executing executeBatchStatement: ", e); TSStatus status = onQueryException( @@ -1695,6 +1716,10 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) { isAllSuccessful = false; results.add(status); } finally { + if (queryId != null) { + // Nobody can fetch or close the query of this statement once the batch has returned. + COORDINATOR.cleanupQueryExecution(queryId, req, t); + } CommonUtils.addStatementExecutionLatency( OperationType.EXECUTE_STATEMENT, type, System.nanoTime() - t2); if (quota != null) { @@ -2511,6 +2536,8 @@ private TSQueryTemplateResp executeTemplateQueryStatement( Statement statement, TSQueryTemplateReq req, TSQueryTemplateResp resp) { long startTime = System.nanoTime(); OperationQuota quota = null; + Long queryId = null; + Throwable t = null; try { IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); // permission check @@ -2527,7 +2554,7 @@ private TSQueryTemplateResp executeTemplateQueryStatement( if (ENABLE_AUDIT_LOG) { AuditLogger.log(String.format("execute Query: %s", statement), statement); } - long queryId = SESSION_MANAGER.requestQueryId(); + queryId = SESSION_MANAGER.requestQueryId(); // create and cache dataset ExecutionResult executionResult = COORDINATOR.executeForTreeModel( @@ -2571,10 +2598,15 @@ private TSQueryTemplateResp executeTemplateQueryStatement( return resp; } } catch (Exception e) { + t = e; resp.setStatus( onQueryException(e, "\"" + statement + "\". " + OperationType.EXECUTE_STATEMENT)); return null; } finally { + if (queryId != null) { + // The whole result is part of the response, so nobody can fetch or close the query later. + COORDINATOR.cleanupQueryExecution(queryId, req, t); + } CommonUtils.addStatementExecutionLatency( OperationType.EXECUTE_STATEMENT, statement.getType().name(),