Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -71,4 +71,8 @@ private DataNodeQueryMessages() {}
"Delete After Loading {} error.";
public static final String LOG_FAILED_TO_AUTHORIZE_CQ_ARG_FOR_USER_ARG_ARG_B436A81F =
"Failed to authorize CQ {} for user {}: {}";
public static final String
EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E =
"Query is not found, it may be killed by others, timeout or some other runtime errors,"
+ " you can see more details in server log.";
}
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,7 @@ private DataNodeQueryMessages() {}
public static final String LOG_DELETE_AFTER_LOADING_ARG_ERROR_C606A060 = "加载后删除 {} 时出错。";
public static final String LOG_FAILED_TO_AUTHORIZE_CQ_ARG_FOR_USER_ARG_ARG_B436A81F =
"CQ {} 的用户 {} 鉴权失败:{}";
public static final String
EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E =
"未找到查询,可能已被其他用户终止、超时或发生其他运行时错误,更多详情请查看服务器日志。";
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.protocol.basic.BasicOpenSessionResp;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
Expand Down Expand Up @@ -250,7 +251,8 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler {
TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes();

private static final String NO_QUERY_EXECUTION_ERR_MSG =
"Query is not found, it may be killed by others, timeout or some other runtime errors, you can see more details in server log.";
DataNodeQueryMessages
.EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E;

@FunctionalInterface
public interface SelectResult {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.iotdb.db.queryengine.plan;

import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.client.ClientPoolFactory;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient;
Expand All @@ -32,6 +33,7 @@
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.query.QueryTimeoutRuntimeException;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.queryengine.common.DataNodeEndPoints;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.QueryId;
Expand All @@ -50,6 +52,8 @@
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement;
import org.apache.iotdb.db.utils.SetThreadName;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -163,6 +167,19 @@ private ExecutionResult execution(
// if it's write request and the result status needs to retry
result.status.setNeedRetry(true);
}
if (execution.isQuery()
&& queryExecutionMap.get(queryId) != execution
&& isSuccessful(result.status)) {
// The query was removed while it was being started, for example by cleanUpStaleQueries
// during a dispatch that took too long. Removing a query does not fail its state, so its
// status still says success, but nobody can fetch its result any more.
return new ExecutionResult(
result.queryId,
RpcUtils.getStatus(
TSStatusCode.QUERY_WAS_KILLED,
DataNodeQueryMessages
.EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E));
}
return result;
} catch (RuntimeException | Error e) {
// Analysis can fail before registration. Only link the query ID to its SQL here; callers
Expand All @@ -178,6 +195,12 @@ private ExecutionResult execution(
}
}

private static boolean isSuccessful(TSStatus status) {
return status != null
&& (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
|| status.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode());
}

/** This method is called by the write method. So it does not set the timeout parameter. */
public ExecutionResult executeForTreeModel(
Statement statement,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.db.queryengine.plan;

import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution;
import org.apache.iotdb.db.queryengine.plan.execution.QueryExecution;
import org.apache.iotdb.db.queryengine.plan.planner.IPlanner;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;

import org.apache.tsfile.read.common.block.TsBlock;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BiFunction;
import java.util.function.Consumer;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

/** What the RPC that executes a statement is told when its query is removed while it starts. */
public class CoordinatorRemovedQueryTest {

// A query id is never reused, as a terminated query releases itself asynchronously by its id
private static final AtomicLong NEXT_QUERY_ID = new AtomicLong(20_001L);

private static final long QUERY_TIMEOUT_IN_MS = 2_000L;
// Coordinator.cleanUpStaleQueries spares a query until it exceeds its timeout by this long
private static final long STALE_QUERY_GRACE_PERIOD_IN_MS = 60_000L;

private static int previousDataNodeId;
private static ExecutorService stateMachineExecutor;

private final List<Long> executedQueryIds = new ArrayList<>();

@BeforeClass
public static void setUp() {
// the coordinator builds its query id generator from the data node id of this node
previousDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId();
IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0);
stateMachineExecutor = Executors.newSingleThreadExecutor();
}

@AfterClass
public static void tearDown() {
stateMachineExecutor.shutdownNow();
IoTDBDescriptor.getInstance().getConfig().setDataNodeId(previousDataNodeId);
}

@After
public void releaseExecutedQueries() {
// a query that had to be left alone is still registered
executedQueryIds.forEach(Coordinator.getInstance()::cleanupQueryExecution);
}

@Test
public void testQueryReclaimedWhileItStartsIsReportedAsNotFound() throws Throwable {
long queryId = nextQueryId();

// The start of the query takes so long, for example in the dispatch of its fragment
// instances, that the stale query cleanup reclaims the query. The start then succeeds.
ExecutionResult result =
startStatement(
queryId,
QueryType.READ,
context -> {
context.setStartTime(
System.currentTimeMillis()
- (QUERY_TIMEOUT_IN_MS + STALE_QUERY_GRACE_PERIOD_IN_MS + 1_000L));
Coordinator.getInstance().cleanUpStaleQueries();
});

assertNull(Coordinator.getInstance().getQueryExecution(queryId));
// nobody can fetch the result of this query, so the statement must not be reported as
// successful
assertEquals(TSStatusCode.QUERY_WAS_KILLED.getStatusCode(), result.status.getCode());
assertEquals(
DataNodeQueryMessages
.EXCEPTION_QUERY_IS_NOT_FOUND_IT_MAY_BE_KILLED_BY_OTHERS_TIMEOUT_OR_SOME_OTHER_RUNTIME_ERRORS_YOU_CAN_SEE_MORE_DETAILS_IN_SERVER_LOG_90BD085E,
result.status.getMessage());
}

@Test
public void testQueryThatStaysRegisteredIsReportedAsSuccessful() throws Throwable {
long queryId = nextQueryId();

ExecutionResult result = startStatement(queryId, QueryType.READ, context -> {});

assertNotNull(Coordinator.getInstance().getQueryExecution(queryId));
assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.status.getCode());
}

@Test
public void testStatementThatIsNotAQueryIsReportedAsSuccessful() throws Throwable {
long queryId = nextQueryId();

// only queries are registered, so a missing execution means nothing for any other statement
ExecutionResult result = startStatement(queryId, QueryType.WRITE, context -> {});

assertNull(Coordinator.getInstance().getQueryExecution(queryId));
assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.status.getCode());
}

@Test
public void testFailureOfQueryRemovedWhileItStartsIsKept() throws Throwable {
long queryId = nextQueryId();
TSStatus failure = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, "dispatch failed");
IQueryExecution execution = mock(IQueryExecution.class);
when(execution.isQuery()).thenReturn(true);
when(execution.getQueryId()).thenReturn("removed_and_failed");
doAnswer(
invocation -> {
Coordinator.getInstance().cleanupQueryExecution(queryId);
return null;
})
.when(execution)
.start();
when(execution.getStatus())
.thenReturn(new ExecutionResult(new QueryId("removed_and_failed"), failure));

ExecutionResult result = execute(queryId, (queryContext, startTime) -> execution);

assertNull(Coordinator.getInstance().getQueryExecution(queryId));
// the reason of the failure tells more than "not found"
assertEquals(failure, result.status);
}

private long nextQueryId() {
long queryId = NEXT_QUERY_ID.getAndIncrement();
executedQueryIds.add(queryId);
return queryId;
}

/**
* Executes a statement the way a client RPC does and returns what that RPC is told. The result of
* the statement is already in memory, so no fragment instance is dispatched. {@code
* whileItStarts} runs after the statement has been registered and before its start returns.
*/
private static ExecutionResult startStatement(
long queryId, QueryType queryType, Consumer<MPPQueryContext> whileItStarts) throws Throwable {
IAnalysis analysis = mock(IAnalysis.class);
when(analysis.canSkipExecute(any(MPPQueryContext.class)))
.thenAnswer(
invocation -> {
whileItStarts.accept(invocation.getArgument(0));
return true;
});
when(analysis.constructResultForMemorySource(any(MPPQueryContext.class)))
.thenReturn(new TsBlock(0));
IPlanner planner = mock(IPlanner.class);
when(planner.analyze(any(MPPQueryContext.class))).thenReturn(analysis);
return execute(
queryId,
(queryContext, startTime) -> {
queryContext.setQueryType(queryType);
queryContext.setTimeOut(QUERY_TIMEOUT_IN_MS);
queryContext.setStartTime(startTime);
return new QueryExecution(planner, queryContext, stateMachineExecutor);
});
}

private static ExecutionResult execute(
long queryId, BiFunction<MPPQueryContext, Long, IQueryExecution> queryExecutionFactory)
throws Throwable {
Method execution =
Coordinator.class.getDeclaredMethod(
"execution",
long.class,
SessionInfo.class,
String.class,
boolean.class,
boolean.class,
BiFunction.class);
execution.setAccessible(true);
try {
return (ExecutionResult)
execution.invoke(
Coordinator.getInstance(),
queryId,
new SessionInfo(0, "test", ZoneId.systemDefault(), "test"),
"SELECT s FROM t1",
false,
false,
queryExecutionFactory);
} catch (InvocationTargetException e) {
throw e.getCause();
}
}
}
Loading