Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,37 @@ public void testJDBCExecuteBatch() {
}
}

@Test
public void testJDBCExecuteBatchWithSelectInto() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
try {
statement.addBatch("insert into root.batch_into.src(timestamp,s1) values(1,1.0)");
statement.addBatch("insert into root.batch_into.src(timestamp,s1) values(2,2.0)");
statement.addBatch("select s1 into root.batch_into.dst(s1) from root.batch_into.src");
statement.executeBatch();
statement.clearBatch();
// A batch has no result set to fetch, so the rows must be written when it returns.
ResultSet resultSet = statement.executeQuery("select s1 from root.batch_into.dst");
int count = 0;

long[] timestamps = {1, 2};
double[] values = {1.0, 2.0};

while (resultSet.next()) {
assertEquals(timestamps[count], resultSet.getLong(ColumnHeaderConstant.TIME));
assertEquals(values[count], resultSet.getDouble("root.batch_into.dst.s1"), 0.0);
count++;
}
assertEquals(timestamps.length, count);
} finally {
executeQuietly(statement, "DELETE DATABASE root.batch_into");
}
} catch (SQLException e) {
fail(e.getMessage());
}
}

@Test
public void testJDBCExecuteBatchForCreateMultiTimeSeriesPlan() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,12 @@
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.sql.Connection;
import java.sql.Statement;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.QUERY_ID;
import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.STATEMENT;
Expand All @@ -60,6 +65,8 @@ public class IoTDBShowQueriesIT {
private static final String KILL_USER = "show_queries_kill_user";
private static final String PASSWORD = "password123456";
private static final String PENDING_QUERY = "SELECT s1 FROM " + DEVICE;
private static final String SHOW_QUERIES = "SHOW QUERIES";
private static final String TEMPLATE = "show_queries_template";
// Leave one default 1,000-row TsBlock after the initial Session fetch.
private static final int STREAMING_QUERY_ROW_COUNT = SessionConfig.DEFAULT_FETCH_SIZE + 1_000;

Expand Down Expand Up @@ -128,6 +135,35 @@ public void killQueryAuthTest() throws Exception {
}
}

@Test
public void queryInBatchReleaseTest() throws Exception {
try (ISession adminSession = EnvFactory.getEnv().getSessionConnection();
Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
Set<String> queryIdsBefore = getOtherQueryIds(adminSession);
statement.addBatch(PENDING_QUERY);
statement.executeBatch();
// A batch only returns the status of its statements, so nobody can fetch or close the query.
assertNoQueryIsLeft(adminSession, queryIdsBefore);
}
}

@Test
public void schemaTemplateQueryReleaseTest() throws Exception {
try (ISession adminSession = EnvFactory.getEnv().getSessionConnection()) {
adminSession.executeNonQueryStatement("CREATE DEVICE TEMPLATE " + TEMPLATE + " (s1 INT32)");
try {
Set<String> queryIdsBefore = getOtherQueryIds(adminSession);
// The measurements come back in the response, so nothing is left to fetch or close.
Assert.assertEquals(
Collections.singletonList("s1"), adminSession.showMeasurementsInTemplate(TEMPLATE));
assertNoQueryIsLeft(adminSession, queryIdsBefore);
} finally {
adminSession.executeNonQueryStatement("DROP DEVICE TEMPLATE " + TEMPLATE);
}
}
}

private static boolean containsQuery(ISession session, String query) throws Exception {
try (SessionDataSet dataSet = session.executeQueryStatement("SHOW QUERIES")) {
SessionDataSet.DataIterator iterator = dataSet.iterator();
Expand All @@ -152,4 +188,25 @@ private static String getQueryId(ISession session, String query) throws Exceptio
Assert.fail("Query not found: " + query);
return null;
}

/** Ids of the registered queries, except the SHOW QUERIES statement that lists them. */
private static Set<String> getOtherQueryIds(ISession session) throws Exception {
Set<String> queryIds = new HashSet<>();
try (SessionDataSet dataSet = session.executeQueryStatement(SHOW_QUERIES)) {
SessionDataSet.DataIterator iterator = dataSet.iterator();
while (iterator.next()) {
if (!SHOW_QUERIES.equals(iterator.getString(STATEMENT))) {
queryIds.add(iterator.getString(QUERY_ID));
}
}
}
return queryIds;
}

private static void assertNoQueryIsLeft(ISession session, Set<String> queryIdsBefore)
throws Exception {
Set<String> leftQueryIds = getOtherQueryIds(session);
leftQueryIds.removeAll(queryIdsBefore);
Assert.assertEquals(Collections.emptySet(), leftQueryIds);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,48 @@ public void testSimple() throws SQLException {
}
}

/** A batch has no result set to fetch, so the rows must be written when the batch returns. */
@Test
public void testInsertQueryInBatch() throws SQLException {
try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
Statement statement = connection.createStatement()) {
List<String> expectedHeader = new ArrayList<>();
List<String> expectedRetArray = new ArrayList<>();

statement.execute("USE test");
ResultSet resultSet =
statement.executeQuery("select * from vehicle0 order by time, deviceId, manufacturer");
buildExpectedResult(resultSet, expectedHeader, expectedRetArray);

statement.execute(String.format(createTableTemplate, 1));
String insertQuery = String.format(insertIntoQuery, 1);
statement.addBatch(insertQuery);
statement.executeBatch();
statement.clearBatch();

// 3 devices with 400 rows each, see prepareData
try (ResultSet countResult = statement.executeQuery("SELECT count(*) FROM vehicle1")) {
Assert.assertTrue(countResult.next());
assertEquals(1200, countResult.getLong(1));
}
ResultSet r1 =
statement.executeQuery(
String.format("SELECT * FROM vehicle%d order by time, deviceId, manufacturer", 1));
resultSetEqualTest(r1, expectedHeader, expectedRetArray);

// Nobody can fetch or close the statement of a batch, so it must not stay registered.
try (ResultSet queries =
statement.executeQuery(
"SELECT statement FROM information_schema.queries WHERE statement = '"
+ insertQuery
+ "'")) {
assertFalse(queries.next());
}

statement.execute(String.format(dropTableTemplate, 1));
}
}

/** Insert a table's own rows at shifted timestamps and verify that its row count doubles. */
@Test
public void testInsertIntoSourceTable() throws SQLException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -588,6 +588,19 @@ private static void clearUp(
clientSession.removeQueryId(statementId, queryId);
}

/** Runs a query to its end and discards its result. */
private static void consumeResult(IQueryExecution queryExecution) throws IoTDBException {
if (queryExecution == null) {
// only queries are registered, any other statement has already been executed
return;
}
while (queryExecution.hasNextResult()) {
if (!queryExecution.getBatchResult().isPresent()) {
break;
}
}
}

/** Adapter that wraps TSExecuteStatementReq to implement NativeStatementRequest. */
private static class TSExecuteStatementReqAdapter implements NativeStatementRequest {
private final TSExecuteStatementReq req;
Expand Down Expand Up @@ -2190,8 +2203,9 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) {
long t2 = System.nanoTime();
String type = null;
OperationQuota quota = null;
Long queryId = null;
Throwable t = null;
try {
long queryId;
ExecutionResult result;
if (clientSession.getSqlDialect() == SqlDialect.TREE) {
Statement s = StatementGenerator.createStatement(statement, clientSession.getZoneId());
Expand Down Expand Up @@ -2314,16 +2328,26 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) {
if (result.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
&& result.status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) {
isAllSuccessful = false;
} else if (queryId != null) {
// A batch returns no result set, so nobody fetches the result of a statement that is a
// query. Without a consumer such a statement is not run to its end, which matters for
// the ones that write, such as SELECT INTO.
consumeResult(COORDINATOR.getQueryExecution(queryId));
}
results.add(result.status);
} catch (Exception e) {
t = e;
LOGGER.warn(DataNodeMiscMessages.ERROR_EXECUTING_BATCH_STATEMENT, e);
TSStatus status =
onQueryException(
e, "\"" + statement + "\". " + OperationType.EXECUTE_BATCH_STATEMENT);
isAllSuccessful = false;
results.add(status);
} finally {
if (queryId != null) {
// Nobody can fetch or close the query of this statement once the batch has returned.
COORDINATOR.cleanupQueryExecution(queryId, req, t);
}
CommonUtils.addStatementExecutionLatency(
OperationType.EXECUTE_STATEMENT, type, System.nanoTime() - t2);
if (quota != null) {
Expand Down Expand Up @@ -3226,6 +3250,8 @@ private TSQueryTemplateResp executeTemplateQueryStatement(
Statement statement, TSQueryTemplateReq req, TSQueryTemplateResp resp) {
long startTime = System.nanoTime();
OperationQuota quota = null;
Long queryId = null;
Throwable t = null;
try {
IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime();
// permission check
Expand All @@ -3245,7 +3271,7 @@ private TSQueryTemplateResp executeTemplateQueryStatement(
DataNodeThrottleQuotaManager.getInstance()
.checkQuota(SESSION_MANAGER.getCurrSession().getUsername(), statement);

long queryId = SESSION_MANAGER.requestQueryId();
queryId = SESSION_MANAGER.requestQueryId();
// create and cache dataset
ExecutionResult executionResult =
COORDINATOR.executeForTreeModel(
Expand Down Expand Up @@ -3292,10 +3318,15 @@ private TSQueryTemplateResp executeTemplateQueryStatement(
return resp;
}
} catch (Exception e) {
t = e;
resp.setStatus(
onQueryException(e, "\"" + statement + "\". " + OperationType.EXECUTE_STATEMENT));
return null;
} finally {
if (queryId != null) {
// The whole result is part of the response, so nobody can fetch or close the query later.
COORDINATOR.cleanupQueryExecution(queryId, req, t);
}
CommonUtils.addStatementExecutionLatency(
OperationType.EXECUTE_STATEMENT,
statement.getType().name(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1024,14 +1024,18 @@ public static void recordQueries(
* We need to reclaim resources from queries that have exceeded their timeout by more than one
* minute. This indicates that the associated clients have failed to perform proper resource
* cleanup.
*
* <p>Whether an RPC of the query is in progress does not matter here. The timeout bounds the
* whole lifetime of a query, so a query without an RPC in progress has been abandoned by its
* client, and an RPC that is still in progress this long after the timeout is stuck, e.g. in a
* dispatch that never returns, and is only unblocked by stopping the query. Sparing the queries
* with an RPC in progress would also keep forever every query whose caller never reports the end
* of its RPC through {@link #recordExecutionTime}.
*/
public void cleanUpStaleQueries() {
long currentTime = System.currentTimeMillis();
queryExecutionMap.forEach(
(queryId, queryExecution) -> {
if (queryExecution.isActive()) {
return;
}
long timeout = queryExecution.getTimeout();
long queryStartTime = queryExecution.getStartExecutionTime();
long executeTime = currentTime - queryStartTime;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ public interface IQueryExecution {
* finishes. On the other hand, if there is no active RPC, it means that the client has finished
* fetching results or has not started fetching results yet, and the QueryExecution can be safely
* cleaned up.
*
* @return true if an RPC of this query is in progress, false otherwise
*/
boolean isActive();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -595,7 +595,7 @@ private void finalizeUDFClassLoader() {
* ClientRPCServiceImpl is responsible for performing the cleanup.
*/
private synchronized void cleanUpCoordinatorContextMapIfNeeded(Throwable t) {
if (isActive()) {
if (!isActive()) {
Coordinator.getInstance()
.cleanupQueryExecution(
context.getLocalQueryId(), (org.apache.thrift.TBase<?, ?>) null, t);
Expand All @@ -611,7 +611,7 @@ private synchronized void cleanUpCoordinatorContextMapIfNeeded(Throwable t) {
* cleaned up.
*/
public synchronized boolean isActive() {
return startTimeOfCurrentRpc == -1;
return startTimeOfCurrentRpc != -1;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ public void updateCurrentRpcStartTime(long startTime) {

@Override
public boolean isActive() {
return startTimeOfCurrentRpc == -1;
return startTimeOfCurrentRpc != -1;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@

import static com.google.common.util.concurrent.Futures.immediateFuture;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class ConfigExecutionTest {
Expand Down Expand Up @@ -184,6 +185,29 @@ public void exceptionAfterInvokeGetStatusTest() {
}
}

@Test
public void isActiveTellsWhetherAnRpcOfTheQueryIsInProgress() {
ExecutorService executor = getExecutor();
try {
IConfigTask task =
(clientManager) -> immediateFuture(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
ConfigExecution execution = new ConfigExecution(genMPPQueryContext(), executor, task);
// the execution is created by the RPC that executes the statement
assertTrue(execution.isActive());
// that RPC returns, and the client has not asked for the result yet
execution.recordExecutionTime(1);
assertFalse(execution.isActive());
// an RPC that fetches the result arrives
execution.updateCurrentRpcStartTime(System.nanoTime());
assertTrue(execution.isActive());
// and returns as well
execution.recordExecutionTime(1);
assertFalse(execution.isActive());
} finally {
executor.shutdownNow();
}
}

private MPPQueryContext genMPPQueryContext() {
MPPQueryContext context = new MPPQueryContext(new QueryId("query1"));
context.setQueryType(QueryType.OTHER);
Expand Down
Loading
Loading