Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,33 @@ public void testJDBCExecuteBatch() {
}
}

@Test
public void testJDBCExecuteBatchWithSelectInto() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
statement.addBatch("insert into root.batch_into.src(timestamp,s1) values(1,1.0)");
statement.addBatch("insert into root.batch_into.src(timestamp,s1) values(2,2.0)");
statement.addBatch("select s1 into root.batch_into.dst(s1) from root.batch_into.src");
statement.executeBatch();
statement.clearBatch();
// A batch has no result set to fetch, so the rows must be written when it returns.
ResultSet resultSet = statement.executeQuery("select s1 from root.batch_into.dst");
int count = 0;

long[] timestamps = {1, 2};
double[] values = {1.0, 2.0};

while (resultSet.next()) {
assertEquals(timestamps[count], resultSet.getLong(ColumnHeaderConstant.TIME));
assertEquals(values[count], resultSet.getDouble("root.batch_into.dst.s1"), 0.0);
count++;
}
assertEquals(timestamps.length, count);
} catch (SQLException e) {
fail(e.getMessage());
}
}

@Test
public void testJDBCExecuteBatchForCreateMultiTimeSeriesPlan() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.db.it;

import org.apache.iotdb.isession.ISession;
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;

import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.sql.Connection;
import java.sql.Statement;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.QUERY_ID;
import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.STATEMENT;

@RunWith(IoTDBTestRunner.class)
@Category({LocalStandaloneIT.class, ClusterIT.class})
public class IoTDBShowQueriesIT {
private static final String DATABASE = "root.show_queries";
private static final String DEVICE = DATABASE + ".d1";
private static final String TIMESERIES = DEVICE + ".s1";
private static final String BATCH_QUERY = "SELECT s1 FROM " + DEVICE;
private static final String SHOW_QUERIES = "SHOW QUERIES";
private static final String TEMPLATE = "show_queries_template";

@BeforeClass
public static void setUp() throws Exception {
EnvFactory.getEnv().initClusterEnvironment();
try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
session.executeNonQueryStatement("CREATE DATABASE " + DATABASE);
session.executeNonQueryStatement("CREATE TIMESERIES " + TIMESERIES + " INT32");
session.executeNonQueryStatement("INSERT INTO " + DEVICE + "(time, s1) VALUES (1, 1)");
}
}

@AfterClass
public static void tearDown() throws Exception {
EnvFactory.getEnv().cleanClusterEnvironment();
}

@Test
public void queryInBatchReleaseTest() throws Exception {
try (ISession adminSession = EnvFactory.getEnv().getSessionConnection();
Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
Set<String> queryIdsBefore = getOtherQueryIds(adminSession);
statement.addBatch(BATCH_QUERY);
statement.executeBatch();
// A batch only returns the status of its statements, so nobody can fetch or close the query.
assertNoQueryIsLeft(adminSession, queryIdsBefore);
}
}

@Test
public void schemaTemplateQueryReleaseTest() throws Exception {
try (ISession adminSession = EnvFactory.getEnv().getSessionConnection()) {
adminSession.executeNonQueryStatement("CREATE DEVICE TEMPLATE " + TEMPLATE + " (s1 INT32)");
try {
Set<String> queryIdsBefore = getOtherQueryIds(adminSession);
// The measurements come back in the response, so nothing is left to fetch or close.
Assert.assertEquals(
Collections.singletonList("s1"), adminSession.showMeasurementsInTemplate(TEMPLATE));
assertNoQueryIsLeft(adminSession, queryIdsBefore);
} finally {
adminSession.executeNonQueryStatement("DROP DEVICE TEMPLATE " + TEMPLATE);
}
}
}

/** Ids of the registered queries, except the SHOW QUERIES statement that lists them. */
private static Set<String> getOtherQueryIds(ISession session) throws Exception {
Set<String> queryIds = new HashSet<>();
try (SessionDataSet dataSet = session.executeQueryStatement(SHOW_QUERIES)) {
SessionDataSet.DataIterator iterator = dataSet.iterator();
while (iterator.next()) {
if (!SHOW_QUERIES.equals(iterator.getString(STATEMENT))) {
queryIds.add(iterator.getString(QUERY_ID));
}
}
}
return queryIds;
}

private static void assertNoQueryIsLeft(ISession session, Set<String> queryIdsBefore)
throws Exception {
Set<String> leftQueryIds = getOtherQueryIds(session);
leftQueryIds.removeAll(queryIdsBefore);
Assert.assertEquals(Collections.emptySet(), leftQueryIds);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1616,6 +1616,19 @@ public TSExecuteStatementResp executeStatement(TSExecuteStatementReq req) {
return executeStatementInternal(req, OLD_SELECT_RESULT);
}

/** Runs a query to its end and discards its result. */
private static void consumeResult(IQueryExecution queryExecution) throws IoTDBException {
if (queryExecution == null) {
// only queries are registered, any other statement has already been executed
return;
}
while (queryExecution.hasNextResult()) {
if (!queryExecution.getBatchResult().isPresent()) {
break;
}
}
}

@Override
public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) {
long t1 = System.nanoTime();
Expand All @@ -1632,6 +1645,8 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) {
long t2 = System.nanoTime();
String type = null;
OperationQuota quota = null;
Long queryId = null;
Throwable t = null;
try {
Statement s = StatementGenerator.createStatement(statement, clientSession.getZoneId());
if (s == null) {
Expand All @@ -1652,7 +1667,7 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) {
AuditLogger.log(statement, s);
}

long queryId = SESSION_MANAGER.requestQueryId();
queryId = SESSION_MANAGER.requestQueryId();
type = s.getType() == null ? null : s.getType().name();
// create and cache dataset

Expand Down Expand Up @@ -1685,16 +1700,26 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) {
if (result.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
&& result.status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) {
isAllSuccessful = false;
} else if (queryId != null) {
// A batch returns no result set, so nobody fetches the result of a statement that is a
// query. Without a consumer such a statement is not run to its end, which matters for
// the ones that write, such as SELECT INTO.
consumeResult(COORDINATOR.getQueryExecution(queryId));
}
results.add(result.status);
} catch (Exception e) {
t = e;
LOGGER.warn("Error occurred when executing executeBatchStatement: ", e);
TSStatus status =
onQueryException(
e, "\"" + statement + "\". " + OperationType.EXECUTE_BATCH_STATEMENT);
isAllSuccessful = false;
results.add(status);
} finally {
if (queryId != null) {
// Nobody can fetch or close the query of this statement once the batch has returned.
COORDINATOR.cleanupQueryExecution(queryId, req, t);
}
CommonUtils.addStatementExecutionLatency(
OperationType.EXECUTE_STATEMENT, type, System.nanoTime() - t2);
if (quota != null) {
Expand Down Expand Up @@ -2511,6 +2536,8 @@ private TSQueryTemplateResp executeTemplateQueryStatement(
Statement statement, TSQueryTemplateReq req, TSQueryTemplateResp resp) {
long startTime = System.nanoTime();
OperationQuota quota = null;
Long queryId = null;
Throwable t = null;
try {
IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime();
// permission check
Expand All @@ -2527,7 +2554,7 @@ private TSQueryTemplateResp executeTemplateQueryStatement(
if (ENABLE_AUDIT_LOG) {
AuditLogger.log(String.format("execute Query: %s", statement), statement);
}
long queryId = SESSION_MANAGER.requestQueryId();
queryId = SESSION_MANAGER.requestQueryId();
// create and cache dataset
ExecutionResult executionResult =
COORDINATOR.executeForTreeModel(
Expand Down Expand Up @@ -2571,10 +2598,15 @@ private TSQueryTemplateResp executeTemplateQueryStatement(
return resp;
}
} catch (Exception e) {
t = e;
resp.setStatus(
onQueryException(e, "\"" + statement + "\". " + OperationType.EXECUTE_STATEMENT));
return null;
} finally {
if (queryId != null) {
// The whole result is part of the response, so nobody can fetch or close the query later.
COORDINATOR.cleanupQueryExecution(queryId, req, t);
}
CommonUtils.addStatementExecutionLatency(
OperationType.EXECUTE_STATEMENT,
statement.getType().name(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -331,14 +331,18 @@ public void cleanupQueryExecution(
* We need to reclaim resources from queries that have exceeded their timeout by more than one
* minute. This indicates that the associated clients have failed to perform proper resource
* cleanup.
*
* <p>Whether an RPC of the query is in progress does not matter here. The timeout bounds the
* whole lifetime of a query, so a query without an RPC in progress has been abandoned by its
* client, and an RPC that is still in progress this long after the timeout is stuck, e.g. in a
* dispatch that never returns, and is only unblocked by stopping the query. Sparing the queries
* with an RPC in progress would also keep forever every query whose caller never reports the end
* of its RPC through {@link #recordExecutionTime}.
*/
public void cleanUpStaleQueries() {
long currentTime = System.currentTimeMillis();
queryExecutionMap.forEach(
(queryId, queryExecution) -> {
if (queryExecution.isActive()) {
return;
}
long timeout = queryExecution.getTimeout();
long queryStartTime = queryExecution.getStartExecutionTime();
long executeTime = currentTime - queryStartTime;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ public interface IQueryExecution {
* finishes. On the other hand, if there is no active RPC, it means that the client has finished
* fetching results or has not started fetching results yet, and the QueryExecution can be safely
* cleaned up.
*
* @return true if an RPC of this query is in progress, false otherwise
*/
boolean isActive();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ private void releaseResource(Throwable t) {
* ClientRPCServiceImpl is responsible for performing the cleanup.
*/
private synchronized void cleanUpCoordinatorContextMapIfNeeded(Throwable t) {
if (isActive()) {
if (!isActive()) {
Coordinator.getInstance().cleanupQueryExecution(context.getLocalQueryId(), null, t);
}
}
Expand All @@ -446,7 +446,7 @@ private synchronized void cleanUpCoordinatorContextMapIfNeeded(Throwable t) {
* cleaned up.
*/
public synchronized boolean isActive() {
return startTimeOfCurrentRpc == -1;
return startTimeOfCurrentRpc != -1;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ public void updateCurrentRpcStartTime(long startTime) {

@Override
public boolean isActive() {
return startTimeOfCurrentRpc == -1;
return startTimeOfCurrentRpc != -1;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@

import static com.google.common.util.concurrent.Futures.immediateFuture;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class ConfigExecutionTest {
Expand Down Expand Up @@ -157,6 +158,29 @@ public void exceptionAfterInvokeGetStatusTest() {
}
}

@Test
public void isActiveTellsWhetherAnRpcOfTheQueryIsInProgress() {
ExecutorService executor = getExecutor();
try {
IConfigTask task =
(clientManager) -> immediateFuture(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
ConfigExecution execution = new ConfigExecution(genMPPQueryContext(), executor, task);
// the execution is created by the RPC that executes the statement
assertTrue(execution.isActive());
// that RPC returns, and the client has not asked for the result yet
execution.recordExecutionTime(1);
assertFalse(execution.isActive());
// an RPC that fetches the result arrives
execution.updateCurrentRpcStartTime(System.nanoTime());
assertTrue(execution.isActive());
// and returns as well
execution.recordExecutionTime(1);
assertFalse(execution.isActive());
} finally {
executor.shutdownNow();
}
}

private MPPQueryContext genMPPQueryContext() {
MPPQueryContext context = new MPPQueryContext(new QueryId("query1"));
context.setQueryType(QueryType.WRITE);
Expand Down
Loading
Loading