Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ public enum TSStatusCode {
TRANSFER_LEADER_ERROR(1008),
GET_CLUSTER_ID_ERROR(1009),
CAN_NOT_CONNECT_CONFIGNODE(1010),
CONFIG_NODE_LEADER_WARMING_UP(1014),

// Sync, Load TsFile
LOAD_FILE_ERROR(1100),
Expand Down
11 changes: 9 additions & 2 deletions iotdb-core/ainode/iotdb/ainode/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def __init__(self, config_leader: TEndPoint):

self._MSG_RECONNECTION_FAIL = "Fail to connect to any config node. Please check status of ConfigNodes"
self._RETRY_NUM = 5
self._STARTUP_RETRY_NUM = 60
self._RETRY_INTERVAL_MS = 1

self._try_to_connect()
Expand Down Expand Up @@ -123,6 +124,12 @@ def _update_config_node_leader(self, status: TSStatus) -> bool:
else:
self._config_leader = None
return True
if status.code == TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.get_status_code():
logger.info(
"ConfigNode leader is warming up before serving AINode, will wait and retry. Reason: {}",
status.message,
)
return True
return False

def node_register(self, cluster_name: str, configuration: TAINodeConfiguration,
Expand All @@ -133,7 +140,7 @@ def node_register(self, cluster_name: str, configuration: TAINodeConfiguration,
versionInfo=version_info
)

for _ in range(0, self._RETRY_NUM):
for _ in range(0, self._STARTUP_RETRY_NUM):
try:
resp = self._client.registerAINode(req)
if not self._update_config_node_leader(resp.status):
Expand All @@ -156,7 +163,7 @@ def node_restart(self, cluster_name: str, configuration: TAINodeConfiguration,
versionInfo=version_info
)

for _ in range(0, self._RETRY_NUM):
for _ in range(0, self._STARTUP_RETRY_NUM):
try:
resp = self._client.restartAINode(req)
if not self._update_config_node_leader(resp.status):
Expand Down
1 change: 1 addition & 0 deletions iotdb-core/ainode/iotdb/ainode/constant.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
class TSStatusCode(Enum):
SUCCESS_STATUS = 200
REDIRECTION_RECOMMEND = 400
CONFIG_NODE_LEADER_WARMING_UP = 1014
AINODE_INTERNAL_ERROR = 1510
INVALID_URI_ERROR = 1511
INVALID_INFERENCE_CONFIG = 1512
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,5 +89,62 @@ public final class ConfigNodeMessages {
public static final String UNEXPECTED_INTERRUPTION_DURING_THE_CLOSE_METHOD_OF_LOGWRITER =
"[ConfigRegion Update] Unexpected interruption during the close method of logWriter";

public static final String
MESSAGE_CURRENT_CONFIGNODE_NOT_LEADER_PLEASE_REDIRECT_NEW_CONFIGNODE_F9AF262D =
"The current ConfigNode is not leader, please redirect to a new ConfigNode.";
public static final String
MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_CONSENSUS_NOT_READY_YET_F1FB7C9A =
"The current ConfigNode is leader but consensus is not ready yet.";
public static final String
MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_LEADER_SERVICES_NOT_READY_YET_ECC25E1A =
"The current ConfigNode is leader but leader services are not ready yet.";
public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_NOT_STARTED_1C75DB82 =
"ConfigNode leader load services are not started.";
public static final String
MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F =
"ConfigNode leader is waiting for cluster heartbeat sampling.";
public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_STOPPED_5884BB0F =
"ConfigNode leader load services are stopped.";
public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_CC9C90E7 =
"ConfigNode leader load services are ready.";
public static final String
MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_FIRST_HEARTBEAT_FROM_REGISTERED_CONFIGNODES_DATANODES_ARG_2FD3F6FA =
"ConfigNode leader is waiting for first heartbeat from registered ConfigNodes/DataNodes: %s";
public static final String
MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_AFTER_WAITING_ARG_MS_FOR_FIRST_HEARTBEAT_MISSING_HEARTBEATS_ARG_A3AEA8B4 =
"ConfigNode leader load services are ready after waiting %dms for first heartbeat. Missing heartbeats: %s";
public static final String
LOG_UNEXPECTED_INTERRUPTION_WHILE_WAITING_FOR_CONFIGNODE_LEADER_LOAD_WARM_UP_BB5AA4F7 =
"Unexpected interruption while waiting for ConfigNode leader load warm-up.";
public static final String
LOG_CURRENT_CONFIGNODE_NODEID_ARG_IP_ARG_FINISHED_STARTING_LEADER_SERVICES_WHILE_LOAD_WARM_UP_IS_STILL_IN_PROGRESS_ARG_17C09A31 =
"Current ConfigNode(nodeId: {}, ip: {}) finished starting leader services while load warm-up is still in progress: {}";
public static final String
LOG_CONFIGNODE_LEADER_WARMING_UP_BEFORE_SERVING_REGISTERING_CONFIGNODE_STATUS_ARG_RETRY_ARG_3C924873 =
"ConfigNode leader is warming up before serving the registering ConfigNode, will wait and retry. Status: {}, retry: {}";
public static final String
LOG_CONFIGNODE_LEADER_ARG_WARMING_UP_BEFORE_SERVING_DATANODE_ARG_WILL_WAIT_AND_RETRY_REASON_ARG_3A2A4163 =
"ConfigNode leader {} is warming up before serving DataNode {}, will wait and retry. Reason: {}";
public static final String
LOG_SKIP_STARTING_CONFIGNODE_LEADER_SERVICES_BECAUSE_THE_LEADER_EPOCH_IS_STALE_9E4467F6 =
"Skip starting ConfigNode leader services because the leader epoch is stale.";
public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_NEW_LEADER_ARG_5C41D7D7 =
"ConfigNode %d at %s is no longer the leader; new leader is node %d.";
public static final String
LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_CLEANING_SERVICES_37CA9A1A =
"ConfigNode %d at %s is no longer the leader; cleaning up leader services.";
public static final String LOG_CONFIGNODE_ARG_AT_ARG_BECOMES_CONFIG_REGION_LEADER_2EB9B9E2 =
"ConfigNode %d at %s becomes ConfigRegion leader.";
public static final String LOG_CONFIGNODE_ARG_AT_ARG_AS_CONFIG_REGION_LEADER_IS_READY_74DFCB26 =
"ConfigNode %d at %s as ConfigRegion leader is ready to work.";
public static final String LOG_CONFIGNODE_ARG_AT_ARG_LEADER_SERVICES_ARE_UNAVAILABLE_0223CF16 =
"ConfigNode %d at %s is no longer leader; all leader services are unavailable.";
public static final String
LOG_FAILED_TO_START_CONFIGNODE_LEADER_SERVICE_ON_NODE_ARG_AT_ARG_ARG_THE_NODE_WILL_FINISH_WARMING_UP_BUT_THIS_SERVICE_REMAINS_UNAVAILABLE_UNTIL_THE_NEXT_LEADERSHIP_TRANSITION_9E0D12F9 =
"Failed to start ConfigNode leader service on node %s at %s: %s; the node will finish warming up, but this service remains unavailable until the next leadership transition.";

public static final String MESSAGE_UNREADY_NODES_ARG_144B31E1 = "nodes=%s";
public static final String MESSAGE_MORE_UNREADY_NODES_ARG_C7CDA960 = "...(%d more)";

private ConfigNodeMessages() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -88,5 +88,62 @@ public final class ConfigNodeMessages {
public static final String UNEXPECTED_INTERRUPTION_DURING_THE_CLOSE_METHOD_OF_LOGWRITER =
"[ConfigRegion Update] 关闭 logWriter 过程中发生意外中断";

public static final String
MESSAGE_CURRENT_CONFIGNODE_NOT_LEADER_PLEASE_REDIRECT_NEW_CONFIGNODE_F9AF262D =
"当前 ConfigNode 不是 Leader,请重定向到新的 ConfigNode。";
public static final String
MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_CONSENSUS_NOT_READY_YET_F1FB7C9A =
"当前 ConfigNode 是 Leader,但共识层尚未就绪。";
public static final String
MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_LEADER_SERVICES_NOT_READY_YET_ECC25E1A =
"当前 ConfigNode 是 Leader,但 Leader 服务尚未就绪。";
public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_NOT_STARTED_1C75DB82 =
"ConfigNode Leader 的负载服务尚未启动。";
public static final String
MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F =
"ConfigNode Leader 正在等待集群心跳采样。";
public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_STOPPED_5884BB0F =
"ConfigNode Leader 的负载服务已停止。";
public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_CC9C90E7 =
"ConfigNode Leader 的负载服务已就绪。";
public static final String
MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_FIRST_HEARTBEAT_FROM_REGISTERED_CONFIGNODES_DATANODES_ARG_2FD3F6FA =
"ConfigNode Leader 正在等待已注册 ConfigNode/DataNode 的首次心跳:%s";
public static final String
MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_AFTER_WAITING_ARG_MS_FOR_FIRST_HEARTBEAT_MISSING_HEARTBEATS_ARG_A3AEA8B4 =
"ConfigNode Leader 等待首次心跳 %d 毫秒后负载服务已就绪。缺少心跳的节点:%s";
public static final String
LOG_UNEXPECTED_INTERRUPTION_WHILE_WAITING_FOR_CONFIGNODE_LEADER_LOAD_WARM_UP_BB5AA4F7 =
"等待 ConfigNode Leader 负载预热时发生意外中断。";
public static final String
LOG_CURRENT_CONFIGNODE_NODEID_ARG_IP_ARG_FINISHED_STARTING_LEADER_SERVICES_WHILE_LOAD_WARM_UP_IS_STILL_IN_PROGRESS_ARG_17C09A31 =
"当前 ConfigNode(nodeId: {}, ip: {}) 已完成 Leader 服务启动,但负载预热仍在进行:{}";
public static final String
LOG_CONFIGNODE_LEADER_WARMING_UP_BEFORE_SERVING_REGISTERING_CONFIGNODE_STATUS_ARG_RETRY_ARG_3C924873 =
"ConfigNode Leader 正在预热,暂不能处理注册中的 ConfigNode,将等待并重试。状态:{},重试次数:{}";
public static final String
LOG_CONFIGNODE_LEADER_ARG_WARMING_UP_BEFORE_SERVING_DATANODE_ARG_WILL_WAIT_AND_RETRY_REASON_ARG_3A2A4163 =
"ConfigNode Leader {} 正在预热,暂不能处理 DataNode {},将等待并重试。原因:{}";
public static final String
LOG_SKIP_STARTING_CONFIGNODE_LEADER_SERVICES_BECAUSE_THE_LEADER_EPOCH_IS_STALE_9E4467F6 =
"Leader 任期已过期,跳过启动 ConfigNode Leader 服务。";
public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_NEW_LEADER_ARG_5C41D7D7 =
"ConfigNode %d(%s)不再是 Leader;新 Leader 为节点 %d。";
public static final String
LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_CLEANING_SERVICES_37CA9A1A =
"ConfigNode %d(%s)不再是 Leader,正在清理 Leader 服务。";
public static final String LOG_CONFIGNODE_ARG_AT_ARG_BECOMES_CONFIG_REGION_LEADER_2EB9B9E2 =
"ConfigNode %d(%s)成为 ConfigRegion Leader。";
public static final String LOG_CONFIGNODE_ARG_AT_ARG_AS_CONFIG_REGION_LEADER_IS_READY_74DFCB26 =
"ConfigNode %d(%s)已作为 ConfigRegion Leader 就绪。";
public static final String LOG_CONFIGNODE_ARG_AT_ARG_LEADER_SERVICES_ARE_UNAVAILABLE_0223CF16 =
"ConfigNode %d(%s)不再是 Leader,所有 Leader 服务均不可用。";
public static final String
LOG_FAILED_TO_START_CONFIGNODE_LEADER_SERVICE_ON_NODE_ARG_AT_ARG_ARG_THE_NODE_WILL_FINISH_WARMING_UP_BUT_THIS_SERVICE_REMAINS_UNAVAILABLE_UNTIL_THE_NEXT_LEADERSHIP_TRANSITION_9E0D12F9 =
"启动节点 %s(地址 %s)的 ConfigNode Leader 服务 %s 失败;节点仍将完成预热,但该服务在下一次 Leader 切换前不可用。";

public static final String MESSAGE_UNREADY_NODES_ARG_144B31E1 = "未就绪节点:%s";
public static final String MESSAGE_MORE_UNREADY_NODES_ARG_C7CDA960 = "...(另有 %d 个)";

private ConfigNodeMessages() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public void onComplete(TAIHeartbeatResp aiHeartbeatResp) {
public void onError(Exception e) {
if (ThriftClient.isConnectionBroken(e)) {
loadManager.forceUpdateNodeCache(
NodeType.DataNode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown));
NodeType.AINode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown));
}
loadManager.getLoadCache().resetHeartbeatProcessing(nodeId);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.iotdb.confignode.client.async.handlers.heartbeat;

import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.commons.client.ThriftClient;
import org.apache.iotdb.commons.cluster.NodeStatus;
Expand All @@ -36,6 +37,7 @@

import org.apache.thrift.async.AsyncMethodCallback;

import java.util.Collections;
import java.util.Map;
import java.util.function.Consumer;

Expand Down Expand Up @@ -82,54 +84,86 @@ public DataNodeHeartbeatHandler(

@Override
public void onComplete(TDataNodeHeartbeatResp heartbeatResp) {
// Update NodeCache
cacheNodeHeartbeatSample(heartbeatResp);
cacheRegionGroupHeartbeatSamples(heartbeatResp);
cacheUsageSamples(heartbeatResp);
cachePipeHeartbeat(heartbeatResp);
cacheConfirmedConfigNodeEndPoints(heartbeatResp);
cacheRegionSizeSamples(heartbeatResp);
}

private void cacheNodeHeartbeatSample(TDataNodeHeartbeatResp heartbeatResp) {
loadManager
.getLoadCache()
.cacheDataNodeHeartbeatSample(nodeId, new NodeHeartbeatSample(heartbeatResp));
}

private void cacheRegionGroupHeartbeatSamples(TDataNodeHeartbeatResp heartbeatResp) {
RegionStatus regionStatus = RegionStatus.valueOf(heartbeatResp.getStatus());

heartbeatResp
.getJudgedLeaders()
.forEach(
(regionGroupId, isLeader) -> {

// Do not allow regions to inherit the Removing state from datanode
RegionStatus nextRegionStatus = regionStatus;
if (nextRegionStatus == RegionStatus.Removing) {
nextRegionStatus =
loadManager
.getLoadCache()
.getRegionCacheLastSampleStatus(regionGroupId, nodeId);
}

// Update RegionGroupCache
loadManager
.getLoadCache()
.cacheRegionHeartbeatSample(
regionGroupId,
nodeId,
new RegionHeartbeatSample(
heartbeatResp.getHeartbeatTimestamp(),
// Region will inherit DataNode's status
nextRegionStatus),
false);

if (((TConsensusGroupType.SchemaRegion.equals(regionGroupId.getType())
&& SCHEMA_REGION_SHOULD_CACHE_CONSENSUS_SAMPLE)
|| (TConsensusGroupType.DataRegion.equals(regionGroupId.getType())
&& DATA_REGION_SHOULD_CACHE_CONSENSUS_SAMPLE))
&& Boolean.TRUE.equals(isLeader)) {
// Update ConsensusGroupCache when necessary
loadManager
.getLoadCache()
.cacheConsensusSample(
regionGroupId,
new ConsensusGroupHeartbeatSample(
heartbeatResp.getConsensusLogicalTimeMap().get(regionGroupId), nodeId));
}
});
Map<TConsensusGroupId, Boolean> judgedLeaders =
heartbeatResp.isSetJudgedLeaders()
? heartbeatResp.getJudgedLeaders()
: Collections.emptyMap();
judgedLeaders.forEach(
(regionGroupId, isLeader) -> {
cacheRegionHeartbeatSample(heartbeatResp, regionStatus, regionGroupId);
cacheConsensusSampleIfNeeded(heartbeatResp, regionGroupId, isLeader);
});
}

private void cacheRegionHeartbeatSample(
TDataNodeHeartbeatResp heartbeatResp,
RegionStatus dataNodeRegionStatus,
TConsensusGroupId regionGroupId) {
loadManager
.getLoadCache()
.cacheRegionHeartbeatSample(
regionGroupId,
nodeId,
new RegionHeartbeatSample(
heartbeatResp.getHeartbeatTimestamp(),
getRegionHeartbeatStatus(regionGroupId, dataNodeRegionStatus)),
false);
}

private RegionStatus getRegionHeartbeatStatus(
TConsensusGroupId regionGroupId, RegionStatus dataNodeRegionStatus) {
return dataNodeRegionStatus == RegionStatus.Removing
? loadManager.getLoadCache().getRegionCacheLastSampleStatus(regionGroupId, nodeId)
: dataNodeRegionStatus;
}

private void cacheConsensusSampleIfNeeded(
TDataNodeHeartbeatResp heartbeatResp, TConsensusGroupId regionGroupId, Boolean isLeader) {
if (!Boolean.TRUE.equals(isLeader)
|| !shouldCacheConsensusSample(regionGroupId)
|| !hasConsensusLogicalTimestamp(heartbeatResp, regionGroupId)) {
return;
}

loadManager
.getLoadCache()
.cacheConsensusSample(
regionGroupId,
new ConsensusGroupHeartbeatSample(
heartbeatResp.getConsensusLogicalTimeMap().get(regionGroupId), nodeId));
}

private boolean shouldCacheConsensusSample(TConsensusGroupId regionGroupId) {
return (TConsensusGroupType.SchemaRegion.equals(regionGroupId.getType())
&& SCHEMA_REGION_SHOULD_CACHE_CONSENSUS_SAMPLE)
|| (TConsensusGroupType.DataRegion.equals(regionGroupId.getType())
&& DATA_REGION_SHOULD_CACHE_CONSENSUS_SAMPLE);
}

private boolean hasConsensusLogicalTimestamp(
TDataNodeHeartbeatResp heartbeatResp, TConsensusGroupId regionGroupId) {
return heartbeatResp.isSetConsensusLogicalTimeMap()
&& heartbeatResp.getConsensusLogicalTimeMap().containsKey(regionGroupId);
}

private void cacheUsageSamples(TDataNodeHeartbeatResp heartbeatResp) {
if (heartbeatResp.getRegionDeviceUsageMap() != null) {
deviceNum.putAll(heartbeatResp.getRegionDeviceUsageMap());
deviceUsageRespProcess.accept(heartbeatResp.getRegionDeviceUsageMap());
Expand All @@ -141,6 +175,9 @@ public void onComplete(TDataNodeHeartbeatResp heartbeatResp) {
if (heartbeatResp.getRegionDisk() != null) {
regionDisk.putAll(heartbeatResp.getRegionDisk());
}
}

private void cachePipeHeartbeat(TDataNodeHeartbeatResp heartbeatResp) {
if (heartbeatResp.getPipeMetaList() != null) {
pipeRuntimeCoordinator.parseHeartbeat(
nodeId,
Expand All @@ -150,12 +187,18 @@ public void onComplete(TDataNodeHeartbeatResp heartbeatResp) {
heartbeatResp.getPipeRemainingTimeList(),
heartbeatResp.getPipeRecentFailureList());
}
}

private void cacheConfirmedConfigNodeEndPoints(TDataNodeHeartbeatResp heartbeatResp) {
if (heartbeatResp.isSetConfirmedConfigNodeEndPoints()) {
loadManager
.getLoadCache()
.updateConfirmedConfigNodeEndPoints(
nodeId, heartbeatResp.getConfirmedConfigNodeEndPoints());
}
}

private void cacheRegionSizeSamples(TDataNodeHeartbeatResp heartbeatResp) {
if (heartbeatResp.isSetRegionDisk()) {
loadManager.getLoadCache().updateRegionSizeMap(nodeId, heartbeatResp.getRegionDisk());
}
Expand Down
Loading
Loading