From 3dce93bab48ec902e0ee83bdfa63b2d652cf2983 Mon Sep 17 00:00:00 2001 From: Yongzao Date: Wed, 10 Jun 2026 14:31:12 +0800 Subject: [PATCH 1/2] [To dev/1.3] Improve ConfigNode leader warm-up (#17821) Backport of apache/iotdb@ddd8faaff23956ac1f51c2e34c722d0db9f47f08. --- .../org/apache/iotdb/rpc/TSStatusCode.java | 1 + iotdb-core/ainode/iotdb/ainode/client.py | 11 +- iotdb-core/ainode/iotdb/ainode/constant.py | 1 + .../confignode/i18n/ConfigNodeMessages.java | 56 +++++ .../confignode/i18n/ConfigNodeMessages.java | 56 +++++ .../heartbeat/AINodeHeartbeatHandler.java | 2 +- .../heartbeat/DataNodeHeartbeatHandler.java | 125 ++++++--- .../ConfigRegionStateMachine.java | 238 +++++++++++++----- .../confignode/manager/ConfigManager.java | 4 + .../confignode/manager/ProcedureManager.java | 13 +- .../manager/consensus/ConsensusManager.java | 80 ++++-- .../confignode/manager/load/LoadManager.java | 79 ++++++ .../manager/load/cache/AbstractLoadCache.java | 4 + .../manager/load/cache/LoadCache.java | 33 +++ .../cache/consensus/ConsensusGroupCache.java | 2 +- .../procedure/ProcedureExecutor.java | 13 +- .../iotdb/confignode/service/ConfigNode.java | 10 +- .../manager/load/LoadManagerTest.java | 70 ++++++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 6 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 6 + .../db/protocol/client/ConfigNodeClient.java | 48 +++- .../iotdb/commons/concurrent/ThreadName.java | 3 +- 22 files changed, 713 insertions(+), 148 deletions(-) diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index 49e75c6da9049..b9b0d82bae513 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -181,6 +181,7 @@ public enum TSStatusCode { TRANSFER_LEADER_ERROR(1008), GET_CLUSTER_ID_ERROR(1009), CAN_NOT_CONNECT_CONFIGNODE(1010), + CONFIG_NODE_LEADER_WARMING_UP(1014), // Sync, Load TsFile LOAD_FILE_ERROR(1100), diff --git a/iotdb-core/ainode/iotdb/ainode/client.py b/iotdb-core/ainode/iotdb/ainode/client.py index e44cdf476760e..1f68277a3bc7c 100644 --- a/iotdb-core/ainode/iotdb/ainode/client.py +++ b/iotdb-core/ainode/iotdb/ainode/client.py @@ -53,6 +53,7 @@ def __init__(self, config_leader: TEndPoint): self._MSG_RECONNECTION_FAIL = "Fail to connect to any config node. Please check status of ConfigNodes" self._RETRY_NUM = 5 + self._STARTUP_RETRY_NUM = 60 self._RETRY_INTERVAL_MS = 1 self._try_to_connect() @@ -123,6 +124,12 @@ def _update_config_node_leader(self, status: TSStatus) -> bool: else: self._config_leader = None return True + if status.code == TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.get_status_code(): + logger.info( + "ConfigNode leader is warming up before serving AINode, will wait and retry. Reason: {}", + status.message, + ) + return True return False def node_register(self, cluster_name: str, configuration: TAINodeConfiguration, @@ -133,7 +140,7 @@ def node_register(self, cluster_name: str, configuration: TAINodeConfiguration, versionInfo=version_info ) - for _ in range(0, self._RETRY_NUM): + for _ in range(0, self._STARTUP_RETRY_NUM): try: resp = self._client.registerAINode(req) if not self._update_config_node_leader(resp.status): @@ -156,7 +163,7 @@ def node_restart(self, cluster_name: str, configuration: TAINodeConfiguration, versionInfo=version_info ) - for _ in range(0, self._RETRY_NUM): + for _ in range(0, self._STARTUP_RETRY_NUM): try: resp = self._client.restartAINode(req) if not self._update_config_node_leader(resp.status): diff --git a/iotdb-core/ainode/iotdb/ainode/constant.py b/iotdb-core/ainode/iotdb/ainode/constant.py index 596cde0b224ac..0a8082db4f124 100644 --- a/iotdb-core/ainode/iotdb/ainode/constant.py +++ b/iotdb-core/ainode/iotdb/ainode/constant.py @@ -66,6 +66,7 @@ class TSStatusCode(Enum): SUCCESS_STATUS = 200 REDIRECTION_RECOMMEND = 400 + CONFIG_NODE_LEADER_WARMING_UP = 1014 AINODE_INTERNAL_ERROR = 1510 INVALID_URI_ERROR = 1511 INVALID_INFERENCE_CONFIG = 1512 diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 4e60e599e1d5c..897c6d3ec63f1 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -89,5 +89,61 @@ public final class ConfigNodeMessages { public static final String UNEXPECTED_INTERRUPTION_DURING_THE_CLOSE_METHOD_OF_LOGWRITER = "[ConfigRegion Update] Unexpected interruption during the close method of logWriter"; + public static final String + MESSAGE_CURRENT_CONFIGNODE_NOT_LEADER_PLEASE_REDIRECT_NEW_CONFIGNODE_F9AF262D = + "The current ConfigNode is not leader, please redirect to a new ConfigNode."; + public static final String + MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_CONSENSUS_NOT_READY_YET_F1FB7C9A = + "The current ConfigNode is leader but consensus is not ready yet."; + public static final String + MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_LEADER_SERVICES_NOT_READY_YET_ECC25E1A = + "The current ConfigNode is leader but leader services are not ready yet."; + public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_NOT_STARTED_1C75DB82 = + "ConfigNode leader load services are not started."; + public static final String + MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F = + "ConfigNode leader is waiting for cluster heartbeat sampling."; + public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_STOPPED_5884BB0F = + "ConfigNode leader load services are stopped."; + public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_CC9C90E7 = + "ConfigNode leader load services are ready."; + public static final String + MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_FIRST_HEARTBEAT_FROM_REGISTERED_CONFIGNODES_DATANODES_ARG_2FD3F6FA = + "ConfigNode leader is waiting for first heartbeat from registered ConfigNodes/DataNodes: %s"; + public static final String + MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_AFTER_WAITING_ARG_MS_FOR_FIRST_HEARTBEAT_MISSING_HEARTBEATS_ARG_A3AEA8B4 = + "ConfigNode leader load services are ready after waiting %dms for first heartbeat. Missing heartbeats: %s"; + public static final String + LOG_UNEXPECTED_INTERRUPTION_WHILE_WAITING_FOR_CONFIGNODE_LEADER_LOAD_WARM_UP_BB5AA4F7 = + "Unexpected interruption while waiting for ConfigNode leader load warm-up."; + public static final String + LOG_CURRENT_CONFIGNODE_NODEID_ARG_IP_ARG_FINISHED_STARTING_LEADER_SERVICES_WHILE_LOAD_WARM_UP_IS_STILL_IN_PROGRESS_ARG_17C09A31 = + "Current ConfigNode(nodeId: {}, ip: {}) finished starting leader services while load warm-up is still in progress: {}"; + public static final String + LOG_CONFIGNODE_LEADER_WARMING_UP_BEFORE_SERVING_REGISTERING_CONFIGNODE_STATUS_ARG_RETRY_ARG_3C924873 = + "ConfigNode leader is warming up before serving the registering ConfigNode, will wait and retry. Status: {}, retry: {}"; + public static final String + LOG_CONFIGNODE_LEADER_ARG_WARMING_UP_BEFORE_SERVING_DATANODE_ARG_WILL_WAIT_AND_RETRY_REASON_ARG_3A2A4163 = + "ConfigNode leader {} is warming up before serving DataNode {}, will wait and retry. Reason: {}"; + public static final String + LOG_SKIP_STARTING_CONFIGNODE_LEADER_SERVICES_BECAUSE_THE_LEADER_EPOCH_IS_STALE_9E4467F6 = + "Skip starting ConfigNode leader services because the leader epoch is stale."; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_NEW_LEADER_ARG_5C41D7D7 = + "ConfigNode %d at %s is no longer the leader; new leader is node %d."; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_CLEANING_SERVICES_37CA9A1A = + "ConfigNode %d at %s is no longer the leader; cleaning up leader services."; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_BECOMES_CONFIG_REGION_LEADER_2EB9B9E2 = + "ConfigNode %d at %s becomes ConfigRegion leader."; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_AS_CONFIG_REGION_LEADER_IS_READY_74DFCB26 = + "ConfigNode %d at %s as ConfigRegion leader is ready to work."; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_LEADER_SERVICES_ARE_UNAVAILABLE_0223CF16 = + "ConfigNode %d at %s is no longer leader; all leader services are unavailable."; + public static final String + LOG_FAILED_TO_START_CONFIGNODE_LEADER_SERVICE_ARG_FOR_NODE_ARG_AT_ARG_THE_NODE_WILL_FINISH_WARMING_UP_BUT_THIS_SERVICE_REMAINS_UNAVAILABLE_UNTIL_THE_NEXT_LEADERSHIP_TRANSITION_82D6EBBD = + "Failed to start ConfigNode leader service %s for node %s at %s; the node will finish warming up, but this service remains unavailable until the next leadership transition."; + + public static final String MESSAGE_UNREADY_NODES_ARG_144B31E1 = "nodes=%s"; + public static final String MESSAGE_MORE_UNREADY_NODES_ARG_C7CDA960 = "...(%d more)"; + private ConfigNodeMessages() {} } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 726a87487f44e..0d165ffa439df 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -88,5 +88,61 @@ public final class ConfigNodeMessages { public static final String UNEXPECTED_INTERRUPTION_DURING_THE_CLOSE_METHOD_OF_LOGWRITER = "[ConfigRegion Update] 关闭 logWriter 过程中发生意外中断"; + public static final String + MESSAGE_CURRENT_CONFIGNODE_NOT_LEADER_PLEASE_REDIRECT_NEW_CONFIGNODE_F9AF262D = + "当前 ConfigNode 不是 Leader,请重定向到新的 ConfigNode。"; + public static final String + MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_CONSENSUS_NOT_READY_YET_F1FB7C9A = + "当前 ConfigNode 是 Leader,但共识层尚未就绪。"; + public static final String + MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_LEADER_SERVICES_NOT_READY_YET_ECC25E1A = + "当前 ConfigNode 是 Leader,但 Leader 服务尚未就绪。"; + public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_NOT_STARTED_1C75DB82 = + "ConfigNode Leader 的负载服务尚未启动。"; + public static final String + MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F = + "ConfigNode Leader 正在等待集群心跳采样。"; + public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_STOPPED_5884BB0F = + "ConfigNode Leader 的负载服务已停止。"; + public static final String MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_CC9C90E7 = + "ConfigNode Leader 的负载服务已就绪。"; + public static final String + MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_FIRST_HEARTBEAT_FROM_REGISTERED_CONFIGNODES_DATANODES_ARG_2FD3F6FA = + "ConfigNode Leader 正在等待已注册 ConfigNode/DataNode 的首次心跳:%s"; + public static final String + MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_AFTER_WAITING_ARG_MS_FOR_FIRST_HEARTBEAT_MISSING_HEARTBEATS_ARG_A3AEA8B4 = + "ConfigNode Leader 等待首次心跳 %d 毫秒后负载服务已就绪。缺少心跳的节点:%s"; + public static final String + LOG_UNEXPECTED_INTERRUPTION_WHILE_WAITING_FOR_CONFIGNODE_LEADER_LOAD_WARM_UP_BB5AA4F7 = + "等待 ConfigNode Leader 负载预热时发生意外中断。"; + public static final String + LOG_CURRENT_CONFIGNODE_NODEID_ARG_IP_ARG_FINISHED_STARTING_LEADER_SERVICES_WHILE_LOAD_WARM_UP_IS_STILL_IN_PROGRESS_ARG_17C09A31 = + "当前 ConfigNode(nodeId: {}, ip: {}) 已完成 Leader 服务启动,但负载预热仍在进行:{}"; + public static final String + LOG_CONFIGNODE_LEADER_WARMING_UP_BEFORE_SERVING_REGISTERING_CONFIGNODE_STATUS_ARG_RETRY_ARG_3C924873 = + "ConfigNode Leader 正在预热,暂不能处理注册中的 ConfigNode,将等待并重试。状态:{},重试次数:{}"; + public static final String + LOG_CONFIGNODE_LEADER_ARG_WARMING_UP_BEFORE_SERVING_DATANODE_ARG_WILL_WAIT_AND_RETRY_REASON_ARG_3A2A4163 = + "ConfigNode Leader {} 正在预热,暂不能处理 DataNode {},将等待并重试。原因:{}"; + public static final String + LOG_SKIP_STARTING_CONFIGNODE_LEADER_SERVICES_BECAUSE_THE_LEADER_EPOCH_IS_STALE_9E4467F6 = + "Leader 任期已过期,跳过启动 ConfigNode Leader 服务。"; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_NEW_LEADER_ARG_5C41D7D7 = + "ConfigNode %d(%s)不再是 Leader;新 Leader 为节点 %d。"; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_CLEANING_SERVICES_37CA9A1A = + "ConfigNode %d(%s)不再是 Leader,正在清理 Leader 服务。"; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_BECOMES_CONFIG_REGION_LEADER_2EB9B9E2 = + "ConfigNode %d(%s)成为 ConfigRegion Leader。"; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_AS_CONFIG_REGION_LEADER_IS_READY_74DFCB26 = + "ConfigNode %d(%s)已作为 ConfigRegion Leader 就绪。"; + public static final String LOG_CONFIGNODE_ARG_AT_ARG_LEADER_SERVICES_ARE_UNAVAILABLE_0223CF16 = + "ConfigNode %d(%s)不再是 Leader,所有 Leader 服务均不可用。"; + public static final String + LOG_FAILED_TO_START_CONFIGNODE_LEADER_SERVICE_ARG_FOR_NODE_ARG_AT_ARG_THE_NODE_WILL_FINISH_WARMING_UP_BUT_THIS_SERVICE_REMAINS_UNAVAILABLE_UNTIL_THE_NEXT_LEADERSHIP_TRANSITION_82D6EBBD = + "启动节点 %s(地址 %s)的 ConfigNode Leader 服务 %s 失败;节点仍将完成预热,但该服务在下一次 Leader 切换前不可用。"; + + public static final String MESSAGE_UNREADY_NODES_ARG_144B31E1 = "未就绪节点:%s"; + public static final String MESSAGE_MORE_UNREADY_NODES_ARG_C7CDA960 = "...(另有 %d 个)"; + private ConfigNodeMessages() {} } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java index 9d8e0b6e8474f..03e6c0bfe3111 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java @@ -50,7 +50,7 @@ public void onComplete(TAIHeartbeatResp aiHeartbeatResp) { public void onError(Exception e) { if (ThriftClient.isConnectionBroken(e)) { loadManager.forceUpdateNodeCache( - NodeType.DataNode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown)); + NodeType.AINode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown)); } loadManager.getLoadCache().resetHeartbeatProcessing(nodeId); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java index 0acfb9ec7c282..27d7dc5773484 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java @@ -19,6 +19,7 @@ package org.apache.iotdb.confignode.client.async.handlers.heartbeat; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.cluster.NodeStatus; @@ -36,6 +37,7 @@ import org.apache.thrift.async.AsyncMethodCallback; +import java.util.Collections; import java.util.Map; import java.util.function.Consumer; @@ -82,54 +84,86 @@ public DataNodeHeartbeatHandler( @Override public void onComplete(TDataNodeHeartbeatResp heartbeatResp) { - // Update NodeCache + cacheNodeHeartbeatSample(heartbeatResp); + cacheRegionGroupHeartbeatSamples(heartbeatResp); + cacheUsageSamples(heartbeatResp); + cachePipeHeartbeat(heartbeatResp); + cacheConfirmedConfigNodeEndPoints(heartbeatResp); + cacheRegionSizeSamples(heartbeatResp); + } + + private void cacheNodeHeartbeatSample(TDataNodeHeartbeatResp heartbeatResp) { loadManager .getLoadCache() .cacheDataNodeHeartbeatSample(nodeId, new NodeHeartbeatSample(heartbeatResp)); + } + private void cacheRegionGroupHeartbeatSamples(TDataNodeHeartbeatResp heartbeatResp) { RegionStatus regionStatus = RegionStatus.valueOf(heartbeatResp.getStatus()); - heartbeatResp - .getJudgedLeaders() - .forEach( - (regionGroupId, isLeader) -> { - - // Do not allow regions to inherit the Removing state from datanode - RegionStatus nextRegionStatus = regionStatus; - if (nextRegionStatus == RegionStatus.Removing) { - nextRegionStatus = - loadManager - .getLoadCache() - .getRegionCacheLastSampleStatus(regionGroupId, nodeId); - } - - // Update RegionGroupCache - loadManager - .getLoadCache() - .cacheRegionHeartbeatSample( - regionGroupId, - nodeId, - new RegionHeartbeatSample( - heartbeatResp.getHeartbeatTimestamp(), - // Region will inherit DataNode's status - nextRegionStatus), - false); - - if (((TConsensusGroupType.SchemaRegion.equals(regionGroupId.getType()) - && SCHEMA_REGION_SHOULD_CACHE_CONSENSUS_SAMPLE) - || (TConsensusGroupType.DataRegion.equals(regionGroupId.getType()) - && DATA_REGION_SHOULD_CACHE_CONSENSUS_SAMPLE)) - && Boolean.TRUE.equals(isLeader)) { - // Update ConsensusGroupCache when necessary - loadManager - .getLoadCache() - .cacheConsensusSample( - regionGroupId, - new ConsensusGroupHeartbeatSample( - heartbeatResp.getConsensusLogicalTimeMap().get(regionGroupId), nodeId)); - } - }); + Map judgedLeaders = + heartbeatResp.isSetJudgedLeaders() + ? heartbeatResp.getJudgedLeaders() + : Collections.emptyMap(); + judgedLeaders.forEach( + (regionGroupId, isLeader) -> { + cacheRegionHeartbeatSample(heartbeatResp, regionStatus, regionGroupId); + cacheConsensusSampleIfNeeded(heartbeatResp, regionGroupId, isLeader); + }); + } + + private void cacheRegionHeartbeatSample( + TDataNodeHeartbeatResp heartbeatResp, + RegionStatus dataNodeRegionStatus, + TConsensusGroupId regionGroupId) { + loadManager + .getLoadCache() + .cacheRegionHeartbeatSample( + regionGroupId, + nodeId, + new RegionHeartbeatSample( + heartbeatResp.getHeartbeatTimestamp(), + getRegionHeartbeatStatus(regionGroupId, dataNodeRegionStatus)), + false); + } + + private RegionStatus getRegionHeartbeatStatus( + TConsensusGroupId regionGroupId, RegionStatus dataNodeRegionStatus) { + return dataNodeRegionStatus == RegionStatus.Removing + ? loadManager.getLoadCache().getRegionCacheLastSampleStatus(regionGroupId, nodeId) + : dataNodeRegionStatus; + } + + private void cacheConsensusSampleIfNeeded( + TDataNodeHeartbeatResp heartbeatResp, TConsensusGroupId regionGroupId, Boolean isLeader) { + if (!Boolean.TRUE.equals(isLeader) + || !shouldCacheConsensusSample(regionGroupId) + || !hasConsensusLogicalTimestamp(heartbeatResp, regionGroupId)) { + return; + } + + loadManager + .getLoadCache() + .cacheConsensusSample( + regionGroupId, + new ConsensusGroupHeartbeatSample( + heartbeatResp.getConsensusLogicalTimeMap().get(regionGroupId), nodeId)); + } + + private boolean shouldCacheConsensusSample(TConsensusGroupId regionGroupId) { + return (TConsensusGroupType.SchemaRegion.equals(regionGroupId.getType()) + && SCHEMA_REGION_SHOULD_CACHE_CONSENSUS_SAMPLE) + || (TConsensusGroupType.DataRegion.equals(regionGroupId.getType()) + && DATA_REGION_SHOULD_CACHE_CONSENSUS_SAMPLE); + } + + private boolean hasConsensusLogicalTimestamp( + TDataNodeHeartbeatResp heartbeatResp, TConsensusGroupId regionGroupId) { + return heartbeatResp.isSetConsensusLogicalTimeMap() + && heartbeatResp.getConsensusLogicalTimeMap().containsKey(regionGroupId); + } + private void cacheUsageSamples(TDataNodeHeartbeatResp heartbeatResp) { if (heartbeatResp.getRegionDeviceUsageMap() != null) { deviceNum.putAll(heartbeatResp.getRegionDeviceUsageMap()); deviceUsageRespProcess.accept(heartbeatResp.getRegionDeviceUsageMap()); @@ -141,6 +175,9 @@ public void onComplete(TDataNodeHeartbeatResp heartbeatResp) { if (heartbeatResp.getRegionDisk() != null) { regionDisk.putAll(heartbeatResp.getRegionDisk()); } + } + + private void cachePipeHeartbeat(TDataNodeHeartbeatResp heartbeatResp) { if (heartbeatResp.getPipeMetaList() != null) { pipeRuntimeCoordinator.parseHeartbeat( nodeId, @@ -150,12 +187,18 @@ public void onComplete(TDataNodeHeartbeatResp heartbeatResp) { heartbeatResp.getPipeRemainingTimeList(), heartbeatResp.getPipeRecentFailureList()); } + } + + private void cacheConfirmedConfigNodeEndPoints(TDataNodeHeartbeatResp heartbeatResp) { if (heartbeatResp.isSetConfirmedConfigNodeEndPoints()) { loadManager .getLoadCache() .updateConfirmedConfigNodeEndPoints( nodeId, heartbeatResp.getConfirmedConfigNodeEndPoints()); } + } + + private void cacheRegionSizeSamples(TDataNodeHeartbeatResp heartbeatResp) { if (heartbeatResp.isSetRegionDisk()) { loadManager.getLoadCache().updateRegionSizeMap(nodeId, heartbeatResp.getRegionDisk()); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java index e96c7e56e684b..ff706427e7015 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java @@ -61,19 +61,29 @@ import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.Comparator; +import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; /** {@link IStateMachine} for ConfigRegion. */ public class ConfigRegionStateMachine implements IStateMachine, IStateMachine.EventApi { private static final Logger LOGGER = LoggerFactory.getLogger(ConfigRegionStateMachine.class); - private static final ExecutorService threadPool = + private static final ExecutorService leaderServicesTransitionExecutor = + IoTDBThreadPoolFactory.newSingleThreadExecutor( + ThreadName.CONFIG_NODE_LEADER_SERVICES_TRANSITION.getName()); + private static final ExecutorService leaderServicesStartupPool = IoTDBThreadPoolFactory.newCachedThreadPool(ThreadName.CONFIG_NODE_RECOVER.getName()); private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); private final ConfigPlanExecutor executor; + private final AtomicBoolean leaderServicesReady; + private final AtomicLong leaderServicesEpoch; + private final Object leaderServicesLock; private ConfigManager configManager; /** Variables for {@link ConfigNode} Simple Consensus. */ @@ -97,6 +107,9 @@ public class ConfigRegionStateMachine implements IStateMachine, IStateMachine.Ev public ConfigRegionStateMachine(ConfigManager configManager, ConfigPlanExecutor executor) { this.executor = executor; + this.leaderServicesReady = new AtomicBoolean(false); + this.leaderServicesEpoch = new AtomicLong(0); + this.leaderServicesLock = new Object(); this.configManager = configManager; this.currentNodeTEndPoint = new TEndPoint() @@ -245,30 +258,103 @@ public void loadSnapshot(File latestSnapshotRootDir) { @Override public void notifyLeaderChanged(ConsensusGroupId groupId, int newLeaderId) { - // We get currentNodeId here because the currentNodeId - // couldn't initialize earlier than the ConfigRegionStateMachine int currentNodeId = ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(); if (currentNodeId != newLeaderId) { LOGGER.info( - "Current node [nodeId:{}, ip:port: {}] is no longer the leader, " - + "the new leader is [nodeId:{}]", + ConfigNodeMessages.LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_NEW_LEADER_ARG_5C41D7D7, currentNodeId, currentNodeTEndPoint, newLeaderId); + resignLeaderAsync(); } } @Override public void notifyNotLeader() { - // We get currentNodeId here because the currentNodeId - // couldn't initialize earlier than the ConfigRegionStateMachine int currentNodeId = ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(); LOGGER.info( - "Current node [nodeId:{}, ip:port: {}] is no longer the leader, " - + "start cleaning up related services", + ConfigNodeMessages.LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_CLEANING_SERVICES_37CA9A1A, currentNodeId, currentNodeTEndPoint); - // Stop leader scheduling services + resignLeaderAsync(); + } + + @Override + public void notifyLeaderReady() { + LOGGER.info( + ConfigNodeMessages.LOG_CONFIGNODE_ARG_AT_ARG_BECOMES_CONFIG_REGION_LEADER_2EB9B9E2, + ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(), + currentNodeTEndPoint); + long epoch = nextLeaderServicesEpoch(); + leaderServicesTransitionExecutor.submit(() -> becomeLeader(epoch)); + } + + private void resignLeaderAsync() { + invalidateLeaderServices(); + leaderServicesTransitionExecutor.submit(this::stopLeaderServices); + } + + private void becomeLeader(final long epoch) { + if (!isCurrentLeaderServicesEpoch(epoch)) { + LOGGER.info( + ConfigNodeMessages + .LOG_SKIP_STARTING_CONFIGNODE_LEADER_SERVICES_BECAUSE_THE_LEADER_EPOCH_IS_STALE_9E4467F6); + return; + } + + configManager.getLoadManager().startLoadServices(); + + CompletableFuture[] startups = + leaderServiceStartups().stream() + .map(startup -> startInParallelIfEpochCurrent(epoch, startup)) + .toArray(CompletableFuture[]::new); + CompletableFuture.allOf(startups).join(); + + if (!isCurrentLeaderServicesEpoch(epoch)) { + return; + } + configManager + .getProcedureManager() + .startExecutor(() -> markLeaderServicesReadyIfEpochCurrent(epoch)); + markLeaderServicesReadyIfEpochCurrent(epoch); + + boolean loadReady = waitForLoadReady(epoch); + if (!isCurrentLeaderServicesEpoch(epoch)) { + return; + } + logLoadWarmUpIfNeeded(loadReady); + LOGGER.info( + ConfigNodeMessages.LOG_CONFIGNODE_ARG_AT_ARG_AS_CONFIG_REGION_LEADER_IS_READY_74DFCB26, + ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(), + currentNodeTEndPoint); + } + + private List leaderServiceStartups() { + return Arrays.asList( + () -> configManager.getProcedureManager().getStore().getProcedureInfo().upgrade(), + () -> configManager.getRetryFailedTasksThread().startRetryFailedTasksService(), + () -> configManager.getPartitionManager().startRegionCleaner(), + configManager::checkUserPathPrivilege, + configManager::addMetrics, + () -> PipeConfigNodeAgent.runtime().notifyLeaderReady(), + () -> configManager.getCQManager().startCQScheduler(), + () -> configManager.getPipeManager().getPipeRuntimeCoordinator().startPipeMetaSync(), + () -> configManager.getPipeManager().getPipeRuntimeCoordinator().startPipeHeartbeat(), + () -> + configManager + .getPipeManager() + .getPipeRuntimeCoordinator() + .onConfigRegionGroupLeaderChanged(), + () -> + configManager + .getSubscriptionManager() + .getSubscriptionCoordinator() + .startSubscriptionMetaSync(), + () -> configManager.getClusterManager().checkClusterId()); + } + + private void stopLeaderServices() { + int currentNodeId = ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(); configManager.getPipeManager().getPipeRuntimeCoordinator().stopPipeMetaSync(); configManager.getPipeManager().getPipeRuntimeCoordinator().stopPipeHeartbeat(); configManager.getSubscriptionManager().getSubscriptionCoordinator().stopSubscriptionMetaSync(); @@ -278,76 +364,94 @@ public void notifyNotLeader() { configManager.getPartitionManager().stopRegionCleaner(); configManager.getCQManager().stopCQScheduler(); configManager.getClusterSchemaManager().clearSchemaQuotaCache(); - // Remove Metric after leader change configManager.removeMetrics(); - - // Shutdown leader related service for config pipe PipeConfigNodeAgent.runtime().notifyLeaderUnavailable(); - - // Clean receiver file dir PipeConfigNodeAgent.receiver().cleanPipeReceiverDir(); - LOGGER.info( - "Current node [nodeId:{}, ip:port: {}] is no longer the leader, " - + "all services on old leader are unavailable now.", + ConfigNodeMessages.LOG_CONFIGNODE_ARG_AT_ARG_LEADER_SERVICES_ARE_UNAVAILABLE_0223CF16, currentNodeId, currentNodeTEndPoint); } - @Override - public void notifyLeaderReady() { - LOGGER.info( - "Current node [nodeId: {}, ip:port: {}] becomes config region leader", - ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(), - currentNodeTEndPoint); + private CompletableFuture startInParallelIfEpochCurrent( + final long epoch, final Runnable startup) { + return CompletableFuture.runAsync( + () -> { + if (isCurrentLeaderServicesEpoch(epoch)) { + startup.run(); + } + }, + leaderServicesStartupPool); + } - // Always start load services first - configManager.getLoadManager().startLoadServices(); + private void markLeaderServicesReadyIfEpochCurrent(final long epoch) { + synchronized (leaderServicesLock) { + if (isCurrentLeaderServicesEpoch(epoch)) { + leaderServicesReady.set(true); + } + } + } - // Start leader scheduling services - configManager.getProcedureManager().startExecutor(); - threadPool.submit( - () -> configManager.getProcedureManager().getStore().getProcedureInfo().upgrade()); - configManager.getRetryFailedTasksThread().startRetryFailedTasksService(); - configManager.getPartitionManager().startRegionCleaner(); - configManager.checkUserPathPrivilege(); - // Add Metric after leader ready - configManager.addMetrics(); - - // Activate leader related service for config pipe - PipeConfigNodeAgent.runtime().notifyLeaderReady(); - - // we do cq recovery async for performance: - // cq recovery may be time-consuming, we use another thread to do it in - // make notifyLeaderChanged not blocked by it - threadPool.submit(() -> configManager.getCQManager().startCQScheduler()); - - threadPool.submit( - () -> configManager.getPipeManager().getPipeRuntimeCoordinator().startPipeMetaSync()); - threadPool.submit( - () -> configManager.getPipeManager().getPipeRuntimeCoordinator().startPipeHeartbeat()); - threadPool.submit( - () -> - configManager - .getPipeManager() - .getPipeRuntimeCoordinator() - .onConfigRegionGroupLeaderChanged()); + private void logLoadWarmUpIfNeeded(final boolean loadReady) { + if (!loadReady) { + LOGGER.info( + ConfigNodeMessages + .LOG_CURRENT_CONFIGNODE_NODEID_ARG_IP_ARG_FINISHED_STARTING_LEADER_SERVICES_WHILE_LOAD_WARM_UP_IS_STILL_IN_PROGRESS_ARG_17C09A31, + ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(), + currentNodeTEndPoint, + configManager.getLoadManager().getLoadReadyReason()); + } + } - threadPool.submit( - () -> - configManager - .getSubscriptionManager() - .getSubscriptionCoordinator() - .startSubscriptionMetaSync()); + private boolean waitForLoadReady(final long epoch) { + long startTime = System.currentTimeMillis(); + long timeout = CommonDescriptor.getInstance().getConfig().getCnConnectionTimeoutInMS() / 2; + while (isCurrentLeaderServicesEpoch(epoch) && System.currentTimeMillis() - startTime < timeout) { + if (configManager.getLoadManager().isLoadReady()) { + return true; + } + if (!sleepForLoadReady()) { + return false; + } + } + return isCurrentLeaderServicesEpoch(epoch) && configManager.getLoadManager().isLoadReady(); + } + + private boolean sleepForLoadReady() { + try { + TimeUnit.MILLISECONDS.sleep(100); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.warn( + ConfigNodeMessages + .LOG_UNEXPECTED_INTERRUPTION_WHILE_WAITING_FOR_CONFIGNODE_LEADER_LOAD_WARM_UP_BB5AA4F7, + e); + return false; + } + } - // To adapt old version, we check cluster ID after state machine has been fully recovered. - // Do check async because sync will be slow and block every other things. - threadPool.submit(() -> configManager.getClusterManager().checkClusterId()); + public boolean areLeaderServicesReady() { + return leaderServicesReady.get(); + } - LOGGER.info( - "Current node [nodeId: {}, ip:port: {}] as config region leader is ready to work", - ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(), - currentNodeTEndPoint); + private long nextLeaderServicesEpoch() { + synchronized (leaderServicesLock) { + leaderServicesReady.set(false); + return leaderServicesEpoch.incrementAndGet(); + } + } + + private void invalidateLeaderServices() { + synchronized (leaderServicesLock) { + leaderServicesReady.set(false); + leaderServicesEpoch.incrementAndGet(); + } + } + + private boolean isCurrentLeaderServicesEpoch(final long epoch) { + return leaderServicesEpoch.get() == epoch + && configManager.getConsensusManager().isLeaderReady(); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index ff11b178aaa17..a9ec2d1f42eb2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -1146,6 +1146,10 @@ protected TSStatus confirmLeader() { "ConsensusManager of target-ConfigNode is not initialized, " + "please make sure the target-ConfigNode has been started successfully."); } + // Procedure recovery replays metadata writes before external load warm-up is complete. + if (procedureManager.isProcedureExecutionThread()) { + return getConsensusManager().confirmLeaderForInternalProcedure(); + } return getConsensusManager().confirmLeader(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 5dc0fbe8eeac0..4bf86f2961a8a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -195,14 +195,21 @@ public ProcedureManager(ConfigManager configManager, ProcedureInfo procedureInfo } public void startExecutor() { + startExecutor(null); + } + + public void startExecutor(final Runnable beforeStartingWorkers) { if (!executor.isRunning()) { executor.init(CONFIG_NODE_CONFIG.getProcedureCoreWorkerThreadsCount()); - executor.startWorkers(); executor.startCompletedCleaner( CONFIG_NODE_CONFIG.getProcedureCompletedCleanInterval(), CONFIG_NODE_CONFIG.getProcedureCompletedEvictTTL()); executor.addInternalProcedure(partitionTableCleaner); store.start(); + if (beforeStartingWorkers != null) { + beforeStartingWorkers.run(); + } + executor.startWorkers(); LOGGER.info("ProcedureManager is started successfully."); } } @@ -219,6 +226,10 @@ public void stopExecutor() { } } + public boolean isProcedureExecutionThread() { + return ProcedureExecutor.isProcedureExecutionThread(); + } + @TestOnly public TSStatus createManyDatabases() { this.executor.submitProcedure(new CreateManyDatabasesProcedure()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java index 67b5654070fde..778ac628b7104 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java @@ -34,6 +34,7 @@ import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; import org.apache.iotdb.confignode.consensus.statemachine.ConfigRegionStateMachine; import org.apache.iotdb.confignode.exception.AddPeerException; +import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.consensus.ConsensusFactory; @@ -76,12 +77,14 @@ public class ConsensusManager { new ConfigRegionId(CONF.getConfigRegionId()); private final IManager configManager; + private final ConfigRegionStateMachine stateMachine; private IConsensus consensusImpl; private boolean isInitialized; public ConsensusManager(IManager configManager, ConfigRegionStateMachine stateMachine) { this.configManager = configManager; + this.stateMachine = stateMachine; setConsensusLayer(stateMachine); } @@ -411,38 +414,61 @@ public boolean isLeaderExist() { * NEED_REDIRECTION otherwise */ public TSStatus confirmLeader() { - TSStatus result = new TSStatus(); - if (isLeaderReady()) { - result.setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - } else { - result.setCode(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()); - if (isLeader()) { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < MAX_WAIT_READY_TIME_MS) { - if (isLeaderReady()) { - result.setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - return result; - } - try { - Thread.sleep(RETRY_WAIT_TIME_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOGGER.warn("Unexpected interruption during waiting for configNode leader ready."); - break; - } - } - result.setMessage( - "The current ConfigNode is leader but not ready yet, please try again later."); - } else { - result.setMessage( - "The current ConfigNode is not leader, please redirect to a new ConfigNode."); - } + return confirmLeader(true); + } + + private TSStatus confirmLeader(final boolean checkLoadReady) { + if (!isLeader()) { + TSStatus result = new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()); + result.setMessage( + ConfigNodeMessages.MESSAGE_CURRENT_CONFIGNODE_NOT_LEADER_PLEASE_REDIRECT_NEW_CONFIGNODE_F9AF262D); TConfigNodeLocation leaderLocation = getLeaderLocation(); if (leaderLocation != null) { result.setRedirectNode(leaderLocation.getInternalEndPoint()); } + return result; + } + + waitForLeaderReady(); + + if (!isLeaderReady()) { + return getLeaderWarmingUpStatus( + ConfigNodeMessages.MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_CONSENSUS_NOT_READY_YET_F1FB7C9A); + } + if (!stateMachine.areLeaderServicesReady()) { + return getLeaderWarmingUpStatus( + ConfigNodeMessages.MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_LEADER_SERVICES_NOT_READY_YET_ECC25E1A); + } + if (checkLoadReady && !configManager.getLoadManager().isLoadReady()) { + return getLeaderWarmingUpStatus(configManager.getLoadManager().getLoadReadyReason()); + } + + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + + public TSStatus confirmLeaderForInternalProcedure() { + return confirmLeader(false); + } + + private void waitForLeaderReady() { + long startTime = System.currentTimeMillis(); + while (!isLeaderReady() && System.currentTimeMillis() - startTime < MAX_WAIT_READY_TIME_MS) { + try { + Thread.sleep(RETRY_WAIT_TIME_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.warn( + ConfigNodeMessages + .LOG_UNEXPECTED_INTERRUPTION_WHILE_WAITING_FOR_CONFIGNODE_LEADER_LOAD_WARM_UP_BB5AA4F7, + e); + return; + } } - return result; + } + + private TSStatus getLeaderWarmingUpStatus(String message) { + return new TSStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()) + .setMessage(message); } public ConsensusGroupId getConsensusGroupId() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java index c2214fa2d11ce..8a83e485af7e2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java @@ -33,6 +33,7 @@ import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; import org.apache.iotdb.confignode.exception.NoAvailableRegionGroupException; import org.apache.iotdb.confignode.exception.NotEnoughDataNodeException; +import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.load.balancer.PartitionBalancer; import org.apache.iotdb.confignode.manager.load.balancer.RegionBalancer; @@ -49,7 +50,10 @@ import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; /** * The {@link LoadManager} at ConfigNodeGroup-Leader is active. It proactively implements the @@ -57,6 +61,8 @@ */ public class LoadManager { + private static final long FIRST_HEARTBEAT_READY_TOLERANCE_MS = TimeUnit.SECONDS.toMillis(30); + protected final IManager configManager; /** Balancers. */ @@ -71,6 +77,11 @@ public class LoadManager { protected HeartbeatService heartbeatService; private final StatisticsService statisticsService; private final EventService eventService; + private final AtomicBoolean loadServicesStarted = new AtomicBoolean(false); + private final AtomicLong loadReadyStartTimeMillis = new AtomicLong(0); + private final AtomicBoolean loadReady = new AtomicBoolean(false); + private volatile String loadReadyReason = + ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_NOT_STARTED_1C75DB82; public LoadManager(IManager configManager) { this.configManager = configManager; @@ -140,7 +151,12 @@ public void reBalanceDataPartitionPolicy(String database) { } public void startLoadServices() { + loadReady.set(false); + loadReadyStartTimeMillis.set(System.currentTimeMillis()); + loadReadyReason = + ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F; loadCache.initHeartbeatCache(configManager); + loadServicesStarted.set(true); heartbeatService.startHeartbeatService(); statisticsService.startLoadStatisticsService(); eventService.startEventService(); @@ -148,6 +164,11 @@ public void startLoadServices() { } public void stopLoadServices() { + loadServicesStarted.set(false); + loadReadyStartTimeMillis.set(0); + loadReady.set(false); + loadReadyReason = + ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_STOPPED_5884BB0F; heartbeatService.stopHeartbeatService(); statisticsService.stopLoadStatisticsService(); eventService.stopEventService(); @@ -156,6 +177,55 @@ public void stopLoadServices() { routeBalancer.clearRegionPriority(); } + public boolean isLoadReady() { + return loadReady.get() || tryUpdateLoadReady(); + } + + public String getLoadReadyReason() { + return loadReadyReason; + } + + private synchronized boolean tryUpdateLoadReady() { + if (loadReady.get()) { + return true; + } + if (!loadServicesStarted.get()) { + loadReadyReason = + ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_NOT_STARTED_1C75DB82; + return false; + } + + loadCache.updateNodeStatistics(false); + eventService.checkAndBroadcastNodeStatisticsChangeEventIfNecessary(); + + List unreadyReasons = loadCache.getNodeHeartbeatUnreadyReasons(); + if (unreadyReasons.isEmpty()) { + loadReadyReason = + ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_CC9C90E7; + loadReady.set(true); + return true; + } + + long elapsedMillis = System.currentTimeMillis() - loadReadyStartTimeMillis.get(); + if (elapsedMillis < FIRST_HEARTBEAT_READY_TOLERANCE_MS) { + loadReadyReason = + String.format( + ConfigNodeMessages + .MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_FIRST_HEARTBEAT_FROM_REGISTERED_CONFIGNODES_DATANODES_ARG_2FD3F6FA, + unreadyReasons); + return false; + } + + loadReadyReason = + String.format( + ConfigNodeMessages + .MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_AFTER_WAITING_ARG_MS_FOR_FIRST_HEARTBEAT_MISSING_HEARTBEATS_ARG_A3AEA8B4, + FIRST_HEARTBEAT_READY_TOLERANCE_MS, + unreadyReasons); + loadReady.set(true); + return true; + } + public void clearDataPartitionPolicyTable(String database) { partitionBalancer.clearDataPartitionPolicyTable(database); } @@ -462,6 +532,15 @@ public RouteBalancer getRouteBalancer() { return routeBalancer; } + @TestOnly + void markLoadServicesStartedForTest(long loadReadyStartTimeMillis) { + loadServicesStarted.set(true); + loadReady.set(false); + this.loadReadyStartTimeMillis.set(loadReadyStartTimeMillis); + loadReadyReason = + ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F; + } + @TestOnly public EventService getEventService() { return eventService; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java index 6355dfdcc8283..8994adc5b1403 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java @@ -97,6 +97,10 @@ public AbstractHeartbeatSample getLastSample() { return slidingWindow.isEmpty() ? null : slidingWindow.get(slidingWindow.size() - 1); } + public boolean hasHeartbeatSample() { + return getLastSample() != null; + } + /** * Update currentStatistics based on the latest heartbeat sample that cached in the slidingWindow. */ diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java index 3796f7ac2b767..4c33cf32d8787 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java @@ -48,12 +48,14 @@ import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.region.RegionStatistics; import org.apache.iotdb.confignode.manager.partition.RegionGroupStatus; +import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -77,6 +79,7 @@ public class LoadCache { Math.max( ProcedureManager.PROCEDURE_WAIT_TIME_OUT - TimeUnit.SECONDS.toMillis(2), TimeUnit.SECONDS.toMillis(10)); + private static final int MAX_UNREADY_ENTITY_PRINT = 10; private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); @@ -482,6 +485,36 @@ public Map getCurrentConsensusGroup return consensusGroupStatisticsMap; } + public List getNodeHeartbeatUnreadyReasons() { + List unreadyNodes = new ArrayList<>(); + nodeCacheMap.forEach( + (nodeId, nodeCache) -> { + if (nodeId == ConfigNodeHeartbeatCache.CURRENT_NODE_ID) { + return; + } + if ((nodeCache instanceof ConfigNodeHeartbeatCache + || nodeCache instanceof DataNodeHeartbeatCache) + && !nodeCache.hasHeartbeatSample()) { + unreadyNodes.add(nodeId); + } + }); + if (unreadyNodes.isEmpty()) { + return Collections.emptyList(); + } + Collections.sort(unreadyNodes); + List nodesToPrint = + unreadyNodes.subList(0, Math.min(MAX_UNREADY_ENTITY_PRINT, unreadyNodes.size())); + String suffix = + unreadyNodes.size() > MAX_UNREADY_ENTITY_PRINT + ? String.format( + ConfigNodeMessages.MESSAGE_MORE_UNREADY_NODES_ARG_C7CDA960, + unreadyNodes.size() - MAX_UNREADY_ENTITY_PRINT) + : ""; + return Collections.singletonList( + String.format(ConfigNodeMessages.MESSAGE_UNREADY_NODES_ARG_144B31E1, nodesToPrint) + + suffix); + } + /** * Safely get NodeStatus by NodeId. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java index aa924dc29b585..7dcacc4fd805c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java @@ -41,7 +41,7 @@ public synchronized void updateCurrentStatistics(boolean forceUpdate) { synchronized (slidingWindow) { lastSample = (ConsensusGroupHeartbeatSample) getLastSample(); } - if (lastSample != null && lastSample.getLeaderId() != UN_READY_LEADER_ID) { + if (lastSample != null) { currentStatistics.set( new ConsensusGroupStatistics(System.nanoTime(), lastSample.getLeaderId())); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java index 8165754293e44..6b8b35a106d36 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java @@ -52,6 +52,8 @@ public class ProcedureExecutor { private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class); + private static final ThreadLocal PROCEDURE_EXECUTION_CONTEXT = + ThreadLocal.withInitial(() -> false); private final ConcurrentHashMap> completed = new ConcurrentHashMap<>(); @@ -102,6 +104,10 @@ public ProcedureExecutor(final Env environment, final IProcedureStore store this(environment, store, new SimpleProcedureScheduler()); } + public static boolean isProcedureExecutionThread() { + return PROCEDURE_EXECUTION_CONTEXT.get(); + } + public void init(int numThreads) { this.corePoolSize = numThreads; this.maxPoolSize = 10 * numThreads; @@ -775,7 +781,12 @@ public void run() { activeExecutorCount.incrementAndGet(); startTime.set(System.currentTimeMillis()); try { - executeProcedure(procedure); + PROCEDURE_EXECUTION_CONTEXT.set(true); + try { + executeProcedure(procedure); + } finally { + PROCEDURE_EXECUTION_CONTEXT.remove(); + } } finally { procedure.releaseExecution(); activeExecutorCount.decrementAndGet(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index 7f0db7a37840b..9e93922f5e8af 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -47,6 +47,7 @@ import org.apache.iotdb.confignode.conf.ConfigNodeConstant; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeStartupCheck; +import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.conf.SystemPropertiesUtils; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; @@ -86,7 +87,7 @@ public class ConfigNode extends ServerCommandLine implements ConfigNodeMBean { private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); - private static final int STARTUP_RETRY_NUM = 10; + private static final int STARTUP_RETRY_NUM = 20; private static final long STARTUP_RETRY_INTERVAL_IN_MS = TimeUnit.SECONDS.toMillis(3); private static final int SCHEDULE_WAITING_RETRY_NUM = (int) (COMMON_CONFIG.getCnConnectionTimeoutInMS() / STARTUP_RETRY_INTERVAL_IN_MS); @@ -394,6 +395,13 @@ private void sendRegisterConfigNodeRequest() throws StartupException, IOExceptio LOGGER.info("ConfigNode need redirect to {}, retry {} ...", seedConfigNode, retry); } else if (status.getCode() == TSStatusCode.INTERNAL_REQUEST_RETRY_ERROR.getStatusCode()) { LOGGER.warn("The result of register self ConfigNode is {}, retry {} ...", status, retry); + } else if (status.getCode() + == TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()) { + LOGGER.info( + ConfigNodeMessages + .LOG_CONFIGNODE_LEADER_WARMING_UP_BEFORE_SERVING_REGISTERING_CONFIGNODE_STATUS_ARG_RETRY_ARG_3C924873, + status, + retry); } else { throw new StartupException(status.getMessage()); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java index fd62c31ae3678..e2c51df9693be 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java @@ -48,6 +48,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -292,4 +293,73 @@ public void testConsensusGroupCache() throws InterruptedException { new Pair<>(new ConsensusGroupStatistics(newLeaderId), null), differentConsensusGroupStatisticsMap.get(regionGroupId)); } + + @Test + public void testLoadWarmUpRequiresOnlyConfigNodeAndDataNodeSamples() { + LoadCache loadCache = new LoadCache(); + TConsensusGroupId regionGroupId = new TConsensusGroupId(TConsensusGroupType.DataRegion, 100); + Set dataNodeIds = Stream.of(11, 12).collect(Collectors.toSet()); + + loadCache.createNodeHeartbeatCache(NodeType.ConfigNode, 10); + dataNodeIds.forEach( + dataNodeId -> loadCache.createNodeHeartbeatCache(NodeType.DataNode, dataNodeId)); + loadCache.createNodeHeartbeatCache(NodeType.AINode, 13); + loadCache.createRegionGroupHeartbeatCache("root.warmup", regionGroupId, dataNodeIds); + + Assert.assertEquals( + Collections.singletonList("nodes=[10, 11, 12]"), + loadCache.getNodeHeartbeatUnreadyReasons()); + + loadCache.cacheConfigNodeHeartbeatSample(10, new NodeHeartbeatSample(NodeStatus.Unknown)); + + dataNodeIds.forEach( + dataNodeId -> + loadCache.cacheDataNodeHeartbeatSample( + dataNodeId, new NodeHeartbeatSample(NodeStatus.Running))); + loadCache.updateNodeStatistics(false); + + Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty()); + } + + @Test + public void testRegionAndConsensusGroupsDoNotBlockLoadWarmUp() { + LoadCache loadCache = new LoadCache(); + TConsensusGroupId regionGroupId = new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 101); + Set dataNodeIds = Stream.of(21, 22).collect(Collectors.toSet()); + + dataNodeIds.forEach( + dataNodeId -> loadCache.createNodeHeartbeatCache(NodeType.DataNode, dataNodeId)); + loadCache.createRegionGroupHeartbeatCache("root.warmup", regionGroupId, dataNodeIds); + dataNodeIds.forEach( + dataNodeId -> { + loadCache.cacheDataNodeHeartbeatSample( + dataNodeId, new NodeHeartbeatSample(NodeStatus.Running)); + }); + loadCache.updateNodeStatistics(false); + loadCache.updateRegionGroupStatistics(); + loadCache.updateConsensusGroupStatistics(); + + Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty()); + } + + @Test + public void testLoadWarmUpToleratesMissingFirstHeartbeatAfterThirtySeconds() { + LOAD_CACHE.clearHeartbeatCache(); + LOAD_CACHE.createNodeHeartbeatCache(NodeType.DataNode, 31); + + try { + LOAD_MANAGER.markLoadServicesStartedForTest(System.currentTimeMillis()); + + Assert.assertFalse(LOAD_MANAGER.isLoadReady()); + Assert.assertTrue(LOAD_MANAGER.getLoadReadyReason().contains("waiting for first heartbeat")); + + LOAD_MANAGER.markLoadServicesStartedForTest( + System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(31)); + + Assert.assertTrue(LOAD_MANAGER.isLoadReady()); + Assert.assertTrue(LOAD_MANAGER.getLoadReadyReason().contains("Missing heartbeats")); + } finally { + LOAD_MANAGER.stopLoadServices(); + } + } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 073473b47423b..d3530f4e5cbc6 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -71,4 +71,10 @@ private DataNodeQueryMessages() {} "Delete After Loading {} error."; public static final String LOG_FAILED_TO_AUTHORIZE_CQ_ARG_FOR_USER_ARG_ARG_B436A81F = "Failed to authorize CQ {} for user {}: {}"; + public static final String + LOG_CONFIGNODE_LEADER_ARG_WARMING_UP_BEFORE_SERVING_DATANODE_ARG_WILL_WAIT_AND_RETRY_REASON_ARG_3A2A4163 = + "ConfigNode leader {} is warming up before serving DataNode {}, will wait and retry. Reason: {}"; + public static final String + LOG_UNEXPECTED_INTERRUPTION_WHILE_WAITING_TO_RETRY_CONFIGNODE_CONNECTION_D4F49D21 = + "Unexpected interruption while waiting to retry a ConfigNode connection."; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 370792a20cc24..e2b70600484d6 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -65,6 +65,12 @@ private DataNodeQueryMessages() {} EXCEPTION_THE_DATABASE_ARG_DOES_NOT_EXIST_AND_AUTO_CREATE_SCHEMA_IS_DISABLED_28F8D49A = "数据库 %s 不存在,且 auto-create-schema 已禁用。"; public static final String LOG_DELETE_AFTER_LOADING_ARG_ERROR_C606A060 = "加载后删除 {} 时出错。"; + public static final String + LOG_CONFIGNODE_LEADER_ARG_WARMING_UP_BEFORE_SERVING_DATANODE_ARG_WILL_WAIT_AND_RETRY_REASON_ARG_3A2A4163 = + "ConfigNode Leader {} 正在预热,暂不能处理 DataNode {},将等待并重试。原因:{}"; + public static final String + LOG_UNEXPECTED_INTERRUPTION_WHILE_WAITING_TO_RETRY_CONFIGNODE_CONNECTION_D4F49D21 = + "等待重试 ConfigNode 连接时发生意外中断。"; public static final String LOG_FAILED_TO_AUTHORIZE_CQ_ARG_FOR_USER_ARG_ARG_B436A81F = "CQ {} 的用户 {} 鉴权失败:{}"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 07d12e5b60733..8078ea87c4db3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -167,6 +167,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory; import org.apache.iotdb.rpc.TSStatusCode; @@ -197,6 +198,7 @@ public class ConfigNodeClient implements IConfigNodeRPCService.Iface, ThriftClie "Failed to connect to ConfigNode %s from DataNode %s when executing %s, Exception:"; private static final long RETRY_INTERVAL_MS = 1000L; private static final long WAIT_CN_LEADER_ELECTION_INTERVAL_MS = 2000L; + private static final long REGISTER_LEADER_WARMING_UP_RETRY_TIMEOUT_MS = 60_000L; private static final String UNSUPPORTED_INVOCATION = "This method is not supported for invocation by DataNode"; @@ -360,12 +362,35 @@ private boolean updateConfigNodeLeader(TSStatus status) { } return true; } + if (status.getCode() == TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()) { + if (!isFirstInitiated) { + logger.info( + DataNodeQueryMessages + .LOG_CONFIGNODE_LEADER_ARG_WARMING_UP_BEFORE_SERVING_DATANODE_ARG_WILL_WAIT_AND_RETRY_REASON_ARG_3A2A4163, + configNode, + config.getAddressAndPort(), + status.getMessage()); + } + try { + Thread.sleep(WAIT_CN_LEADER_ELECTION_INTERVAL_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn( + DataNodeQueryMessages + .LOG_UNEXPECTED_INTERRUPTION_WHILE_WAITING_TO_RETRY_CONFIGNODE_CONNECTION_D4F49D21); + } + return true; + } return false; } finally { isFirstInitiated = false; } } + private boolean isConfigNodeLeaderWarmingUp(TSStatus status) { + return status.getCode() == TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode(); + } + /** * The frame of execute RPC, include logic of retry and exception handling. * @@ -441,20 +466,33 @@ public TGetClusterIdResp getClusterId() throws TException { @Override public TDataNodeRegisterResp registerDataNode(TDataNodeRegisterReq req) throws TException { - for (int i = 0; i < RETRY_NUM; i++) { + boolean leaderWarmingUpObserved = false; + long leaderWarmingUpRetryDeadline = 0; + for (int i = 0; + i < RETRY_NUM + || (leaderWarmingUpObserved + && System.currentTimeMillis() < leaderWarmingUpRetryDeadline); + i++) { try { TDataNodeRegisterResp resp = client.registerDataNode(req); if (!updateConfigNodeLeader(resp.status)) { return resp; } + if (isConfigNodeLeaderWarmingUp(resp.status) && !leaderWarmingUpObserved) { + leaderWarmingUpObserved = true; + leaderWarmingUpRetryDeadline = + System.currentTimeMillis() + REGISTER_LEADER_WARMING_UP_RETRY_TIMEOUT_MS; + } // set latest config node list - List newConfigNodes = new ArrayList<>(); - for (TConfigNodeLocation configNodeLocation : resp.getConfigNodeList()) { - newConfigNodes.add(configNodeLocation.getInternalEndPoint()); + if (resp.isSetConfigNodeList()) { + List newConfigNodes = new ArrayList<>(); + for (TConfigNodeLocation configNodeLocation : resp.getConfigNodeList()) { + newConfigNodes.add(configNodeLocation.getInternalEndPoint()); + } + configNodes = newConfigNodes; } - configNodes = newConfigNodes; } catch (TException e) { String message = String.format( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java index 49061c2ef2221..84740f3c0b214 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java @@ -93,6 +93,7 @@ public enum ThreadName { CONFIG_NODE_REGION_MAINTAINER("IoTDB-Region-Maintainer"), // -------------------------- ConfigNode-Recover -------------------------- CONFIG_NODE_RECOVER("ConfigNode-Manager-Recovery"), + CONFIG_NODE_LEADER_SERVICES_TRANSITION("ConfigNode-Leader-Services-Transition"), // -------------------------- ConfigNode-Procedure ------------------------ // TODO: Use Thread Pool to manage the procedure thread @Potato CONFIG_NODE_PROCEDURE_WORKER("ProcedureWorkerGroup"), @@ -353,7 +354,7 @@ public enum ThreadName { new HashSet<>(Arrays.asList(CONFIG_NODE_REGION_MAINTAINER)); private static final Set configNodeRecoverThreadNames = - new HashSet<>(Arrays.asList(CONFIG_NODE_RECOVER)); + new HashSet<>(Arrays.asList(CONFIG_NODE_RECOVER, CONFIG_NODE_LEADER_SERVICES_TRANSITION)); private static final Set configNodeProcedureThreadNames = new HashSet<>( From a4b7a0a95ab8e3fcbdfdeff37291fd7ecfa69bad Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:18:00 +0800 Subject: [PATCH 2/2] Fix Pipe leader-change startup ordering (#19318) * Fix Pipe leader-change procedure ordering * Fix procedure recovery during leader startup * Fix leader readiness after procedure workers start * Fix internal procedure recovery readiness (cherry picked from commit 727b83af26099403d973ac01a061438c6aeb16bd) --- .../confignode/i18n/ConfigNodeMessages.java | 9 +- .../confignode/i18n/ConfigNodeMessages.java | 7 +- .../ConfigRegionStateMachine.java | 88 +++++++++++++++---- .../manager/consensus/ConsensusManager.java | 19 ++-- .../confignode/manager/load/LoadManager.java | 12 +-- .../manager/load/cache/LoadCache.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 5 +- 7 files changed, 100 insertions(+), 42 deletions(-) diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 897c6d3ec63f1..94b8ee079a542 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -130,8 +130,9 @@ public final class ConfigNodeMessages { "Skip starting ConfigNode leader services because the leader epoch is stale."; public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_NEW_LEADER_ARG_5C41D7D7 = "ConfigNode %d at %s is no longer the leader; new leader is node %d."; - public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_CLEANING_SERVICES_37CA9A1A = - "ConfigNode %d at %s is no longer the leader; cleaning up leader services."; + public static final String + LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_CLEANING_SERVICES_37CA9A1A = + "ConfigNode %d at %s is no longer the leader; cleaning up leader services."; public static final String LOG_CONFIGNODE_ARG_AT_ARG_BECOMES_CONFIG_REGION_LEADER_2EB9B9E2 = "ConfigNode %d at %s becomes ConfigRegion leader."; public static final String LOG_CONFIGNODE_ARG_AT_ARG_AS_CONFIG_REGION_LEADER_IS_READY_74DFCB26 = @@ -139,8 +140,8 @@ public final class ConfigNodeMessages { public static final String LOG_CONFIGNODE_ARG_AT_ARG_LEADER_SERVICES_ARE_UNAVAILABLE_0223CF16 = "ConfigNode %d at %s is no longer leader; all leader services are unavailable."; public static final String - LOG_FAILED_TO_START_CONFIGNODE_LEADER_SERVICE_ARG_FOR_NODE_ARG_AT_ARG_THE_NODE_WILL_FINISH_WARMING_UP_BUT_THIS_SERVICE_REMAINS_UNAVAILABLE_UNTIL_THE_NEXT_LEADERSHIP_TRANSITION_82D6EBBD = - "Failed to start ConfigNode leader service %s for node %s at %s; the node will finish warming up, but this service remains unavailable until the next leadership transition."; + LOG_FAILED_TO_START_CONFIGNODE_LEADER_SERVICE_ON_NODE_ARG_AT_ARG_ARG_THE_NODE_WILL_FINISH_WARMING_UP_BUT_THIS_SERVICE_REMAINS_UNAVAILABLE_UNTIL_THE_NEXT_LEADERSHIP_TRANSITION_9E0D12F9 = + "Failed to start ConfigNode leader service on node %s at %s: %s; the node will finish warming up, but this service remains unavailable until the next leadership transition."; public static final String MESSAGE_UNREADY_NODES_ARG_144B31E1 = "nodes=%s"; public static final String MESSAGE_MORE_UNREADY_NODES_ARG_C7CDA960 = "...(%d more)"; diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 0d165ffa439df..f0576df523708 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -129,8 +129,9 @@ public final class ConfigNodeMessages { "Leader 任期已过期,跳过启动 ConfigNode Leader 服务。"; public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_NEW_LEADER_ARG_5C41D7D7 = "ConfigNode %d(%s)不再是 Leader;新 Leader 为节点 %d。"; - public static final String LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_CLEANING_SERVICES_37CA9A1A = - "ConfigNode %d(%s)不再是 Leader,正在清理 Leader 服务。"; + public static final String + LOG_CONFIGNODE_ARG_AT_ARG_IS_NO_LONGER_LEADER_CLEANING_SERVICES_37CA9A1A = + "ConfigNode %d(%s)不再是 Leader,正在清理 Leader 服务。"; public static final String LOG_CONFIGNODE_ARG_AT_ARG_BECOMES_CONFIG_REGION_LEADER_2EB9B9E2 = "ConfigNode %d(%s)成为 ConfigRegion Leader。"; public static final String LOG_CONFIGNODE_ARG_AT_ARG_AS_CONFIG_REGION_LEADER_IS_READY_74DFCB26 = @@ -138,7 +139,7 @@ public final class ConfigNodeMessages { public static final String LOG_CONFIGNODE_ARG_AT_ARG_LEADER_SERVICES_ARE_UNAVAILABLE_0223CF16 = "ConfigNode %d(%s)不再是 Leader,所有 Leader 服务均不可用。"; public static final String - LOG_FAILED_TO_START_CONFIGNODE_LEADER_SERVICE_ARG_FOR_NODE_ARG_AT_ARG_THE_NODE_WILL_FINISH_WARMING_UP_BUT_THIS_SERVICE_REMAINS_UNAVAILABLE_UNTIL_THE_NEXT_LEADERSHIP_TRANSITION_82D6EBBD = + LOG_FAILED_TO_START_CONFIGNODE_LEADER_SERVICE_ON_NODE_ARG_AT_ARG_ARG_THE_NODE_WILL_FINISH_WARMING_UP_BUT_THIS_SERVICE_REMAINS_UNAVAILABLE_UNTIL_THE_NEXT_LEADERSHIP_TRANSITION_9E0D12F9 = "启动节点 %s(地址 %s)的 ConfigNode Leader 服务 %s 失败;节点仍将完成预热,但该服务在下一次 Leader 切换前不可用。"; public static final String MESSAGE_UNREADY_NODES_ARG_144B31E1 = "未就绪节点:%s"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java index ff706427e7015..0868474a22539 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java @@ -66,7 +66,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; /** {@link IStateMachine} for ConfigRegion. */ @@ -81,9 +80,30 @@ public class ConfigRegionStateMachine implements IStateMachine, IStateMachine.Ev IoTDBThreadPoolFactory.newCachedThreadPool(ThreadName.CONFIG_NODE_RECOVER.getName()); private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); private final ConfigPlanExecutor executor; - private final AtomicBoolean leaderServicesReady; + + /** + * Readiness of leader services in the {@link #leaderServicesEpoch current epoch}. Recovered + * procedures can run once the Pipe callback has completed, but external requests must also wait + * for worker startup to finish. + */ + private volatile LeaderServicesReadiness leaderServicesReadiness = + LeaderServicesReadiness.WARMING_UP; + + /** + * Monotonically increasing leadership generation. Every become-leader / resign-leader transition + * bumps it, so any work submitted for an older epoch can detect it is stale and bail out. + */ private final AtomicLong leaderServicesEpoch; + + /** Guards {@link #leaderServicesReadiness} and {@link #leaderServicesEpoch} as a unit. */ private final Object leaderServicesLock; + + private enum LeaderServicesReadiness { + WARMING_UP, + READY_FOR_PROCEDURES, + READY + } + private ConfigManager configManager; /** Variables for {@link ConfigNode} Simple Consensus. */ @@ -107,7 +127,6 @@ public class ConfigRegionStateMachine implements IStateMachine, IStateMachine.Ev public ConfigRegionStateMachine(ConfigManager configManager, ConfigPlanExecutor executor) { this.executor = executor; - this.leaderServicesReady = new AtomicBoolean(false); this.leaderServicesEpoch = new AtomicLong(0); this.leaderServicesLock = new Object(); this.configManager = configManager; @@ -313,10 +332,23 @@ private void becomeLeader(final long epoch) { if (!isCurrentLeaderServicesEpoch(epoch)) { return; } + final boolean procedureExecutorWasRunning = + configManager.getProcedureManager().getExecutor().isRunning(); configManager .getProcedureManager() - .startExecutor(() -> markLeaderServicesReadyIfEpochCurrent(epoch)); - markLeaderServicesReadyIfEpochCurrent(epoch); + .startExecutor( + () -> { + startPipeOnLeaderChangedIfEpochCurrent(epoch); + markLeaderServicesReadyIfEpochCurrent( + epoch, LeaderServicesReadiness.READY_FOR_PROCEDURES); + }); + if (!isCurrentLeaderServicesEpoch(epoch)) { + return; + } + if (procedureExecutorWasRunning) { + startPipeOnLeaderChangedIfEpochCurrent(epoch); + } + markLeaderServicesReadyIfEpochCurrent(epoch, LeaderServicesReadiness.READY); boolean loadReady = waitForLoadReady(epoch); if (!isCurrentLeaderServicesEpoch(epoch)) { @@ -340,11 +372,6 @@ private List leaderServiceStartups() { () -> configManager.getCQManager().startCQScheduler(), () -> configManager.getPipeManager().getPipeRuntimeCoordinator().startPipeMetaSync(), () -> configManager.getPipeManager().getPipeRuntimeCoordinator().startPipeHeartbeat(), - () -> - configManager - .getPipeManager() - .getPipeRuntimeCoordinator() - .onConfigRegionGroupLeaderChanged(), () -> configManager .getSubscriptionManager() @@ -384,10 +411,34 @@ private CompletableFuture startInParallelIfEpochCurrent( leaderServicesStartupPool); } - private void markLeaderServicesReadyIfEpochCurrent(final long epoch) { + private void startPipeOnLeaderChangedIfEpochCurrent(final long epoch) { + startInParallelIfEpochCurrent( + epoch, + () -> { + try { + configManager + .getPipeManager() + .getPipeRuntimeCoordinator() + .onConfigRegionGroupLeaderChanged(); + } catch (final Exception e) { + LOGGER.error( + String.format( + ConfigNodeMessages + .LOG_FAILED_TO_START_CONFIGNODE_LEADER_SERVICE_ON_NODE_ARG_AT_ARG_ARG_THE_NODE_WILL_FINISH_WARMING_UP_BUT_THIS_SERVICE_REMAINS_UNAVAILABLE_UNTIL_THE_NEXT_LEADERSHIP_TRANSITION_9E0D12F9, + ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(), + currentNodeTEndPoint, + "PipeOnLeaderChanged"), + e); + } + }) + .join(); + } + + private void markLeaderServicesReadyIfEpochCurrent( + final long epoch, final LeaderServicesReadiness readiness) { synchronized (leaderServicesLock) { if (isCurrentLeaderServicesEpoch(epoch)) { - leaderServicesReady.set(true); + leaderServicesReadiness = readiness; } } } @@ -406,7 +457,8 @@ private void logLoadWarmUpIfNeeded(final boolean loadReady) { private boolean waitForLoadReady(final long epoch) { long startTime = System.currentTimeMillis(); long timeout = CommonDescriptor.getInstance().getConfig().getCnConnectionTimeoutInMS() / 2; - while (isCurrentLeaderServicesEpoch(epoch) && System.currentTimeMillis() - startTime < timeout) { + while (isCurrentLeaderServicesEpoch(epoch) + && System.currentTimeMillis() - startTime < timeout) { if (configManager.getLoadManager().isLoadReady()) { return true; } @@ -431,20 +483,22 @@ private boolean sleepForLoadReady() { } } - public boolean areLeaderServicesReady() { - return leaderServicesReady.get(); + public boolean areLeaderServicesReady(final boolean forInternalProcedure) { + final LeaderServicesReadiness readiness = leaderServicesReadiness; + return readiness == LeaderServicesReadiness.READY + || (forInternalProcedure && readiness == LeaderServicesReadiness.READY_FOR_PROCEDURES); } private long nextLeaderServicesEpoch() { synchronized (leaderServicesLock) { - leaderServicesReady.set(false); + leaderServicesReadiness = LeaderServicesReadiness.WARMING_UP; return leaderServicesEpoch.incrementAndGet(); } } private void invalidateLeaderServices() { synchronized (leaderServicesLock) { - leaderServicesReady.set(false); + leaderServicesReadiness = LeaderServicesReadiness.WARMING_UP; leaderServicesEpoch.incrementAndGet(); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java index 778ac628b7104..b0a203ad5aac3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java @@ -414,14 +414,15 @@ public boolean isLeaderExist() { * NEED_REDIRECTION otherwise */ public TSStatus confirmLeader() { - return confirmLeader(true); + return confirmLeader(false); } - private TSStatus confirmLeader(final boolean checkLoadReady) { + private TSStatus confirmLeader(final boolean forInternalProcedure) { if (!isLeader()) { TSStatus result = new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()); result.setMessage( - ConfigNodeMessages.MESSAGE_CURRENT_CONFIGNODE_NOT_LEADER_PLEASE_REDIRECT_NEW_CONFIGNODE_F9AF262D); + ConfigNodeMessages + .MESSAGE_CURRENT_CONFIGNODE_NOT_LEADER_PLEASE_REDIRECT_NEW_CONFIGNODE_F9AF262D); TConfigNodeLocation leaderLocation = getLeaderLocation(); if (leaderLocation != null) { result.setRedirectNode(leaderLocation.getInternalEndPoint()); @@ -433,13 +434,15 @@ private TSStatus confirmLeader(final boolean checkLoadReady) { if (!isLeaderReady()) { return getLeaderWarmingUpStatus( - ConfigNodeMessages.MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_CONSENSUS_NOT_READY_YET_F1FB7C9A); + ConfigNodeMessages + .MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_CONSENSUS_NOT_READY_YET_F1FB7C9A); } - if (!stateMachine.areLeaderServicesReady()) { + if (!stateMachine.areLeaderServicesReady(forInternalProcedure)) { return getLeaderWarmingUpStatus( - ConfigNodeMessages.MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_LEADER_SERVICES_NOT_READY_YET_ECC25E1A); + ConfigNodeMessages + .MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_LEADER_SERVICES_NOT_READY_YET_ECC25E1A); } - if (checkLoadReady && !configManager.getLoadManager().isLoadReady()) { + if (!forInternalProcedure && !configManager.getLoadManager().isLoadReady()) { return getLeaderWarmingUpStatus(configManager.getLoadManager().getLoadReadyReason()); } @@ -447,7 +450,7 @@ private TSStatus confirmLeader(final boolean checkLoadReady) { } public TSStatus confirmLeaderForInternalProcedure() { - return confirmLeader(false); + return confirmLeader(true); } private void waitForLeaderReady() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java index 8a83e485af7e2..22314ca2cfa33 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java @@ -154,7 +154,8 @@ public void startLoadServices() { loadReady.set(false); loadReadyStartTimeMillis.set(System.currentTimeMillis()); loadReadyReason = - ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F; + ConfigNodeMessages + .MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F; loadCache.initHeartbeatCache(configManager); loadServicesStarted.set(true); heartbeatService.startHeartbeatService(); @@ -167,8 +168,7 @@ public void stopLoadServices() { loadServicesStarted.set(false); loadReadyStartTimeMillis.set(0); loadReady.set(false); - loadReadyReason = - ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_STOPPED_5884BB0F; + loadReadyReason = ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_STOPPED_5884BB0F; heartbeatService.stopHeartbeatService(); statisticsService.stopLoadStatisticsService(); eventService.stopEventService(); @@ -200,8 +200,7 @@ private synchronized boolean tryUpdateLoadReady() { List unreadyReasons = loadCache.getNodeHeartbeatUnreadyReasons(); if (unreadyReasons.isEmpty()) { - loadReadyReason = - ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_CC9C90E7; + loadReadyReason = ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_LOAD_SERVICES_READY_CC9C90E7; loadReady.set(true); return true; } @@ -538,7 +537,8 @@ void markLoadServicesStartedForTest(long loadReadyStartTimeMillis) { loadReady.set(false); this.loadReadyStartTimeMillis.set(loadReadyStartTimeMillis); loadReadyReason = - ConfigNodeMessages.MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F; + ConfigNodeMessages + .MESSAGE_CONFIGNODE_LEADER_WAITING_FOR_CLUSTER_HEARTBEAT_SAMPLING_EDB0331F; } @TestOnly diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java index 4c33cf32d8787..f7a56cf0876ce 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java @@ -32,6 +32,7 @@ import org.apache.iotdb.commons.cluster.RegionStatus; import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.ProcedureManager; import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupCache; @@ -48,7 +49,6 @@ import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.region.RegionStatistics; import org.apache.iotdb.confignode.manager.partition.RegionGroupStatus; -import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index 9e93922f5e8af..be315e61814f7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -47,8 +47,8 @@ import org.apache.iotdb.confignode.conf.ConfigNodeConstant; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeStartupCheck; -import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.conf.SystemPropertiesUtils; +import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent; @@ -395,8 +395,7 @@ private void sendRegisterConfigNodeRequest() throws StartupException, IOExceptio LOGGER.info("ConfigNode need redirect to {}, retry {} ...", seedConfigNode, retry); } else if (status.getCode() == TSStatusCode.INTERNAL_REQUEST_RETRY_ERROR.getStatusCode()) { LOGGER.warn("The result of register self ConfigNode is {}, retry {} ...", status, retry); - } else if (status.getCode() - == TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()) { + } else if (status.getCode() == TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()) { LOGGER.info( ConfigNodeMessages .LOG_CONFIGNODE_LEADER_WARMING_UP_BEFORE_SERVING_REGISTERING_CONFIGNODE_STATUS_ARG_RETRY_ARG_3C924873,