Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -681,13 +681,20 @@ public TSStatus visitPipeEnrichedNonWritePlanNode(
public TSStatus visitPipeOperateSchemaQueueNode(
final PipeOperateSchemaQueueNode node, final ISchemaRegion schemaRegion) {
final SchemaRegionId id = schemaRegion.getSchemaRegionId();
final SchemaRegionListeningQueue queue = PipeDataNodeAgent.runtime().schemaListener(id);
if (node.isOpen() && !queue.isOpened()) {
logger.info("Opened pipe listening queue on schema region {}", id);
queue.open();
} else if (!node.isOpen() && queue.isOpened()) {
logger.info("Closed pipe listening queue on schema region {}", id);
queue.close();
if (node.isOpen()) {
final SchemaRegionListeningQueue queue = PipeDataNodeAgent.runtime().schemaListener(id);
if (!queue.isOpened()) {
logger.info("Opened pipe listening queue on schema region {}", id);
queue.open();
}
} else {
final SchemaRegionListeningQueue queue =
PipeDataNodeAgent.runtime().schemaListenerIfPresent(id);
if (queue != null && queue.isOpened()) {
logger.info("Closed pipe listening queue on schema region {}", id);
queue.close();
PipeDataNodeAgent.runtime().cleanupSchemaListenerIfUnused(id);
}
}
return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,25 +121,36 @@ public boolean isReadOnly() {

@Override
public boolean takeSnapshot(File snapshotDir) {
if (schemaRegion.createSnapshot(snapshotDir)
&& PipeDataNodeAgent.runtime()
.schemaListener(schemaRegion.getSchemaRegionId())
.createSnapshot(snapshotDir)) {
listen2Snapshot4PipeListener(true);
if (!schemaRegion.createSnapshot(snapshotDir)) {
return false;
}

final SchemaRegionListeningQueue listener =
PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId());
if (listener == null) {
return true;
}
return false;

if (!listener.createSnapshot(snapshotDir)) {
return false;
}
listen2Snapshot4PipeListener(true);
return true;
}

@Override
public void loadSnapshot(File latestSnapshotRootDir) {
schemaRegion.loadSnapshot(latestSnapshotRootDir);
PipeDataNodeAgent.runtime()
.schemaListener(schemaRegion.getSchemaRegionId())
.loadSnapshot(latestSnapshotRootDir);
// We recompute the snapshot for pipe listener when loading snapshot
// to recover the newest snapshot in cache
listen2Snapshot4PipeListener(false);
final SchemaRegionListeningQueue listener =
PipeDataNodeAgent.runtime()
.schemaListenerForSnapshotLoad(schemaRegion.getSchemaRegionId(), latestSnapshotRootDir);
if (listener != null) {
listener.loadSnapshot(latestSnapshotRootDir);
PipeDataNodeAgent.runtime().cleanupSchemaListenerIfUnused(schemaRegion.getSchemaRegionId());
// We recompute the snapshot for pipe listener when loading snapshot
// to recover the newest snapshot in cache
listen2Snapshot4PipeListener(false);
}
}

public void listen2Snapshot4PipeListener(boolean isTmp) {
Expand All @@ -149,8 +160,11 @@ public void listen2Snapshot4PipeListener(boolean isTmp) {
.getUuid()
.toString(),
isTmp);
SchemaRegionListeningQueue listener =
PipeDataNodeAgent.runtime().schemaListener(schemaRegion.getSchemaRegionId());
final SchemaRegionListeningQueue listener =
PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId());
if (listener == null) {
return;
}
if (Objects.isNull(snapshotPaths) || Objects.isNull(snapshotPaths.getLeft())) {
if (listener.isOpened()) {
logger.warn(
Expand All @@ -173,9 +187,11 @@ public TSStatus write(IConsensusRequest request) {
try {
TSStatus result = ((PlanNode) request).accept(new SchemaExecutionVisitor(), schemaRegion);
if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
PipeDataNodeAgent.runtime()
.schemaListener(schemaRegion.getSchemaRegionId())
.tryListenToNode((PlanNode) request);
final SchemaRegionListeningQueue listener =
PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId());
if (listener != null) {
listener.tryListenToNode((PlanNode) request);
}
}
return result;
} catch (IllegalArgumentException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

Expand Down Expand Up @@ -151,6 +152,15 @@ public SchemaRegionListeningQueue schemaListener(SchemaRegionId schemaRegionId)
return regionListenerManager.listener(schemaRegionId);
}

public SchemaRegionListeningQueue schemaListenerIfPresent(SchemaRegionId schemaRegionId) {
return regionListenerManager.listenerIfPresent(schemaRegionId);
}

public SchemaRegionListeningQueue schemaListenerForSnapshotLoad(
SchemaRegionId schemaRegionId, File snapshotDir) {
return regionListenerManager.listenerForSnapshotLoad(schemaRegionId, snapshotDir);
}

public int increaseAndGetSchemaListenerReferenceCount(SchemaRegionId schemaRegionId) {
return regionListenerManager.increaseAndGetReferenceCount(schemaRegionId);
}
Expand All @@ -159,6 +169,14 @@ public int decreaseAndGetSchemaListenerReferenceCount(SchemaRegionId schemaRegio
return regionListenerManager.decreaseAndGetReferenceCount(schemaRegionId);
}

public void cleanupSchemaListenerIfUnused(SchemaRegionId schemaRegionId) {
regionListenerManager.cleanupListenerIfUnused(schemaRegionId);
}

public void clearSchemaRegionState(SchemaRegionId schemaRegionId) {
regionListenerManager.clearSchemaRegionState(schemaRegionId);
}

public void notifySchemaLeaderReady(SchemaRegionId schemaRegionId) {
regionListenerManager.notifyLeaderReady(schemaRegionId);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
package org.apache.iotdb.db.pipe.agent.runtime;

import org.apache.iotdb.commons.consensus.SchemaRegionId;
import org.apache.iotdb.commons.pipe.agent.task.PipeTask;
import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionListenerMetrics;
import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue;

import java.io.File;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
Expand All @@ -34,6 +34,7 @@ public class PipeSchemaRegionListenerManager {

private final Map<SchemaRegionId, PipeSchemaRegionListener> id2ListenerMap =
new ConcurrentHashMap<>();
private final Map<SchemaRegionId, AtomicBoolean> id2LeaderReadyMap = new ConcurrentHashMap<>();

public synchronized Set<SchemaRegionId> regionIds() {
return id2ListenerMap.keySet();
Expand All @@ -44,6 +45,27 @@ public synchronized SchemaRegionListeningQueue listener(final SchemaRegionId sch
.listeningQueue;
}

public synchronized SchemaRegionListeningQueue listenerIfPresent(
final SchemaRegionId schemaRegionId) {
final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId);
return listener == null ? null : listener.listeningQueue;
}

public synchronized SchemaRegionListeningQueue listenerForSnapshotLoad(
final SchemaRegionId schemaRegionId, final File snapshotDir) {
final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId);
if (!SchemaRegionListeningQueue.hasSnapshot(snapshotDir)) {
// The snapshot can replace a follower that missed the queue-close log entry. Absence of a
// queue snapshot represents a closed queue, so discard the follower's previous contents.
if (listener != null) {
listener.listeningQueue.close();
cleanupListenerIfUnused(schemaRegionId, listener);
}
return null;
}
return listener == null ? listener(schemaRegionId) : listener.listeningQueue;
}

public synchronized int increaseAndGetReferenceCount(final SchemaRegionId schemaRegionId) {
return id2ListenerMap
.computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
Expand All @@ -52,62 +74,65 @@ public synchronized int increaseAndGetReferenceCount(final SchemaRegionId schema
}

public synchronized int decreaseAndGetReferenceCount(final SchemaRegionId schemaRegionId) {
return id2ListenerMap
.computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
.listeningQueueReferenceCount
.updateAndGet(v -> v > 0 ? v - 1 : 0);
final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId);
if (listener == null) {
return 0;
}

final int referenceCount =
listener.listeningQueueReferenceCount.updateAndGet(v -> v > 0 ? v - 1 : 0);
if (referenceCount == 0 && !listener.listeningQueue.isOpened()) {
cleanupListenerIfUnused(schemaRegionId, listener);
}
return referenceCount;
}

public synchronized void notifyLeaderReady(final SchemaRegionId schemaRegionId) {
id2ListenerMap
.computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
.notifyLeaderReady();
id2LeaderReadyMap.computeIfAbsent(schemaRegionId, id -> new AtomicBoolean()).set(true);
}

public synchronized void notifyLeaderUnavailable(final SchemaRegionId schemaRegionId) {
id2ListenerMap
.computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
.notifyLeaderUnavailable();
id2LeaderReadyMap.computeIfAbsent(schemaRegionId, id -> new AtomicBoolean()).set(false);
}

public synchronized boolean isLeaderReady(final SchemaRegionId schemaRegionId) {
return id2ListenerMap
.computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
.isLeaderReady();
final AtomicBoolean isLeaderReady = id2LeaderReadyMap.get(schemaRegionId);
return isLeaderReady != null && isLeaderReady.get();
}

public synchronized void cleanupListenerIfUnused(final SchemaRegionId schemaRegionId) {
final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId);
if (listener != null) {
cleanupListenerIfUnused(schemaRegionId, listener);
}
}

public synchronized void clearSchemaRegionState(final SchemaRegionId schemaRegionId) {
final PipeSchemaRegionListener listener = id2ListenerMap.remove(schemaRegionId);
if (listener != null) {
PipeSchemaRegionListenerMetrics.getInstance().deregister(schemaRegionId.getId());
}
id2LeaderReadyMap.remove(schemaRegionId);
}

private void cleanupListenerIfUnused(
final SchemaRegionId schemaRegionId, final PipeSchemaRegionListener listener) {
if (listener.listeningQueueReferenceCount.get() > 0 || listener.listeningQueue.isOpened()) {
return;
}
if (id2ListenerMap.remove(schemaRegionId, listener)) {
PipeSchemaRegionListenerMetrics.getInstance().deregister(schemaRegionId.getId());
}
}

private static class PipeSchemaRegionListener {

private final SchemaRegionListeningQueue listeningQueue = new SchemaRegionListeningQueue();
private final AtomicInteger listeningQueueReferenceCount = new AtomicInteger(0);

private final AtomicBoolean isLeaderReady = new AtomicBoolean(false);

protected PipeSchemaRegionListener(final SchemaRegionId schemaRegionId) {
PipeSchemaRegionListenerMetrics.getInstance()
.register(listeningQueue, schemaRegionId.getId());
}

/**
* Get leader ready state, DO NOT use consensus layer's leader ready flag because
* SimpleConsensus' ready flag is always {@code true}. Note that this flag has nothing to do
* with listening and a {@link PipeTask} starts only iff the current node is a leader and ready.
*
* @return {@code true} iff the current node is a leader and ready
*/
private boolean isLeaderReady() {
return isLeaderReady.get();
}

// Leader ready flag has the following effect
// 1. The linked list starts serving only after leader gets ready
// 2. Config pipe task is only created after leader gets ready
private void notifyLeaderReady() {
isLeaderReady.set(true);
}

private void notifyLeaderUnavailable() {
isLeaderReady.set(false);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter;
import org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener;
import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningFilter;
import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
Expand Down Expand Up @@ -349,10 +350,14 @@ private Set<Integer> clearSchemaRegionListeningQueueIfNecessary(
}

schemaRegionId2ListeningQueueNewFirstIndex.forEach(
(schemaRegionId, listeningQueueNewFirstIndex) ->
PipeDataNodeAgent.runtime()
.schemaListener(new SchemaRegionId(schemaRegionId))
.removeBefore(listeningQueueNewFirstIndex));
(schemaRegionId, listeningQueueNewFirstIndex) -> {
final SchemaRegionListeningQueue listeningQueue =
PipeDataNodeAgent.runtime()
.schemaListenerIfPresent(new SchemaRegionId(schemaRegionId));
if (listeningQueue != null) {
listeningQueue.removeBefore(listeningQueueNewFirstIndex);
}
});

return schemaRegionId2ListeningQueueNewFirstIndex.keySet();
}
Expand Down
Loading
Loading