Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,19 @@
private static final Logger LOGGER = LoggerFactory.getLogger(WriterSafeFrontierTracker.class);

private final Map<Integer, WriterFrontierState> states = new HashMap<>();
private volatile long effectiveSafePtVersion;

public synchronized void recordAppliedProgress(
final long physicalTime, final int writerNodeId, final long appliedLocalSeq) {
final WriterFrontierState state =
states.computeIfAbsent(writerNodeId, ignored -> new WriterFrontierState());
state.appliedLocalSeq = Math.max(state.appliedLocalSeq, appliedLocalSeq);
if (physicalTime > 0) {
state.effectiveSafePt = Math.max(state.effectiveSafePt, physicalTime);
long effectiveSafePt = Math.max(state.effectiveSafePt, physicalTime);
if (state.hasPendingSafeTime() && state.appliedLocalSeq >= state.pendingSafeLocalSeq) {
effectiveSafePt = Math.max(effectiveSafePt, state.pendingSafePhysicalTime);
state.clearPendingSafeTime();
}
promotePendingIfReady(state);
advanceEffectiveSafePt(state, effectiveSafePt);
}

public synchronized void observePendingWriterSafeTimeBarrier(
Expand All @@ -59,7 +62,7 @@
final WriterFrontierState state =
states.computeIfAbsent(writerNodeId, ignored -> new WriterFrontierState());
if (state.appliedLocalSeq >= barrierLocalSeq) {
state.effectiveSafePt = Math.max(state.effectiveSafePt, safePhysicalTime);
advanceEffectiveSafePt(state, safePhysicalTime);
state.clearPendingSafeTime();
return;
}
Expand Down Expand Up @@ -107,6 +110,44 @@
return Collections.unmodifiableMap(snapshot);
}

public synchronized Snapshot snapshotEffectiveSafePtsWithVersion() {
final Map<Integer, Long> snapshot = new HashMap<>();
for (final Map.Entry<Integer, WriterFrontierState> entry : states.entrySet()) {
snapshot.put(entry.getKey(), entry.getValue().effectiveSafePt);
}
return new Snapshot(effectiveSafePtVersion, Collections.unmodifiableMap(snapshot));
}

public long getEffectiveSafePtVersion() {
return effectiveSafePtVersion;
}

private void advanceEffectiveSafePt(
final WriterFrontierState state, final long safePhysicalTime) {
if (safePhysicalTime > state.effectiveSafePt) {
state.effectiveSafePt = safePhysicalTime;
effectiveSafePtVersion++;

Check warning on line 129 in iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/WriterSafeFrontierTracker.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Use an "AtomicLong" for this field; its operations are atomic.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHyy1kWqAoMpLQXAwzV&open=AaHyy1kWqAoMpLQXAwzV&pullRequest=18845
}
}

public static final class Snapshot {
private final long version;
private final Map<Integer, Long> effectiveSafePts;

private Snapshot(final long version, final Map<Integer, Long> effectiveSafePts) {
this.version = version;
this.effectiveSafePts = effectiveSafePts;
}

public long getVersion() {
return version;
}

public Map<Integer, Long> getEffectiveSafePts() {
return effectiveSafePts;
}
}

/**
* Returns each safe time together with the applied sequence it covers. A subscription may use the
* safe time only after consuming that sequence; applying records to the state machine does not
Expand Down Expand Up @@ -140,16 +181,6 @@
}
}

private void promotePendingIfReady(final WriterFrontierState state) {
if (!state.hasPendingSafeTime()) {
return;
}
if (state.appliedLocalSeq >= state.pendingSafeLocalSeq) {
state.effectiveSafePt = Math.max(state.effectiveSafePt, state.pendingSafePhysicalTime);
state.clearPendingSafeTime();
}
}

private static boolean dominates(
final long leftSafePhysicalTime,
final long leftSafeLocalSeq,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,25 @@ public void testSameWriterKeepsOnlyNewestPendingWriterSafeTimeBarrier() {
tracker.recordAppliedProgress(0L, 9, 35L);
assertEquals(220L, tracker.getEffectiveSafePt(9));
}

@Test
public void testEffectiveSafePtSnapshotIncludesAtomicVersion() {
final WriterSafeFrontierTracker tracker = new WriterSafeFrontierTracker();

tracker.observePendingWriterSafeTimeBarrier(200L, 9, 30L);
assertEquals(0L, tracker.getEffectiveSafePtVersion());

tracker.recordAppliedProgress(150L, 9, 30L);
assertEquals(200L, tracker.getEffectiveSafePt(9));
assertEquals(1L, tracker.getEffectiveSafePtVersion());

tracker.recordAppliedProgress(160L, 9, 31L);
assertEquals(1L, tracker.getEffectiveSafePtVersion());

tracker.recordAppliedProgress(250L, 9, 32L);
final WriterSafeFrontierTracker.Snapshot snapshot =
tracker.snapshotEffectiveSafePtsWithVersion();
assertEquals(2L, snapshot.getVersion());
assertEquals(Long.valueOf(250L), snapshot.getEffectiveSafePts().get(9));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,17 @@ public WALMetaData() {
}

public WALMetaData(long firstSearchIndex, List<Integer> buffersSize, Set<Long> memTablesId) {
this(firstSearchIndex, buffersSize, memTablesId, 0);
}

private WALMetaData(
long firstSearchIndex, List<Integer> buffersSize, Set<Long> memTablesId, int entriesNum) {
this.firstSearchIndex = firstSearchIndex;
this.buffersSize = buffersSize;
this.memTablesId = memTablesId;
this.physicalTimes = new ArrayList<>();
this.nodeIds = new ArrayList<>();
this.localSeqs = new ArrayList<>();
this.physicalTimes = new ArrayList<>(entriesNum);
this.nodeIds = new ArrayList<>(entriesNum);
this.localSeqs = new ArrayList<>(entriesNum);
}

/** Adds an entry without explicit writer progress metadata. */
Expand Down Expand Up @@ -249,7 +254,12 @@ public static WALMetaData deserialize(ByteBuffer buffer, WALFileVersion version)
memTablesId.add(buffer.getLong());
}
}
WALMetaData result = new WALMetaData(firstSearchIndex, buffersSize, memTablesId);
WALMetaData result =
new WALMetaData(
firstSearchIndex,
buffersSize,
memTablesId,
version == WALFileVersion.V3 ? entriesNum : 0);
// V3 extension: file-level timestamp range + per-entry writer progress metadata
if (version == WALFileVersion.V3 && buffer.hasRemaining()) {
result.minDataTs = buffer.getLong();
Expand Down
Loading
Loading