Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
e6e0872
Batch scan TimeSeriesMetadata for wide-table queries
ColinLeeo Nov 10, 2026
e8d85ee
Fix NPE in batch TimeSeriesMetadata read by loading chunk metadata
ColinLeeo Nov 10, 2026
54b4326
Clean up batch TimeSeriesMetadata scan path
ColinLeeo Nov 10, 2026
33530b6
Expand batch TimeSeriesMetadata scan to device-level time index files
ColinLeeo Nov 10, 2026
a5643b0
Hardcode device metadata batch scan ratio to 0.6
ColinLeeo Nov 10, 2026
139a360
Synchronize batch metadata read and include device in corruption error
ColinLeeo Nov 11, 2026
27ad764
Fix batch scan ratio denominator and cache empty series
ColinLeeo Nov 11, 2026
84dd40b
Add unit test for batch scan absent-device throw logic
ColinLeeo Nov 11, 2026
098cd15
Add end-to-end test for wide-table batch metadata scan
ColinLeeo Nov 11, 2026
5793d3e
Don't use batch metadata scan for single-column devices
ColinLeeo Nov 11, 2026
19dbd62
Reuse the metadata cache for batch device scans
ColinLeeo Nov 12, 2026
a8c6939
Keep DEBUG queries on the per-column metadata path
ColinLeeo Nov 12, 2026
fcdb3d3
Check the bloom filter before the batch device read
ColinLeeo Nov 12, 2026
9f62d5d
Address wide-scan batch review: partial cache reuse and diagnostics
ColinLeeo Nov 13, 2026
2c81bfd
Refine the batch metadata path after review
ColinLeeo Nov 13, 2026
ce3a4cd
Document the selective-read limitation of the batch path
ColinLeeo Nov 13, 2026
7a2fbe1
Address the follow-up review of the wide-scan batch path
ColinLeeo Nov 16, 2026
42172e4
Account the pre-fallback phase as disk time on a cold batch miss
ColinLeeo Nov 16, 2026
92b5638
Refactor wide-device metadata loading into a set-based cache API
ColinLeeo Nov 17, 2026
5ec3abe
Skip a device once its time column is proven absent
ColinLeeo Nov 17, 2026
fb1df4e
Seed pending measurements in the cache-disabled metadata path
ColinLeeo Nov 17, 2026
20716ec
Restore per-series load counters and trim disabled-cache reads
ColinLeeo Nov 17, 2026
caadbe5
Inline one-use metadata load helpers
ColinLeeo Nov 17, 2026
6f35269
Use the selective tsfile metadata reader
ColinLeeo Nov 17, 2026
ae97170
Depend on the tsfile snapshot that carries the selective reader
ColinLeeo Nov 17, 2026
96a2298
Pin the dated tsfile snapshot instead of the moving one
ColinLeeo Nov 17, 2026
ecb4ce6
Cut cache-key and returned-set allocations on the batch path
ColinLeeo Nov 18, 2026
0cad7de
Check the bloom filter before the batch read and remember absent series
ColinLeeo Nov 19, 2026
37ca9c7
Trim the per-call cost and close the gaps found in review
ColinLeeo Nov 19, 2026
6bd6f96
Address the remaining cache review comments
ColinLeeo Nov 19, 2026
d156631
Inline the aligned metadata builder and name the recovery flag
ColinLeeo Nov 19, 2026
6a3ba99
Check the device before the cache pass and count rejected lookups
ColinLeeo Nov 19, 2026
7041acb
Split BloomFilter load time by cache phase
ColinLeeo Nov 19, 2026
f34c294
Fix tablet serialization for newer TsFile API
Caideyipi Nov 20, 2026
73b07a8
Use the snapshot cut from the merged tsfile develop
ColinLeeo Nov 20, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.record.Tablet;

import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
Expand Down Expand Up @@ -293,13 +295,14 @@ private static long calculateTabletSizeInBytes(final Tablet tablet) {
}

private static ByteBuffer serializeTabletWithAlignment(
final Tablet tablet, final boolean isAligned) {
final ByteBuffer buffer =
ByteBuffer.allocate(Math.addExact(tablet.serializedSize(), Byte.BYTES));
tablet.serialize(buffer);
ReadWriteIOUtils.write(isAligned, buffer);
buffer.flip();
return buffer;
final Tablet tablet, final boolean isAligned) throws IOException {
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(Math.addExact(tablet.serializedSize(), Byte.BYTES));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
tablet.serialize(outputStream);
ReadWriteIOUtils.write(isAligned, outputStream);
return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
}
}

static boolean mayAppendTablet(final Tablet target, final Tablet source) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -272,12 +272,15 @@ public byte[] toTPipeTransferBytes() throws IOException {
throw new IOException(DataNodePipeMessages.CANNOT_SERIALIZE_BOTH_TABLET_AND_STATEMENT_ARE);
}

final ByteBuffer buffer = ByteBuffer.allocate(calculateAirGapSerializedSize(tabletToSerialize));
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), buffer);
tabletToSerialize.serialize(buffer);
ReadWriteIOUtils.write(isAlignedToSerialize, buffer);
return buffer.array();
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateAirGapSerializedSize(tabletToSerialize));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), outputStream);
tabletToSerialize.serialize(outputStream);
ReadWriteIOUtils.write(isAlignedToSerialize, outputStream);
return byteArrayOutputStream.toByteArray();
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,14 +174,16 @@ public static PipeTransferTabletRawReqV2 fromTPipeTransferReq(

public static byte[] toTPipeTransferBytes(
final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException {
final ByteBuffer buffer =
ByteBuffer.allocate(calculateAirGapSerializedSize(tablet, dataBaseName));
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), buffer);
tablet.serialize(buffer);
ReadWriteIOUtils.write(isAligned, buffer);
ReadWriteIOUtils.write(dataBaseName, buffer);
return buffer.array();
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateAirGapSerializedSize(tablet, dataBaseName));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), outputStream);
tablet.serialize(outputStream);
ReadWriteIOUtils.write(isAligned, outputStream);
ReadWriteIOUtils.write(dataBaseName, outputStream);
return byteArrayOutputStream.toByteArray();
}
}

static int calculateSerializedSize(final Tablet tablet, final String dataBaseName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,10 @@ public class FragmentInstanceContext extends QueryContext {
// empty for zero time partitions
private List<Long> timePartitions;

// An optimization during restart changes the time index from FILE TIME INDEX
// to DEVICE TIME INDEX, which may cause a related validation false positive.
private boolean ignoreNotExistsDevice = false;
// Queries can start before the data regions finish recovering after a restart. During that
// window the time index may still change from FILE TIME INDEX to DEVICE TIME INDEX, so an
// absent device must not be treated as corruption or remembered as absent.
private boolean isStorageEngineRecovering = false;

private QueryDataSourceType queryDataSourceType = QueryDataSourceType.SERIES_SCAN;

Expand Down Expand Up @@ -456,7 +457,7 @@ protected FragmentInstanceContext(

public void start() {
long now = System.currentTimeMillis();
ignoreNotExistsDevice = !StorageEngine.getInstance().isReadyForNonReadWriteFunctions();
isStorageEngineRecovering = !StorageEngine.getInstance().isReadyForNonReadWriteFunctions();
executionStartTime.compareAndSet(null, now);
startNanos.compareAndSet(0, System.nanoTime());

Expand Down Expand Up @@ -1642,8 +1643,9 @@ public long getUnclosedSeqFileNum() {
return unclosedSeqFileNum.get();
}

public boolean ignoreNotExistsDevice() {
return ignoreNotExistsDevice;
/** Whether this query started while the storage engine was still recovering its data regions. */
public boolean isStorageEngineRecovering() {
return isStorageEngineRecovering;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAlignedChunkLoader;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskChunkLoader;
Expand All @@ -47,7 +46,6 @@
import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
Expand All @@ -62,6 +60,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -112,7 +111,7 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(
seriesPath.getDeviceId(),
seriesPath.getMeasurement()),
allSensors,
context.ignoreNotExistsDevice()
context.isStorageEngineRecovering()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
context.isDebug(),
context);
Expand Down Expand Up @@ -341,23 +340,30 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
// the order of timeSeriesMetadata list is same as subSensorList's order
TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
List<String> valueMeasurementList = alignedPath.getMeasurementList();
// resolves the time column and every selected sensor in one call: the cache reads each leaf
// group that holds one of them once and answers the whole group from that read, so a wide
// projection does not walk the device metadata index once per column
Set<String> allSensors = alignedPath.getAllSensors();
boolean isDebug = context.isDebug();
String filePath = resource.getTsFilePath();
IDeviceID deviceId = alignedPath.getDeviceId();

// when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
// we should not ignore the non-exist of device in TsFileMetadata
TimeseriesMetadata timeColumn =
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""),
boolean ignoreNotExists =
context.isStorageEngineRecovering()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE;

Map<String, TimeseriesMetadata> metadata =
cache.getTimeseriesMetadata(
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
resource.getTsFilePath(),
resource.getTsFileID(),
alignedPath.getDeviceId(),
allSensors,
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
isDebug,
ignoreNotExists,
context.isDebug(),
context,
rootMeasurementMetadataIndexNodeOffset);

TimeseriesMetadata timeColumn = metadata.get("");
if (timeColumn != null) {
// only need time column, like count_time aggregation
if (valueMeasurementList.isEmpty()) {
Expand All @@ -375,21 +381,13 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
} else {
List<TimeseriesMetadata> valueTimeSeriesMetadataList =
new ArrayList<>(valueMeasurementList.size());
// if all the queried aligned sensors does not exist, we will return null
boolean exist = false;
for (String measurement : valueMeasurementList) {
TimeseriesMetadata valueColumn =
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement),
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
isDebug,
context,
rootMeasurementMetadataIndexNodeOffset);
valueTimeSeriesMetadataList.add(metadata.get(measurement));
}
// if all the queried aligned sensors do not exist, skip the device in this file
boolean exist = false;
for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) {
exist = (exist || (valueColumn != null));
valueTimeSeriesMetadataList.add(valueColumn);
}
if (!ignoreAllNullRows || exist) {
alignedTimeSeriesMetadata =
Expand Down
Loading
Loading