diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java index 66baaed98bd1c..92efffcb4558c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java @@ -32,10 +32,12 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; +import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -293,13 +295,14 @@ private static long calculateTabletSizeInBytes(final Tablet tablet) { } private static ByteBuffer serializeTabletWithAlignment( - final Tablet tablet, final boolean isAligned) { - final ByteBuffer buffer = - ByteBuffer.allocate(Math.addExact(tablet.serializedSize(), Byte.BYTES)); - tablet.serialize(buffer); - ReadWriteIOUtils.write(isAligned, buffer); - buffer.flip(); - return buffer; + final Tablet tablet, final boolean isAligned) throws IOException { + try (final PublicBAOS byteArrayOutputStream = + new PublicBAOS(Math.addExact(tablet.serializedSize(), Byte.BYTES)); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + tablet.serialize(outputStream); + ReadWriteIOUtils.write(isAligned, outputStream); + return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } } static boolean mayAppendTablet(final Tablet target, final Tablet source) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java index 7faf5449f1f77..00907ed8302f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java @@ -272,12 +272,15 @@ public byte[] toTPipeTransferBytes() throws IOException { throw new IOException(DataNodePipeMessages.CANNOT_SERIALIZE_BOTH_TABLET_AND_STATEMENT_ARE); } - final ByteBuffer buffer = ByteBuffer.allocate(calculateAirGapSerializedSize(tabletToSerialize)); - ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer); - ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), buffer); - tabletToSerialize.serialize(buffer); - ReadWriteIOUtils.write(isAlignedToSerialize, buffer); - return buffer.array(); + try (final PublicBAOS byteArrayOutputStream = + new PublicBAOS(calculateAirGapSerializedSize(tabletToSerialize)); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), outputStream); + tabletToSerialize.serialize(outputStream); + ReadWriteIOUtils.write(isAlignedToSerialize, outputStream); + return byteArrayOutputStream.toByteArray(); + } } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java index df83276fdf7a0..dd6941d7f7d8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java @@ -174,14 +174,16 @@ public static PipeTransferTabletRawReqV2 fromTPipeTransferReq( public static byte[] toTPipeTransferBytes( final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException { - final ByteBuffer buffer = - ByteBuffer.allocate(calculateAirGapSerializedSize(tablet, dataBaseName)); - ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer); - ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), buffer); - tablet.serialize(buffer); - ReadWriteIOUtils.write(isAligned, buffer); - ReadWriteIOUtils.write(dataBaseName, buffer); - return buffer.array(); + try (final PublicBAOS byteArrayOutputStream = + new PublicBAOS(calculateAirGapSerializedSize(tablet, dataBaseName)); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), outputStream); + tablet.serialize(outputStream); + ReadWriteIOUtils.write(isAligned, outputStream); + ReadWriteIOUtils.write(dataBaseName, outputStream); + return byteArrayOutputStream.toByteArray(); + } } static int calculateSerializedSize(final Tablet tablet, final String dataBaseName) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 6d33ea28a6066..5ff5dca4c1e6d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -161,9 +161,10 @@ public class FragmentInstanceContext extends QueryContext { // empty for zero time partitions private List timePartitions; - // An optimization during restart changes the time index from FILE TIME INDEX - // to DEVICE TIME INDEX, which may cause a related validation false positive. - private boolean ignoreNotExistsDevice = false; + // Queries can start before the data regions finish recovering after a restart. During that + // window the time index may still change from FILE TIME INDEX to DEVICE TIME INDEX, so an + // absent device must not be treated as corruption or remembered as absent. + private boolean isStorageEngineRecovering = false; private QueryDataSourceType queryDataSourceType = QueryDataSourceType.SERIES_SCAN; @@ -456,7 +457,7 @@ protected FragmentInstanceContext( public void start() { long now = System.currentTimeMillis(); - ignoreNotExistsDevice = !StorageEngine.getInstance().isReadyForNonReadWriteFunctions(); + isStorageEngineRecovering = !StorageEngine.getInstance().isReadyForNonReadWriteFunctions(); executionStartTime.compareAndSet(null, now); startNanos.compareAndSet(0, System.nanoTime()); @@ -1642,8 +1643,9 @@ public long getUnclosedSeqFileNum() { return unclosedSeqFileNum.get(); } - public boolean ignoreNotExistsDevice() { - return ignoreNotExistsDevice; + /** Whether this query started while the storage engine was still recovering its data regions. */ + public boolean isStorageEngineRecovering() { + return isStorageEngineRecovering; } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 4374e974db5bc..f43f0578450f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -30,7 +30,6 @@ import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; -import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAlignedChunkLoader; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskChunkLoader; @@ -47,7 +46,6 @@ import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; -import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -62,6 +60,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -112,7 +111,7 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( seriesPath.getDeviceId(), seriesPath.getMeasurement()), allSensors, - context.ignoreNotExistsDevice() + context.isStorageEngineRecovering() || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, context.isDebug(), context); @@ -341,23 +340,30 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // the order of timeSeriesMetadata list is same as subSensorList's order TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); List valueMeasurementList = alignedPath.getMeasurementList(); + // resolves the time column and every selected sensor in one call: the cache reads each leaf + // group that holds one of them once and answers the whole group from that read, so a wide + // projection does not walk the device metadata index once per column Set allSensors = alignedPath.getAllSensors(); - boolean isDebug = context.isDebug(); - String filePath = resource.getTsFilePath(); - IDeviceID deviceId = alignedPath.getDeviceId(); // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata - TimeseriesMetadata timeColumn = - cache.get( - filePath, - new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""), + boolean ignoreNotExists = + context.isStorageEngineRecovering() + || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + + Map metadata = + cache.getTimeseriesMetadata( + resource.getTsFilePath(), + resource.getTsFileID(), + alignedPath.getDeviceId(), + allSensors, allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, - isDebug, + ignoreNotExists, + context.isDebug(), context, rootMeasurementMetadataIndexNodeOffset); + + TimeseriesMetadata timeColumn = metadata.get(""); if (timeColumn != null) { // only need time column, like count_time aggregation if (valueMeasurementList.isEmpty()) { @@ -375,21 +381,13 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr } else { List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); - // if all the queried aligned sensors does not exist, we will return null - boolean exist = false; for (String measurement : valueMeasurementList) { - TimeseriesMetadata valueColumn = - cache.get( - filePath, - new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), - allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, - isDebug, - context, - rootMeasurementMetadataIndexNodeOffset); + valueTimeSeriesMetadataList.add(metadata.get(measurement)); + } + // if all the queried aligned sensors do not exist, skip the device in this file + boolean exist = false; + for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) { exist = (exist || (valueColumn != null)); - valueTimeSeriesMetadataList.add(valueColumn); } if (!ignoreAllNullRows || exist) { alignedTimeSeriesMetadata = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 68457df0ebbcd..5f0524db0e177 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.execution.fragment.QueryStatistics; import org.apache.iotdb.db.queryengine.metric.TimeSeriesMetadataCacheMetrics; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; @@ -35,7 +36,6 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Weigher; -import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -48,13 +48,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.lang.ref.WeakReference; +import java.util.Collection; import java.util.Collections; -import java.util.List; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.WeakHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongConsumer; @@ -86,10 +87,17 @@ public class TimeSeriesMetadataCache { private final AtomicLong entryAverageSize = new AtomicLong(0); - private final Map> devices = - Collections.synchronizedMap(new WeakHashMap<>()); private static final String SEPARATOR = "$"; + private static final int DEVICE_LOCK_STRIPES = 1024; + private static final Object[] DEVICE_LOCKS = new Object[DEVICE_LOCK_STRIPES]; + + static { + for (int i = 0; i < DEVICE_LOCK_STRIPES; i++) { + DEVICE_LOCKS[i] = new Object(); + } + } + private final AtomicLong evictedExistingEntryCount = new AtomicLong(0); private final AtomicLong evictedNonExistingEntryCount = new AtomicLong(0); @@ -137,7 +145,6 @@ public static TimeSeriesMetadataCache getInstance() { return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE; } - @SuppressWarnings({"squid:S1860", "squid:S6541", "squid:S3776"}) // Suppress synchronize warning public TimeseriesMetadata get( String filePath, TimeSeriesMetadataCacheKey key, @@ -149,7 +156,6 @@ public TimeseriesMetadata get( return get(filePath, key, allSensors, ignoreNotExists, debug, queryContext, null); } - @SuppressWarnings({"squid:S1860", "squid:S6541", "squid:S3776"}) // Suppress synchronize warning public TimeseriesMetadata get( String filePath, TimeSeriesMetadataCacheKey key, @@ -159,163 +165,515 @@ public TimeseriesMetadata get( QueryContext queryContext, long[] deviceMetadataIndexNodeOffset) throws IOException { + // a cache hit is the common case for a single series: answer it without building the batch + // request (a singleton set, a result map, a pending set) or the device prefix + if (CACHE_ENABLE && !queryContext.isExternalTsFileScan()) { + long cacheStartTime = System.nanoTime(); + TimeseriesMetadata cached = lruCache.getIfPresent(key); + if (cached != null) { + QueryStatistics statistics = queryContext.getQueryStatistics(); + statistics.getLoadTimeSeriesMetadataFromCacheCount().getAndAdd(1); + statistics + .getLoadTimeSeriesMetadataFromCacheTime() + .getAndAdd(System.nanoTime() - cacheStartTime); + if (cached == NULL_EXISTS_CACHE_PLACE_HOLDER) { + if (debug) { + DEBUG_LOGGER.info(StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, key); + } + return null; + } + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.STORAGE_LOG_GET_TIMESERIES_METADATA_IN_FILE_FROM_CACHE_36652729, + key.device, + key.measurement, + filePath, + cached); + } + return new TimeseriesMetadata(cached); + } + } + return getTimeseriesMetadata( + filePath, + key.tsFileID, + key.device, + Collections.singleton(key.measurement), + allSensors, + ignoreNotExists, + debug, + queryContext, + deviceMetadataIndexNodeOffset) + .get(key.measurement); + } + + /** + * Resolve the metadata of {@code requested} measurements of a device in one file. + * + *

Cached measurements are returned directly. Each remaining measurement is first checked + * against the file's bloom filter: a measurement the filter rules out is answered with null and + * not remembered, because probing the filter again is cheaper than a cache entry. The + * measurements the filter cannot rule out are loaded together in one reader call, which visits + * every leaf group that holds one of them once. A measurement that call does not return, or + * returns with an empty statistics count, is absent from the file and is remembered with a + * negative cache entry, so a later query does not walk its leaf group again. + * + *

Offset-addressed external TsFiles may not have a bloom filter that covers their series, so + * bloom filtering is skipped for them and the reader settles every requested measurement. + * + *

{@code allSensors} is the full selected set; it filters each group read and warms the cache, + * so a later single-measurement {@link #get} can reuse the group. {@code requested} must be a + * subset of {@code allSensors}. + * + *

The returned map holds copies of the cached (or freshly read) metadata, so the caller may + * apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached entries. + * Load statistics are counted per requested measurement. + * + * @param deviceMetadataIndexNodeOffset non-null for external tsfiles, which address the device + * metadata index by offset; bloom filtering is skipped in that case + * @return a map with an entry for every measurement in {@code requested}: a metadata copy when + * the series exists and is non-empty, null otherwise + */ + public Map getTimeseriesMetadata( + String filePath, + TsFileID tsFileID, + IDeviceID device, + Set requested, + Set allSensors, + boolean ignoreNotExists, + boolean debug, + QueryContext queryContext, + long[] deviceMetadataIndexNodeOffset) + throws IOException { long startTime = System.nanoTime(); + long cachePhaseEndTime = 0; long loadBloomFilterTime = 0; + long bloomFilterTimeBeforeCachePhase = 0; + long bloomFilterTimeAfterCachePhase = 0; + long cacheHitCount = 0; + long diskLookupCount = 0; + boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; LongConsumer bloomFilterIoSizeRecorder = queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; - boolean cacheHit = true; - boolean externalTsFile = queryContext.isExternalTsFileScan(); + + Map result = new HashMap<>(requested.size()); + Set pending = new HashSet<>(requested.size()); + try { - if (!CACHE_ENABLE || externalTsFile) { - String deviceStringFormat = key.device.toString(); - cacheHit = false; - - // bloom filter part - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, key.tsFileID, true, bloomFilterIoSizeRecorder, externalTsFile); - if (deviceMetadataIndexNodeOffset == null) { - BloomFilter bloomFilter = reader.readBloomFilter(bloomFilterIoSizeRecorder); - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount().incrementAndGet(); - if (bloomFilter != null - && !bloomFilter.contains( - deviceStringFormat + IoTDBConstant.PATH_SEPARATOR + key.measurement)) { - loadBloomFilterTime = System.nanoTime() - startTime; - return null; + if (CACHE_ENABLE && !externalTsFile) { + String deviceName = device.toString(); + String devicePrefix = deviceName + IoTDBConstant.PATH_SEPARATOR; + BloomFilter bloomFilter = null; + // an aligned device always has its time column, so check it before the per-column cache + // pass: a file without data for this device then costs one lookup and at most one + // BloomFilter probe instead of one lookup per selected measurement + if (requested.contains("")) { + TimeseriesMetadata cachedTime = + lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, "")); + if (cachedTime == NULL_EXISTS_CACHE_PLACE_HOLDER) { + cacheHitCount = 1; + cachePhaseEndTime = System.nanoTime(); + fillMissingWithNull(requested, result); + return result; + } + if (cachedTime != null) { + result.put("", new TimeseriesMetadata(cachedTime)); + } else if (deviceMetadataIndexNodeOffset == null) { + long bloomFilterStartTime = System.nanoTime(); + bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, debug, bloomFilterIoSizeRecorder, false); + bloomFilterTimeBeforeCachePhase = System.nanoTime() - bloomFilterStartTime; + loadBloomFilterTime += bloomFilterTimeBeforeCachePhase; + if (bloomFilter != null && !bloomFilter.contains(devicePrefix)) { + fillMissingWithNull(requested, result); + return result; + } + } + } + collectCachedMetadata(tsFileID, device, requested, result, pending, filePath, debug); + cacheHitCount = result.size(); + if (pending.isEmpty()) { + cachePhaseEndTime = System.nanoTime(); + return result; + } + cachePhaseEndTime = System.nanoTime(); + if (bloomFilter == null && deviceMetadataIndexNodeOffset == null) { + long bloomFilterStartTime = System.nanoTime(); + bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, debug, bloomFilterIoSizeRecorder, false); + bloomFilterTimeAfterCachePhase = System.nanoTime() - bloomFilterStartTime; + loadBloomFilterTime += bloomFilterTimeAfterCachePhase; + } + // the filter and the cache are thread-safe, so a rejected measurement never enters the + // device lock; only the survivors need the locked recheck and the reader + bloomAbsentMeasurements( + tsFileID, device, pending, requested, result, bloomFilter, devicePrefix, debug); + if (pending.isEmpty()) { + fillMissingWithNull(requested, result); + return result; + } + // serialize reads of the same device+file (same lock key as get) so threads don't each + // walk the device metadata index and race on the shared reader + synchronized (deviceLock(deviceName, filePath)) { + // another thread may have loaded some of the survivors while this one waited + Iterator pendingIterator = pending.iterator(); + while (pendingIterator.hasNext()) { + String measurement = pendingIterator.next(); + TimeseriesMetadata cached = + lruCache.getIfPresent( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); + if (cached == null) { + continue; + } + pendingIterator.remove(); + cacheHitCount++; + result.put( + measurement, + cached == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(cached)); + } + if (pending.isEmpty()) { + cachePhaseEndTime = System.nanoTime(); + return result; } - loadBloomFilterTime = System.nanoTime() - startTime; + if (debug) { + for (String measurement : pending) { + DEBUG_LOGGER.info( + StorageEngineMessages.CACHE_MISS_IN_FILE, device, measurement, filePath); + } + DEBUG_LOGGER.info(StorageEngineMessages.DEVICE_ALL_SENSORS, device, allSensors); + } + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); + readAndMerge( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + pending, + requested, + allSensors, + ignoreNotExists, + true, + debug, + result, + timeSeriesMetadataIoSizeRecorder); + return result; } + } - TimeseriesMetadata timeseriesMetadata = - reader.readTimeseriesMetadata( - key.device, - deviceMetadataIndexNodeOffset, - key.measurement, - ignoreNotExists, - timeSeriesMetadataIoSizeRecorder); - return (timeseriesMetadata == null || timeseriesMetadata.getStatistics().getCount() == 0) - ? null - : timeseriesMetadata; + // cache disabled or external tsfiles: nothing was resolved from the cache + pending.addAll(requested); + String devicePrefix = device.toString() + IoTDBConstant.PATH_SEPARATOR; + // offset-addressed external files may not have a bloom filter that covers their series + BloomFilter bloomFilter = null; + if (deviceMetadataIndexNodeOffset == null) { + long bloomFilterStartTime = System.nanoTime(); + bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, debug, bloomFilterIoSizeRecorder, externalTsFile); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; + } + if (deviceAbsent(requested, result, bloomFilter, devicePrefix)) { + fillMissingWithNull(requested, result); + return result; } + bloomAbsentMeasurements( + tsFileID, device, pending, requested, result, bloomFilter, devicePrefix, debug); + if (pending.isEmpty()) { + fillMissingWithNull(requested, result); + return result; + } + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + readAndMerge( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + pending, + requested, + allSensors, + ignoreNotExists, + false, + debug, + result, + timeSeriesMetadataIoSizeRecorder); + return result; + } finally { + long endTime = System.nanoTime(); + // every requested measurement that was not served by the cache is a lookup, including the + // ones an early return rejected with the BloomFilter + diskLookupCount = requested.size() - cacheHitCount; + long cacheLookupTime; + long diskLookupTime; + if (cacheHitCount == 0 || cachePhaseEndTime == 0) { + cacheLookupTime = 0; + diskLookupTime = endTime - startTime - loadBloomFilterTime; + } else if (diskLookupCount == 0) { + cacheLookupTime = endTime - startTime - loadBloomFilterTime; + diskLookupTime = 0; + } else { + // the pre-cache BloomFilter load belongs to the cache interval; only a load after the + // cache phase belongs to the disk interval + cacheLookupTime = cachePhaseEndTime - startTime - bloomFilterTimeBeforeCachePhase; + diskLookupTime = endTime - cachePhaseEndTime - bloomFilterTimeAfterCachePhase; + } + recordLoadStatistics( + queryContext, + cacheHitCount, + cacheLookupTime, + diskLookupCount, + diskLookupTime, + loadBloomFilterTime); + } + } + + private BloomFilter loadBloomFilter( + String filePath, + TsFileID tsFileID, + QueryContext queryContext, + boolean debug, + LongConsumer bloomFilterIoSizeRecorder, + boolean externalTsFile) + throws IOException { + return BloomFilterCache.getInstance() + .get( + new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), + debug, + bloomFilterIoSizeRecorder, + queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, + queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, + externalTsFile); + } - TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key); + /** + * An aligned device always has its time column. When the cache or the bloom filter proves it + * absent, no selected series of the device can be read from this file. + */ + private static boolean deviceAbsent( + Set requested, + Map result, + BloomFilter bloomFilter, + String devicePrefix) { + if (!requested.contains("")) { + return false; + } + if (result.containsKey("")) { + return result.get("") == null; + } + return bloomFilter != null && !bloomFilter.contains(devicePrefix); + } - if (timeseriesMetadata == null) { + /** + * Removes the pending measurements the bloom filter proves absent and answers them with null. + * They are not remembered: a later query probes the filter again instead of keeping a negative + * cache entry, which would add one entry per absent column per file. + */ + private static void bloomAbsentMeasurements( + TsFileID tsFileID, + IDeviceID device, + Set pending, + Set requested, + Map result, + BloomFilter bloomFilter, + String devicePrefix, + boolean debug) { + if (bloomFilter == null || pending.isEmpty()) { + return; + } + Iterator iterator = pending.iterator(); + while (iterator.hasNext()) { + String measurement = iterator.next(); + // the device gate already probed the time column + if (measurement.isEmpty() || bloomFilter.contains(devicePrefix + measurement)) { + continue; + } + iterator.remove(); + if (requested.contains(measurement)) { if (debug) { DEBUG_LOGGER.info( - StorageEngineMessages.CACHE_MISS_IN_FILE, key.device, key.measurement, filePath); - DEBUG_LOGGER.info(StorageEngineMessages.DEVICE_ALL_SENSORS, key.device, allSensors); + StorageEngineMessages.TS_METADATA_FILTERED_BY_BLOOM_FILTER, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); } - String deviceStringFormat = key.device.toString(); - // allow for the parallelism of different devices - synchronized ( - devices.computeIfAbsent( - deviceStringFormat + SEPARATOR + filePath, WeakReference::new)) { - // double check - timeseriesMetadata = lruCache.getIfPresent(key); - if (timeseriesMetadata == null) { - cacheHit = false; - - if (deviceMetadataIndexNodeOffset == null) { - long loadBloomFilterStartTime = System.nanoTime(); - // bloom filter part - BloomFilter bloomFilter = - BloomFilterCache.getInstance() - .get( - new BloomFilterCache.BloomFilterCacheKey(filePath, key.tsFileID), - debug, - bloomFilterIoSizeRecorder, - queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount() - ::addAndGet, - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount() - ::addAndGet, - false); - if (bloomFilter != null - && !bloomFilter.contains( - deviceStringFormat + TsFileConstant.PATH_SEPARATOR + key.measurement)) { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.TS_METADATA_FILTERED_BY_BLOOM_FILTER, key); - } - loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime; - return null; - } - - loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime; - } - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, key.tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); - List timeSeriesMetadataList = - reader.readTimeseriesMetadata( - key.device, - deviceMetadataIndexNodeOffset, - key.measurement, - allSensors, - ignoreNotExists, - timeSeriesMetadataIoSizeRecorder); - // put TimeSeriesMetadata of all sensors used in this read into cache - for (TimeseriesMetadata metadata : timeSeriesMetadataList) { - TimeSeriesMetadataCacheKey k = - new TimeSeriesMetadataCacheKey( - key.tsFileID, key.device, metadata.getMeasurementId()); - if (metadata.getStatistics().getCount() != 0) { - lruCache.put(k, metadata); - } - if (metadata.getMeasurementId().equals(key.measurement)) { - timeseriesMetadata = metadata.getStatistics().getCount() == 0 ? null : metadata; - } - } - if (timeseriesMetadata == null - && !ignoreNotExists - && memoryConfig.isMayCacheNonExistSeries()) { - lruCache.put(key, NULL_EXISTS_CACHE_PLACE_HOLDER); - } - } + result.put(measurement, null); + } + } + } + + /** + * Reads the pending measurements from the reader and merges the result into {@code result}. + * + *

The three sets have different roles. {@code pending} holds the measurements that are neither + * in the LRU nor ruled out by the bloom filter; it selects the leaf groups the reader visits. + * {@code requested} holds the measurements whose metadata the caller wants in the result. {@code + * allSensors} is the full selected set of the scan; when it is the same object as {@code + * requested} (the aligned batch call) the reader deserializes only the misses, otherwise (the + * single-column path) it deserializes the whole selected set so the group of the requested column + * is warmed for later lookups. + * + *

The reader returns the measurements of the visited groups that are in {@code returned}; + * {@code pending} is always part of that set. A pending measurement the reader does not return + * was routed to a group that was read, so it is absent from the file and is remembered with a + * negative cache entry. + */ + private void readAndMerge( + TsFileSequenceReader reader, + TsFileID tsFileID, + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + Set pending, + Set requested, + Set allSensors, + boolean ignoreNotExists, + boolean cacheEnabled, + boolean debug, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + // cached measurements are already in the result: no need to deserialize them again. `pending` + // must be part of what the reader returns, otherwise a miss outside `allSensors` would be + // mistaken for an absent series and pollute the negative cache. + Set returned; + if (!cacheEnabled || allSensors == requested) { + // the aligned batch caller passes the same set for both, so equal contents are already known + returned = pending; + } else if (result.isEmpty() && allSensors.containsAll(pending)) { + // nothing cached to exclude and the selected set covers the misses: use it without copying + returned = allSensors; + } else { + returned = new HashSet<>(allSensors); + returned.addAll(pending); + returned.removeAll(result.keySet()); + } + Map loaded = + reader.readTimeseriesMetadataForMeasurements( + device, + deviceMetadataIndexNodeOffset, + pending, + returned, + ignoreNotExists, + ioSizeRecorder); + for (Map.Entry entry : loaded.entrySet()) { + String measurement = entry.getKey(); + pending.remove(measurement); + TimeseriesMetadata metadata = entry.getValue(); + if (metadata.getStatistics().getCount() == 0) { + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); + } + if (requested.contains(measurement)) { + result.put(measurement, null); + } + cacheNonExistenceIfNeeded(tsFileID, device, measurement, cacheEnabled, ignoreNotExists); + } else { + if (cacheEnabled) { + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), metadata); } + if (requested.contains(measurement)) { + result.put(measurement, new TimeseriesMetadata(metadata)); + } + } + } + // what is left in pending was routed to a leaf group that was read and not found there + for (String measurement : pending) { + if (requested.contains(measurement)) { + result.put(measurement, null); } - if (timeseriesMetadata == null || timeseriesMetadata == NULL_EXISTS_CACHE_PLACE_HOLDER) { + cacheNonExistenceIfNeeded(tsFileID, device, measurement, cacheEnabled, ignoreNotExists); + } + } + + private static void fillMissingWithNull( + Set requested, Map result) { + for (String measurement : requested) { + result.putIfAbsent(measurement, null); + } + } + + private void cacheNonExistenceIfNeeded( + TsFileID tsFileID, + IDeviceID device, + String measurement, + boolean cacheEnabled, + boolean ignoreNotExists) { + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } + } + + private static void recordLoadStatistics( + QueryContext queryContext, + long cacheHitCount, + long cacheLookupTime, + long diskLookupCount, + long diskLookupTime, + long bloomFilterTime) { + QueryStatistics statistics = queryContext.getQueryStatistics(); + statistics.getLoadBloomFilterTime().getAndAdd(bloomFilterTime); + if (cacheHitCount > 0) { + statistics.getLoadTimeSeriesMetadataFromCacheCount().getAndAdd(cacheHitCount); + statistics.getLoadTimeSeriesMetadataFromCacheTime().getAndAdd(cacheLookupTime); + } + if (diskLookupCount > 0 || diskLookupTime > 0) { + statistics.getLoadTimeSeriesMetadataFromDiskCount().getAndAdd(diskLookupCount); + statistics.getLoadTimeSeriesMetadataFromDiskTime().getAndAdd(diskLookupTime); + } + } + + /** + * Resolve cached measurements into {@code resolved} and collect the rest into {@code missing}. + */ + private void collectCachedMetadata( + TsFileID tsFileID, + IDeviceID device, + Collection measurements, + Map resolved, + Collection missing, + String filePath, + boolean debug) { + for (String measurement : measurements) { + TimeSeriesMetadataCacheKey key = + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement); + TimeseriesMetadata metadata = lruCache.getIfPresent(key); + if (metadata == null) { + missing.add(measurement); + } else if (metadata == NULL_EXISTS_CACHE_PLACE_HOLDER) { if (debug) { DEBUG_LOGGER.info(StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, key); } - return null; + resolved.put(measurement, null); } else { if (debug) { DEBUG_LOGGER.info( StorageEngineMessages.STORAGE_LOG_GET_TIMESERIES_METADATA_IN_FILE_FROM_CACHE_36652729, - key.device, - key.measurement, + device, + measurement, filePath, - timeseriesMetadata); + metadata); } - return new TimeseriesMetadata(timeseriesMetadata); - } - } finally { - queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); - if (cacheHit) { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } else { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); + resolved.put(measurement, new TimeseriesMetadata(metadata)); } } } + /** + * Striped, strongly referenced monitor for one device+file. A WeakHashMap entry can be reclaimed + * by GC between calls, which would let two threads hold "the same" lock simultaneously. + */ + private static Object deviceLock(String deviceString, String filePath) { + return DEVICE_LOCKS[ + Math.floorMod((deviceString + SEPARATOR + filePath).hashCode(), DEVICE_LOCK_STRIPES)]; + } + public double calculateTimeSeriesMetadataHitRatio() { return lruCache.stats().hitRate(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 87526eb594b9f..27724b1305d47 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -22,17 +22,27 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSourceType; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.LazyTsFileDeviceIterator; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.BloomFilter; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Ignore; import org.junit.Test; @@ -41,8 +51,12 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -51,9 +65,11 @@ import java.util.function.LongConsumer; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class TimeSeriesMetadataCacheTest { @@ -241,20 +257,20 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { fileReaderManager.setReaderForTest(tsFileID, true, reader); Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(null); Mockito.when( - reader.readTimeseriesMetadata( + reader.readTimeseriesMetadataForMeasurements( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(true), Mockito.any(LongConsumer.class))) - .thenReturn(Collections.emptyList()); + .thenReturn(Collections.emptyMap()); Mockito.when( - reader.readTimeseriesMetadata( + reader.readTimeseriesMetadataForMeasurements( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class))) .thenThrow(new IOException("missing device")); @@ -264,11 +280,11 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { IOException.class, () -> cache.get(filePath, key, allSensors, false, false, new QueryContext(false, false))); Mockito.verify(reader) - .readTimeseriesMetadata( + .readTimeseriesMetadataForMeasurements( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class)); } finally { @@ -280,4 +296,815 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { .setMayCacheNonExistSeries(mayCacheNonExistSeries); } } + + private static Map getTimeseriesMetadata( + TimeSeriesMetadataCache cache, + String filePath, + TsFileID tsFileID, + IDeviceID deviceID, + Set allSensors, + boolean ignoreNotExists) + throws IOException { + return cache.getTimeseriesMetadata( + filePath, + tsFileID, + deviceID, + allSensors, + allSensors, + ignoreNotExists, + false, + new QueryContext(false, false), + null); + } + + private static void stubGroupRead( + TsFileSequenceReader reader, + IDeviceID deviceID, + boolean ignoreNotExists, + List group) + throws IOException { + Map loaded = new HashMap<>(); + for (TimeseriesMetadata metadata : group) { + loaded.put(metadata.getMeasurementId(), metadata); + } + Mockito.when( + reader.readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(ignoreNotExists), + Mockito.any(LongConsumer.class))) + .thenReturn(loaded); + } + + @Test + public void testGetTimeseriesMetadata() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-device.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + TimeseriesMetadata timeColumn = + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS()); + + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + TimeseriesMetadata s1 = + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS()); + + // an empty series (count == 0) is mapped to null and cached as the null placeholder + TimeseriesMetadata emptyS2 = + new TimeseriesMetadata( + (byte) 0, + 0, + "s2", + TSDataType.INT32, + Statistics.getStatsByType(TSDataType.INT32), + new PublicBAOS()); + + List group = new ArrayList<>(); + group.add(timeColumn); + group.add(s1); + group.add(emptyS2); + // the group read must carry chunk metadata, otherwise the downstream chunk loader NPEs + s1.setChunkMetadataList(Collections.singletonList(new ChunkMetadata())); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, false, group); + + Map result = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertNotNull(result.get("")); + assertEquals("", result.get("").getMeasurementId()); + assertNotNull(result.get("s1")); + assertEquals("s1", result.get("s1").getMeasurementId()); + // chunk metadata must flow through to the result (and the cached copy) unchanged + assertNotNull(result.get("s1").getChunkMetadataList()); + assertNull(result.get("s2")); + + // a second lookup of cached measurements must be served without touching the reader again + Set cachedSensors = new HashSet<>(Arrays.asList("", "s1")); + Map cachedResult = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, cachedSensors, false); + assertNotNull(cachedResult.get("s1")); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadataForMeasurements( + Mockito.any(), + Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + + // the group read also populated the per-measurement cache + TimeseriesMetadata cached = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cached); + assertEquals("s1", cached.getMeasurementId()); + + // the cache hands out copies: mutating a returned entry must not poison later reads + result.get("s1").setModified(true); + TimeseriesMetadata cachedAgain = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cachedAgain); + assertFalse(cachedAgain.isModified()); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-absent-device-bloom.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = Collections.singleton(""); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // the bloom filter proves the device is missing, so the device must not be read at all + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) + .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); + + Map result = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(result.get("")); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadataForMeasurements( + Mockito.any(), + Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-absent-device-throw.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, true, Collections.emptyList()); + Mockito.when( + reader.readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenThrow(new IOException("device absent")); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + bloomFilter.add(deviceID.toString() + ".s2"); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + // ignoreNotExists keeps the missing device silent, but caches nothing + Map ignored = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + assertNull(ignored.get("s2")); + + // a normal query must surface the reader's failure instead of a silently empty device + assertThrows( + IOException.class, + () -> getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchMetadataMatchesPerColumn() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + int columnCount = 700; + File file = new File("target/batch-compare.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.compare.d1"); + List schemas = new ArrayList<>(columnCount); + for (int i = 0; i < columnCount; i++) { + schemas.add(new MeasurementSchema("s" + i, TSDataType.INT32)); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries(deviceID, schemas); + for (long time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + for (int i = 0; i < columnCount; i++) { + record.addPoint("s" + i, (int) time); + } + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = file.getAbsolutePath(); + Set allSensors = new HashSet<>(); + for (int i = 0; i < columnCount; i++) { + allSensors.add("s" + i); + } + allSensors.add(""); + allSensors.add("absent"); + + Map perColumn = new HashMap<>(); + for (String measurement : allSensors) { + perColumn.put( + measurement, + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement), + allSensors, + false, + false, + new QueryContext(false, false))); + } + cache.clear(); + + Map batch = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertEquals(allSensors.size(), batch.size()); + for (String measurement : allSensors) { + TimeseriesMetadata expected = perColumn.get(measurement); + TimeseriesMetadata actual = batch.get(measurement); + if (expected == null) { + assertNull(measurement, actual); + continue; + } + assertNotNull(measurement, actual); + assertEquals(measurement, actual.getMeasurementId()); + assertEquals( + measurement, expected.getStatistics().getCount(), actual.getStatistics().getCount()); + assertEquals( + measurement, + expected.getStatistics().getStartTime(), + actual.getStatistics().getStartTime()); + assertEquals( + measurement, + expected.getStatistics().getEndTime(), + actual.getStatistics().getEndTime()); + assertEquals( + measurement, + expected.getChunkMetadataList().size(), + actual.getChunkMetadataList().size()); + for (int i = 0; i < expected.getChunkMetadataList().size(); i++) { + ChunkMetadata expectedChunk = (ChunkMetadata) expected.getChunkMetadataList().get(i); + ChunkMetadata actualChunk = (ChunkMetadata) actual.getChunkMetadataList().get(i); + assertEquals( + measurement, + expectedChunk.getOffsetOfChunkHeader(), + actualChunk.getOffsetOfChunkHeader()); + assertEquals( + measurement, + expectedChunk.getStatistics().getStartTime(), + actualChunk.getStatistics().getStartTime()); + assertEquals( + measurement, + expectedChunk.getStatistics().getEndTime(), + actualChunk.getStatistics().getEndTime()); + } + } + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + assertTrue(file.delete()); + } + } + + @Test + public void testBloomFilterExcludedMeasurementsSkipRead() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-time-column-filtered.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // both the time column and s1 are excluded, so no leaf group needs to be read + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Map result = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertNull(result.get("")); + assertNull(result.get("s1")); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadataForMeasurements( + Mockito.any(), + Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testEvictedMeasurementReloadsOnlyItsGroup() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-single-miss.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics statistics = Statistics.getStatsByType(TSDataType.INT32); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + TimeseriesMetadata s2 = + new TimeseriesMetadata((byte) 0, 0, "s2", TSDataType.INT32, statistics, new PublicBAOS()); + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + + List group = new ArrayList<>(); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + group.add(s2); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNotNull(first.get("s2")); + + // evict one measurement: the next call reads only the group that holds it, and that one + // read also resolves the other members of the group + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s2")); + Map second = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertNotNull(second.get("s2")); + Mockito.verify(reader, Mockito.times(2)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-ignore-not-exists.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, true, Collections.emptyList()); + Mockito.when( + reader.readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenThrow(new IOException("device absent")); + + Map ignored = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + + // ignoreNotExists must not write NULL placeholders: a normal query still has to read + assertThrows( + IOException.class, + () -> getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false)); + // the empty group read resolves the time column as absent, which ends the load for the + // device, plus the failing normal query + Mockito.verify(reader, Mockito.times(2)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testEvictedMeasurementsShareOneGroupRead() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-few-missing.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2", "s3", "s4", "s5")); + + List group = new ArrayList<>(); + for (String measurement : allSensors) { + TSDataType dataType = measurement.isEmpty() ? TSDataType.INT64 : TSDataType.INT32; + Statistics statistics = Statistics.getStatsByType(dataType); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, measurement, dataType, statistics, new PublicBAOS())); + } + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNotNull(first.get("s5")); + + // evict three measurements: they share one leaf group, so a single read must resolve them + for (String measurement : Arrays.asList("s2", "s3", "s4")) { + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement)); + } + + Map second = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNotNull(second.get("s2")); + assertNotNull(second.get("s3")); + assertNotNull(second.get("s4")); + Mockito.verify(reader, Mockito.times(2)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testReaderConfirmedAbsenceIsRemembered() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-confirmed-absent.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 3); + bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + bloomFilter.add(deviceID.toString() + ".s2"); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + List group = new ArrayList<>(); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(first.get("s2")); + + // the reader confirmed s2 absent, so the second query is answered from the cache alone + Map second = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(second.get("s2")); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBloomExcludedMeasurementIsNotRemembered() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-bloom-excluded.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // the device exists, but s1 is not in the file + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + "."); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + List group = new ArrayList<>(); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(first.get("s1")); + + // s1 was excluded by the bloom filter, not read, so it is probed again instead of being + // remembered; the second query still needs no reader call + Map second = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(second.get("s1")); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBloomLoadBeforeCachePhaseIsNotSubtractedFromDiskTime() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-bloom-phase.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 2); + bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + // a slow load makes the accounting visible: if the pre-cache load is subtracted from the + // disk interval, the recorded disk time turns negative + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) + .thenAnswer( + invocation -> { + Thread.sleep(20); + return bloomFilter; + }); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + List group = new ArrayList<>(); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + stubGroupRead(reader, deviceID, false, group); + + // warm both entries + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + // evict only the time column, and drop the bloom cache so the next call reloads it + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "")); + bloomFilterCache.clear(); + + QueryContext queryContext = new QueryContext(false, false); + Map result = + cache.getTimeseriesMetadata( + filePath, + tsFileID, + deviceID, + allSensors, + allSensors, + false, + false, + queryContext, + null); + assertNotNull(result.get("")); + assertNotNull(result.get("s1")); + assertTrue( + "cache time: " + + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromCacheTime().get(), + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromCacheTime().get() >= 0); + assertTrue( + "disk time: " + + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskTime().get(), + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskTime().get() >= 0); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testExternalTsFileReadsAllRequestedMeasurements() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + File file = new File("target/external-cache-disabled.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.external.d1"); + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries( + deviceID, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32))); + for (int time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + record.addPoint("s1", time); + record.addPoint("s2", time); + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + long[] offset; + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + LazyTsFileDeviceIterator iterator = new LazyTsFileDeviceIterator(reader); + assertTrue(iterator.hasNext()); + iterator.next(); + offset = iterator.getCurrentDeviceMeasurementNodeOffset(); + } + // external tsfiles bypass the metadata cache: every requested measurement must still be read + TsFileResource resource = new TsFileResource(file); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + String filePath = file.getAbsolutePath(); + // the external registry keys readers by path and closes one when its last reference is + // released, so hold one for the duration of the test + FileReaderManager.getInstance().increaseExternalFileReaderReference(filePath); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setQueryDataSourceType( + QueryDataSourceType.EXTERNAL_TSFILE_SCAN); + try { + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + Map result = + cache.getTimeseriesMetadata( + filePath, + resource.getTsFileID(), + deviceID, + allSensors, + allSensors, + false, + false, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT, + offset); + assertEquals(allSensors.size(), result.size()); + for (String measurement : allSensors) { + assertNotNull(measurement, result.get(measurement)); + assertEquals(measurement, result.get(measurement).getMeasurementId()); + } + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setQueryDataSourceType( + QueryDataSourceType.SERIES_SCAN); + cache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().decreaseExternalFileReaderReference(filePath); + assertTrue(file.delete()); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java new file mode 100644 index 0000000000000..91835b8b30f2d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.commons.path.AlignedPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; +import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.constant.TestConstant; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.chunk.IChunkWriter; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.function.LongConsumer; + +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter; +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk; + +/** + * Verifies metadata loading on a real device wider than one measurement leaf group: each selected + * column is resolved from the leaf group that holds it, and one group read serves every selected + * column of that group. + */ +public class AlignedSeriesScanBatchTest { + + private static final String DATABASE = "root.wide_pd"; + private static final IDeviceID DEVICE = + IDeviceID.Factory.DEFAULT_FACTORY.create(DATABASE + ".d1"); + private static final int COLUMN_COUNT = 300; + private static final int PRESENT_COLUMN_COUNT = 30; + private static final int ROW_COUNT = 10; + + @Test + public void testScanReadsWideAlignedDevice() throws Exception { + Assert.assertTrue( + "the fixture must span more than one measurement leaf group", + COLUMN_COUNT + 1 > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()); + File file = newFile(1); + TsFileResource resource = writeWideDevice(file, COLUMN_COUNT); + TsFileSequenceReader fileReader = new TsFileSequenceReader(file.getAbsolutePath()); + TsFileSequenceReader reader = Mockito.spy(fileReader); + FileReaderManager.getInstance().setReaderForTest(resource.getTsFileID(), true, reader); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + metadataCache.clear(); + bloomFilterCache.clear(); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + // the unified path is only reachable from table dialect, which forces this flag to false + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + try { + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); + + TsBlock block = scan(alignedPath(columnNames(COLUMN_COUNT)), resource); + + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertFalse("s" + column + " row " + row, valueColumn.isNull(row)); + Assert.assertEquals("s" + column + " row " + row, row, valueColumn.getInt(row)); + } + } + + // a cold cache: every selected series is a disk lookup, counted per series + Assert.assertEquals( + cacheHitsBefore, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get()); + // per-series counters: a cold scan counts one disk lookup per selected measurement + long diskReads = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get() + - diskReadsBefore; + Assert.assertTrue("disk lookups: " + diskReads, diskReads >= 1); + Assert.assertTrue("disk lookups: " + diskReads, diskReads <= COLUMN_COUNT + 1); + // the whole projection is resolved with one reader call for this file, not one per column + Mockito.verify(reader) + .readTimeseriesMetadataForMeasurements( + Mockito.any(), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testSparseProjectionFiltersAbsentColumns() throws Exception { + File file = newFile(2); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + AlignedFullPath scanPath = alignedPath(columnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + try { + metadataCache.clear(); + bloomFilterCache.clear(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); + + TsBlock block = scan(scanPath, resource); + + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); + boolean present = column < PRESENT_COLUMN_COUNT; + for (int row = 0; row < ROW_COUNT; row++) { + if (present) { + Assert.assertFalse(valueColumn.isNull(row)); + Assert.assertEquals(row, valueColumn.getInt(row)); + } else { + Assert.assertTrue("s" + column + " row " + row, valueColumn.isNull(row)); + } + } + } + + // the bloom filter excludes the 270 absent columns; a few may survive as false positives, + // but the sparse projection must still look up far fewer series than it selects + long diskReads = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get() + - diskReadsBefore; + // per-series counters: bloom-excluded measurements count as disk lookups, as they did on the + // per-column path + Assert.assertTrue("disk lookups: " + diskReads, diskReads >= 1); + Assert.assertTrue("disk lookups: " + diskReads, diskReads <= COLUMN_COUNT + 1); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testAllNullProjection() throws Exception { + File file = newFile(3); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + // every selected field is absent from the file + AlignedFullPath absentPath = alignedPath(absentColumnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + try { + // table dialect: all-null rows are kept + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + metadataCache.clear(); + bloomFilterCache.clear(); + TsBlock block = scan(absentPath, resource); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + for (int column = 0; column < COLUMN_COUNT; column++) { + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertTrue("absent" + column + " row " + row, block.getColumn(column).isNull(row)); + } + } + + // tree dialect: an all-null projection skips the file + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); + metadataCache.clear(); + bloomFilterCache.clear(); + Assert.assertFalse(hasFile(absentPath, resource)); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + private static TsBlock scan(AlignedFullPath scanPath, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, resource); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + return seriesScanUtil.nextPage(); + } + + private static boolean hasFile(AlignedFullPath scanPath, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, resource); + for (int attempt = 0; attempt < 1000; attempt++) { + Optional hasNextFile = seriesScanUtil.hasNextFile(); + if (hasNextFile.isPresent()) { + return hasNextFile.get(); + } + } + throw new IllegalStateException("series scan did not advance after 1000 attempts"); + } + + private static AlignedSeriesScanUtil newScanUtil( + AlignedFullPath scanPath, TsFileResource resource) throws IOException { + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource( + new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); + return seriesScanUtil; + } + + private static AlignedFullPath alignedPath(List measurementNames) { + List schemas = new ArrayList<>(measurementNames.size()); + for (String measurement : measurementNames) { + schemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + } + return new AlignedFullPath(DEVICE, measurementNames, schemas); + } + + private static List columnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("s" + i); + } + return names; + } + + private static List absentColumnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("absent" + i); + } + return names; + } + + private static File newFile(int version) { + File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, version)); + if (!file.getParentFile().exists()) { + Assert.assertTrue(file.getParentFile().mkdirs()); + } + return file; + } + + private static TsFileResource writeWideDevice(File file, int presentColumnCount) + throws IOException, IllegalPathException { + TsFileResource resource = new TsFileResource(file); + List writtenPaths = new ArrayList<>(presentColumnCount); + List dataTypes = new ArrayList<>(presentColumnCount); + List encodings = new ArrayList<>(presentColumnCount); + List compressionTypes = new ArrayList<>(presentColumnCount); + for (int i = 0; i < presentColumnCount; i++) { + writtenPaths.add(new AlignedPath(DEVICE.toString(), "s" + i)); + dataTypes.add(TSDataType.INT32); + encodings.add(TSEncoding.PLAIN); + compressionTypes.add(CompressionType.UNCOMPRESSED); + } + try (TsFileIOWriter writer = new TsFileIOWriter(file)) { + writer.startChunkGroup(DEVICE); + for (IChunkWriter chunkWriter : + createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { + writeAlignedChunk( + (AlignedChunkWriterImpl) chunkWriter, + writer, + Collections.singletonList(new TimeRange(0L, ROW_COUNT - 1L)), + true); + } + writer.endChunkGroup(); + writer.endFile(); + } + resource.updateStartTime(DEVICE, 0); + resource.updateEndTime(DEVICE, ROW_COUNT - 1L); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + return resource; + } + + private static void cleanup( + File file, + TsFileResource resource, + TimeSeriesMetadataCache metadataCache, + BloomFilterCache bloomFilterCache) + throws IOException { + metadataCache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); + Assert.assertTrue(file.delete()); + } +} diff --git a/pom.xml b/pom.xml index 425530778fe66..7458fec64e943 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 0.25.0 1.9 1.5.6-3 - 2.5.1-261118-SNAPSHOT + 2.5.1-261120-SNAPSHOT en