diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java index f4ae6c679d034..a50d6d1defdd5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java @@ -102,7 +102,8 @@ public ClusterSchemaTree fetchSchema( if (withTemplate && pattern.hasExplicitDevice() && templateManager.checkTemplateSetInfo(pattern) != null) { - explicitDevicePatternList.add(pattern.getDevicePath()); + explicitDevicePatternList.add( + new PartialPath(Arrays.copyOf(pattern.getNodes(), pattern.getNodeLength() - 1))); explicitDevicePatternCount++; } else if (!pattern.hasWildcard()) { explicitPathList.add(pattern); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java index c7f67307a7019..e5b16ade049d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java @@ -124,9 +124,21 @@ null, new TreeDeviceNormalSchema(database, deviceSchemaInfo.isAligned())) .update(deviceSchemaInfo.getMeasurementSchemaInfoList()) : 0; } else { - return deviceSchema.compareAndSet( - null, new TreeDeviceTemplateSchema(database, deviceSchemaInfo.getTemplateId())) - ? TreeDeviceTemplateSchema.INSTANCE_SIZE + // A last-cache declaration from a fragment of a remotely coordinated query may have created + // a normal schema for this template device first; the fetched template schema supersedes it. + final TreeDeviceTemplateSchema templateSchema = + new TreeDeviceTemplateSchema(database, deviceSchemaInfo.getTemplateId()); + final IDeviceSchema previous = + deviceSchema.getAndUpdate( + schema -> + Objects.isNull(schema) || schema instanceof TreeDeviceNormalSchema + ? templateSchema + : schema); + if (Objects.isNull(previous)) { + return TreeDeviceTemplateSchema.INSTANCE_SIZE; + } + return previous instanceof TreeDeviceNormalSchema + ? TreeDeviceTemplateSchema.INSTANCE_SIZE - previous.estimateSize() : 0; } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java index 59cdf62056126..70ce45ad20ff2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java @@ -462,4 +462,47 @@ public void testPut() throws Exception { TableDeviceSchemaCache.getInstance().invalidateAll(); Assert.assertEquals(0, TableDeviceSchemaCache.getInstance().getMemoryUsage()); } + + @Test + public void testPutTemplateSchemaAfterLastCacheDeclaration() throws Exception { + final String database = "root.sg1"; + final PartialPath device = new PartialPath("root.sg1.d1"); + final Template template = + new Template( + "t1", + Arrays.asList("s1", "s2"), + Arrays.asList(TSDataType.DOUBLE, TSDataType.INT32), + Arrays.asList(TSEncoding.RLE, TSEncoding.RLE), + Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY)); + template.setId(1); + ClusterTemplateManager.getInstance().putTemplate(template); + final MeasurementPath s1Path = + new MeasurementPath(device.concatNode("s1"), template.getSchema("s1")); + final TimeValuePair tv1 = new TimeValuePair(1, new TsPrimitiveType.TsDouble(1)); + + // A DataNode running the fragment of a SELECT LAST coordinated elsewhere declares the last + // cache before it has fetched this device's template schema. + treeDeviceSchemaCacheManager.declareLastCache(database, s1Path); + treeDeviceSchemaCacheManager.updateLastCacheIfExists( + database, + IDeviceID.Factory.DEFAULT_FACTORY.create( + StringArrayDeviceID.splitDeviceIdString(device.getNodes())), + new String[] {"s1"}, + new TimeValuePair[] {tv1}, + false, + new MeasurementSchema[] {(MeasurementSchema) template.getSchema("s1")}); + + final ClusterSchemaTree clusterSchemaTree = new ClusterSchemaTree(); + clusterSchemaTree.appendTemplateDevice(device, false, 1, template); + clusterSchemaTree.setDatabases(Collections.singleton(database)); + treeDeviceSchemaCacheManager.put(clusterSchemaTree); + + final List measurementPaths = + treeDeviceSchemaCacheManager + .getMatchedTemplateSchema(device) + .searchMeasurementPaths(ALL_MATCH_PATTERN) + .left; + Assert.assertEquals(2, measurementPaths.size()); + Assert.assertEquals(tv1, treeDeviceSchemaCacheManager.getLastCache(s1Path)); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcherTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcherTest.java new file mode 100644 index 0000000000000..a3c59c3b8bfe0 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcherTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.schema; + +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; +import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; +import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; + +public class ClusterSchemaFetcherTest { + + private Template template; + private int previousDataNodeId; + + @Before + public void setUp() throws Exception { + // ClusterSchemaFetcher creates the Coordinator, whose query id generator needs a DataNode id. + previousDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0); + template = + new Template( + "t1", + Arrays.asList("s1", "s2"), + Arrays.asList(TSDataType.DOUBLE, TSDataType.INT32), + Arrays.asList(TSEncoding.RLE, TSEncoding.RLE), + Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY)); + template.setId(1); + ClusterTemplateManager.getInstance() + .addTemplateSetInfo( + TemplateInternalRPCUtil.generateAddTemplateSetInfoBytes(template, "root.sg1")); + } + + @After + public void tearDown() { + TreeDeviceSchemaCacheManager.getInstance().cleanUp(); + ClusterTemplateManager.getInstance().clear(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(previousDataNodeId); + } + + @Test + public void testFetchTemplateDeviceSchemaFromCache() throws Exception { + final ClusterSchemaTree cached = new ClusterSchemaTree(); + cached.appendTemplateDevice(new PartialPath("root.sg1.d1"), false, 1, template); + cached.setDatabases(Collections.singleton("root.sg1")); + TreeDeviceSchemaCacheManager.getInstance().put(cached); + + final PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendFullPath(new PartialPath("root.sg1.d1.s1")); + patternTree.appendFullPath(new PartialPath("root.sg1.d1.s2")); + + // A cache hit returns without a remote fetch. A miss falls through to + // fetchSchemaOfPreciseMatchOrPreciseDeviceUsingTemplate, which fails here without a cluster. + final ClusterSchemaTree fetched = + ClusterSchemaFetcher.getInstance() + .fetchSchema(patternTree, true, new MPPQueryContext(new QueryId("test")), true); + + final List paths = + fetched.searchMeasurementPaths(ALL_MATCH_PATTERN).left.stream() + .map(MeasurementPath::getFullPath) + .sorted() + .collect(Collectors.toList()); + Assert.assertEquals(Arrays.asList("root.sg1.d1.s1", "root.sg1.d1.s2"), paths); + Assert.assertEquals(Collections.singleton("root.sg1"), fetched.getDatabases()); + } +}