From 09bef00f982081751cbd74322948d27683eebcbb Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Wed, 18 Nov 2026 23:17:35 +0800 Subject: [PATCH 1/2] [Pipe] Avoid creating DAL managers during region cleanup and suppress expected close warnings --- .../deletion/DeletionResourceManager.java | 21 +++- .../persist/PageCacheDeletionBuffer.java | 6 +- .../storageengine/dataregion/DataRegion.java | 7 +- .../DeletionResourceManagerCleanupTest.java | 114 ++++++++++++++++++ .../persist/PageCacheDeletionBufferTest.java | 75 ++++++++++++ 5 files changed, 215 insertions(+), 8 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java index 98f447a4b12a3..416fe2776597a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java @@ -176,7 +176,26 @@ public List getAllDeletionResources() { } } - public void removeDAL() { + /** Close an existing manager and remove its DAL without creating a manager for cleanup. */ + public static void closeAndRemoveDAL(int groupId) { + if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == null) { + return; + } + final DeletionResourceManager manager = + DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.get(groupId); + if (manager != null) { + manager.close(); + removeDAL(groupId, manager.storageDir); + } else { + removeDAL( + groupId, + new File( + IoTDBDescriptor.getInstance().getConfig().getIotConsensusV2DeletionFileDir(), + Integer.toString(groupId))); + } + } + + private static void removeDAL(int dataRegionId, File storageDir) { if (storageDir.exists()) { FileUtils.deleteFileOrDirectory(storageDir); LOGGER.info( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java index 6fe2246627f1f..945b70e7c380c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java @@ -390,7 +390,11 @@ private void persistDeletion() throws IOException { maxProgressIndexInCurrentFile.updateToMinimumEqualOrIsAfterProgressIndex( firstDeletionResource.getProgressIndex()); } catch (InterruptedException e) { - LOGGER.warn(DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM); + // shutdownNow interrupts the idle queue wait during normal close. + if (!isClosed) { + LOGGER.warn( + DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM); + } Thread.currentThread().interrupt(); return; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 94f44022ceffe..d19a9e0813e63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2456,12 +2456,7 @@ public void shutdownUpgradeModFileThreadPool() { } public void deleteDALFolderAndClose() { - Optional.ofNullable(DeletionResourceManager.getInstance(dataRegionId.getId())) - .ifPresent( - manager -> { - manager.close(); - manager.removeDAL(); - }); + DeletionResourceManager.closeAndRemoveDAL(dataRegionId.getId()); } /** close all tsfile resource */ diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java new file mode 100644 index 0000000000000..7eff2b86e004d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.consensus.deletion; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.pipe.consensus.deletion.persist.PageCacheDeletionBuffer; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Appender; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Files; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +public class DeletionResourceManagerCleanupTest { + private static final int REGION_ID = 91821; + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + private String previousDirectory; + private int previousBufferSize; + private Logger bufferLogger; + private Appender appender; + + @Before + @SuppressWarnings("unchecked") + public void setUp() throws Exception { + previousDirectory = + IoTDBDescriptor.getInstance().getConfig().getIotConsensusV2DeletionFileDir(); + previousBufferSize = PageCacheDeletionBuffer.DAL_BUFFER_SIZE; + IoTDBDescriptor.getInstance() + .getConfig() + .setIotConsensusV2DeletionFileDir(temporaryFolder.getRoot().getAbsolutePath()); + PageCacheDeletionBuffer.setDalBufferSize(1024); + DeletionResourceManager.buildForTest(); + bufferLogger = (Logger) LoggerFactory.getLogger(PageCacheDeletionBuffer.class); + appender = mock(Appender.class); + bufferLogger.addAppender(appender); + } + + @After + public void tearDown() { + try { + DeletionResourceManager.closeAndRemoveDAL(REGION_ID); + } finally { + bufferLogger.detachAppender(appender); + IoTDBDescriptor.getInstance().getConfig().setIotConsensusV2DeletionFileDir(previousDirectory); + PageCacheDeletionBuffer.setDalBufferSize(previousBufferSize); + } + } + + @Test + public void testCleanupWithoutManagerDoesNotStartBuffer() { + DeletionResourceManager.closeAndRemoveDAL(REGION_ID); + + Assert.assertFalse(regionDirectory().exists()); + verify(appender, never()).doAppend(any(ILoggingEvent.class)); + } + + @Test + public void testCleanupRemovesOrphanedDALWithoutStartingBuffer() throws Exception { + final File directory = regionDirectory(); + Assert.assertTrue(directory.mkdir()); + Files.write(new File(directory, "_0-0.deletion").toPath(), new byte[] {1, 2, 3}); + + DeletionResourceManager.closeAndRemoveDAL(REGION_ID); + + Assert.assertFalse(directory.exists()); + verify(appender, never()).doAppend(any(ILoggingEvent.class)); + } + + @Test + public void testCleanupClosesExistingManagerAndAllowsRecreation() { + final DeletionResourceManager manager = DeletionResourceManager.getInstance(REGION_ID); + Assert.assertTrue(regionDirectory().exists()); + + DeletionResourceManager.closeAndRemoveDAL(REGION_ID); + + Assert.assertFalse(regionDirectory().exists()); + Assert.assertNotSame(manager, DeletionResourceManager.getInstance(REGION_ID)); + Assert.assertTrue(regionDirectory().exists()); + } + + private File regionDirectory() { + return new File(temporaryFolder.getRoot(), Integer.toString(REGION_ID)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java index 59d0d2862bd7c..9891f86052062 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java @@ -24,11 +24,16 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResource; import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager; import org.apache.iotdb.db.pipe.consensus.deletion.recover.DeletionReader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Appender; import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; @@ -37,6 +42,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.mockito.InOrder; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -45,11 +51,16 @@ import java.nio.channels.FileChannel; import java.util.Collections; import java.util.List; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import static org.mockito.AdditionalAnswers.delegatesTo; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; @@ -75,6 +86,70 @@ public void tearDown() { PageCacheDeletionBuffer.setDalBufferSize(previousDalBufferSize); } + @Test + public void testIdleCloseDoesNotWarnAboutQueueInterruption() throws Exception { + assertQueueInterruptionLogging(true); + } + + @Test + public void testUnexpectedQueueInterruptionStillWarns() throws Exception { + assertQueueInterruptionLogging(false); + } + + @SuppressWarnings("unchecked") + private void assertQueueInterruptionLogging(boolean closeNormally) throws Exception { + final PageCacheDeletionBuffer deletionBuffer = + new PageCacheDeletionBuffer(1, temporaryFolder.newFolder("interrupt").getAbsolutePath()); + final CountDownLatch taking = new CountDownLatch(1); + final AtomicReference worker = new AtomicReference<>(); + final Field queueField = PageCacheDeletionBuffer.class.getDeclaredField("deletionResources"); + queueField.setAccessible(true); + final BlockingQueue originalQueue = + (BlockingQueue) queueField.get(deletionBuffer); + final BlockingQueue queue = + mock(BlockingQueue.class, delegatesTo(originalQueue)); + doAnswer( + invocation -> { + worker.set(Thread.currentThread()); + taking.countDown(); + return originalQueue.take(); + }) + .when(queue) + .take(); + queueField.set(deletionBuffer, queue); + final Logger logger = (Logger) LoggerFactory.getLogger(PageCacheDeletionBuffer.class); + final Appender appender = mock(Appender.class); + logger.addAppender(appender); + deletionBuffer.start(); + try { + Assert.assertTrue(taking.await(10, TimeUnit.SECONDS)); + if (!closeNormally) { + worker.get().interrupt(); + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .untilAsserted( + () -> + verify(appender, atLeastOnce()) + .doAppend(argThat(this::isQueueInterruptionWarning))); + } + } finally { + try { + deletionBuffer.close(); + if (closeNormally) { + verify(appender, never()).doAppend(argThat(this::isQueueInterruptionWarning)); + } + } finally { + logger.detachAppender(appender); + } + } + } + + private boolean isQueueInterruptionWarning(ILoggingEvent event) { + return event.getLevel() == Level.WARN + && DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM.equals( + event.getMessage()); + } + @Test public void testDequeuedDeletionIsNotReportedAsFlushed() throws Exception { final CountDownLatch serializationStarted = new CountDownLatch(1); From dd1a1211842c123403a125f088ece4b72ba106c2 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 19 Nov 2026 18:05:15 +0800 Subject: [PATCH 2/2] [Pipe] Serialize DAL lifecycle and drain region producers before cleanup --- .../iotdb/db/i18n/DataNodePipeMessages.java | 4 + .../iotdb/db/i18n/DataNodePipeMessages.java | 4 + .../consensus/deletion/DeletionResource.java | 5 +- .../deletion/DeletionResourceManager.java | 235 ++++++++++++------ .../persist/PageCacheDeletionBuffer.java | 50 +++- ...icalDataRegionTsFileAndDeletionSource.java | 6 +- .../assigner/PipeDataRegionAssigner.java | 2 +- .../PipeInsertionDataNodeListener.java | 22 ++ .../iotdb/db/storageengine/StorageEngine.java | 10 +- .../DeletionResourceManagerCleanupTest.java | 226 ++++++++++++++++- .../persist/PageCacheDeletionBufferTest.java | 116 ++++++++- .../PipeInsertionDataNodeListenerTest.java | 31 +++ 12 files changed, 623 insertions(+), 88 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 0dccb0fb04424..110971f8538d7 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -3180,4 +3180,8 @@ private DataNodePipeMessages() {} EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68 = "Failed to pin the TsFile %s by creating a hardlink, and the TsFile can not be copied" + " because it is not closed yet"; + public static final String EXCEPTION_DELETION_RESOURCE_MANAGER_FOR_DATAREGION_ARG_IS_CLOSED_OR_DELETED_951C0622 = + "Deletion resource manager for DataRegion %s is closed or deleted."; + public static final String LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D = + "Deletion persist-{}: timed out waiting for {} deletions to flush before shutdown."; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 579172c77d088..cd82aae446aa5 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2954,4 +2954,8 @@ private DataNodePipeMessages() {} public static final String EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68 = "通过创建硬链接固化 TsFile %s 失败,且该 TsFile 尚未关闭,无法改为拷贝"; + public static final String EXCEPTION_DELETION_RESOURCE_MANAGER_FOR_DATAREGION_ARG_IS_CLOSED_OR_DELETED_951C0622 = + "DataRegion %s 的删除资源管理器已关闭或已删除。"; + public static final String LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D = + "Deletion persist-{}:关闭前等待 {} 条删除记录持久化超时。"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java index 051f224890cc5..18010f067cb34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java @@ -70,7 +70,10 @@ public DeletionResource( DataRegionConsensusImpl.getInstance().getReplicationNum(consensusGroupId) - 1); } - public synchronized void decreaseReference() { + // The atomic counter selects one release callback. Do not hold this resource's monitor while + // the callback acquires the region lifecycle lock: close waits for synchronized persist + // callbacks. + public void decreaseReference() { if (pipeTaskReferenceCount.decrementAndGet() == 0) { removeSelf(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java index 416fe2776597a..597177b39dae8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Striped; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +45,7 @@ import java.nio.file.Paths; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; @@ -62,6 +64,11 @@ public class DeletionResourceManager implements AutoCloseable { String.format( "^_(?<%s>\\d+)-(?<%s>\\d+)\\%s$", REBOOT_TIME, MEM_TABLE_FLUSH_ORDER, DELETION_FILE_SUFFIX); + // Stable, bounded locks are shared by creation, ordinary close and region deletion. Never remove + // a lock while a waiter may still hold its reference. + private static final Striped REGION_LOCKS = Striped.lock(1024); + private static final Set DELETED_REGIONS = ConcurrentHashMap.newKeySet(); + private boolean closed; private final int dataRegionId; private final DeletionBuffer deletionBuffer; private final File storageDir; @@ -132,25 +139,50 @@ private void initAndRecover() throws IOException { @Override public void close() { - LOGGER.info(DataNodePipeMessages.CLOSING_DELETION_RESOURCE_MANAGER_FOR, dataRegionId); - this.deleteNode2ResourcesMap.clear(); - this.deletionBuffer.close(); - if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP != null) { - DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.remove(dataRegionId, this); + final Lock lock = REGION_LOCKS.get(dataRegionId); + lock.lock(); + try { + if (closed) { + return; + } + closed = true; + LOGGER.info(DataNodePipeMessages.CLOSING_DELETION_RESOURCE_MANAGER_FOR, dataRegionId); + this.deletionBuffer.close(); + this.deleteNode2ResourcesMap.clear(); + if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP != null) { + DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.remove(dataRegionId, this); + } + LOGGER.info( + DataNodePipeMessages.DELETION_RESOURCE_MANAGER_FOR_HAS_BEEN_SUCCESSFULLY, dataRegionId); + } finally { + lock.unlock(); } - LOGGER.info( - DataNodePipeMessages.DELETION_RESOURCE_MANAGER_FOR_HAS_BEEN_SUCCESSFULLY, dataRegionId); } public DeletionResource registerDeletionResource(AbstractDeleteDataNode deleteDataNode) { - DeletionResource deletionResource = - deleteNode2ResourcesMap.computeIfAbsent( - deleteDataNode, - key -> - new DeletionResource(deleteDataNode, this::removeDeletionResource, dataRegionId)); - // register a persist task for current deletionResource - deletionBuffer.registerDeletionResource(deletionResource); - return deletionResource; + final Lock lock = REGION_LOCKS.get(dataRegionId); + lock.lock(); + try { + final DeletionResource deletionResource = + new DeletionResource(deleteDataNode, this::removeDeletionResource, dataRegionId); + // A producer may have obtained this manager before the region deletion barrier. + // Complete its result instead of leaving it waiting on an already closed buffer. + if (closed || DELETED_REGIONS.contains(dataRegionId)) { + deletionResource.onPersistFailed( + new IOException( + String.format( + DataNodePipeMessages + .EXCEPTION_DELETION_RESOURCE_MANAGER_FOR_DATAREGION_ARG_IS_CLOSED_OR_DELETED_951C0622, + dataRegionId))); + return deletionResource; + } + final DeletionResource resource = + deleteNode2ResourcesMap.computeIfAbsent(deleteDataNode, key -> deletionResource); + deletionBuffer.registerDeletionResource(resource); + return resource; + } finally { + lock.unlock(); + } } public DeletionResource getDeletionResource(AbstractDeleteDataNode deleteDataNode) { @@ -176,22 +208,51 @@ public List getAllDeletionResources() { } } - /** Close an existing manager and remove its DAL without creating a manager for cleanup. */ - public static void closeAndRemoveDAL(int groupId) { - if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == null) { - return; + /** Block new DAL creation before stopping region producers. Does not wait for producer tasks. */ + public static void markRegionDeleted(int groupId) { + final Lock lock = REGION_LOCKS.get(groupId); + lock.lock(); + try { + DELETED_REGIONS.add(groupId); + } finally { + lock.unlock(); } - final DeletionResourceManager manager = - DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.get(groupId); - if (manager != null) { - manager.close(); - removeDAL(groupId, manager.storageDir); - } else { + } + + /** + * Only the storage lifecycle may permit DAL creation again, after old-region cleanup finishes. + */ + public static void onRegionCreated(int groupId) { + final Lock lock = REGION_LOCKS.get(groupId); + lock.lock(); + try { + DELETED_REGIONS.remove(groupId); + } finally { + lock.unlock(); + } + } + + /** Close and remove DAL atomically with respect to creation, without constructing a manager. */ + public static void closeAndRemoveDAL(int groupId) { + final Lock lock = REGION_LOCKS.get(groupId); + lock.lock(); + try { + DELETED_REGIONS.add(groupId); + if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == null) { + return; + } + final DeletionResourceManager manager = + DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.get(groupId); + if (manager != null) { + manager.close(); + } removeDAL( groupId, new File( IoTDBDescriptor.getInstance().getConfig().getIotConsensusV2DeletionFileDir(), Integer.toString(groupId))); + } finally { + lock.unlock(); } } @@ -214,43 +275,53 @@ private static void removeDAL(int dataRegionId, File storageDir) { * This is a hook function, which will be automatically invoked when deletionResource's reference * count returns to 0. */ - private synchronized void removeDeletionResource(DeletionResource deletionResource) { - // Clean memory - deleteNode2ResourcesMap.remove(deletionResource.getDeleteDataNode()); - // Clean disk - ProgressIndex currentProgressIndex = - ReplicateProgressDataNodeManager.extractLocalSimpleProgressIndex( - deletionResource.getProgressIndex()); + private void removeDeletionResource(DeletionResource deletionResource) { + final Lock lock = REGION_LOCKS.get(dataRegionId); + lock.lock(); + try { + // Late reference releases from an old manager must not touch a new lifecycle's DAL. + if (closed || DELETED_REGIONS.contains(dataRegionId)) { + return; + } + // Clean memory + deleteNode2ResourcesMap.remove(deletionResource.getDeleteDataNode()); + // Clean disk + ProgressIndex currentProgressIndex = + ReplicateProgressDataNodeManager.extractLocalSimpleProgressIndex( + deletionResource.getProgressIndex()); - try (Stream pathStream = Files.walk(Paths.get(storageDir.getPath()), 1)) { - Path[] deletionPaths = - pathStream - .filter(Files::isRegularFile) - .filter(path -> path.getFileName().toString().matches(DELETION_FILE_NAME_PATTERN)) - .filter( - path -> - isFileProgressCoveredByGivenProgress( - path.getFileName().toString(), currentProgressIndex)) - .sorted(this::compareFileProgressIndex) - .toArray(Path[]::new); - // File name represents the max progressIndex in its previous file. If currentProgressIndex is - // larger than a fileName's progressIndex, it means that the file before this file has been - // fully synchronized and can be deleted. - // So here we cannot guarantee that the last file can be deleted, we can only guarantee that - // the first n-1 files can be deleted (if the length of deletionPaths is n) - for (int i = 0; i < deletionPaths.length - 1; i++) { - File fileToDelete = deletionPaths[i].toFile(); - FileUtils.deleteFileOrDirectory(fileToDelete); - LOGGER.info( - DataNodePipeMessages.DELETIONMANAGER_DELETE_DELETION_FILE_IN_DIR, + try (Stream pathStream = Files.walk(Paths.get(storageDir.getPath()), 1)) { + Path[] deletionPaths = + pathStream + .filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().matches(DELETION_FILE_NAME_PATTERN)) + .filter( + path -> + isFileProgressCoveredByGivenProgress( + path.getFileName().toString(), currentProgressIndex)) + .sorted(this::compareFileProgressIndex) + .toArray(Path[]::new); + // File names describe the previous file's max progressIndex. A larger currentProgressIndex + // means that the file before this file has been + // fully synchronized and can be deleted. + // So here we cannot guarantee that the last file can be deleted, we can only guarantee that + // the first n-1 files can be deleted (if the length of deletionPaths is n) + for (int i = 0; i < deletionPaths.length - 1; i++) { + File fileToDelete = deletionPaths[i].toFile(); + FileUtils.deleteFileOrDirectory(fileToDelete); + LOGGER.info( + DataNodePipeMessages.DELETIONMANAGER_DELETE_DELETION_FILE_IN_DIR, + dataRegionId, + fileToDelete); + } + } catch (IOException e) { + LOGGER.warn( + DataNodePipeMessages.DELETIONMANAGER_FAILED_TO_DELETE_FILE_IN_DIR, dataRegionId, - fileToDelete); + storageDir); } - } catch (IOException e) { - LOGGER.warn( - DataNodePipeMessages.DELETIONMANAGER_FAILED_TO_DELETE_FILE_IN_DIR, - dataRegionId, - storageDir); + } finally { + lock.unlock(); } } @@ -310,21 +381,41 @@ public static void build() { } } + /** Obtain an existing manager without creating DAL resources for a lookup-only caller. */ + public static DeletionResourceManager getInstanceIfPresent(int groupId) { + final Lock lock = REGION_LOCKS.get(groupId); + lock.lock(); + try { + return DELETED_REGIONS.contains(groupId) + || DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == null + ? null + : DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.get(groupId); + } finally { + lock.unlock(); + } + } + public static DeletionResourceManager getInstance(int groupId) { - // If consensusImpl is not IoTConsensusV2. - if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == null) { - return null; + final Lock lock = REGION_LOCKS.get(groupId); + lock.lock(); + try { + if (DELETED_REGIONS.contains(groupId) + || DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == null) { + return null; + } + return DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.computeIfAbsent( + groupId, + key -> { + try { + return new DeletionResourceManager(groupId); + } catch (IOException e) { + LOGGER.error(DataNodePipeMessages.FAILED_TO_INITIALIZE_DELETIONRESOURCEMANAGER, e); + throw new RuntimeException(e); + } + }); + } finally { + lock.unlock(); } - return DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.computeIfAbsent( - groupId, - key -> { - try { - return new DeletionResourceManager(groupId); - } catch (IOException e) { - LOGGER.error(DataNodePipeMessages.FAILED_TO_INITIALIZE_DELETIONRESOURCEMANAGER, e); - throw new RuntimeException(e); - } - }); } // Only when consensus protocol is IoTConsensusV2, will this class be initialized. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java index 945b70e7c380c..956edd80ef006 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java @@ -62,6 +62,7 @@ public class PageCacheDeletionBuffer implements DeletionBuffer { private static final double FSYNC_BUFFER_RATIO = 0.95; private static final int QUEUE_CAPACITY = config.getDeletionAheadLogBufferQueueCapacity(); private static final long MAX_WAIT_CLOSE_TIME_IN_MS = 10000; + private static final long CLOSE_CHECK_INTERVAL_MS = 100; // Buffer config keep consistent with WAL. public static int DAL_BUFFER_SIZE = config.getWalBufferSize() / 3; @@ -282,11 +283,20 @@ public void close() { } // Force sync existing data in memory to disk. // first waiting serialize and sync tasks finished, then release all resources - waitUntilFlushAllDeletionsOrTimeOut(); + final boolean flushed = waitUntilFlushAllDeletionsOrTimeOut(); if (persistThread != null) { lifecycleLock.lock(); try { - persistThread.shutdownNow(); + if (flushed) { + persistThread.shutdown(); + } else { + LOGGER.warn( + DataNodePipeMessages + .LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D, + dataRegionId, + unflushedDeletionCount.get()); + persistThread.shutdownNow(); + } } finally { lifecycleLock.unlock(); } @@ -310,7 +320,7 @@ public void close() { serializeBuffer = null; } - private void waitUntilFlushAllDeletionsOrTimeOut() { + private boolean waitUntilFlushAllDeletionsOrTimeOut() { long currentTime = System.currentTimeMillis(); while (!isAllDeletionFlushed() && System.currentTimeMillis() - currentTime < MAX_WAIT_CLOSE_TIME_IN_MS) { @@ -319,8 +329,10 @@ private void waitUntilFlushAllDeletionsOrTimeOut() { } catch (InterruptedException e) { LOGGER.error(DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_ALL_DELETIONS_FLUSHED); Thread.currentThread().interrupt(); + return false; } } + return isAllDeletionFlushed(); } private class PersistTask implements Runnable { @@ -374,9 +386,13 @@ private boolean serializeDeletionToBatchBuffer(DeletionResource deletionResource } private void persistDeletion() throws IOException { - // For first deletion we use blocking take() method. + // Bounded waiting lets an idle task observe close without using interruption as control flow. try { - DeletionResource firstDeletionResource = deletionResources.take(); + DeletionResource firstDeletionResource = + deletionResources.poll(CLOSE_CHECK_INTERVAL_MS, TimeUnit.MILLISECONDS); + if (firstDeletionResource == null) { + return; + } pendingDeletionsInOneTask.add(firstDeletionResource); currentTaskDeletionCount++; // A deletion can exceed the DAL buffer even when it is the first entry in a batch. @@ -390,7 +406,7 @@ private void persistDeletion() throws IOException { maxProgressIndexInCurrentFile.updateToMinimumEqualOrIsAfterProgressIndex( firstDeletionResource.getProgressIndex()); } catch (InterruptedException e) { - // shutdownNow interrupts the idle queue wait during normal close. + // Forced shutdown after a flush timeout may still interrupt the queue wait. if (!isClosed) { LOGGER.warn( DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM); @@ -405,10 +421,26 @@ private void persistDeletion() throws IOException { DeletionResource deletionResource = null; try { // Timeout config keep consistent with WAL async mode. - deletionResource = - deletionResources.poll(config.getWalAsyncModeFsyncDelayInMs(), TimeUnit.MILLISECONDS); + final long deadline = + System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos(config.getWalAsyncModeFsyncDelayInMs()); + do { + deletionResource = deletionResources.poll(); + if (deletionResource != null || isClosed) { + break; + } + final long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + break; + } + deletionResource = + deletionResources.poll( + Math.min(remaining, TimeUnit.MILLISECONDS.toNanos(CLOSE_CHECK_INTERVAL_MS)), + TimeUnit.NANOSECONDS); + } while (deletionResource == null); } catch (InterruptedException e) { - LOGGER.warn(DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_WALENTRY_FROM); + LOGGER.warn( + DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM); Thread.currentThread().interrupt(); } // If timeout, flush deletions to disk. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java index 19c78fa993b2f..bc34b4acd01e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java @@ -568,6 +568,10 @@ public synchronized void start() { DataNodePipeMessages .MESSAGE_PIPE_START_TO_EXTRACT_HISTORICAL_TSFILE_AND_DELETION_IF_USES_IOTCONSENSUSV2_C1603968); try { + if (dataRegion.isDeleted()) { + pendingQueue = new ArrayDeque<>(); + return; + } List originalResourceList = new ArrayList<>(); if (shouldExtractInsertion) { @@ -1427,7 +1431,7 @@ private Event supplyDeletionEvent(final DeletionResource deletionResource) { dataRegionId, event); } else { - Optional.ofNullable(DeletionResourceManager.getInstance(dataRegionId)) + Optional.ofNullable(DeletionResourceManager.getInstanceIfPresent(dataRegionId)) .ifPresent( manager -> event.setDeletionResource( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java index 6e308b5f1591c..bdf0fd652f26d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java @@ -387,7 +387,7 @@ private void assignToSourceInternal(final PipeRealtimeEvent event) { final PipeDeleteDataNodeEvent deleteDataNodeEvent = (PipeDeleteDataNodeEvent) innerEvent; final DeletionResourceManager manager = - DeletionResourceManager.getInstance(source.getDataRegionId()); + DeletionResourceManager.getInstanceIfPresent(source.getDataRegionId()); // increase deletion resource's reference and bind real deleteEvent if (Objects.nonNull(manager) && DeletionResource.isDeleteNodeGeneratedInLocalByIoTV2( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java index 9c2a4bee090c5..effbc014f9d41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import java.util.Objects; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReference; @@ -52,10 +53,15 @@ public class PipeInsertionDataNodeListener { private final ConcurrentMap dataRegionId2Assigner = new ConcurrentHashMap<>(); + private final Set deletedRegions = ConcurrentHashMap.newKeySet(); + //////////////////////////// start & stop //////////////////////////// public synchronized void startListenAndAssign( final int dataRegionId, final PipeRealtimeDataRegionSource source) { + if (deletedRegions.contains(dataRegionId)) { + return; + } // Keep registration inside compute so the assigner is fully started before it becomes visible // to concurrent listeners. dataRegionId2Assigner.compute( @@ -68,6 +74,22 @@ public synchronized void startListenAndAssign( }); } + /** Stop admission and drain the region assigner before its DAL is closed. */ + public void releaseDataRegion(final int dataRegionId) { + final PipeDataRegionAssigner assigner; + synchronized (this) { + deletedRegions.add(dataRegionId); + assigner = dataRegionId2Assigner.remove(dataRegionId); + } + if (assigner != null) { + assigner.close(); + } + } + + public synchronized void onRegionCreated(final int dataRegionId) { + deletedRegions.remove(dataRegionId); + } + public void stopListenAndAssign( final int dataRegionId, final PipeRealtimeDataRegionSource source) { PipeDataRegionAssigner assignerToClose = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 6c128c027a441..75f1834dab0c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -57,6 +57,7 @@ import org.apache.iotdb.db.exception.runtime.StorageEngineFailureException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager; import org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeDataRegionAssigner.CompletionToken; import org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; @@ -542,6 +543,9 @@ public DataRegion buildNewDataRegion(String databaseName, DataRegionId dataRegio WRITING_METRICS.createActiveMemtableCounterMetrics(dataRegionId); dataRegion.setCustomFlushListeners(customFlushListeners); dataRegion.setCustomCloseFileListeners(customCloseFileListeners); + // The previous region's cleanup has completed before a new storage lifecycle starts. + DeletionResourceManager.onRegionCreated(dataRegionId.getId()); + PipeInsertionDataNodeListener.getInstance().onRegionCreated(dataRegionId.getId()); return dataRegion; } @@ -892,11 +896,15 @@ public TSStatus deleteDataRegion(DataRegionId regionId) { LOGGER.info(StorageEngineMessages.REMOVING_DATA_REGION, regionId); region.markDeleted(); try { + // Install the creation barrier before draining tasks. Do not hold the DAL lock while + // waiting for receivers/assigners, whose in-flight work may access the manager. + DeletionResourceManager.markRegionDeleted(regionId.getId()); + PipeDataNodeAgent.receiver().iotConsensusV2().releaseReceiverResource(regionId); + PipeInsertionDataNodeListener.getInstance().releaseDataRegion(regionId.getId()); region.abortCompaction(); region.syncDeleteDataFiles(); region.deleteFolder(systemDir); region.deleteDALFolderAndClose(); - PipeDataNodeAgent.receiver().iotConsensusV2().releaseReceiverResource(regionId); switch (CONFIG.getDataRegionConsensusProtocolClass()) { case ConsensusFactory.IOT_CONSENSUS: case ConsensusFactory.IOT_CONSENSUS_V2: diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java index 7eff2b86e004d..3c433e1c97a8b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java @@ -19,8 +19,18 @@ package org.apache.iotdb.db.pipe.consensus.deletion; +import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.consensus.deletion.persist.PageCacheDeletionBuffer; +import org.apache.iotdb.db.pipe.receiver.protocol.iotconsensusv2.IoTConsensusV2ReceiverAgent; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeDataRegionAssigner; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode; +import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.rpc.TSStatusCode; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; @@ -34,12 +44,24 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.lang.reflect.Field; import java.nio.file.Files; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class DeletionResourceManagerCleanupTest { private static final int REGION_ID = 91821; @@ -47,6 +69,7 @@ public class DeletionResourceManagerCleanupTest { private String previousDirectory; private int previousBufferSize; private Logger bufferLogger; + private Logger managerLogger; private Appender appender; @Before @@ -60,9 +83,11 @@ public void setUp() throws Exception { .setIotConsensusV2DeletionFileDir(temporaryFolder.getRoot().getAbsolutePath()); PageCacheDeletionBuffer.setDalBufferSize(1024); DeletionResourceManager.buildForTest(); + DeletionResourceManager.onRegionCreated(REGION_ID); bufferLogger = (Logger) LoggerFactory.getLogger(PageCacheDeletionBuffer.class); appender = mock(Appender.class); bufferLogger.addAppender(appender); + managerLogger = (Logger) LoggerFactory.getLogger(DeletionResourceManager.class); } @After @@ -71,6 +96,8 @@ public void tearDown() { DeletionResourceManager.closeAndRemoveDAL(REGION_ID); } finally { bufferLogger.detachAppender(appender); + managerLogger.detachAppender(appender); + DeletionResourceManager.onRegionCreated(REGION_ID); IoTDBDescriptor.getInstance().getConfig().setIotConsensusV2DeletionFileDir(previousDirectory); PageCacheDeletionBuffer.setDalBufferSize(previousBufferSize); } @@ -97,17 +124,214 @@ public void testCleanupRemovesOrphanedDALWithoutStartingBuffer() throws Exceptio } @Test - public void testCleanupClosesExistingManagerAndAllowsRecreation() { + public void testCleanupBlocksRecreationUntilNewRegionLifecycle() { final DeletionResourceManager manager = DeletionResourceManager.getInstance(REGION_ID); Assert.assertTrue(regionDirectory().exists()); DeletionResourceManager.closeAndRemoveDAL(REGION_ID); Assert.assertFalse(regionDirectory().exists()); + Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID)); + Assert.assertNull(DeletionResourceManager.getInstanceIfPresent(REGION_ID)); + DeletionResourceManager.onRegionCreated(REGION_ID); Assert.assertNotSame(manager, DeletionResourceManager.getInstance(REGION_ID)); Assert.assertTrue(regionDirectory().exists()); } + @Test + public void testLookupDoesNotCreateManager() { + Assert.assertNull(DeletionResourceManager.getInstanceIfPresent(REGION_ID)); + Assert.assertFalse(regionDirectory().exists()); + verify(appender, never()).doAppend(any(ILoggingEvent.class)); + } + + @Test + public void testDeletionBarrierRejectsRetainedManagerRegistration() { + final DeletionResourceManager manager = DeletionResourceManager.getInstance(REGION_ID); + DeletionResourceManager.markRegionDeleted(REGION_ID); + Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID)); + final DeletionResource resource = + manager.registerDeletionResource(mock(AbstractDeleteDataNode.class)); + Assert.assertSame(DeletionResource.Status.FAILURE, resource.waitForResult()); + } + + @Test + public void testOldManagerCannotWriteOrRemoveNewLifecycleDAL() throws Exception { + final DeletionResourceManager old = DeletionResourceManager.getInstance(REGION_ID); + DeletionResourceManager.closeAndRemoveDAL(REGION_ID); + DeletionResourceManager.onRegionCreated(REGION_ID); + DeletionResourceManager.getInstance(REGION_ID); + final File marker = new File(regionDirectory(), "_0-1.deletion"); + Files.write(marker.toPath(), new byte[] {1}); + final DeletionResource rejected = + old.registerDeletionResource(mock(AbstractDeleteDataNode.class)); + Assert.assertSame(DeletionResource.Status.FAILURE, rejected.waitForResult()); + rejected.removeSelf(); + Assert.assertTrue(marker.exists()); + } + + @Test + public void testReferenceReleaseDoesNotHoldPersistCallbackMonitor() { + final AtomicBoolean released = new AtomicBoolean(); + final DeletionResource resource = + new DeletionResource( + mock(AbstractDeleteDataNode.class), + deletion -> { + Assert.assertFalse(Thread.holdsLock(deletion)); + released.set(true); + }, + REGION_ID); + resource.setPipeTaskReferenceCount(1); + resource.decreaseReference(); + Assert.assertTrue(released.get()); + } + + @Test(timeout = 30000) + public void testCleanupWaitsForInFlightInitialization() throws Exception { + final CountDownLatch initializing = new CountDownLatch(1); + final CountDownLatch finishInitialization = new CountDownLatch(1); + doAnswer( + invocation -> { + final ILoggingEvent event = invocation.getArgument(0); + if (DataNodePipeMessages.DELETION_PERSIST_STARTING_TO_PERSIST_CURRENT_WRITING.equals( + event.getMessage())) { + initializing.countDown(); + Assert.assertTrue(finishInitialization.await(10, TimeUnit.SECONDS)); + } + return null; + }) + .when(appender) + .doAppend(any(ILoggingEvent.class)); + final ExecutorService executor = Executors.newFixedThreadPool(2); + try { + final Future creation = + executor.submit(() -> DeletionResourceManager.getInstance(REGION_ID)); + Assert.assertTrue(initializing.await(10, TimeUnit.SECONDS)); + final CountDownLatch cleanupStarted = new CountDownLatch(1); + final Future cleanup = + executor.submit( + () -> { + cleanupStarted.countDown(); + DeletionResourceManager.closeAndRemoveDAL(REGION_ID); + }); + Assert.assertTrue(cleanupStarted.await(10, TimeUnit.SECONDS)); + assertWaiting(cleanup); + Assert.assertTrue(regionDirectory().exists()); + finishInitialization.countDown(); + creation.get(10, TimeUnit.SECONDS); + cleanup.get(10, TimeUnit.SECONDS); + Assert.assertFalse(regionDirectory().exists()); + Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID)); + } finally { + finishInitialization.countDown(); + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } + + @Test(timeout = 30000) + public void testCreationCannotEnterBetweenCloseAndDirectoryRemoval() throws Exception { + DeletionResourceManager.getInstance(REGION_ID); + final CountDownLatch managerClosed = new CountDownLatch(1); + final CountDownLatch removeDirectory = new CountDownLatch(1); + managerLogger.addAppender(appender); + doAnswer( + invocation -> { + final ILoggingEvent event = invocation.getArgument(0); + if (DataNodePipeMessages.DELETION_RESOURCE_MANAGER_FOR_HAS_BEEN_SUCCESSFULLY.equals( + event.getMessage())) { + managerClosed.countDown(); + Assert.assertTrue(removeDirectory.await(10, TimeUnit.SECONDS)); + } + return null; + }) + .when(appender) + .doAppend(any(ILoggingEvent.class)); + final ExecutorService executor = Executors.newFixedThreadPool(2); + try { + final Future cleanup = + executor.submit(() -> DeletionResourceManager.closeAndRemoveDAL(REGION_ID)); + Assert.assertTrue(managerClosed.await(10, TimeUnit.SECONDS)); + final CountDownLatch creationStarted = new CountDownLatch(1); + final Future creation = + executor.submit( + () -> { + creationStarted.countDown(); + return DeletionResourceManager.getInstance(REGION_ID); + }); + Assert.assertTrue(creationStarted.await(10, TimeUnit.SECONDS)); + assertWaiting(creation); + removeDirectory.countDown(); + cleanup.get(10, TimeUnit.SECONDS); + Assert.assertNull(creation.get(10, TimeUnit.SECONDS)); + Assert.assertFalse(regionDirectory().exists()); + } finally { + removeDirectory.countDown(); + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testStorageDeletionStopsReceiverAndAssignerBeforeDALRemoval() throws Exception { + final DataRegionId regionId = new DataRegionId(REGION_ID); + final StorageEngine engine = StorageEngine.getInstance(); + final Field regionsField = StorageEngine.class.getDeclaredField("dataRegionMap"); + regionsField.setAccessible(true); + final Map regions = + (Map) regionsField.get(engine); + final DataRegion region = mock(DataRegion.class); + when(region.getDatabaseName()).thenReturn("root.dal_cleanup"); + when(region.getDataRegionIdString()).thenReturn(Integer.toString(REGION_ID)); + final PipeInsertionDataNodeListener listener = PipeInsertionDataNodeListener.getInstance(); + final Field assignerField = + PipeInsertionDataNodeListener.class.getDeclaredField("dataRegionId2Assigner"); + assignerField.setAccessible(true); + final Map assigners = + (Map) assignerField.get(listener); + final PipeDataRegionAssigner assigner = mock(PipeDataRegionAssigner.class); + final IoTConsensusV2ReceiverAgent receiver = PipeDataNodeAgent.receiver().iotConsensusV2(); + final Field staleField = IoTConsensusV2ReceiverAgent.class.getDeclaredField("staleRegions"); + staleField.setAccessible(true); + final Set stale = (Set) staleField.get(receiver); + DeletionResourceManager.getInstance(REGION_ID); + doAnswer( + invocation -> { + verify(assigner).close(); + Assert.assertFalse(assigners.containsKey(REGION_ID)); + Assert.assertTrue(stale.contains(REGION_ID)); + Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID)); + Assert.assertTrue(regionDirectory().exists()); + DeletionResourceManager.closeAndRemoveDAL(REGION_ID); + return null; + }) + .when(region) + .deleteDALFolderAndClose(); + regions.put(regionId, region); + assigners.put(REGION_ID, assigner); + try { + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), engine.deleteDataRegion(regionId).getCode()); + verify(region).deleteDALFolderAndClose(); + Assert.assertFalse(regionDirectory().exists()); + } finally { + regions.remove(regionId, region); + assigners.remove(REGION_ID, assigner); + listener.onRegionCreated(REGION_ID); + stale.remove(REGION_ID); + } + } + + private void assertWaiting(Future operation) throws Exception { + try { + operation.get(200, TimeUnit.MILLISECONDS); + Assert.fail("Operation must wait for the concurrent lifecycle transition"); + } catch (TimeoutException expected) { + // The other operation is deliberately paused while holding the per-region lock. + } + } + private File regionDirectory() { return new File(temporaryFolder.getRoot(), Integer.toString(REGION_ID)); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java index 9891f86052062..fdc11c91d1d2e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java @@ -53,12 +53,16 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import static org.mockito.AdditionalAnswers.delegatesTo; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; @@ -101,6 +105,7 @@ private void assertQueueInterruptionLogging(boolean closeNormally) throws Except final PageCacheDeletionBuffer deletionBuffer = new PageCacheDeletionBuffer(1, temporaryFolder.newFolder("interrupt").getAbsolutePath()); final CountDownLatch taking = new CountDownLatch(1); + final AtomicBoolean queueInterrupted = new AtomicBoolean(); final AtomicReference worker = new AtomicReference<>(); final Field queueField = PageCacheDeletionBuffer.class.getDeclaredField("deletionResources"); queueField.setAccessible(true); @@ -112,10 +117,15 @@ private void assertQueueInterruptionLogging(boolean closeNormally) throws Except invocation -> { worker.set(Thread.currentThread()); taking.countDown(); - return originalQueue.take(); + try { + return originalQueue.poll(invocation.getArgument(0), invocation.getArgument(1)); + } catch (InterruptedException e) { + queueInterrupted.set(true); + throw e; + } }) .when(queue) - .take(); + .poll(anyLong(), any(TimeUnit.class)); queueField.set(deletionBuffer, queue); final Logger logger = (Logger) LoggerFactory.getLogger(PageCacheDeletionBuffer.class); final Appender appender = mock(Appender.class); @@ -137,6 +147,7 @@ private void assertQueueInterruptionLogging(boolean closeNormally) throws Except deletionBuffer.close(); if (closeNormally) { verify(appender, never()).doAppend(argThat(this::isQueueInterruptionWarning)); + Assert.assertFalse(queueInterrupted.get()); } } finally { logger.detachAppender(appender); @@ -144,6 +155,107 @@ private void assertQueueInterruptionLogging(boolean closeNormally) throws Except } } + @Test(timeout = 30000) + @SuppressWarnings("unchecked") + public void testCloseWhileBatchPollingFlushesWithoutInterruptingWorker() throws Exception { + final PageCacheDeletionBuffer buffer = + new PageCacheDeletionBuffer(1, temporaryFolder.newFolder("batch-close").getAbsolutePath()); + final Field queueField = PageCacheDeletionBuffer.class.getDeclaredField("deletionResources"); + queueField.setAccessible(true); + final BlockingQueue original = + (BlockingQueue) queueField.get(buffer); + final BlockingQueue queue = mock(BlockingQueue.class, delegatesTo(original)); + final CountDownLatch collectingBatch = new CountDownLatch(1); + final CountDownLatch batchCanContinue = new CountDownLatch(1); + final AtomicBoolean queueInterrupted = new AtomicBoolean(); + doAnswer( + invocation -> { + try { + if (invocation.getArgument(1) == TimeUnit.NANOSECONDS) { + collectingBatch.countDown(); + Assert.assertTrue(batchCanContinue.await(10, TimeUnit.SECONDS)); + } + return original.poll(invocation.getArgument(0), invocation.getArgument(1)); + } catch (InterruptedException e) { + queueInterrupted.set(true); + throw e; + } + }) + .when(queue) + .poll(anyLong(), any(TimeUnit.class)); + queueField.set(buffer, queue); + final DeletionResource deletion = createDeletion(1, 1); + final ExecutorService closer = Executors.newSingleThreadExecutor(); + buffer.start(); + try { + buffer.registerDeletionResource(deletion); + Assert.assertTrue(collectingBatch.await(10, TimeUnit.SECONDS)); + final Future close = closer.submit(buffer::close); + final Field closed = PageCacheDeletionBuffer.class.getDeclaredField("isClosed"); + closed.setAccessible(true); + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> closed.getBoolean(buffer)); + batchCanContinue.countDown(); + close.get(10, TimeUnit.SECONDS); + } finally { + batchCanContinue.countDown(); + closer.shutdownNow(); + Assert.assertTrue(closer.awaitTermination(10, TimeUnit.SECONDS)); + } + Assert.assertSame(DeletionResource.Status.SUCCESS, deletion.waitForResult()); + // ThreadPoolExecutor may interrupt an idle worker during shutdown; assert the DAL wait itself. + Assert.assertFalse(queueInterrupted.get()); + Assert.assertTrue(buffer.isAllDeletionFlushed()); + } + + @Test(timeout = 30000) + @SuppressWarnings("unchecked") + public void testFlushTimeoutWarnsAndForceInterruptsBatchPoll() throws Exception { + final PageCacheDeletionBuffer buffer = + new PageCacheDeletionBuffer( + 1, temporaryFolder.newFolder("timeout-close").getAbsolutePath()); + final Field queueField = PageCacheDeletionBuffer.class.getDeclaredField("deletionResources"); + queueField.setAccessible(true); + final BlockingQueue original = + (BlockingQueue) queueField.get(buffer); + final BlockingQueue queue = mock(BlockingQueue.class, delegatesTo(original)); + final CountDownLatch batchPollStarted = new CountDownLatch(1); + final CountDownLatch blockBatch = new CountDownLatch(1); + doAnswer( + invocation -> { + if (invocation.getArgument(1) == TimeUnit.NANOSECONDS) { + batchPollStarted.countDown(); + blockBatch.await(); + } + return original.poll(invocation.getArgument(0), invocation.getArgument(1)); + }) + .when(queue) + .poll(anyLong(), any(TimeUnit.class)); + queueField.set(buffer, queue); + final Logger logger = (Logger) LoggerFactory.getLogger(PageCacheDeletionBuffer.class); + final Appender appender = mock(Appender.class); + logger.addAppender(appender); + final DeletionResource deletion = createDeletion(1, 1); + buffer.start(); + try { + buffer.registerDeletionResource(deletion); + Assert.assertTrue(batchPollStarted.await(10, TimeUnit.SECONDS)); + buffer.close(); + verify(appender) + .doAppend( + argThat( + event -> + event.getLevel() == Level.WARN + && DataNodePipeMessages + .LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D + .equals(event.getMessage()))); + Assert.assertSame(DeletionResource.Status.FAILURE, deletion.waitForResult()); + Assert.assertNotNull(deletion.getCause()); + } finally { + blockBatch.countDown(); + logger.detachAppender(appender); + } + } + private boolean isQueueInterruptionWarning(ILoggingEvent event) { return event.getLevel() == Level.WARN && DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM.equals( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java index 08240ae24f76a..f084c63129ffd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java @@ -19,8 +19,10 @@ package org.apache.iotdb.db.pipe.source.dataregion.realtime.listener; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource; import org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeDataRegionAssigner; +import org.junit.Assert; import org.junit.Test; import java.lang.reflect.Field; @@ -35,6 +37,35 @@ public class PipeInsertionDataNodeListenerTest { + @Test + @SuppressWarnings("unchecked") + public void testRegionDeletionDrainsAssignerAndBlocksLateRegistration() throws Exception { + final PipeInsertionDataNodeListener listener = PipeInsertionDataNodeListener.getInstance(); + final Field field = + PipeInsertionDataNodeListener.class.getDeclaredField("dataRegionId2Assigner"); + field.setAccessible(true); + final ConcurrentMap assigners = + (ConcurrentMap) field.get(listener); + final int id = Integer.MIN_VALUE + 2; + final PipeDataRegionAssigner assigner = mock(PipeDataRegionAssigner.class); + assigners.put(id, assigner); + try { + listener.releaseDataRegion(id); + verify(assigner).close(); + Assert.assertFalse(assigners.containsKey(id)); + listener.startListenAndAssign(id, mock(PipeRealtimeDataRegionSource.class)); + Assert.assertFalse(assigners.containsKey(id)); + listener.onRegionCreated(id); + assigners.put(id, assigner); + final PipeRealtimeDataRegionSource source = mock(PipeRealtimeDataRegionSource.class); + listener.startListenAndAssign(id, source); + verify(assigner).startAssignTo(source); + } finally { + assigners.remove(id); + listener.onRegionCreated(id); + } + } + @Test @SuppressWarnings("unchecked") public void testInsertUsesBarrierInvalidatingPublicationPath() throws Exception {