From 3dcb2572de0e893c70b92bcf29b91d4c4893f84b Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 20 Nov 2026 10:35:30 +0800 Subject: [PATCH 1/4] feat(load): load single-region TsFiles as whole-file pieces through consensus A TsFile whose devices all map to one data region and one time partition no longer has to be split into chunk pieces when the region has multiple replicas. The coordinator copies the source file in 4 MiB whole-file slices, submits each slice as a PIECE in the existing two-phase consensus LOAD transaction, and the receiver reassembles the exact bytes, validates them, and imports the complete TsFile on COMMIT. The local whole-file fast path is kept for a single local replica, and multi-region or time-partition-crossing files keep the split path. --- .../iotdb/db/it/IoTDBLoadTsFileClusterIT.java | 32 ++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 29 +- .../iotdb/db/i18n/StorageEngineMessages.java | 63 +++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 30 +- .../iotdb/db/i18n/StorageEngineMessages.java | 63 +++ .../plan/node/load/LoadSingleTsFileNode.java | 94 +++- .../node/load/LoadTsFileConsensusNode.java | 11 + .../plan/node/load/LoadTsFilePieceNode.java | 22 + .../scheduler/load/LoadTsFileScheduler.java | 79 ++- .../scheduler/load/TsFileLoadStrategy.java | 3 +- .../load/TwoPhaseConsensusLoadStrategy.java | 281 +++++++++- .../storageengine/load/LoadTsFileCleaner.java | 5 + .../storageengine/load/LoadTsFileManager.java | 5 +- .../load/LoadTsFileSnapshot.java | 4 + .../load/TsFileWriterManager.java | 479 +++++++++++++++++- .../load/splitter/TsFileData.java | 2 + .../load/splitter/TsFileDataType.java | 3 +- .../load/splitter/WholeFileData.java | 246 +++++++++ .../LoadSingleTsFileNodeLoadModeTest.java | 145 ++++++ .../TsFileWriterManagerWholeFileTest.java | 108 ++++ .../load/splitter/WholeFileDataTest.java | 85 ++++ 21 files changed, 1753 insertions(+), 36 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java index 99cfba2520ee4..e85ee0c7be7f2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java @@ -237,6 +237,38 @@ public void testLoadTsFileInCluster() throws Exception { } } + /** + * A TsFile whose devices all fall into one time partition and one data region is loaded through + * the whole-file consensus path: three replicas, no chunk splitting, and every replica must end + * up with the same data. + */ + @Test + public void testLoadWholeTsFileIntoSingleRegion() throws Exception { + final long writtenPointCount; + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "load-cluster-whole-file.tsfile"))) { + final List schemas = + Arrays.asList( + new MeasurementSchema(MEASUREMENTS.get(0), TSDataType.INT64, TSEncoding.PLAIN), + new MeasurementSchema(MEASUREMENTS.get(1), TSDataType.INT64, TSEncoding.PLAIN), + new MeasurementSchema(MEASUREMENTS.get(2), TSDataType.INT64, TSEncoding.PLAIN), + new MeasurementSchema(MEASUREMENTS.get(3), TSDataType.INT64, TSEncoding.PLAIN)); + generator.registerTimeseries(DEVICES.get(0), schemas); + generator.generateData(DEVICES.get(0), 1_000, 1L, false); + writtenPointCount = generator.getTotalNumber(); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + createDatabaseAndSchema(statement); + statement.execute("load \"" + tmpDir.getAbsolutePath() + "\""); + Assert.assertEquals(writtenPointCount, queryPointCount(statement)); + } + + // The whole-file transaction must reach all three replicas, not only the coordinator. + awaitAllReplicasHaveData(); + } + /** Four concurrent LOADs on separate time partitions must expose their combined point count. */ @Test public void testLoadConcurrently() throws Exception { diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 378a6381dcc99..25eccf68856f6 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -2773,8 +2773,11 @@ public final class DataNodeQueryMessages { "Device {} is not in the tsFileDevice2IsAligned cache {}."; public static final String LOADTSFILEANALYZER_CURRENT_DATANODE_IS_READ_ONLY_WILL_TRY_TO_CONVERT_TO_TABLETS_AND = "LoadTsFileAnalyzer: Current datanode is read only, will try to convert to tablets and insert later."; - public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT = - "Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%"; + public static final String LOG_LOAD_ANALYSIS_STAGE_ANALYZING_TSFILE_ARG_ARG_ARG_31359828 = + "Load - Analysis Stage: analyzing TsFile {} ({}/{})"; + public static final String + LOG_LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_FILE_ARG_3339452F = + "Load - Analysis Stage: {}/{} TsFiles have been analyzed, progress: {}%, file: {}"; public static final String THE_FILE_ARG_IS_NOT_A_VALID_TSFILE_PLEASE_CHECK_THE_INPUT_FILE = "The file {} is not a valid tsfile. Please check the input file."; public static final String TSFILE_ARG_IS_A_ARG_MODEL_FILE = @@ -2878,6 +2881,25 @@ public final class DataNodeQueryMessages { "The prefix of sourceKey is not 'source.'. Please check the parameters passed in: {}"; public static final String LOADTSFILESCHEDULER_REGION_MIGRATION_WAS_DETECTED_DURING_LOADING_TSFILE_ARG_WILL_CONVERT = "LoadTsFileScheduler: Region migration was detected during loading TsFile {}, will convert to insertion to avoid data loss"; + public static final String LOG_START_LOADING_ARG_TSFILES_BDB79D79 = "Start loading {} TsFiles"; + public static final String LOG_LOAD_TSFILE_ARG_SPLIT_PATH_ARG_A9F2E219 = + "Load TsFile {}, split path={}"; + public static final String LOG_LOADING_TSFILE_ARG_ARG_ARG_F29B2E19 = "Loading TsFile {} ({}/{})"; + public static final String + LOG_LOADING_TSFILE_ARG_ARG_ARG_WITH_STRATEGY_ARG_CURRENTLY_LOADING_ARG_FEA695E6 = + "Loading TsFile {} ({}/{}) with strategy {}, currently loading: {}"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_START_FILESIZE_ARG_D7E5D5F4 = + "Load TsFile {}, phase=dispatch start, fileSize={}"; + public static final String + LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_DISPATCHEDBYTES_ARG_FILESIZE_ARG_PROGRESS_ARG_668540F9 = + "Load TsFile {}, phase=dispatch, dispatchedBytes={}, fileSize={}, progress={}%"; + public static final String + LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_END_PIECES_ARG_BYTES_ARG_ELAPSEDMS_ARG_DB1D69B6 = + "Load TsFile {}, phase=dispatch end, pieces={}, bytes={}, elapsedMs={}"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_START_B225907D = + "Load TsFile {}, phase={} start"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_END_SUCCESS_ARG_ELAPSEDMS_ARG_6126B0D0 = + "Load TsFile {}, phase={} end, success={}, elapsedMs={}"; public static final String LOAD_TSFILE_ARG_SUCCESSFULLY_LOAD_PROCESS_ARG_ARG = "Load TsFile {} Successfully, load process [{}/{}]"; public static final String @@ -4983,4 +5005,7 @@ private DataNodeQueryMessages() {} public static final String LOG_QUERYFAILED_QUERYID_ARG_SQL_ARG_249264E9 = "[QueryFailed] queryId: {}, sql: {}"; + public static final String EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC = + "Whole-file load has no single resolved region for %s."; + } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 91802ff30b0d1..5fb18b3be6ff2 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1452,6 +1452,15 @@ private StorageEngineMessages() {} "Failed to convert data types for tree model statement {}."; public static final String STORAGE_LOG_LOAD_INSERTING_TABLET_TO_CASTING_TYPE_FROM_TO_AE808A8B = "Load: Inserting tablet to {}.{}. Casting type from {} to {}."; + public static final String + LOG_SPLIT_TSFILE_ARG_START_FILESIZE_ARG_CHUNKMETADATA_ARG_FORCEDECODE_ARG_243A182F = + "Split TsFile {} start, fileSize={}, chunkMetadata={}, forceDecode={}"; + public static final String + LOG_SPLIT_TSFILE_ARG_END_COMPLETED_ARG_DEVICES_ARG_ENTIRECHUNKS_ARG_DECODEDCHUNKS_ARG_SKIPPEDCHUNKS_ARG_TIMEPARTITIONS_ARG_DELETIONS_ARG_ELAPSEDMS_ARG_063A3789 = + "Split TsFile {} end, completed={}, devices={}, entireChunks={}, decodedChunks={}, " + + "skippedChunks={}, timePartitions={}, deletions={}, elapsedMs={}"; + public static final String LOG_SKIP_CHUNK_WITHOUT_METADATA_IN_TSFILE_ARG_OFFSET_ARG_DATASIZE_ARG_41F9ED2B = + "Skip chunk without metadata in TsFile {}, offset={}, dataSize={}"; public static final String STORAGE_LOG_TRY_TO_LOAD_TSFILE_V3_INTO_CURRENT_VERSION_V4_FILE_PATH_B8D38E22 = "try to load TsFile V3 into current version (V4), file path: {}"; public static final String STORAGE_LOG_THE_FILE_S_VERSION_NUMBER_IS_HIGHER_THAN_CURRENT_FILE_PATH_6D17349F = @@ -1482,6 +1491,9 @@ private StorageEngineMessages() {} + "will retry later."; public static final String STORAGE_LOG_SUCCESSFULLY_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_ADB5FEC9 = "Successfully auto load tsfile {} (isGeneratedByPipe = {})"; + public static final String + LOG_START_AUTO_LOADING_TSFILE_ARG_ISGENERATEDBYPIPE_ARG_PENDING_ARG_LOADING_ARG_7B71F0D6 = + "Start auto loading TsFile {} (isGeneratedByPipe = {}), pending={}, loading={}"; public static final String STORAGE_LOG_ERROR_OCCURRED_DURING_CREATING_FAIL_DIRECTORY_FOR_ACTIVE_7D3BEB38 = "Error occurred during creating fail directory {} for active load."; public static final String STORAGE_LOG_FAILED_TO_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_STATUS_FILE_F43E9EF7 = @@ -1906,4 +1918,55 @@ private StorageEngineMessages() {} "source files are cleaned up"; public static final String EXCEPTION_TASK_FINISHED_A075ACB1 = "task finished"; + public static final String EXCEPTION_WHOLE_FILE_SLICE_KIND_CANNOT_BE_NULL_1B93752F = + "Whole-file slice kind cannot be null."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878 = + "Whole-file slice payload cannot be empty."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_RANGE_IS_INVALID_OFFSET_D_LENGTH_D_SOURCE_LENGTH_D_402A1599 = + "Whole-file slice range is invalid: offset %d, length %d, source length %d."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24 = + "Whole-file slice time partition is missing."; + + public static final String EXCEPTION_UNKNOWN_WHOLE_FILE_SLICE_KIND_D_WHEN_DESERIALIZING_A_LOAD_PIECE_AE5381F6 = + "Unknown whole-file slice kind %d when deserializing a LOAD piece."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1 = + "Whole-file slice has invalid payload length %d."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_ENDED_AFTER_D_OF_D_PAYLOAD_BYTES_39E892F4 = + "Whole-file slice ended after %d of %d payload bytes."; + + public static final String EXCEPTION_A_WHOLE_FILE_LOAD_PIECE_CANNOT_MIX_WHOLE_FILE_SLICES_AND_CHUNK_DATA_B1297688 = + "A whole-file LOAD piece cannot mix whole-file slices and chunk data."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_CHECKSUM_MISMATCH_AT_OFFSET_D_EXPECTED_D_ACTUAL_D_48238B7E = + "Whole-file slice checksum mismatch at offset %d: expected %d, actual %d."; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_5F80C60D = + "Whole-file staging %s already exists but cannot be resumed."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB = + "Whole-file slice (source length %d, time partition %d) does not match the staged task (source length %d)."; + + public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_DATA_ARRIVED_BEFORE_THE_TSFILE_18FCDD22 = + "Whole-file modification data arrived before the TsFile."; + + public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_SLICE_KIND_S_DOES_NOT_MATCH_THE_STAGED_TASK_KIND_S_7D2A42AA = + "Whole-file modification slice kind %s does not match the staged task kind %s."; + + public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_ADE1CCB8 = + "Whole-file modification staging %s already exists but cannot be resumed."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5 = + "Whole-file slice offset %d exceeds the staged length %d for task %s."; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C = + "Whole-file staging is missing the TsFile of task %s."; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_LENGTH_D_DOES_NOT_MATCH_THE_EXPECTED_LENGTH_D_FOR_TASK_S_89AC32B6 = + "Whole-file staging length %d does not match the expected length %d for task %s."; + } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 02cf8edc0a469..cdcabc28b32b7 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3196,9 +3196,11 @@ public final class DataNodeQueryMessages { public static final String LOADTSFILEANALYZER_CURRENT_DATANODE_IS_READ_ONLY_WILL_TRY_TO_CONVERT_TO_TABLETS_AND = "LoadTsFileAnalyzer: 当前 DataNode 为只读,将尝试转换为 tablet 后再插入。"; - public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT = - - "加载 - 分析阶段:已分析 {}/{} 个 TsFile,进度:{}%"; + public static final String LOG_LOAD_ANALYSIS_STAGE_ANALYZING_TSFILE_ARG_ARG_ARG_31359828 = + "加载 - 分析阶段:正在分析 TsFile {}({}/{})"; + public static final String + LOG_LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_FILE_ARG_3339452F = + "加载 - 分析阶段:已分析 {}/{} 个 TsFile,进度:{}%,文件:{}"; public static final String THE_FILE_ARG_IS_NOT_A_VALID_TSFILE_PLEASE_CHECK_THE_INPUT_FILE = "文件 {} 不是有效的 TsFile,请检查输入文件。"; @@ -3354,6 +3356,25 @@ public final class DataNodeQueryMessages { "LoadTsFileScheduler:加载 TsFile {} 期间检测到 Region 迁移,将转换为 insertion 以避免数据丢失。"; + public static final String LOG_START_LOADING_ARG_TSFILES_BDB79D79 = "开始加载 {} 个 TsFile"; + public static final String LOG_LOAD_TSFILE_ARG_SPLIT_PATH_ARG_A9F2E219 = + "加载 TsFile {},拆分路径={}"; + public static final String LOG_LOADING_TSFILE_ARG_ARG_ARG_F29B2E19 = "正在加载 TsFile {}({}/{})"; + public static final String + LOG_LOADING_TSFILE_ARG_ARG_ARG_WITH_STRATEGY_ARG_CURRENTLY_LOADING_ARG_FEA695E6 = + "正在加载 TsFile {}({}/{}),策略 {},当前正在加载的文件:{}"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_START_FILESIZE_ARG_D7E5D5F4 = + "加载 TsFile {},dispatch 阶段开始,fileSize={}"; + public static final String + LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_DISPATCHEDBYTES_ARG_FILESIZE_ARG_PROGRESS_ARG_668540F9 = + "加载 TsFile {},dispatch 阶段,dispatchedBytes={},fileSize={},进度={}%"; + public static final String + LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_END_PIECES_ARG_BYTES_ARG_ELAPSEDMS_ARG_DB1D69B6 = + "加载 TsFile {},dispatch 阶段结束,pieces={},bytes={},elapsedMs={}"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_START_B225907D = + "加载 TsFile {},阶段 {} 开始"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_END_SUCCESS_ARG_ELAPSEDMS_ARG_6126B0D0 = + "加载 TsFile {},阶段 {} 结束,success={},elapsedMs={}"; public static final String LOAD_TSFILE_ARG_SUCCESSFULLY_LOAD_PROCESS_ARG_ARG = "成功加载 TsFile {},加载进度 [{}/{}]"; public static final String @@ -5684,4 +5705,7 @@ private DataNodeQueryMessages() {} public static final String LOG_QUERYFAILED_QUERYID_ARG_SQL_ARG_249264E9 = "[QueryFailed] queryId: {}, SQL:{}"; + public static final String EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC = + "Whole-file load 未解析出唯一的 region:%s。"; + } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index a2429b5438cd3..29a18cf28d0a0 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1386,6 +1386,15 @@ private StorageEngineMessages() {} "无法转换树模型语句 {} 的数据类型。"; public static final String STORAGE_LOG_LOAD_INSERTING_TABLET_TO_CASTING_TYPE_FROM_TO_AE808A8B = "Load:正在向 {}.{} 插入 tablet。将类型从 {} 转换为 {}。"; + public static final String + LOG_SPLIT_TSFILE_ARG_START_FILESIZE_ARG_CHUNKMETADATA_ARG_FORCEDECODE_ARG_243A182F = + "开始拆分 TsFile {},fileSize={},chunkMetadata={},forceDecode={}"; + public static final String + LOG_SPLIT_TSFILE_ARG_END_COMPLETED_ARG_DEVICES_ARG_ENTIRECHUNKS_ARG_DECODEDCHUNKS_ARG_SKIPPEDCHUNKS_ARG_TIMEPARTITIONS_ARG_DELETIONS_ARG_ELAPSEDMS_ARG_063A3789 = + "结束拆分 TsFile {},completed={},devices={},entireChunks={},decodedChunks={}," + + "skippedChunks={},timePartitions={},deletions={},elapsedMs={}"; + public static final String LOG_SKIP_CHUNK_WITHOUT_METADATA_IN_TSFILE_ARG_OFFSET_ARG_DATASIZE_ARG_41F9ED2B = + "跳过 TsFile {} 中缺少 ChunkMetadata 的 chunk,offset={},dataSize={}"; public static final String STORAGE_LOG_TRY_TO_LOAD_TSFILE_V3_INTO_CURRENT_VERSION_V4_FILE_PATH_B8D38E22 = "尝试将 TsFile V3 加载到当前版本 (V4),文件路径:{}"; public static final String STORAGE_LOG_THE_FILE_S_VERSION_NUMBER_IS_HIGHER_THAN_CURRENT_FILE_PATH_6D17349F = @@ -1410,6 +1419,9 @@ private StorageEngineMessages() {} "拒绝自动加载 TsFile {} (isGeneratedByPipe = {}),原因:当前查询已超时,稍后重试。"; public static final String STORAGE_LOG_SUCCESSFULLY_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_ADB5FEC9 = "成功自动加载 TsFile {} (isGeneratedByPipe = {})"; + public static final String + LOG_START_AUTO_LOADING_TSFILE_ARG_ISGENERATEDBYPIPE_ARG_PENDING_ARG_LOADING_ARG_7B71F0D6 = + "开始自动加载 TsFile {} (isGeneratedByPipe = {}),pending={},loading={}"; public static final String STORAGE_LOG_ERROR_OCCURRED_DURING_CREATING_FAIL_DIRECTORY_FOR_ACTIVE_7D3BEB38 = "为 active load 创建失败目录 {} 时发生错误。"; public static final String STORAGE_LOG_FAILED_TO_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_STATUS_FILE_F43E9EF7 = @@ -1808,4 +1820,55 @@ private StorageEngineMessages() {} public static final String EXCEPTION_SOURCE_FILES_ARE_CLEANED_UP_83638D68 = "源文件已清理"; public static final String EXCEPTION_TASK_FINISHED_A075ACB1 = "任务已完成"; + public static final String EXCEPTION_WHOLE_FILE_SLICE_KIND_CANNOT_BE_NULL_1B93752F = + "Whole-file 切片类型不能为空。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878 = + "Whole-file 切片内容不能为空。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_RANGE_IS_INVALID_OFFSET_D_LENGTH_D_SOURCE_LENGTH_D_402A1599 = + "Whole-file 切片范围非法:offset %d,length %d,source length %d。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24 = + "Whole-file 切片缺少时间分区。"; + + public static final String EXCEPTION_UNKNOWN_WHOLE_FILE_SLICE_KIND_D_WHEN_DESERIALIZING_A_LOAD_PIECE_AE5381F6 = + "反序列化 LOAD piece 时遇到未知的 whole-file 切片类型 %d。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1 = + "Whole-file 切片的 payload 长度非法:%d。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_ENDED_AFTER_D_OF_D_PAYLOAD_BYTES_39E892F4 = + "Whole-file 切片在读取到 %d/%d 字节时提前结束。"; + + public static final String EXCEPTION_A_WHOLE_FILE_LOAD_PIECE_CANNOT_MIX_WHOLE_FILE_SLICES_AND_CHUNK_DATA_B1297688 = + "Whole-file LOAD piece 不能同时包含 whole-file 切片和 chunk 数据。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_CHECKSUM_MISMATCH_AT_OFFSET_D_EXPECTED_D_ACTUAL_D_48238B7E = + "Whole-file 切片在 offset %d 处校验和不一致:期望 %d,实际 %d。"; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_5F80C60D = + "Whole-file staging 文件 %s 已存在,但无法恢复。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB = + "Whole-file 切片(source length %d,time partition %d)与已 staging 的任务(source length %d)不匹配。"; + + public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_DATA_ARRIVED_BEFORE_THE_TSFILE_18FCDD22 = + "Whole-file 修改数据先于 TsFile 到达。"; + + public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_SLICE_KIND_S_DOES_NOT_MATCH_THE_STAGED_TASK_KIND_S_7D2A42AA = + "Whole-file 修改切片类型 %s 与已 staging 的任务类型 %s 不匹配。"; + + public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_ADE1CCB8 = + "Whole-file 修改文件 %s 已存在,但无法恢复。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5 = + "Whole-file 切片 offset %d 超过已 staging 的长度 %d(任务 %s)。"; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C = + "Whole-file staging 缺少任务 %s 的 TsFile。"; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_LENGTH_D_DOES_NOT_MATCH_THE_EXPECTED_LENGTH_D_FOR_TASK_S_89AC32B6 = + "Whole-file staging 长度 %d 与期望长度 %d 不匹配(任务 %s)。"; + } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index c71d98f72504f..aae2e95101c14 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -68,6 +68,9 @@ public class LoadSingleTsFileNode extends WritePlanNode { private TRegionReplicaSet localRegionReplicaSet; + /** The one time partition of a whole-file load, or null when the file has to be split. */ + private TTimePartitionSlot wholeFileTimePartitionSlot; + public LoadSingleTsFileNode( final PlanNodeId id, final TsFileResource resource, @@ -119,19 +122,7 @@ public boolean needDecodeTsFile( return true; } - List> slotList = - new ArrayList<>(resource.getDevices().size() << 1); - for (final IDeviceID device : resource.getDevices()) { - // iterating the index, must present - final TTimePartitionSlot startSlot = - TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(device).get()); - final TTimePartitionSlot endSlot = - TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(device).get()); - slotList.add(new Pair<>(device, startSlot)); - if (!startSlot.equals(endSlot)) { - slotList.add(new Pair<>(device, endSlot)); - } - } + final List> slotList = buildSlotList(); if (slotList.isEmpty()) { throw new IllegalStateException( @@ -153,6 +144,83 @@ public boolean needDecodeTsFile( return needDecodeTsFile; } + /** + * Resolves how this file has to be loaded. A file whose devices all fall into one time slot and + * one data region does not need to be split by chunks even when the region has several replicas: + * the consensus LOAD transaction can carry it as whole-file slices. + */ + public LoadMode resolveLoadMode( + final Function>, List> + partitionFetcher) { + if (needDecodeTsFile) { + return LoadMode.CONSENSUS_SPLIT; + } + + final List> slotList = buildSlotList(); + if (slotList.isEmpty()) { + return LoadMode.CONSENSUS_SPLIT; + } + + // A whole-file load needs exactly one time slot and exactly one target region. Several device + // slots are fine when the partition placement puts all of them into that one region. + final Set timePartitionSlots = new HashSet<>(); + for (final Pair slot : slotList) { + timePartitionSlots.add(slot.right); + } + if (timePartitionSlots.size() != 1) { + return LoadMode.CONSENSUS_SPLIT; + } + + final TTimePartitionSlot firstSlot = timePartitionSlots.iterator().next(); + final Set replicaSets = new HashSet<>(partitionFetcher.apply(slotList)); + if (replicaSets.size() != 1) { + return LoadMode.CONSENSUS_SPLIT; + } + final TRegionReplicaSet replicaSet = replicaSets.iterator().next(); + localRegionReplicaSet = replicaSet; + wholeFileTimePartitionSlot = firstSlot; + + if (replicaSet.getDataNodeLocations().size() == 1) { + return isDispatchedToLocal(replicaSet.getDataNodeLocations().get(0).getInternalEndPoint()) + ? LoadMode.LOCAL_WHOLE_FILE + : LoadMode.CONSENSUS_WHOLE_FILE; + } + return LoadMode.CONSENSUS_WHOLE_FILE; + } + + private List> buildSlotList() { + final List> slotList = + new ArrayList<>(resource.getDevices().size() << 1); + for (final IDeviceID device : resource.getDevices()) { + // iterating the index, must present + final TTimePartitionSlot startSlot = + TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(device).get()); + final TTimePartitionSlot endSlot = + TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(device).get()); + slotList.add(new Pair<>(device, startSlot)); + if (!startSlot.equals(endSlot)) { + slotList.add(new Pair<>(device, endSlot)); + } + } + return slotList; + } + + public TTimePartitionSlot getWholeFileTimePartitionSlot() { + return wholeFileTimePartitionSlot; + } + + /** The load paths a source TsFile can take, resolved before the strategy is created. */ + public enum LoadMode { + /** One local replica and one region: import the local file directly, as before. */ + LOCAL_WHOLE_FILE, + /** + * One region with several replicas: copy the whole file through the consensus LOAD protocol. + */ + CONSENSUS_WHOLE_FILE, + /** More than one region or a file that has to be decoded: split into chunk pieces. */ + CONSENSUS_SPLIT + } + private boolean isDispatchedToLocal(Set replicaSets) { if (replicaSets.size() > 1) { return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileConsensusNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileConsensusNode.java index 44e016bc8804a..b002f6925dd87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileConsensusNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileConsensusNode.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.load.splitter.SerializedSizes; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; +import org.apache.iotdb.db.storageengine.load.splitter.TsFileDataType; import org.apache.tsfile.exception.NotImplementedException; import org.apache.tsfile.exception.write.PageException; @@ -373,6 +374,16 @@ public boolean hasChunkData() { return !tsFileDataList.isEmpty(); } + /** Whether this PIECE carries slices of a whole source file instead of staged TsFileData. */ + public boolean hasWholeFileData() { + for (final TsFileData data : tsFileDataList) { + if (data.getType() == TsFileDataType.WHOLE_FILE) { + return true; + } + } + return false; + } + public Map getTimePartition2ProgressIndex() { return timePartition2ProgressIndex; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java index 745643678bae6..b0a6842635a3b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -53,6 +53,13 @@ public class LoadTsFilePieceNode extends WritePlanNode { private File tsFile; private long pieceIndex; + /** + * Whether this piece carries slices of the complete source TsFile instead of decoded chunk data. + * The receiver recognises the same mode from the {@code WholeFileData} it receives; the flag + * makes the coordinator side explicit and keeps logs/diagnostics unambiguous. + */ + private boolean wholeFile; + private long dataSize; private List tsFileDataList; @@ -76,6 +83,21 @@ public long getPieceIndex() { return pieceIndex; } + public boolean isWholeFilePiece() { + return wholeFile; + } + + public LoadTsFilePieceNode markAsWholeFilePiece() { + this.wholeFile = true; + return this; + } + + /** A piece of a complete source TsFile, carried as whole-file slices instead of chunks. */ + public static LoadTsFilePieceNode wholeFilePiece( + final PlanNodeId id, final File tsFile, final long pieceIndex) { + return new LoadTsFilePieceNode(id, tsFile, pieceIndex).markAsWholeFilePiece(); + } + public long getDataSize() { return dataSize; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index 24c78410dd5a5..bfd4c002eef4a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -278,6 +278,14 @@ public class LoadTsFileScheduler implements IScheduler { /** Global thread-safe set tracking file paths actively being loaded across all schedulers. */ private static final Set ACTIVE_LOADING_FILES = ConcurrentHashMap.newKeySet(); + private static final String LOCAL_SPLIT_PATH = "local"; + + private static final String DISTRIBUTED_SPLIT_PATH = "distributed"; + + private static final String TABLET_REWRITE_SPLIT_PATH = "tablet-rewrite"; + + private static final String WHOLE_FILE_SPLIT_PATH = "whole-file"; + private final MPPQueryContext queryContext; private final QueryStateMachine stateMachine; private final LoadTsFileDispatcherImpl dispatcher; @@ -364,6 +372,7 @@ public void start() { boolean allFilesSucceeded = true; final int totalFiles = tsFileNodeList.size(); + logLoadProgress(DataNodeQueryMessages.LOG_START_LOADING_ARG_TSFILES_BDB79D79, totalFiles); for (int i = 0; i < totalFiles; ++i) { if (stopped) { return; @@ -438,6 +447,7 @@ private boolean processSingleNode( } final TsFileLoadStrategy strategy = resolveStrategy(node, userName); + logLoadFileStart(filePath, index + 1, totalFiles, strategy.getClass().getSimpleName()); final boolean loadSuccess = strategy.execute(node); if (isRegionMigrating(startTimeMs)) { @@ -445,12 +455,12 @@ private boolean processSingleNode( DataNodeQueryMessages .LOADTSFILESCHEDULER_REGION_MIGRATION_WAS_DETECTED_DURING_LOADING_TSFILE_ARG_WILL_CONVERT, filePath); - logCannotLoad(node, index, totalFiles); + logCannotLoad(node, index + 1, totalFiles); return false; } if (!loadSuccess) { - logCannotLoad(node, index, totalFiles); + logCannotLoad(node, index + 1, totalFiles); return false; } @@ -471,20 +481,48 @@ private boolean processSingleNode( */ private TsFileLoadStrategy resolveStrategy(final LoadSingleTsFileNode node, final String userName) throws Exception { + final String filePath = node.getTsFileResource().getTsFilePath(); if (EncryptDBUtils.requiresFileRewrite()) { + logSplitPath(filePath, TABLET_REWRITE_SPLIT_PATH); return new TabletRewriteLoadStrategy(queryContext, isGeneratedByPipe); } // The judgement resolves the partitions of the devices anyway, so the routing reuses them. final DataPartitionResolver partitionResolver = new DataPartitionResolver(partitionFetcher, userName); - final boolean needDecode = node.needDecodeTsFile(partitionResolver::resolve); + final LoadSingleTsFileNode.LoadMode loadMode = node.resolveLoadMode(partitionResolver::resolve); + + switch (loadMode) { + case LOCAL_WHOLE_FILE: + logSplitPath(filePath, LOCAL_SPLIT_PATH); + return new LocalLoadStrategy(queryContext, fragmentId, dispatcher); + case CONSENSUS_WHOLE_FILE: + logSplitPath(filePath, WHOLE_FILE_SPLIT_PATH); + return new TwoPhaseConsensusLoadStrategy( + dispatcher, + partitionResolver, + memoryBlock, + consensusSubmitter, + isGeneratedByPipe, + true); + case CONSENSUS_SPLIT: + default: + logSplitPath(filePath, DISTRIBUTED_SPLIT_PATH); + return new TwoPhaseConsensusLoadStrategy( + dispatcher, + partitionResolver, + memoryBlock, + consensusSubmitter, + isGeneratedByPipe, + false); + } + } - if (!needDecode) { - return new LocalLoadStrategy(queryContext, fragmentId, dispatcher); - } else { - return new TwoPhaseConsensusLoadStrategy( - dispatcher, partitionResolver, memoryBlock, consensusSubmitter, isGeneratedByPipe); + private void logSplitPath(final String filePath, final String splitPath) { + if (!LOGGER.isDebugEnabled()) { + return; } + LOGGER.debug( + DataNodeQueryMessages.LOG_LOAD_TSFILE_ARG_SPLIT_PATH_ARG_A9F2E219, filePath, splitPath); } private boolean isRegionMigrating(final long taskStartTimeMs) { @@ -561,6 +599,31 @@ private void teardown() { } } + private void logLoadFileStart( + final String filePath, final int current, final int total, final String strategy) { + logLoadProgress( + DataNodeQueryMessages.LOG_LOADING_TSFILE_ARG_ARG_ARG_F29B2E19, filePath, current, total); + if (!LOGGER.isDebugEnabled()) { + return; + } + LOGGER.debug( + DataNodeQueryMessages + .LOG_LOADING_TSFILE_ARG_ARG_ARG_WITH_STRATEGY_ARG_CURRENTLY_LOADING_ARG_FEA695E6, + filePath, + current, + total, + strategy, + String.join(", ", ACTIVE_LOADING_FILES)); + } + + private void logLoadProgress(final String message, final Object... args) { + if (isGeneratedByPipe) { + LOGGER.debug(message, args); + } else { + LOGGER.info(message, args); + } + } + private void logLoadSuccess(final String filePath, final int current, final int total) { if (isGeneratedByPipe) { LOGGER.debug( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TsFileLoadStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TsFileLoadStrategy.java index 97b89f8208af1..cb0c9a4b8b7f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TsFileLoadStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TsFileLoadStrategy.java @@ -32,7 +32,8 @@ *
    *
  • {@link LocalLoadStrategy} - no decode needed, dispatch the file to the local region; *
  • {@link TwoPhaseConsensusLoadStrategy} - decode, stream PIECE batches through consensus, - * then PREPARE+COMMIT or ABORT. + * then PREPARE+COMMIT or ABORT. When the whole file belongs to one data region it is copied + * as whole-file pieces instead of being split into chunks. *
*/ public interface TsFileLoadStrategy { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java index aa0ec86cddd07..b465876c71989 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java @@ -32,23 +32,33 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.load.memory.LoadTsFileDataCacheMemoryBlock; import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet; import org.apache.iotdb.db.storageengine.load.splitter.ChunkData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileSplitter; +import org.apache.iotdb.db.storageengine.load.splitter.WholeFileData; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.zip.CRC32C; /** * Two-phase consensus LOAD strategy for files that need decoding. @@ -88,6 +98,15 @@ public class TwoPhaseConsensusLoadStrategy implements TsFileLoadStrategy { private static final long LOAD_CONSENSUS_SUBMIT_RETRY_BACKOFF_MS = 100L; + /** Progress of one file is logged on the first piece, each further 10%, and at least every 5s. */ + static final long LOAD_DISPATCH_PROGRESS_LOG_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(5); + + static final int LOAD_DISPATCH_PROGRESS_BUCKETS = 10; + + private static final String PREPARE_COMMIT_PHASE = "prepare-commit"; + + private static final String ABORT_PHASE = "abort"; + private final LoadTsFileDispatcherImpl dispatcher; /** Resolves the data partitions of the file, reusing the ones the judgement already resolved. */ @@ -97,6 +116,9 @@ public class TwoPhaseConsensusLoadStrategy implements TsFileLoadStrategy { private final LoadConsensusSubmitter consensusSubmitter; private final boolean isGeneratedByPipe; + /** When true the source file is copied as whole-file slices instead of being split by chunks. */ + private final boolean wholeFile; + /** The source file being loaded, kept for the phase-two commands. */ private LoadSingleTsFileNode currentNode; @@ -129,17 +151,40 @@ public class TwoPhaseConsensusLoadStrategy implements TsFileLoadStrategy { private final Map timePartitionSlotToProgressIndex = new HashMap<>(); + private String sourceFilePath = ""; + + private long sourceFileLength; + + private long dispatchedPayloadBytes; + + private long dispatchedPieceCount; + + private int lastLoggedProgressBucket = -1; + + private long lastProgressLogNanos; + public TwoPhaseConsensusLoadStrategy( LoadTsFileDispatcherImpl dispatcher, DataPartitionResolver partitionResolver, LoadTsFileDataCacheMemoryBlock block, LoadConsensusSubmitter consensusSubmitter, boolean isGeneratedByPipe) { + this(dispatcher, partitionResolver, block, consensusSubmitter, isGeneratedByPipe, false); + } + + public TwoPhaseConsensusLoadStrategy( + LoadTsFileDispatcherImpl dispatcher, + DataPartitionResolver partitionResolver, + LoadTsFileDataCacheMemoryBlock block, + LoadConsensusSubmitter consensusSubmitter, + boolean isGeneratedByPipe, + boolean wholeFile) { this.dispatcher = dispatcher; this.partitionResolver = partitionResolver; this.block = block; this.consensusSubmitter = consensusSubmitter; this.isGeneratedByPipe = isGeneratedByPipe; + this.wholeFile = wholeFile; } @Override @@ -152,29 +197,130 @@ public boolean execute(LoadSingleTsFileNode node) { regionTotalBytes.clear(); regionExpectedDataEnd.clear(); timePartitionSlotToProgressIndex.clear(); + dispatchedPayloadBytes = 0L; + dispatchedPieceCount = 0L; + lastLoggedProgressBucket = -1; + lastProgressLogNanos = 0L; + final File sourceFile = node.getTsFileResource().getTsFile(); + sourceFilePath = sourceFile.getAbsolutePath(); + sourceFileLength = sourceFile.length(); + + logLoadProgress( + DataNodeQueryMessages.LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_START_FILESIZE_ARG_D7E5D5F4, + sourceFilePath, + sourceFileLength); long startTime = System.nanoTime(); final boolean isFirstPhaseSuccess; try { isFirstPhaseSuccess = firstPhase(node); } finally { + final long elapsedNanos = System.nanoTime() - startTime; LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost( - LoadTsFileCostMetricsSet.FIRST_PHASE, System.nanoTime() - startTime); + LoadTsFileCostMetricsSet.FIRST_PHASE, elapsedNanos); + logLoadProgress( + DataNodeQueryMessages + .LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_END_PIECES_ARG_BYTES_ARG_ELAPSEDMS_ARG_DB1D69B6, + sourceFilePath, + dispatchedPieceCount, + dispatchedPayloadBytes, + TimeUnit.NANOSECONDS.toMillis(elapsedNanos)); } + final String secondPhaseName = isFirstPhaseSuccess ? PREPARE_COMMIT_PHASE : ABORT_PHASE; + logLoadProgress( + DataNodeQueryMessages.LOG_LOAD_TSFILE_ARG_PHASE_ARG_START_B225907D, + sourceFilePath, + secondPhaseName); startTime = System.nanoTime(); - final boolean isSecondPhaseSuccess; + boolean isSecondPhaseSuccess = false; try { isSecondPhaseSuccess = secondPhase(isFirstPhaseSuccess); + return isFirstPhaseSuccess && isSecondPhaseSuccess; } finally { + final long elapsedNanos = System.nanoTime() - startTime; LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost( - LoadTsFileCostMetricsSet.SECOND_PHASE, System.nanoTime() - startTime); + LoadTsFileCostMetricsSet.SECOND_PHASE, elapsedNanos); + logLoadProgress( + DataNodeQueryMessages + .LOG_LOAD_TSFILE_ARG_PHASE_ARG_END_SUCCESS_ARG_ELAPSEDMS_ARG_6126B0D0, + sourceFilePath, + secondPhaseName, + isSecondPhaseSuccess, + TimeUnit.NANOSECONDS.toMillis(elapsedNanos)); } + } + + private void logDispatchProgress(final long pieceBytes) { + dispatchedPayloadBytes += pieceBytes; + if (!isLoadProgressLogEnabled()) { + return; + } + final long nowNanos = System.nanoTime(); + if (!shouldLogDispatchProgress( + sourceFileLength, + dispatchedPayloadBytes, + lastLoggedProgressBucket, + lastProgressLogNanos, + nowNanos)) { + return; + } + lastLoggedProgressBucket = progressBucket(sourceFileLength, dispatchedPayloadBytes); + lastProgressLogNanos = nowNanos; + logLoadProgress( + DataNodeQueryMessages + .LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_DISPATCHEDBYTES_ARG_FILESIZE_ARG_PROGRESS_ARG_668540F9, + sourceFilePath, + dispatchedPayloadBytes, + sourceFileLength, + progressPercent(dispatchedPayloadBytes, sourceFileLength)); + } + + private void logLoadProgress(final String message, final Object... args) { + if (!LOGGER.isDebugEnabled()) { + return; + } + LOGGER.debug(message, args); + } + + private boolean isLoadProgressLogEnabled() { + return LOGGER.isDebugEnabled(); + } + + static boolean shouldLogDispatchProgress( + final long fileSize, + final long dispatchedBytes, + final int lastBucket, + final long lastLogNanos, + final long nowNanos) { + if (lastLogNanos == 0L) { + return true; + } + if (nowNanos - lastLogNanos >= LOAD_DISPATCH_PROGRESS_LOG_INTERVAL_NANOS) { + return true; + } + return progressBucket(fileSize, dispatchedBytes) > lastBucket; + } - return isFirstPhaseSuccess && isSecondPhaseSuccess; + static int progressBucket(final long fileSize, final long dispatchedBytes) { + if (fileSize <= 0L || dispatchedBytes <= 0L) { + return 0; + } + if (dispatchedBytes >= fileSize) { + return LOAD_DISPATCH_PROGRESS_BUCKETS; + } + return (int) (dispatchedBytes * LOAD_DISPATCH_PROGRESS_BUCKETS / fileSize); + } + + private static String progressPercent(final long dispatchedBytes, final long fileSize) { + final double percent = fileSize <= 0L ? 0.0 : dispatchedBytes * 100.0 / fileSize; + return String.format("%.3f", percent); } private boolean firstPhase(LoadSingleTsFileNode node) { + if (wholeFile) { + return dispatchWholeFile(node); + } final TsFileSplitConsumer pipeline = new TsFileSplitConsumer( node, @@ -205,6 +351,131 @@ private boolean firstPhase(LoadSingleTsFileNode node) { } } + /** How many source bytes one whole-file PIECE carries. Kept well below the RPC frame limit. */ + private static final int WHOLE_FILE_SLICE_SIZE = 4 * 1024 * 1024; + + /** + * Copies one source file through the consensus LOAD transaction: every slice is a PIECE, so the + * consensus layer replicates it and PREPARE/COMMIT still decide the transaction atomically. + */ + private boolean dispatchWholeFile(final LoadSingleTsFileNode node) { + final TRegionReplicaSet replicaSet = node.getLocalRegionReplicaSet(); + final TTimePartitionSlot timePartitionSlot = node.getWholeFileTimePartitionSlot(); + final File sourceFile = node.getTsFileResource().getTsFile(); + if (replicaSet == null || timePartitionSlot == null || sourceFile == null) { + LOGGER.warn( + String.format( + DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, + sourceFile == null ? node.getTsFileResource().getTsFilePath() : sourceFile), + new IllegalStateException( + String.format( + DataNodeQueryMessages + .EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC, + node.getTsFileResource().getTsFilePath()))); + return false; + } + + final TConsensusGroupId regionId = replicaSet.getRegionId(); + final String loadId = + regionLoadIds.computeIfAbsent(regionId, ignored -> UUID.randomUUID().toString()); + regionRoutes.putIfAbsent(regionId, replicaSet); + computeTimePartitionSlotToProgressIndexIfAbsent(timePartitionSlot); + + final long sourceLength = sourceFile.length(); + if (sourceLength <= 0L) { + LOGGER.warn( + String.format( + DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, sourceFile.getAbsolutePath())); + return false; + } + regionExpectedDataEnd + .computeIfAbsent(regionId, ignored -> new HashMap<>()) + .put(timePartitionSlot, sourceLength); + + if (!dispatchWholeFileKind( + sourceFile, WholeFileData.Kind.TSFILE, sourceLength, timePartitionSlot, replicaSet, node)) { + return false; + } + return dispatchWholeFileMods(sourceFile, timePartitionSlot, replicaSet, node); + } + + private boolean dispatchWholeFileMods( + final File sourceFile, + final TTimePartitionSlot timePartitionSlot, + final TRegionReplicaSet replicaSet, + final LoadSingleTsFileNode node) { + File modsFile = ModificationFile.getExclusiveMods(sourceFile); + WholeFileData.Kind kind = WholeFileData.Kind.MODS_V2; + if (!modsFile.isFile()) { + modsFile = ModificationFileV1.getNormalMods(sourceFile); + kind = WholeFileData.Kind.MODS_V1; + } + if (!modsFile.isFile()) { + return true; + } + return dispatchWholeFileKind( + modsFile, kind, modsFile.length(), timePartitionSlot, replicaSet, node); + } + + private boolean dispatchWholeFileKind( + final File file, + final WholeFileData.Kind kind, + final long sourceLength, + final TTimePartitionSlot timePartitionSlot, + final TRegionReplicaSet replicaSet, + final LoadSingleTsFileNode node) { + long offset = 0L; + long pieceIndex = regionPieceCounts.getOrDefault(replicaSet.getRegionId(), 0L); + try (final FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { + while (offset < sourceLength) { + final int expected = (int) Math.min(WHOLE_FILE_SLICE_SIZE, sourceLength - offset); + final ByteBuffer buffer = ByteBuffer.allocate(expected); + int read = 0; + while (read < expected) { + final int current = channel.read(buffer, offset + read); + if (current < 0) { + break; + } + read += current; + } + if (read <= 0) { + LOGGER.warn( + String.format( + DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, file.getAbsolutePath())); + return false; + } + final byte[] payload = Arrays.copyOf(buffer.array(), read); + final LoadTsFilePieceNode pieceNode = + LoadTsFilePieceNode.wholeFilePiece(node.getPlanNodeId(), file, pieceIndex++); + pieceNode.addTsFileData( + new WholeFileData( + kind, + offset, + sourceLength, + timePartitionSlot.getStartTime(), + isGeneratedByPipe, + crc32c(payload), + payload)); + if (!dispatchOnePieceNode(pieceNode, replicaSet)) { + return false; + } + offset += read; + } + } catch (final IOException e) { + LOGGER.warn( + String.format(DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, file.getAbsolutePath()), + e); + return false; + } + return offset == sourceLength; + } + + private static long crc32c(final byte[] payload) { + final CRC32C crc32c = new CRC32C(); + crc32c.update(payload, 0, payload.length); + return crc32c.getValue(); + } + private boolean dispatchOnePieceNode( LoadTsFilePieceNode pieceNode, TRegionReplicaSet replicaSet) { // The first piece of a region pins the route the splitter resolved; a route that a re-resolve @@ -381,6 +652,8 @@ private boolean dispatchConsensusPiece( piece.getDataSize()); regionPieceCounts.merge(regionId, 1L, Long::sum); regionTotalBytes.merge(regionId, piece.getDataSize(), Long::sum); + dispatchedPieceCount++; + logDispatchProgress(piece.getDataSize()); final Map expectedDataEnd = regionExpectedDataEnd.computeIfAbsent(regionId, id -> new HashMap<>()); for (final TsFileData data : pieceNode.getAllTsFileData()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java index 93a537c1e4aca..bafc0c8584dd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java @@ -326,6 +326,11 @@ static long lastRecordedSearchIndex(final File taskDir) throws IOException { /** Verifies that every staged TsFile in the directory has no missing chunks. */ private static boolean isComplete(final File taskDir) { + // A whole-file task is complete once its marker exists: its bytes were validated by PREPARE, + // and it has no chunk records to check. + if (new File(taskDir, TsFileWriterManager.WHOLE_FILE_META_NAME).isFile()) { + return true; + } final File[] files = taskDir.listFiles(LoadTsFileProgress.PROGRESS_FILE_FILTER); if (files == null) { return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 7e373d774668c..4df8cfe852cf5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -350,11 +350,12 @@ public List writePiece(final LoadTsFileConsens } finally { readLock.unlock(); } - if (node.getPieceRefs().isEmpty()) { + if (!node.hasWholeFileData() && node.getPieceRefs().isEmpty()) { // The chunks of this piece have just been written into the staged file, which recorded where // the payload of each of them landed. The WAL entry keeps everything the leader received - // device, headers, statistics and layout - and only replaces the payload with those - // references, so a replica can be given the very same piece later on + // references, so a replica can be given the very same piece later on. A whole-file slice has + // no reference form and is already in the WAL, so it must not be logged a second time. node.setPieceRefs(refs); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java index 64e3e44a524fc..dedb766ce87fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java @@ -167,6 +167,10 @@ private static void copyTask(final File taskDir, final File targetDir) throws IO } private static void discardUnrecordedStagedFiles(final File snapshotTaskDir) throws IOException { + // A whole-file task has no chunk records by design: its marker and its bytes are the state. + if (new File(snapshotTaskDir, TsFileWriterManager.WHOLE_FILE_META_NAME).isFile()) { + return; + } for (final File stagedFile : listFiles(snapshotTaskDir)) { if (!stagedFile.isFile() || !stagedFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java index ca30a78b6df23..a11934f2144c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java @@ -37,13 +37,17 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; import org.apache.iotdb.db.storageengine.load.metrics.LoadPointCountMetrics; import org.apache.iotdb.db.storageengine.load.splitter.ChunkData; import org.apache.iotdb.db.storageengine.load.splitter.ChunkPayloadRef; import org.apache.iotdb.db.storageengine.load.splitter.DeletionData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; +import org.apache.iotdb.db.storageengine.load.splitter.TsFileDataType; +import org.apache.iotdb.db.storageengine.load.splitter.WholeFileData; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.tsfile.common.conf.TSFileConfig; @@ -60,19 +64,28 @@ import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.TsFilePrecalculatedChunkWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -84,6 +97,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import java.util.zip.CRC32C; /** * Manages the physical writing, crash recovery, and consensus transitions (PIECE, PREPARE, COMMIT, @@ -115,6 +129,27 @@ final class TsFileWriterManager { private final Map timePartition2ModificationFile = new ConcurrentHashMap<>(); + private static final String WHOLE_FILE_MAGIC = "LTWHOLE1"; + private static final int WHOLE_FILE_META_VERSION = 1; + static final String WHOLE_FILE_TSFILE_NAME = "whole-file.tsfile"; + static final String WHOLE_FILE_META_NAME = "whole-file.meta"; + + /** Whole-file staging: the source TsFile is copied byte for byte, not rebuilt from chunks. */ + private File wholeFile; + + private File wholeFileMods; + private FileChannel wholeFileChannel; + private FileChannel wholeFileModsChannel; + private long wholeFileSourceLength = -1L; + private long wholeFileSealedLength = -1L; + private long wholeFileModsSourceLength = -1L; + private long wholeFileTimePartitionStart = Long.MIN_VALUE; + private boolean wholeFileGeneratedByPipe; + private WholeFileData.Kind wholeFileModsKind; + private TsFileResource wholeFileResource; + private LoadTsFileProgress wholeFileProgress; + private TTimePartitionSlot wholeFileTimePartition; + private volatile long currentSearchIndex = -1L; private volatile boolean isClosed; @@ -138,11 +173,14 @@ File getTaskDir() { } boolean hasPendingTsFiles() { - return !timePartition2Resource.isEmpty(); + return wholeFile != null || wholeFileChannel != null || !timePartition2Resource.isEmpty(); } boolean hasStagedData() { - return !timePartition2Writer.isEmpty() || !timePartition2ModificationFile.isEmpty(); + return wholeFileChannel != null + || wholeFileResource != null + || !timePartition2Writer.isEmpty() + || !timePartition2ModificationFile.isEmpty(); } // ------------------------------------------------------------------------- @@ -183,6 +221,11 @@ List writePiece(final List tsFileD .EXCEPTION_PRECALCULATED_CHUNK_STAGING_IS_NOT_SUPPORTED_BY_THE_FILE_ENCRYPTION_POLICY_13DA8C32); } + if (containsWholeFileData(tsFileDataList)) { + writeWholeFilePiece(tsFileDataList); + return Collections.emptyList(); + } + for (final TsFileData tsFileData : tsFileDataList) { switch (tsFileData.getType()) { case CHUNK: @@ -205,6 +248,208 @@ List writePiece(final List tsFileD return publishPieceRefs(); } + // ------------------------------------------------------------------------- + // Whole-file staging + // ------------------------------------------------------------------------- + + private static boolean containsWholeFileData(final List tsFileDataList) { + for (final TsFileData data : tsFileDataList) { + if (data.getType() == TsFileDataType.WHOLE_FILE) { + return true; + } + } + return false; + } + + private void writeWholeFilePiece(final List tsFileDataList) throws IOException { + for (final TsFileData data : tsFileDataList) { + if (data.getType() != TsFileDataType.WHOLE_FILE) { + throw new IOException( + StorageEngineMessages + .EXCEPTION_A_WHOLE_FILE_LOAD_PIECE_CANNOT_MIX_WHOLE_FILE_SLICES_AND_CHUNK_DATA_B1297688); + } + writeWholeFileData((WholeFileData) data); + } + if (wholeFileChannel != null) { + wholeFileChannel.force(false); + } + if (wholeFileModsChannel != null) { + wholeFileModsChannel.force(false); + } + } + + private void writeWholeFileData(final WholeFileData data) throws IOException { + final CRC32C crc32c = new CRC32C(); + crc32c.update(data.getPayload(), 0, data.getPayload().length); + if (crc32c.getValue() != data.getChecksum()) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_CHECKSUM_MISMATCH_AT_OFFSET_D_EXPECTED_D_ACTUAL_D_48238B7E, + data.getOffset(), + data.getChecksum(), + crc32c.getValue())); + } + + switch (data.getKind()) { + case TSFILE: + ensureWholeFile(data); + appendWholeFile(wholeFile, wholeFileChannel, data); + return; + case MODS_V1: + case MODS_V2: + ensureWholeFileMods(data); + appendWholeFile(wholeFileMods, wholeFileModsChannel, data); + return; + default: + throw new IOException(StorageEngineMessages.UNSUPPORTED_TSFILE_DATA_TYPE + data.getType()); + } + } + + private void ensureWholeFile(final WholeFileData data) throws IOException { + if (wholeFileChannel != null) { + validateWholeFile(data); + return; + } + if (wholeFile != null) { + // Recovered from disk: only the metadata still has to match the slices that arrive next. + validateWholeFile(data); + if (wholeFileChannel == null) { + wholeFileChannel = + FileChannel.open(wholeFile.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); + } + return; + } + wholeFile = new File(taskDir, WHOLE_FILE_TSFILE_NAME); + if (!wholeFile.createNewFile()) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_5F80C60D, + wholeFile.getAbsolutePath())); + } + wholeFileChannel = + FileChannel.open(wholeFile.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); + wholeFileSourceLength = data.getSourceLength(); + wholeFileTimePartitionStart = data.getTimePartitionStart(); + wholeFileTimePartition = new TTimePartitionSlot(wholeFileTimePartitionStart); + wholeFileGeneratedByPipe = data.isGeneratedByPipe(); + wholeFileProgress = new LoadTsFileProgress(wholeFile); + writeWholeFileMeta(); + } + + private void validateWholeFile(final WholeFileData data) throws IOException { + if (wholeFileSourceLength != data.getSourceLength() + || wholeFileTimePartitionStart != data.getTimePartitionStart() + || wholeFileGeneratedByPipe != data.isGeneratedByPipe()) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB, + data.getSourceLength(), + data.getTimePartitionStart(), + wholeFileSourceLength)); + } + } + + private void ensureWholeFileMods(final WholeFileData data) throws IOException { + if (wholeFile == null) { + throw new IOException( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_MODIFICATION_DATA_ARRIVED_BEFORE_THE_TSFILE_18FCDD22); + } + if (wholeFileModsChannel != null) { + if (wholeFileModsKind != data.getKind() + || wholeFileModsSourceLength != data.getSourceLength()) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_MODIFICATION_SLICE_KIND_S_DOES_NOT_MATCH_THE_STAGED_TASK_KIND_S_7D2A42AA, + data.getKind(), + wholeFileModsKind)); + } + return; + } + wholeFileModsKind = data.getKind(); + wholeFileMods = + data.getKind() == WholeFileData.Kind.MODS_V1 + ? ModificationFileV1.getNormalMods(wholeFile) + : ModificationFile.getExclusiveMods(wholeFile); + if (!wholeFileMods.createNewFile()) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_MODIFICATION_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_ADE1CCB8, + wholeFileMods.getAbsolutePath())); + } + wholeFileModsChannel = + FileChannel.open(wholeFileMods.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); + wholeFileModsSourceLength = data.getSourceLength(); + } + + private void appendWholeFile(final File file, final FileChannel channel, final WholeFileData data) + throws IOException { + final long length = file.length(); + if (data.getOffset() > length) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5, + data.getOffset(), + length, + taskDir.getAbsolutePath())); + } + final long alreadyStagedLong = Math.max(0L, length - data.getOffset()); + if (alreadyStagedLong >= data.getPayload().length) { + return; + } + final int alreadyStaged = (int) alreadyStagedLong; + final ByteBuffer buffer = + ByteBuffer.wrap(data.getPayload(), alreadyStaged, data.getPayload().length - alreadyStaged); + channel.position(length); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.force(false); + } + + private void writeWholeFileMeta() throws IOException { + final File meta = new File(taskDir, WHOLE_FILE_META_NAME); + try (final FileOutputStream fos = new FileOutputStream(meta, false); + final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(fos))) { + ReadWriteIOUtils.write(WHOLE_FILE_MAGIC, out); + ReadWriteIOUtils.write(WHOLE_FILE_META_VERSION, out); + ReadWriteIOUtils.write(wholeFileSourceLength, out); + ReadWriteIOUtils.write(wholeFileTimePartitionStart, out); + ReadWriteIOUtils.write(wholeFileGeneratedByPipe, out); + ReadWriteIOUtils.write(wholeFile.getName(), out); + out.flush(); + fos.getFD().sync(); + } + } + + private static WholeFileMeta readWholeFileMeta(final File meta) throws IOException { + if (!meta.isFile()) { + return null; + } + try (final FileInputStream fis = new FileInputStream(meta); + final DataInputStream in = new DataInputStream(new BufferedInputStream(fis))) { + final String magic = ReadWriteIOUtils.readString(in); + if (!WHOLE_FILE_MAGIC.equals(magic)) { + return null; + } + final int version = ReadWriteIOUtils.readInt(in); + if (version != WHOLE_FILE_META_VERSION) { + return null; + } + return new WholeFileMeta( + ReadWriteIOUtils.readLong(in), + ReadWriteIOUtils.readLong(in), + ReadWriteIOUtils.readBool(in), + ReadWriteIOUtils.readString(in)); + } + } + /** One fsync per staged file of the piece, after every chunk of that piece is written. */ private void forceStagedFiles() throws IOException { for (final TsFilePrecalculatedChunkWriter writer : timePartition2Writer.values()) { @@ -228,6 +473,24 @@ void recordTerminal( final long searchIndex, final Map timePartition2ExpectedDataEnd) throws IOException { + if (wholeFile != null) { + final long expectedDataEnd = + expectedDataEndOf(timePartition2ExpectedDataEnd, wholeFileTimePartition); + if (wholeFileProgress == null + || !canMarkWholeFileTerminal( + terminalOp, + wholeFileSealedLength >= 0L ? wholeFileSealedLength : wholeFile.length(), + expectedDataEnd)) { + LOGGER.warn( + StorageEngineMessages + .LOG_STAGED_FILE_ARG_OF_THE_LOAD_TASK_IS_NOT_CONTINUOUS_SO_IT_CANNOT_BE_MARKED_ARG_AND_THE_TERMINAL_RECORD_IS_WITHHELD_7B21F365, + wholeFile.getAbsolutePath(), + terminalOp); + return; + } + wholeFileProgress.recordTerminal(terminalOp, searchIndex); + return; + } for (final Map.Entry entry : timePartition2Progress.entrySet()) { final TsFilePrecalculatedChunkWriter writer = timePartition2Writer.get(entry.getKey()); @@ -279,6 +542,17 @@ static boolean canMarkTerminal( : progress.getFirstHoleOffset() < 0L && fileLength >= progress.getTotalLength(); } + /** Whether a whole-file staging has the bytes the terminal command of the task may describe. */ + static boolean canMarkWholeFileTerminal( + final LoadTsFileConsensusOp terminalOp, final long fileLength, final long expectedDataEnd) { + if (expectedDataEnd >= 0L && fileLength < expectedDataEnd) { + return false; + } + return terminalOp != LoadTsFileConsensusOp.COMMIT + || expectedDataEnd < 0L + || fileLength == expectedDataEnd; + } + /** Writes a chunk into the target staged file or skips if the byte range was already recorded. */ private void write(final TTimePartitionSlot timePartitionSlot, final ChunkData chunkData) throws IOException, PageException { @@ -519,6 +793,12 @@ void prepare( checkNotClosed(); touch(); + if (wholeFile != null) { + prepareWholeFile( + isGeneratedByPipe, timePartitionProgressIndexMap, timePartition2ExpectedDataEnd); + return; + } + // Validate every announced partition before sealing any file. A missing partition or a file // without durable chunk records must not turn into a successful partial import. validateStagedFiles(timePartition2ExpectedDataEnd); @@ -560,9 +840,83 @@ void prepare( } } + /** + * Seals a whole-file staging: the copied TsFile is validated and its resource is rebuilt. Nothing + * is imported here; COMMIT does that through {@link #loadAll}. + */ + private void prepareWholeFile( + final boolean isGeneratedByPipe, + final Map timePartitionProgressIndexMap, + final Map timePartition2ExpectedDataEnd) + throws IOException, LoadFileException { + if (wholeFile == null || wholeFileProgress == null) { + throw new LoadFileException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C, + taskDir.getAbsolutePath())); + } + + if (wholeFileChannel != null) { + wholeFileChannel.force(false); + } + final long fileLength = wholeFile.length(); + wholeFileSealedLength = fileLength; + final long expectedDataEnd = + expectedDataEndOf(timePartition2ExpectedDataEnd, wholeFileTimePartition); + if ((wholeFileSourceLength >= 0L && fileLength != wholeFileSourceLength) + || (expectedDataEnd >= 0L && fileLength != expectedDataEnd)) { + throw new LoadFileException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_LENGTH_D_DOES_NOT_MATCH_THE_EXPECTED_LENGTH_D_FOR_TASK_S_89AC32B6, + fileLength, + expectedDataEnd >= 0L ? expectedDataEnd : wholeFileSourceLength, + taskDir.getAbsolutePath())); + } + + if (wholeFileChannel != null) { + wholeFileChannel.close(); + wholeFileChannel = null; + } + if (wholeFileModsChannel != null) { + wholeFileModsChannel.force(false); + wholeFileModsChannel.close(); + wholeFileModsChannel = null; + } + + final TsFileResource resource = new TsFileResource(wholeFile); + try (final TsFileSequenceReader reader = + new TsFileSequenceReader(wholeFile.getAbsolutePath())) { + resource.updatePlanIndexes(reader.getMinPlanIndex()); + resource.updatePlanIndexes(reader.getMaxPlanIndex()); + TsFileResourceUtils.updateTsFileResource(reader, resource); + } + resource.setStatus(TsFileResourceStatus.NORMAL); + resource.setGeneratedByPipe(isGeneratedByPipe); + resource.setProgressIndex( + timePartitionProgressIndexMap.getOrDefault( + wholeFileTimePartition, MinimumProgressIndex.INSTANCE)); + resource.serialize(); + + wholeFileResource = resource; + timePartition2Resource.put(wholeFileTimePartition, resource); + timePartition2Progress.put(wholeFileTimePartition, wholeFileProgress); + } + /** Formally imports sealed staged TsFiles into the target storage engine data regions. */ void loadAll(final boolean isGeneratedByPipe, final boolean deleteStagedSource) throws IOException, LoadFileException { + if (wholeFile != null && wholeFileSealedLength < 0L && wholeFile.isFile()) { + wholeFileSealedLength = wholeFile.length(); + } + if (wholeFile != null && wholeFileResource == null) { + throw new LoadFileException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C, + taskDir.getAbsolutePath())); + } for (final Map.Entry entry : timePartition2Resource.entrySet()) { final DataRegion targetDataRegion = dataRegion; @@ -708,6 +1062,11 @@ private long getTsFileWritePointCount(final TsFilePrecalculatedChunkWriter write * writers. */ TsFileWriterManager recoverFromDisk() throws IOException { + final File wholeFileMeta = new File(taskDir, WHOLE_FILE_META_NAME); + if (wholeFileMeta.isFile()) { + return recoverWholeFileFromDisk(wholeFileMeta); + } + final File[] files = taskDir.listFiles(LoadTsFileProgress.PROGRESS_FILE_FILTER); if (files == null) { return this; @@ -830,6 +1189,52 @@ TsFileWriterManager recoverFromDisk() throws IOException { return this; } + private TsFileWriterManager recoverWholeFileFromDisk(final File metaFile) throws IOException { + final WholeFileMeta meta = readWholeFileMeta(metaFile); + if (meta == null) { + return this; + } + wholeFile = new File(taskDir, meta.tsFileName); + if (!wholeFile.isFile()) { + return this; + } + wholeFileChannel = + FileChannel.open(wholeFile.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); + wholeFileSourceLength = meta.sourceLength; + wholeFileSealedLength = -1L; + wholeFileTimePartitionStart = meta.timePartitionStart; + wholeFileTimePartition = new TTimePartitionSlot(wholeFileTimePartitionStart); + wholeFileGeneratedByPipe = meta.generatedByPipe; + wholeFileProgress = new LoadTsFileProgress(wholeFile); + + final File modsV2 = ModificationFile.getExclusiveMods(wholeFile); + final File modsV1 = ModificationFileV1.getNormalMods(wholeFile); + if (modsV2.isFile()) { + wholeFileMods = modsV2; + wholeFileModsKind = WholeFileData.Kind.MODS_V2; + } else if (modsV1.isFile()) { + wholeFileMods = modsV1; + wholeFileModsKind = WholeFileData.Kind.MODS_V1; + } + if (wholeFileMods != null) { + wholeFileModsChannel = + FileChannel.open( + wholeFileMods.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); + wholeFileModsSourceLength = wholeFileMods.length(); + } + + if (new File(wholeFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).isFile()) { + wholeFileSealedLength = wholeFile.length(); + final TsFileResource resource = new TsFileResource(wholeFile); + resource.deserialize(); + resource.setStatus(TsFileResourceStatus.NORMAL); + wholeFileResource = resource; + timePartition2Resource.put(wholeFileTimePartition, resource); + timePartition2Progress.put(wholeFileTimePartition, wholeFileProgress); + } + return this; + } + /** * Replays serialized chunk headers and statistics to restore writer metadata state without * physical pages. @@ -884,6 +1289,53 @@ void close(final boolean retainStagedFiles) { } isClosed = true; + if (wholeFileChannel != null) { + try { + wholeFileChannel.close(); + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.CLOSE_TSFILE_IO_WRITER_ERROR, + wholeFile == null ? taskDir.getPath() : wholeFile.getPath(), + e); + } + wholeFileChannel = null; + } + if (wholeFileModsChannel != null) { + try { + wholeFileModsChannel.close(); + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.CLOSE_MODIFICATION_FILE_ERROR, + wholeFileMods == null ? taskDir.getPath() : wholeFileMods, + e); + } + wholeFileModsChannel = null; + } + if (!retainStagedFiles) { + for (final File staged : + new File[] {wholeFile, wholeFileMods, new File(taskDir, WHOLE_FILE_META_NAME)}) { + if (staged == null) { + continue; + } + try { + deleteIfExistsWithRetry(staged.toPath()); + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.LOG_FAILED_TO_DELETE_ARG_3A7BD6FD, staged.getAbsolutePath(), e); + } + } + if (wholeFileProgress != null) { + try { + deleteIfExistsWithRetry(wholeFileProgress.getProgressFile().toPath()); + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.LOG_FAILED_TO_DELETE_ARG_3A7BD6FD, + wholeFileProgress.getProgressFile().getAbsolutePath(), + e); + } + } + } + for (final TsFilePrecalculatedChunkWriter writer : timePartition2Writer.values()) { try { writer.getOutput().close(); @@ -938,6 +1390,11 @@ void close(final boolean retainStagedFiles) { timePartition2Resource.clear(); timePartition2Progress.clear(); timePartition2ModificationFile.clear(); + wholeFile = null; + wholeFileMods = null; + wholeFileResource = null; + wholeFileProgress = null; + wholeFileTimePartition = null; } // ------------------------------------------------------------------------- @@ -995,6 +1452,24 @@ private long parseTimePartitionStart(final String tsFileName) { tsFileName.length() - TsFileConstant.TSFILE_SUFFIX.length())); } + private static final class WholeFileMeta { + private final long sourceLength; + private final long timePartitionStart; + private final boolean generatedByPipe; + private final String tsFileName; + + private WholeFileMeta( + final long sourceLength, + final long timePartitionStart, + final boolean generatedByPipe, + final String tsFileName) { + this.sourceLength = sourceLength; + this.timePartitionStart = timePartitionStart; + this.generatedByPipe = generatedByPipe; + this.tsFileName = tsFileName; + } + } + /** The bytes this piece added to each staged file, as the refs a replica reads them back with. */ private List publishPieceRefs() { final List refs = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java index 19bc95756b01a..abe0316fd790c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java @@ -70,6 +70,8 @@ static TsFileData deserialize(InputStream stream) return ChunkData.deserialize(stream); case DELETION: return DeletionData.deserialize(stream); + case WHOLE_FILE: + return WholeFileData.deserialize(stream); default: throw new UnsupportedOperationException( StorageEngineMessages.UNKNOWN_TSFILE_DATA_TYPE + type); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileDataType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileDataType.java index 9e3f73f724dd5..1e3e81e0eff3b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileDataType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileDataType.java @@ -21,5 +21,6 @@ public enum TsFileDataType { CHUNK, - DELETION + DELETION, + WHOLE_FILE } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java new file mode 100644 index 0000000000000..4d0e03f4446d3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.load.splitter; + +import org.apache.iotdb.db.i18n.StorageEngineMessages; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +/** + * One slice of a source file that a LOAD task transfers byte for byte. + * + *

A {@link #kind} of {@link Kind#TSFILE} carries a slice of the source TsFile; the modification + * kinds carry a slice of its deletion file. Slices of one file are identified by {@code offset} and + * are applied in order, so a retried slice is skipped instead of being appended twice. + * + *

The source length, time partition and pipe flag are repeated in every slice: the receiver + * creates the staging state on the first slice and needs them to recognise the task and to rebuild + * its resource on PREPARE. + */ +public class WholeFileData implements TsFileData { + + /** The file a slice belongs to. The order is part of the wire format and must not change. */ + public enum Kind { + TSFILE, + MODS_V1, + MODS_V2 + } + + private static final int TYPE_ORDINAL_BYTES = Integer.BYTES; + private static final int KIND_BYTES = Integer.BYTES; + private static final int LONG_BYTES = Long.BYTES; + private static final int BOOLEAN_BYTES = Byte.BYTES; + private static final int PAYLOAD_LENGTH_BYTES = Integer.BYTES; + private static final int METADATA_BYTES = + TYPE_ORDINAL_BYTES + KIND_BYTES + 4 * LONG_BYTES + BOOLEAN_BYTES + PAYLOAD_LENGTH_BYTES; + + private final Kind kind; + private final long offset; + private final long sourceLength; + private final long timePartitionStart; + private final boolean generatedByPipe; + private final long checksum; + private final byte[] payload; + + public WholeFileData( + final Kind kind, + final long offset, + final long sourceLength, + final long timePartitionStart, + final boolean generatedByPipe, + final long checksum, + final byte[] payload) { + if (kind == null) { + throw new IllegalArgumentException( + StorageEngineMessages.EXCEPTION_WHOLE_FILE_SLICE_KIND_CANNOT_BE_NULL_1B93752F); + } + if (payload == null || payload.length == 0) { + throw new IllegalArgumentException( + StorageEngineMessages.EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878); + } + if (offset < 0 || sourceLength < 0 || offset > sourceLength - payload.length) { + throw new IllegalArgumentException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_RANGE_IS_INVALID_OFFSET_D_LENGTH_D_SOURCE_LENGTH_D_402A1599, + offset, + payload.length, + sourceLength)); + } + if (generatedByPipe && timePartitionStart == Long.MIN_VALUE) { + throw new IllegalArgumentException( + StorageEngineMessages.EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24); + } + this.kind = kind; + this.offset = offset; + this.sourceLength = sourceLength; + this.timePartitionStart = timePartitionStart; + this.generatedByPipe = generatedByPipe; + this.checksum = checksum; + this.payload = payload; + } + + public Kind getKind() { + return kind; + } + + public long getOffset() { + return offset; + } + + public long getSourceLength() { + return sourceLength; + } + + public long getTimePartitionStart() { + return timePartitionStart; + } + + public boolean isGeneratedByPipe() { + return generatedByPipe; + } + + public long getChecksum() { + return checksum; + } + + public byte[] getPayload() { + return payload; + } + + @Override + public long getDataSize() { + return payload.length; + } + + @Override + public TsFileDataType getType() { + return TsFileDataType.WHOLE_FILE; + } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + serialize(stream, true); + } + + @Override + public void serialize(final DataOutputStream stream, final boolean includeContent) + throws IOException { + // A whole-file slice has no reference form: its bytes are the payload, so they are always + // written. A caller that asks for references therefore still gets a self-contained slice. + ReadWriteIOUtils.write(getType().ordinal(), stream); + ReadWriteIOUtils.write(kind.ordinal(), stream); + ReadWriteIOUtils.write(offset, stream); + ReadWriteIOUtils.write(sourceLength, stream); + ReadWriteIOUtils.write(timePartitionStart, stream); + ReadWriteIOUtils.write(generatedByPipe, stream); + ReadWriteIOUtils.write(checksum, stream); + ReadWriteIOUtils.write(payload.length, stream); + stream.write(payload); + } + + @Override + public int serializedSize(final boolean includeContent) { + return Math.toIntExact(METADATA_BYTES + (long) payload.length); + } + + public static WholeFileData deserialize(final InputStream stream) throws IOException { + final int kindOrdinal = ReadWriteIOUtils.readInt(stream); + if (kindOrdinal < 0 || kindOrdinal >= Kind.values().length) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_UNKNOWN_WHOLE_FILE_SLICE_KIND_D_WHEN_DESERIALIZING_A_LOAD_PIECE_AE5381F6, + kindOrdinal)); + } + final Kind kind = Kind.values()[kindOrdinal]; + final long offset = ReadWriteIOUtils.readLong(stream); + final long sourceLength = ReadWriteIOUtils.readLong(stream); + final long timePartitionStart = ReadWriteIOUtils.readLong(stream); + final boolean generatedByPipe = ReadWriteIOUtils.readBool(stream); + final long checksum = ReadWriteIOUtils.readLong(stream); + final int payloadLength = ReadWriteIOUtils.readInt(stream); + if (payloadLength <= 0) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1, + payloadLength)); + } + final byte[] payload = new byte[payloadLength]; + int read = 0; + while (read < payloadLength) { + final int current = stream.read(payload, read, payloadLength - read); + if (current < 0) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_ENDED_AFTER_D_OF_D_PAYLOAD_BYTES_39E892F4, + read, + payloadLength)); + } + read += current; + } + return new WholeFileData( + kind, offset, sourceLength, timePartitionStart, generatedByPipe, checksum, payload); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WholeFileData)) { + return false; + } + final WholeFileData that = (WholeFileData) o; + return offset == that.offset + && sourceLength == that.sourceLength + && timePartitionStart == that.timePartitionStart + && generatedByPipe == that.generatedByPipe + && checksum == that.checksum + && kind == that.kind + && java.util.Arrays.equals(payload, that.payload); + } + + @Override + public int hashCode() { + return Objects.hash(kind, offset, sourceLength, timePartitionStart, generatedByPipe, checksum); + } + + @Override + public String toString() { + return "WholeFileData{" + + "kind=" + + kind + + ", offset=" + + offset + + ", length=" + + payload.length + + ", sourceLength=" + + sourceLength + + '}'; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java new file mode 100644 index 0000000000000..40d5b4393c661 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.node.load; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadSingleTsFileNode; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; + +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.Collections; + +public class LoadSingleTsFileNodeLoadModeTest { + + @Test + public void singleDeviceSlotAndTimeSlotUsesWholeFileConsensus() { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final LoadSingleTsFileNode node = nodeWithDevices(0L, 1L, false, device("root.sg.d1")); + + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_WHOLE_FILE, mode); + Assert.assertEquals(region, node.getLocalRegionReplicaSet()); + Assert.assertNotNull(node.getWholeFileTimePartitionSlot()); + } + + @Test + public void severalDeviceSlotsInOneRegionUseWholeFileConsensus() { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final StringArrayDeviceID first = device("root.sg.d1"); + final SeriesPartitionExecutor executor = + SeriesPartitionExecutor.getSeriesPartitionExecutor( + IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionExecutorClass(), + IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionSlotNum()); + + StringArrayDeviceID second = null; + for (int i = 2; i < 10_000 && second == null; i++) { + final StringArrayDeviceID candidate = device("root.sg.d" + i); + if (!executor + .getSeriesPartitionSlot(first) + .equals(executor.getSeriesPartitionSlot(candidate))) { + second = candidate; + } + } + Assert.assertNotNull( + "the test needs two device ids that map to different series slots", second); + + final LoadSingleTsFileNode node = nodeWithDevices(0L, 100L, false, first, second); + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_WHOLE_FILE, mode); + } + + @Test + public void severalTimeSlotsKeepTheSplitPath() { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final long interval = TimePartitionUtils.getTimePartitionInterval(); + final LoadSingleTsFileNode node = + nodeWithDevices(1L, 2 * interval, false, device("root.sg.d1")); + + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_SPLIT, mode); + } + + @Test + public void severalRegionsKeepTheSplitPath() { + final TRegionReplicaSet first = region(1, 1, 2); + final TRegionReplicaSet second = region(2, 3, 4); + final LoadSingleTsFileNode node = nodeWithDevices(0L, 100L, false, device("root.sg.d1")); + + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Arrays.asList(first, second)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_SPLIT, mode); + } + + @Test + public void timeColumnConversionKeepsTheSplitPath() { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final LoadSingleTsFileNode node = nodeWithDevices(0L, 100L, true, device("root.sg.d1")); + + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_SPLIT, mode); + } + + private static StringArrayDeviceID device(final String deviceId) { + return new StringArrayDeviceID(deviceId); + } + + private static LoadSingleTsFileNode nodeWithDevices( + final long startTime, + final long endTime, + final boolean needDecodeTsFile, + final StringArrayDeviceID... devices) { + final TsFileResource resource = new TsFileResource(new File("1.tsfile")); + for (final StringArrayDeviceID device : devices) { + resource.updateStartTime(device, startTime); + resource.updateEndTime(device, endTime); + } + return new LoadSingleTsFileNode( + new PlanNodeId(""), resource, false, "root.sg", false, 1L, needDecodeTsFile); + } + + private static TRegionReplicaSet region(final int regionId, final int... dataNodeIds) { + final TDataNodeLocation[] locations = new TDataNodeLocation[dataNodeIds.length]; + for (int i = 0; i < dataNodeIds.length; i++) { + locations[i] = + new TDataNodeLocation() + .setDataNodeId(dataNodeIds[i]) + .setInternalEndPoint(new TEndPoint("127.0.0.1", 6667 + dataNodeIds[i])); + } + return new TRegionReplicaSet( + new TConsensusGroupId(TConsensusGroupType.DataRegion, regionId), Arrays.asList(locations)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java new file mode 100644 index 0000000000000..affbe2aecf05f --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.load; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.load.splitter.WholeFileData; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Collections; +import java.util.List; +import java.util.zip.CRC32C; + +public class TsFileWriterManagerWholeFileTest { + + private File taskDir; + private TsFileWriterManager writerManager; + + @Before + public void setUp() throws Exception { + taskDir = Files.createTempDirectory("whole-file-staging-test").toFile(); + final DataRegion dataRegion = Mockito.mock(DataRegion.class); + writerManager = new TsFileWriterManager(dataRegion, taskDir); + } + + @After + public void tearDown() throws Exception { + if (writerManager != null) { + writerManager.close(); + } + deleteRecursively(taskDir); + } + + @Test + public void testSlicesAreAppendedInOrderAndDuplicatesAreSkipped() throws Exception { + final byte[] source = new byte[] {0, 1, 2, 3, 4, 5, 6, 7}; + final List firstRefs = + writerManager.writePiece(Collections.singletonList(slice(0, source, 0, 3)), 1L); + Assert.assertTrue(firstRefs.isEmpty()); + final List secondRefs = + writerManager.writePiece(Collections.singletonList(slice(3, source, 3, 5)), 2L); + Assert.assertTrue(secondRefs.isEmpty()); + + // A retried first slice must not append its bytes a second time. + writerManager.writePiece(Collections.singletonList(slice(0, source, 0, 3)), 3L); + + final File staged = new File(taskDir, "whole-file.tsfile"); + Assert.assertTrue(staged.isFile()); + Assert.assertArrayEquals(source, Files.readAllBytes(staged.toPath())); + Assert.assertTrue(writerManager.hasStagedData()); + } + + @Test + public void testHoleIsRejected() throws Exception { + final byte[] source = new byte[] {0, 1, 2, 3}; + try { + writerManager.writePiece(Collections.singletonList(slice(2, source, 2, 2)), 1L); + Assert.fail("a slice that leaves a hole must be rejected"); + } catch (final IOException expected) { + // expected: the receiver cannot append a slice whose predecessor never arrived + } + } + + private static WholeFileData slice( + final int offset, final byte[] source, final int from, final int length) { + final byte[] payload = new byte[length]; + System.arraycopy(source, from, payload, 0, length); + final CRC32C crc32c = new CRC32C(); + crc32c.update(payload, 0, payload.length); + return new WholeFileData( + WholeFileData.Kind.TSFILE, offset, source.length, 0L, false, crc32c.getValue(), payload); + } + + private static void deleteRecursively(final File file) { + final File[] children = file.listFiles(); + if (children != null) { + for (final File child : children) { + deleteRecursively(child); + } + } + file.delete(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java new file mode 100644 index 0000000000000..77ca1d63952fd --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.load.splitter; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.util.zip.CRC32C; + +public class WholeFileDataTest { + + @Test + public void testRoundTripKeepsEveryField() throws Exception { + final byte[] payload = new byte[] {1, 2, 3, 4, 5}; + final CRC32C crc32c = new CRC32C(); + crc32c.update(payload, 0, payload.length); + final WholeFileData data = + new WholeFileData( + WholeFileData.Kind.TSFILE, 10L, 100L, 1_000L, true, crc32c.getValue(), payload); + + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (final DataOutputStream out = new DataOutputStream(baos)) { + data.serialize(out); + } + Assert.assertEquals(data.serializedSize(true), baos.size()); + + final TsFileData deserialized = + TsFileData.deserialize(new ByteArrayInputStream(baos.toByteArray())); + Assert.assertTrue(deserialized instanceof WholeFileData); + final WholeFileData wholeFileData = (WholeFileData) deserialized; + Assert.assertEquals(data, wholeFileData); + Assert.assertArrayEquals(payload, wholeFileData.getPayload()); + Assert.assertEquals(WholeFileData.Kind.TSFILE, wholeFileData.getKind()); + Assert.assertEquals(10L, wholeFileData.getOffset()); + Assert.assertEquals(100L, wholeFileData.getSourceLength()); + Assert.assertEquals(1_000L, wholeFileData.getTimePartitionStart()); + Assert.assertTrue(wholeFileData.isGeneratedByPipe()); + } + + @Test + public void testModificationSliceRoundTrip() throws Exception { + final byte[] payload = new byte[] {9, 8, 7}; + final WholeFileData data = + new WholeFileData(WholeFileData.Kind.MODS_V2, 0L, 3L, -1L, false, 0L, payload); + + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (final DataOutputStream out = new DataOutputStream(baos)) { + data.serialize(out, false); + } + final WholeFileData deserialized = + (WholeFileData) TsFileData.deserialize(new ByteArrayInputStream(baos.toByteArray())); + Assert.assertEquals(data, deserialized); + Assert.assertEquals(WholeFileData.Kind.MODS_V2, deserialized.getKind()); + } + + @Test + public void testRejectsSliceOutsideSourceLength() { + try { + new WholeFileData(WholeFileData.Kind.TSFILE, 95L, 100L, 0L, false, 0L, new byte[10]); + Assert.fail(); + } catch (final IllegalArgumentException expected) { + // expected: the slice must not extend past the declared source length + } + } +} From 9e5323141ea4851764136bfa2ce576dad439c851 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 20 Nov 2026 10:45:58 +0800 Subject: [PATCH 2/4] fix(load): keep the legacy analysis progress message for master callers The branch is based on apache/master, whose LoadTsFileAnalyzer still logs LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT. The working-tree i18n refactor had renamed it, so the two locales must keep the old key as well until the caller is migrated. --- .../en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java | 3 +++ .../zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 25eccf68856f6..42963afba06b3 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -5008,4 +5008,7 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC = "Whole-file load has no single resolved region for %s."; + public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT = + "Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%"; + } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index cdcabc28b32b7..1915cf1a9a3b9 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -5708,4 +5708,7 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC = "Whole-file load 未解析出唯一的 region:%s。"; + public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT = + "加载 - 分析阶段:已分析 {}/{} 个 TsFile,进度:{}%"; + } From 9d44b2a4439e5bd38d29c8f383dd133f9260416f Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 20 Nov 2026 11:22:41 +0800 Subject: [PATCH 3/4] refactor(load): carry only the TsFile in whole-file pieces Deletions are stored in a .mods sidecar and are not part of the TsFile bytes. The whole-file path therefore no longer defines MODS_V1/MODS_V2 slice kinds: a TsFile with a deletion sidecar falls back to the chunk-splitting path, which already transfers its deletions, and the whole-file path only carries the complete .tsfile. --- .../iotdb/db/i18n/StorageEngineMessages.java | 10 -- .../iotdb/db/i18n/StorageEngineMessages.java | 10 -- .../plan/node/load/LoadSingleTsFileNode.java | 11 +++ .../load/TwoPhaseConsensusLoadStrategy.java | 30 +----- .../load/TsFileWriterManager.java | 94 +------------------ .../load/splitter/WholeFileData.java | 59 +++--------- .../LoadSingleTsFileNodeLoadModeTest.java | 30 +++++- .../TsFileWriterManagerWholeFileTest.java | 3 +- .../load/splitter/WholeFileDataTest.java | 22 +---- 9 files changed, 63 insertions(+), 206 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 5fb18b3be6ff2..d8699c8144a52 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1918,8 +1918,6 @@ private StorageEngineMessages() {} "source files are cleaned up"; public static final String EXCEPTION_TASK_FINISHED_A075ACB1 = "task finished"; - public static final String EXCEPTION_WHOLE_FILE_SLICE_KIND_CANNOT_BE_NULL_1B93752F = - "Whole-file slice kind cannot be null."; public static final String EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878 = "Whole-file slice payload cannot be empty."; @@ -1930,8 +1928,6 @@ private StorageEngineMessages() {} public static final String EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24 = "Whole-file slice time partition is missing."; - public static final String EXCEPTION_UNKNOWN_WHOLE_FILE_SLICE_KIND_D_WHEN_DESERIALIZING_A_LOAD_PIECE_AE5381F6 = - "Unknown whole-file slice kind %d when deserializing a LOAD piece."; public static final String EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1 = "Whole-file slice has invalid payload length %d."; @@ -1951,14 +1947,8 @@ private StorageEngineMessages() {} public static final String EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB = "Whole-file slice (source length %d, time partition %d) does not match the staged task (source length %d)."; - public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_DATA_ARRIVED_BEFORE_THE_TSFILE_18FCDD22 = - "Whole-file modification data arrived before the TsFile."; - public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_SLICE_KIND_S_DOES_NOT_MATCH_THE_STAGED_TASK_KIND_S_7D2A42AA = - "Whole-file modification slice kind %s does not match the staged task kind %s."; - public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_ADE1CCB8 = - "Whole-file modification staging %s already exists but cannot be resumed."; public static final String EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5 = "Whole-file slice offset %d exceeds the staged length %d for task %s."; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 29a18cf28d0a0..057231d10e731 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1820,8 +1820,6 @@ private StorageEngineMessages() {} public static final String EXCEPTION_SOURCE_FILES_ARE_CLEANED_UP_83638D68 = "源文件已清理"; public static final String EXCEPTION_TASK_FINISHED_A075ACB1 = "任务已完成"; - public static final String EXCEPTION_WHOLE_FILE_SLICE_KIND_CANNOT_BE_NULL_1B93752F = - "Whole-file 切片类型不能为空。"; public static final String EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878 = "Whole-file 切片内容不能为空。"; @@ -1832,8 +1830,6 @@ private StorageEngineMessages() {} public static final String EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24 = "Whole-file 切片缺少时间分区。"; - public static final String EXCEPTION_UNKNOWN_WHOLE_FILE_SLICE_KIND_D_WHEN_DESERIALIZING_A_LOAD_PIECE_AE5381F6 = - "反序列化 LOAD piece 时遇到未知的 whole-file 切片类型 %d。"; public static final String EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1 = "Whole-file 切片的 payload 长度非法:%d。"; @@ -1853,14 +1849,8 @@ private StorageEngineMessages() {} public static final String EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB = "Whole-file 切片(source length %d,time partition %d)与已 staging 的任务(source length %d)不匹配。"; - public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_DATA_ARRIVED_BEFORE_THE_TSFILE_18FCDD22 = - "Whole-file 修改数据先于 TsFile 到达。"; - public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_SLICE_KIND_S_DOES_NOT_MATCH_THE_STAGED_TASK_KIND_S_7D2A42AA = - "Whole-file 修改切片类型 %s 与已 staging 的任务类型 %s 不匹配。"; - public static final String EXCEPTION_WHOLE_FILE_MODIFICATION_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_ADE1CCB8 = - "Whole-file 修改文件 %s 已存在,但无法恢复。"; public static final String EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5 = "Whole-file 切片 offset %d 超过已 staging 的长度 %d(任务 %s)。"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index aae2e95101c14..a19b1c616a02e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.load.util.LoadUtil; @@ -170,6 +171,11 @@ public LoadMode resolveLoadMode( if (timePartitionSlots.size() != 1) { return LoadMode.CONSENSUS_SPLIT; } + // A complete TsFile has no deletion sidecar. When one exists it is not part of the TsFile + // bytes, so the whole-file path cannot carry it: use the chunk path, which sends deletions. + if (hasModificationFile(resource.getTsFile())) { + return LoadMode.CONSENSUS_SPLIT; + } final TTimePartitionSlot firstSlot = timePartitionSlots.iterator().next(); final Set replicaSets = new HashSet<>(partitionFetcher.apply(slotList)); @@ -188,6 +194,11 @@ public LoadMode resolveLoadMode( return LoadMode.CONSENSUS_WHOLE_FILE; } + private static boolean hasModificationFile(final File tsFile) { + return ModificationFile.getExclusiveMods(tsFile).isFile() + || ModificationFileV1.getNormalMods(tsFile).isFile(); + } + private List> buildSlotList() { final List> slotList = new ArrayList<>(resource.getDevices().size() << 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java index b465876c71989..ffe9b506ed7ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java @@ -32,8 +32,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; -import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.load.memory.LoadTsFileDataCacheMemoryBlock; import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet; import org.apache.iotdb.db.storageengine.load.splitter.ChunkData; @@ -392,34 +390,11 @@ private boolean dispatchWholeFile(final LoadSingleTsFileNode node) { .computeIfAbsent(regionId, ignored -> new HashMap<>()) .put(timePartitionSlot, sourceLength); - if (!dispatchWholeFileKind( - sourceFile, WholeFileData.Kind.TSFILE, sourceLength, timePartitionSlot, replicaSet, node)) { - return false; - } - return dispatchWholeFileMods(sourceFile, timePartitionSlot, replicaSet, node); - } - - private boolean dispatchWholeFileMods( - final File sourceFile, - final TTimePartitionSlot timePartitionSlot, - final TRegionReplicaSet replicaSet, - final LoadSingleTsFileNode node) { - File modsFile = ModificationFile.getExclusiveMods(sourceFile); - WholeFileData.Kind kind = WholeFileData.Kind.MODS_V2; - if (!modsFile.isFile()) { - modsFile = ModificationFileV1.getNormalMods(sourceFile); - kind = WholeFileData.Kind.MODS_V1; - } - if (!modsFile.isFile()) { - return true; - } - return dispatchWholeFileKind( - modsFile, kind, modsFile.length(), timePartitionSlot, replicaSet, node); + return dispatchWholeFileSlices(sourceFile, sourceLength, timePartitionSlot, replicaSet, node); } - private boolean dispatchWholeFileKind( + private boolean dispatchWholeFileSlices( final File file, - final WholeFileData.Kind kind, final long sourceLength, final TTimePartitionSlot timePartitionSlot, final TRegionReplicaSet replicaSet, @@ -449,7 +424,6 @@ private boolean dispatchWholeFileKind( LoadTsFilePieceNode.wholeFilePiece(node.getPlanNodeId(), file, pieceIndex++); pieceNode.addTsFileData( new WholeFileData( - kind, offset, sourceLength, timePartitionSlot.getStartTime(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java index a11934f2144c1..7d4b06d9c1b1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java @@ -37,7 +37,6 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; @@ -137,15 +136,11 @@ final class TsFileWriterManager { /** Whole-file staging: the source TsFile is copied byte for byte, not rebuilt from chunks. */ private File wholeFile; - private File wholeFileMods; private FileChannel wholeFileChannel; - private FileChannel wholeFileModsChannel; private long wholeFileSourceLength = -1L; private long wholeFileSealedLength = -1L; - private long wholeFileModsSourceLength = -1L; private long wholeFileTimePartitionStart = Long.MIN_VALUE; private boolean wholeFileGeneratedByPipe; - private WholeFileData.Kind wholeFileModsKind; private TsFileResource wholeFileResource; private LoadTsFileProgress wholeFileProgress; private TTimePartitionSlot wholeFileTimePartition; @@ -273,9 +268,6 @@ private void writeWholeFilePiece(final List tsFileDataList) throws I if (wholeFileChannel != null) { wholeFileChannel.force(false); } - if (wholeFileModsChannel != null) { - wholeFileModsChannel.force(false); - } } private void writeWholeFileData(final WholeFileData data) throws IOException { @@ -291,19 +283,8 @@ private void writeWholeFileData(final WholeFileData data) throws IOException { crc32c.getValue())); } - switch (data.getKind()) { - case TSFILE: - ensureWholeFile(data); - appendWholeFile(wholeFile, wholeFileChannel, data); - return; - case MODS_V1: - case MODS_V2: - ensureWholeFileMods(data); - appendWholeFile(wholeFileMods, wholeFileModsChannel, data); - return; - default: - throw new IOException(StorageEngineMessages.UNSUPPORTED_TSFILE_DATA_TYPE + data.getType()); - } + ensureWholeFile(data); + appendWholeFile(wholeFile, wholeFileChannel, data); } private void ensureWholeFile(final WholeFileData data) throws IOException { @@ -352,41 +333,6 @@ private void validateWholeFile(final WholeFileData data) throws IOException { } } - private void ensureWholeFileMods(final WholeFileData data) throws IOException { - if (wholeFile == null) { - throw new IOException( - StorageEngineMessages - .EXCEPTION_WHOLE_FILE_MODIFICATION_DATA_ARRIVED_BEFORE_THE_TSFILE_18FCDD22); - } - if (wholeFileModsChannel != null) { - if (wholeFileModsKind != data.getKind() - || wholeFileModsSourceLength != data.getSourceLength()) { - throw new IOException( - String.format( - StorageEngineMessages - .EXCEPTION_WHOLE_FILE_MODIFICATION_SLICE_KIND_S_DOES_NOT_MATCH_THE_STAGED_TASK_KIND_S_7D2A42AA, - data.getKind(), - wholeFileModsKind)); - } - return; - } - wholeFileModsKind = data.getKind(); - wholeFileMods = - data.getKind() == WholeFileData.Kind.MODS_V1 - ? ModificationFileV1.getNormalMods(wholeFile) - : ModificationFile.getExclusiveMods(wholeFile); - if (!wholeFileMods.createNewFile()) { - throw new IOException( - String.format( - StorageEngineMessages - .EXCEPTION_WHOLE_FILE_MODIFICATION_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_ADE1CCB8, - wholeFileMods.getAbsolutePath())); - } - wholeFileModsChannel = - FileChannel.open(wholeFileMods.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); - wholeFileModsSourceLength = data.getSourceLength(); - } - private void appendWholeFile(final File file, final FileChannel channel, final WholeFileData data) throws IOException { final long length = file.length(); @@ -879,11 +825,6 @@ private void prepareWholeFile( wholeFileChannel.close(); wholeFileChannel = null; } - if (wholeFileModsChannel != null) { - wholeFileModsChannel.force(false); - wholeFileModsChannel.close(); - wholeFileModsChannel = null; - } final TsFileResource resource = new TsFileResource(wholeFile); try (final TsFileSequenceReader reader = @@ -1207,22 +1148,6 @@ private TsFileWriterManager recoverWholeFileFromDisk(final File metaFile) throws wholeFileGeneratedByPipe = meta.generatedByPipe; wholeFileProgress = new LoadTsFileProgress(wholeFile); - final File modsV2 = ModificationFile.getExclusiveMods(wholeFile); - final File modsV1 = ModificationFileV1.getNormalMods(wholeFile); - if (modsV2.isFile()) { - wholeFileMods = modsV2; - wholeFileModsKind = WholeFileData.Kind.MODS_V2; - } else if (modsV1.isFile()) { - wholeFileMods = modsV1; - wholeFileModsKind = WholeFileData.Kind.MODS_V1; - } - if (wholeFileMods != null) { - wholeFileModsChannel = - FileChannel.open( - wholeFileMods.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); - wholeFileModsSourceLength = wholeFileMods.length(); - } - if (new File(wholeFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).isFile()) { wholeFileSealedLength = wholeFile.length(); final TsFileResource resource = new TsFileResource(wholeFile); @@ -1300,20 +1225,8 @@ void close(final boolean retainStagedFiles) { } wholeFileChannel = null; } - if (wholeFileModsChannel != null) { - try { - wholeFileModsChannel.close(); - } catch (final IOException e) { - LOGGER.warn( - StorageEngineMessages.CLOSE_MODIFICATION_FILE_ERROR, - wholeFileMods == null ? taskDir.getPath() : wholeFileMods, - e); - } - wholeFileModsChannel = null; - } if (!retainStagedFiles) { - for (final File staged : - new File[] {wholeFile, wholeFileMods, new File(taskDir, WHOLE_FILE_META_NAME)}) { + for (final File staged : new File[] {wholeFile, new File(taskDir, WHOLE_FILE_META_NAME)}) { if (staged == null) { continue; } @@ -1391,7 +1304,6 @@ void close(final boolean retainStagedFiles) { timePartition2Progress.clear(); timePartition2ModificationFile.clear(); wholeFile = null; - wholeFileMods = null; wholeFileResource = null; wholeFileProgress = null; wholeFileTimePartition = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java index 4d0e03f4446d3..c0abcf29becf2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java @@ -26,37 +26,31 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; import java.util.Objects; /** - * One slice of a source file that a LOAD task transfers byte for byte. + * One slice of a complete source TsFile that a LOAD task transfers byte for byte. * - *

A {@link #kind} of {@link Kind#TSFILE} carries a slice of the source TsFile; the modification - * kinds carry a slice of its deletion file. Slices of one file are identified by {@code offset} and - * are applied in order, so a retried slice is skipped instead of being appended twice. + *

The deletion sidecar of a TsFile is not part of this record. A file that has modifications is + * not loaded through the whole-file path at all: it falls back to the chunk-splitting path, which + * already transfers its deletions. The whole-file path therefore only ever carries the {@code + * .tsfile} bytes, and a receiver can rebuild the exact file without any extra artifact. * - *

The source length, time partition and pipe flag are repeated in every slice: the receiver - * creates the staging state on the first slice and needs them to recognise the task and to rebuild - * its resource on PREPARE. + *

Slices are identified by {@code offset} and are applied in order, so a retried slice is + * skipped instead of being appended twice. The source length, time partition and pipe flag are + * repeated in every slice: the receiver creates the staging state on the first slice and needs them + * to recognise the task and to rebuild its resource on PREPARE. */ public class WholeFileData implements TsFileData { - /** The file a slice belongs to. The order is part of the wire format and must not change. */ - public enum Kind { - TSFILE, - MODS_V1, - MODS_V2 - } - private static final int TYPE_ORDINAL_BYTES = Integer.BYTES; - private static final int KIND_BYTES = Integer.BYTES; private static final int LONG_BYTES = Long.BYTES; private static final int BOOLEAN_BYTES = Byte.BYTES; private static final int PAYLOAD_LENGTH_BYTES = Integer.BYTES; private static final int METADATA_BYTES = - TYPE_ORDINAL_BYTES + KIND_BYTES + 4 * LONG_BYTES + BOOLEAN_BYTES + PAYLOAD_LENGTH_BYTES; + TYPE_ORDINAL_BYTES + 4 * LONG_BYTES + BOOLEAN_BYTES + PAYLOAD_LENGTH_BYTES; - private final Kind kind; private final long offset; private final long sourceLength; private final long timePartitionStart; @@ -65,17 +59,12 @@ public enum Kind { private final byte[] payload; public WholeFileData( - final Kind kind, final long offset, final long sourceLength, final long timePartitionStart, final boolean generatedByPipe, final long checksum, final byte[] payload) { - if (kind == null) { - throw new IllegalArgumentException( - StorageEngineMessages.EXCEPTION_WHOLE_FILE_SLICE_KIND_CANNOT_BE_NULL_1B93752F); - } if (payload == null || payload.length == 0) { throw new IllegalArgumentException( StorageEngineMessages.EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878); @@ -93,7 +82,6 @@ public WholeFileData( throw new IllegalArgumentException( StorageEngineMessages.EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24); } - this.kind = kind; this.offset = offset; this.sourceLength = sourceLength; this.timePartitionStart = timePartitionStart; @@ -102,10 +90,6 @@ public WholeFileData( this.payload = payload; } - public Kind getKind() { - return kind; - } - public long getOffset() { return offset; } @@ -151,7 +135,6 @@ public void serialize(final DataOutputStream stream, final boolean includeConten // A whole-file slice has no reference form: its bytes are the payload, so they are always // written. A caller that asks for references therefore still gets a self-contained slice. ReadWriteIOUtils.write(getType().ordinal(), stream); - ReadWriteIOUtils.write(kind.ordinal(), stream); ReadWriteIOUtils.write(offset, stream); ReadWriteIOUtils.write(sourceLength, stream); ReadWriteIOUtils.write(timePartitionStart, stream); @@ -167,15 +150,6 @@ public int serializedSize(final boolean includeContent) { } public static WholeFileData deserialize(final InputStream stream) throws IOException { - final int kindOrdinal = ReadWriteIOUtils.readInt(stream); - if (kindOrdinal < 0 || kindOrdinal >= Kind.values().length) { - throw new IOException( - String.format( - StorageEngineMessages - .EXCEPTION_UNKNOWN_WHOLE_FILE_SLICE_KIND_D_WHEN_DESERIALIZING_A_LOAD_PIECE_AE5381F6, - kindOrdinal)); - } - final Kind kind = Kind.values()[kindOrdinal]; final long offset = ReadWriteIOUtils.readLong(stream); final long sourceLength = ReadWriteIOUtils.readLong(stream); final long timePartitionStart = ReadWriteIOUtils.readLong(stream); @@ -204,7 +178,7 @@ public static WholeFileData deserialize(final InputStream stream) throws IOExcep read += current; } return new WholeFileData( - kind, offset, sourceLength, timePartitionStart, generatedByPipe, checksum, payload); + offset, sourceLength, timePartitionStart, generatedByPipe, checksum, payload); } @Override @@ -221,21 +195,18 @@ public boolean equals(final Object o) { && timePartitionStart == that.timePartitionStart && generatedByPipe == that.generatedByPipe && checksum == that.checksum - && kind == that.kind - && java.util.Arrays.equals(payload, that.payload); + && Arrays.equals(payload, that.payload); } @Override public int hashCode() { - return Objects.hash(kind, offset, sourceLength, timePartitionStart, generatedByPipe, checksum); + return Objects.hash(offset, sourceLength, timePartitionStart, generatedByPipe, checksum); } @Override public String toString() { return "WholeFileData{" - + "kind=" - + kind - + ", offset=" + + "offset=" + offset + ", length=" + payload.length diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java index 40d5b4393c661..f5ee0feef1c5f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadSingleTsFileNode; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.file.metadata.StringArrayDeviceID; @@ -36,6 +37,7 @@ import org.junit.Test; import java.io.File; +import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; @@ -92,6 +94,23 @@ public void severalTimeSlotsKeepTheSplitPath() { Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_SPLIT, mode); } + @Test + public void deletionSidecarKeepsTheSplitPath() throws Exception { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final File tsFile = Files.createTempFile("load-mode", ".tsfile").toFile(); + final File mods = ModificationFile.getExclusiveMods(tsFile); + Assert.assertTrue(mods.createNewFile()); + try { + final LoadSingleTsFileNode node = nodeWithFile(tsFile, 0L, 1L, false, device("root.sg.d1")); + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_SPLIT, mode); + } finally { + Files.deleteIfExists(mods.toPath()); + Files.deleteIfExists(tsFile.toPath()); + } + } + @Test public void severalRegionsKeepTheSplitPath() { final TRegionReplicaSet first = region(1, 1, 2); @@ -122,7 +141,16 @@ private static LoadSingleTsFileNode nodeWithDevices( final long endTime, final boolean needDecodeTsFile, final StringArrayDeviceID... devices) { - final TsFileResource resource = new TsFileResource(new File("1.tsfile")); + return nodeWithFile(new File("1.tsfile"), startTime, endTime, needDecodeTsFile, devices); + } + + private static LoadSingleTsFileNode nodeWithFile( + final File tsFile, + final long startTime, + final long endTime, + final boolean needDecodeTsFile, + final StringArrayDeviceID... devices) { + final TsFileResource resource = new TsFileResource(tsFile); for (final StringArrayDeviceID device : devices) { resource.updateStartTime(device, startTime); resource.updateEndTime(device, endTime); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java index affbe2aecf05f..7181108a266a3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java @@ -92,8 +92,7 @@ private static WholeFileData slice( System.arraycopy(source, from, payload, 0, length); final CRC32C crc32c = new CRC32C(); crc32c.update(payload, 0, payload.length); - return new WholeFileData( - WholeFileData.Kind.TSFILE, offset, source.length, 0L, false, crc32c.getValue(), payload); + return new WholeFileData(offset, source.length, 0L, false, crc32c.getValue(), payload); } private static void deleteRecursively(final File file) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java index 77ca1d63952fd..cc1c70d0dfb6b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java @@ -35,8 +35,7 @@ public void testRoundTripKeepsEveryField() throws Exception { final CRC32C crc32c = new CRC32C(); crc32c.update(payload, 0, payload.length); final WholeFileData data = - new WholeFileData( - WholeFileData.Kind.TSFILE, 10L, 100L, 1_000L, true, crc32c.getValue(), payload); + new WholeFileData(10L, 100L, 1_000L, true, crc32c.getValue(), payload); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (final DataOutputStream out = new DataOutputStream(baos)) { @@ -50,33 +49,16 @@ public void testRoundTripKeepsEveryField() throws Exception { final WholeFileData wholeFileData = (WholeFileData) deserialized; Assert.assertEquals(data, wholeFileData); Assert.assertArrayEquals(payload, wholeFileData.getPayload()); - Assert.assertEquals(WholeFileData.Kind.TSFILE, wholeFileData.getKind()); Assert.assertEquals(10L, wholeFileData.getOffset()); Assert.assertEquals(100L, wholeFileData.getSourceLength()); Assert.assertEquals(1_000L, wholeFileData.getTimePartitionStart()); Assert.assertTrue(wholeFileData.isGeneratedByPipe()); } - @Test - public void testModificationSliceRoundTrip() throws Exception { - final byte[] payload = new byte[] {9, 8, 7}; - final WholeFileData data = - new WholeFileData(WholeFileData.Kind.MODS_V2, 0L, 3L, -1L, false, 0L, payload); - - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (final DataOutputStream out = new DataOutputStream(baos)) { - data.serialize(out, false); - } - final WholeFileData deserialized = - (WholeFileData) TsFileData.deserialize(new ByteArrayInputStream(baos.toByteArray())); - Assert.assertEquals(data, deserialized); - Assert.assertEquals(WholeFileData.Kind.MODS_V2, deserialized.getKind()); - } - @Test public void testRejectsSliceOutsideSourceLength() { try { - new WholeFileData(WholeFileData.Kind.TSFILE, 95L, 100L, 0L, false, 0L, new byte[10]); + new WholeFileData(95L, 100L, 0L, false, 0L, new byte[10]); Assert.fail(); } catch (final IllegalArgumentException expected) { // expected: the slice must not extend past the declared source length From 11015ba1eac4b34464ac55560596b3c41d613c4c Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 20 Nov 2026 11:32:10 +0800 Subject: [PATCH 4/4] feat(load): carry whole-file deletions as DeletionData pieces The deletion sidecar is not part of the TsFile bytes, but it does not need a separate MODS slice kind. The coordinator now reads the source .mods with ModificationFile.readAllModifications and sends every entry as the existing DeletionData in the same two-phase LOAD transaction. The receiver stages those entries in the .mods sidecar of the whole-file staging, and loadNewTsFile applies them on COMMIT. --- .../plan/node/load/LoadSingleTsFileNode.java | 11 ---- .../load/TwoPhaseConsensusLoadStrategy.java | 53 ++++++++++++++++ .../storageengine/load/LoadTsFileManager.java | 9 +-- .../load/TsFileWriterManager.java | 62 ++++++++++++++++++- .../LoadSingleTsFileNodeLoadModeTest.java | 4 +- 5 files changed, 120 insertions(+), 19 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index a19b1c616a02e..aae2e95101c14 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -32,7 +32,6 @@ import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.load.util.LoadUtil; @@ -171,11 +170,6 @@ public LoadMode resolveLoadMode( if (timePartitionSlots.size() != 1) { return LoadMode.CONSENSUS_SPLIT; } - // A complete TsFile has no deletion sidecar. When one exists it is not part of the TsFile - // bytes, so the whole-file path cannot carry it: use the chunk path, which sends deletions. - if (hasModificationFile(resource.getTsFile())) { - return LoadMode.CONSENSUS_SPLIT; - } final TTimePartitionSlot firstSlot = timePartitionSlots.iterator().next(); final Set replicaSets = new HashSet<>(partitionFetcher.apply(slotList)); @@ -194,11 +188,6 @@ public LoadMode resolveLoadMode( return LoadMode.CONSENSUS_WHOLE_FILE; } - private static boolean hasModificationFile(final File tsFile) { - return ModificationFile.getExclusiveMods(tsFile).isFile() - || ModificationFileV1.getNormalMods(tsFile).isFile(); - } - private List> buildSlotList() { final List> slotList = new ArrayList<>(resource.getDevices().size() << 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java index ffe9b506ed7ee..80488837d7d0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java @@ -32,9 +32,12 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.load.memory.LoadTsFileDataCacheMemoryBlock; import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet; import org.apache.iotdb.db.storageengine.load.splitter.ChunkData; +import org.apache.iotdb.db.storageengine.load.splitter.DeletionData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileSplitter; import org.apache.iotdb.db.storageengine.load.splitter.WholeFileData; @@ -53,6 +56,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -393,6 +397,55 @@ private boolean dispatchWholeFile(final LoadSingleTsFileNode node) { return dispatchWholeFileSlices(sourceFile, sourceLength, timePartitionSlot, replicaSet, node); } + /** How many deletion bytes one PIECE carries; the mods themselves are usually small. */ + private static final int WHOLE_FILE_DELETION_PIECE_SIZE = 1024 * 1024; + + /** + * Sends the deletion sidecar of the source TsFile as the existing {@code DeletionData} records. + * They are part of the same consensus LOAD transaction, so the receiver stages them next to the + * whole-file bytes and {@code loadNewTsFile} applies them on COMMIT. + */ + private boolean dispatchWholeFileDeletions( + final File sourceFile, + final TTimePartitionSlot timePartitionSlot, + final TRegionReplicaSet replicaSet, + final LoadSingleTsFileNode node) { + final List deletions; + try { + deletions = ModificationFile.readAllModifications(sourceFile, true); + } catch (final IOException e) { + LOGGER.warn( + String.format( + DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, sourceFile.getAbsolutePath()), + e); + return false; + } + if (deletions.isEmpty()) { + return true; + } + + long pieceIndex = regionPieceCounts.getOrDefault(replicaSet.getRegionId(), 0L); + LoadTsFilePieceNode pieceNode = null; + long pieceBytes = 0L; + for (final ModEntry deletion : deletions) { + final DeletionData data = new DeletionData(deletion); + if (pieceNode == null) { + pieceNode = + LoadTsFilePieceNode.wholeFilePiece(node.getPlanNodeId(), sourceFile, pieceIndex++); + } else if (pieceBytes + data.getDataSize() > WHOLE_FILE_DELETION_PIECE_SIZE) { + if (!dispatchOnePieceNode(pieceNode, replicaSet)) { + return false; + } + pieceNode = + LoadTsFilePieceNode.wholeFilePiece(node.getPlanNodeId(), sourceFile, pieceIndex++); + pieceBytes = 0L; + } + pieceNode.addTsFileData(data); + pieceBytes += data.getDataSize(); + } + return pieceNode == null || dispatchOnePieceNode(pieceNode, replicaSet); + } + private boolean dispatchWholeFileSlices( final File file, final long sourceLength, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 4df8cfe852cf5..ab5f5bd29f76a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -338,19 +338,20 @@ public List writePiece(final LoadTsFileConsens // of // a staged file can be read back command by command. final List refs; + final boolean wholeFileTask; if (stopped) { throw newStoppedIOException(); } final Lock readLock = writersLock.readLock(); readLock.lock(); try { - refs = - getOrCreateWriterManager(node.getLoadId()) - .writePiece(node.getTsFileDataList(), node.getSearchIndex()); + final TsFileWriterManager writerManager = getOrCreateWriterManager(node.getLoadId()); + wholeFileTask = writerManager.isWholeFileTask(); + refs = writerManager.writePiece(node.getTsFileDataList(), node.getSearchIndex()); } finally { readLock.unlock(); } - if (!node.hasWholeFileData() && node.getPieceRefs().isEmpty()) { + if (!wholeFileTask && !node.hasWholeFileData() && node.getPieceRefs().isEmpty()) { // The chunks of this piece have just been written into the staged file, which recorded where // the payload of each of them landed. The WAL entry keeps everything the leader received - // device, headers, statistics and layout - and only replaces the payload with those diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java index 7d4b06d9c1b1a..ee4674256177a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java @@ -141,6 +141,7 @@ final class TsFileWriterManager { private long wholeFileSealedLength = -1L; private long wholeFileTimePartitionStart = Long.MIN_VALUE; private boolean wholeFileGeneratedByPipe; + private ModificationFile wholeFileModificationFile; private TsFileResource wholeFileResource; private LoadTsFileProgress wholeFileProgress; private TTimePartitionSlot wholeFileTimePartition; @@ -167,6 +168,11 @@ File getTaskDir() { return taskDir; } + /** Whether this task stages a complete source TsFile instead of per-partition chunks. */ + boolean isWholeFileTask() { + return wholeFile != null; + } + boolean hasPendingTsFiles() { return wholeFile != null || wholeFileChannel != null || !timePartition2Resource.isEmpty(); } @@ -228,7 +234,11 @@ List writePiece(final List tsFileD write(chunkData.getTimePartitionSlot(), chunkData); break; case DELETION: - writeDeletion((DeletionData) tsFileData); + if (wholeFile != null) { + writeWholeFileDeletion((DeletionData) tsFileData); + } else { + writeDeletion((DeletionData) tsFileData); + } break; default: throw new IOException( @@ -287,6 +297,29 @@ private void writeWholeFileData(final WholeFileData data) throws IOException { appendWholeFile(wholeFile, wholeFileChannel, data); } + /** + * Writes one deletion of a whole-file task to the sidecar of the staged TsFile. {@code + * loadNewTsFile} picks that sidecar up on COMMIT, exactly as it does for a chunk-staged file. + */ + private void writeWholeFileDeletion(final DeletionData deletionData) throws IOException { + if (wholeFile == null) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C, + taskDir.getAbsolutePath())); + } + if (wholeFileModificationFile == null) { + final File modsFile = ModificationFile.getExclusiveMods(wholeFile); + if (!modsFile.exists() && !modsFile.createNewFile()) { + throw new IOException( + StorageEngineMessages.EXCEPTION_FAILED_TO_CREATE_MODIFICATION_FILE_0D020A0C); + } + wholeFileModificationFile = new ModificationFile(modsFile, false); + } + deletionData.writeToModificationFile(wholeFileModificationFile); + } + private void ensureWholeFile(final WholeFileData data) throws IOException { if (wholeFileChannel != null) { validateWholeFile(data); @@ -825,6 +858,10 @@ private void prepareWholeFile( wholeFileChannel.close(); wholeFileChannel = null; } + if (wholeFileModificationFile != null) { + wholeFileModificationFile.close(); + wholeFileModificationFile = null; + } final TsFileResource resource = new TsFileResource(wholeFile); try (final TsFileSequenceReader reader = @@ -1147,6 +1184,10 @@ private TsFileWriterManager recoverWholeFileFromDisk(final File metaFile) throws wholeFileTimePartition = new TTimePartitionSlot(wholeFileTimePartitionStart); wholeFileGeneratedByPipe = meta.generatedByPipe; wholeFileProgress = new LoadTsFileProgress(wholeFile); + final File modsFile = ModificationFile.getExclusiveMods(wholeFile); + if (modsFile.isFile()) { + wholeFileModificationFile = new ModificationFile(modsFile, false); + } if (new File(wholeFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).isFile()) { wholeFileSealedLength = wholeFile.length(); @@ -1225,8 +1266,24 @@ void close(final boolean retainStagedFiles) { } wholeFileChannel = null; } + if (wholeFileModificationFile != null) { + try { + wholeFileModificationFile.close(); + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.CLOSE_MODIFICATION_FILE_ERROR, + wholeFileModificationFile.getFile(), + e); + } + wholeFileModificationFile = null; + } if (!retainStagedFiles) { - for (final File staged : new File[] {wholeFile, new File(taskDir, WHOLE_FILE_META_NAME)}) { + for (final File staged : + new File[] { + wholeFile, + wholeFile == null ? null : ModificationFile.getExclusiveMods(wholeFile), + new File(taskDir, WHOLE_FILE_META_NAME) + }) { if (staged == null) { continue; } @@ -1304,6 +1361,7 @@ void close(final boolean retainStagedFiles) { timePartition2Progress.clear(); timePartition2ModificationFile.clear(); wholeFile = null; + wholeFileModificationFile = null; wholeFileResource = null; wholeFileProgress = null; wholeFileTimePartition = null; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java index f5ee0feef1c5f..0333e118e3970 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java @@ -95,7 +95,7 @@ public void severalTimeSlotsKeepTheSplitPath() { } @Test - public void deletionSidecarKeepsTheSplitPath() throws Exception { + public void deletionSidecarStillUsesWholeFileConsensus() throws Exception { final TRegionReplicaSet region = region(1, 1, 2, 3); final File tsFile = Files.createTempFile("load-mode", ".tsfile").toFile(); final File mods = ModificationFile.getExclusiveMods(tsFile); @@ -104,7 +104,7 @@ public void deletionSidecarKeepsTheSplitPath() throws Exception { final LoadSingleTsFileNode node = nodeWithFile(tsFile, 0L, 1L, false, device("root.sg.d1")); final LoadSingleTsFileNode.LoadMode mode = node.resolveLoadMode(ignored -> Collections.singletonList(region)); - Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_SPLIT, mode); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_WHOLE_FILE, mode); } finally { Files.deleteIfExists(mods.toPath()); Files.deleteIfExists(tsFile.toPath());