diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java index 99cfba2520ee4..e85ee0c7be7f2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java @@ -237,6 +237,38 @@ public void testLoadTsFileInCluster() throws Exception { } } + /** + * A TsFile whose devices all fall into one time partition and one data region is loaded through + * the whole-file consensus path: three replicas, no chunk splitting, and every replica must end + * up with the same data. + */ + @Test + public void testLoadWholeTsFileIntoSingleRegion() throws Exception { + final long writtenPointCount; + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "load-cluster-whole-file.tsfile"))) { + final List schemas = + Arrays.asList( + new MeasurementSchema(MEASUREMENTS.get(0), TSDataType.INT64, TSEncoding.PLAIN), + new MeasurementSchema(MEASUREMENTS.get(1), TSDataType.INT64, TSEncoding.PLAIN), + new MeasurementSchema(MEASUREMENTS.get(2), TSDataType.INT64, TSEncoding.PLAIN), + new MeasurementSchema(MEASUREMENTS.get(3), TSDataType.INT64, TSEncoding.PLAIN)); + generator.registerTimeseries(DEVICES.get(0), schemas); + generator.generateData(DEVICES.get(0), 1_000, 1L, false); + writtenPointCount = generator.getTotalNumber(); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + createDatabaseAndSchema(statement); + statement.execute("load \"" + tmpDir.getAbsolutePath() + "\""); + Assert.assertEquals(writtenPointCount, queryPointCount(statement)); + } + + // The whole-file transaction must reach all three replicas, not only the coordinator. + awaitAllReplicasHaveData(); + } + /** Four concurrent LOADs on separate time partitions must expose their combined point count. */ @Test public void testLoadConcurrently() throws Exception { diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 378a6381dcc99..42963afba06b3 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -2773,8 +2773,11 @@ public final class DataNodeQueryMessages { "Device {} is not in the tsFileDevice2IsAligned cache {}."; public static final String LOADTSFILEANALYZER_CURRENT_DATANODE_IS_READ_ONLY_WILL_TRY_TO_CONVERT_TO_TABLETS_AND = "LoadTsFileAnalyzer: Current datanode is read only, will try to convert to tablets and insert later."; - public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT = - "Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%"; + public static final String LOG_LOAD_ANALYSIS_STAGE_ANALYZING_TSFILE_ARG_ARG_ARG_31359828 = + "Load - Analysis Stage: analyzing TsFile {} ({}/{})"; + public static final String + LOG_LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_FILE_ARG_3339452F = + "Load - Analysis Stage: {}/{} TsFiles have been analyzed, progress: {}%, file: {}"; public static final String THE_FILE_ARG_IS_NOT_A_VALID_TSFILE_PLEASE_CHECK_THE_INPUT_FILE = "The file {} is not a valid tsfile. Please check the input file."; public static final String TSFILE_ARG_IS_A_ARG_MODEL_FILE = @@ -2878,6 +2881,25 @@ public final class DataNodeQueryMessages { "The prefix of sourceKey is not 'source.'. Please check the parameters passed in: {}"; public static final String LOADTSFILESCHEDULER_REGION_MIGRATION_WAS_DETECTED_DURING_LOADING_TSFILE_ARG_WILL_CONVERT = "LoadTsFileScheduler: Region migration was detected during loading TsFile {}, will convert to insertion to avoid data loss"; + public static final String LOG_START_LOADING_ARG_TSFILES_BDB79D79 = "Start loading {} TsFiles"; + public static final String LOG_LOAD_TSFILE_ARG_SPLIT_PATH_ARG_A9F2E219 = + "Load TsFile {}, split path={}"; + public static final String LOG_LOADING_TSFILE_ARG_ARG_ARG_F29B2E19 = "Loading TsFile {} ({}/{})"; + public static final String + LOG_LOADING_TSFILE_ARG_ARG_ARG_WITH_STRATEGY_ARG_CURRENTLY_LOADING_ARG_FEA695E6 = + "Loading TsFile {} ({}/{}) with strategy {}, currently loading: {}"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_START_FILESIZE_ARG_D7E5D5F4 = + "Load TsFile {}, phase=dispatch start, fileSize={}"; + public static final String + LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_DISPATCHEDBYTES_ARG_FILESIZE_ARG_PROGRESS_ARG_668540F9 = + "Load TsFile {}, phase=dispatch, dispatchedBytes={}, fileSize={}, progress={}%"; + public static final String + LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_END_PIECES_ARG_BYTES_ARG_ELAPSEDMS_ARG_DB1D69B6 = + "Load TsFile {}, phase=dispatch end, pieces={}, bytes={}, elapsedMs={}"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_START_B225907D = + "Load TsFile {}, phase={} start"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_END_SUCCESS_ARG_ELAPSEDMS_ARG_6126B0D0 = + "Load TsFile {}, phase={} end, success={}, elapsedMs={}"; public static final String LOAD_TSFILE_ARG_SUCCESSFULLY_LOAD_PROCESS_ARG_ARG = "Load TsFile {} Successfully, load process [{}/{}]"; public static final String @@ -4983,4 +5005,10 @@ private DataNodeQueryMessages() {} public static final String LOG_QUERYFAILED_QUERYID_ARG_SQL_ARG_249264E9 = "[QueryFailed] queryId: {}, sql: {}"; + public static final String EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC = + "Whole-file load has no single resolved region for %s."; + + public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT = + "Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%"; + } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 91802ff30b0d1..d8699c8144a52 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1452,6 +1452,15 @@ private StorageEngineMessages() {} "Failed to convert data types for tree model statement {}."; public static final String STORAGE_LOG_LOAD_INSERTING_TABLET_TO_CASTING_TYPE_FROM_TO_AE808A8B = "Load: Inserting tablet to {}.{}. Casting type from {} to {}."; + public static final String + LOG_SPLIT_TSFILE_ARG_START_FILESIZE_ARG_CHUNKMETADATA_ARG_FORCEDECODE_ARG_243A182F = + "Split TsFile {} start, fileSize={}, chunkMetadata={}, forceDecode={}"; + public static final String + LOG_SPLIT_TSFILE_ARG_END_COMPLETED_ARG_DEVICES_ARG_ENTIRECHUNKS_ARG_DECODEDCHUNKS_ARG_SKIPPEDCHUNKS_ARG_TIMEPARTITIONS_ARG_DELETIONS_ARG_ELAPSEDMS_ARG_063A3789 = + "Split TsFile {} end, completed={}, devices={}, entireChunks={}, decodedChunks={}, " + + "skippedChunks={}, timePartitions={}, deletions={}, elapsedMs={}"; + public static final String LOG_SKIP_CHUNK_WITHOUT_METADATA_IN_TSFILE_ARG_OFFSET_ARG_DATASIZE_ARG_41F9ED2B = + "Skip chunk without metadata in TsFile {}, offset={}, dataSize={}"; public static final String STORAGE_LOG_TRY_TO_LOAD_TSFILE_V3_INTO_CURRENT_VERSION_V4_FILE_PATH_B8D38E22 = "try to load TsFile V3 into current version (V4), file path: {}"; public static final String STORAGE_LOG_THE_FILE_S_VERSION_NUMBER_IS_HIGHER_THAN_CURRENT_FILE_PATH_6D17349F = @@ -1482,6 +1491,9 @@ private StorageEngineMessages() {} + "will retry later."; public static final String STORAGE_LOG_SUCCESSFULLY_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_ADB5FEC9 = "Successfully auto load tsfile {} (isGeneratedByPipe = {})"; + public static final String + LOG_START_AUTO_LOADING_TSFILE_ARG_ISGENERATEDBYPIPE_ARG_PENDING_ARG_LOADING_ARG_7B71F0D6 = + "Start auto loading TsFile {} (isGeneratedByPipe = {}), pending={}, loading={}"; public static final String STORAGE_LOG_ERROR_OCCURRED_DURING_CREATING_FAIL_DIRECTORY_FOR_ACTIVE_7D3BEB38 = "Error occurred during creating fail directory {} for active load."; public static final String STORAGE_LOG_FAILED_TO_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_STATUS_FILE_F43E9EF7 = @@ -1906,4 +1918,45 @@ private StorageEngineMessages() {} "source files are cleaned up"; public static final String EXCEPTION_TASK_FINISHED_A075ACB1 = "task finished"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878 = + "Whole-file slice payload cannot be empty."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_RANGE_IS_INVALID_OFFSET_D_LENGTH_D_SOURCE_LENGTH_D_402A1599 = + "Whole-file slice range is invalid: offset %d, length %d, source length %d."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24 = + "Whole-file slice time partition is missing."; + + + public static final String EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1 = + "Whole-file slice has invalid payload length %d."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_ENDED_AFTER_D_OF_D_PAYLOAD_BYTES_39E892F4 = + "Whole-file slice ended after %d of %d payload bytes."; + + public static final String EXCEPTION_A_WHOLE_FILE_LOAD_PIECE_CANNOT_MIX_WHOLE_FILE_SLICES_AND_CHUNK_DATA_B1297688 = + "A whole-file LOAD piece cannot mix whole-file slices and chunk data."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_CHECKSUM_MISMATCH_AT_OFFSET_D_EXPECTED_D_ACTUAL_D_48238B7E = + "Whole-file slice checksum mismatch at offset %d: expected %d, actual %d."; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_5F80C60D = + "Whole-file staging %s already exists but cannot be resumed."; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB = + "Whole-file slice (source length %d, time partition %d) does not match the staged task (source length %d)."; + + + + + public static final String EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5 = + "Whole-file slice offset %d exceeds the staged length %d for task %s."; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C = + "Whole-file staging is missing the TsFile of task %s."; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_LENGTH_D_DOES_NOT_MATCH_THE_EXPECTED_LENGTH_D_FOR_TASK_S_89AC32B6 = + "Whole-file staging length %d does not match the expected length %d for task %s."; + } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 02cf8edc0a469..1915cf1a9a3b9 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3196,9 +3196,11 @@ public final class DataNodeQueryMessages { public static final String LOADTSFILEANALYZER_CURRENT_DATANODE_IS_READ_ONLY_WILL_TRY_TO_CONVERT_TO_TABLETS_AND = "LoadTsFileAnalyzer: 当前 DataNode 为只读,将尝试转换为 tablet 后再插入。"; - public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT = - - "加载 - 分析阶段:已分析 {}/{} 个 TsFile,进度:{}%"; + public static final String LOG_LOAD_ANALYSIS_STAGE_ANALYZING_TSFILE_ARG_ARG_ARG_31359828 = + "加载 - 分析阶段:正在分析 TsFile {}({}/{})"; + public static final String + LOG_LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_FILE_ARG_3339452F = + "加载 - 分析阶段:已分析 {}/{} 个 TsFile,进度:{}%,文件:{}"; public static final String THE_FILE_ARG_IS_NOT_A_VALID_TSFILE_PLEASE_CHECK_THE_INPUT_FILE = "文件 {} 不是有效的 TsFile,请检查输入文件。"; @@ -3354,6 +3356,25 @@ public final class DataNodeQueryMessages { "LoadTsFileScheduler:加载 TsFile {} 期间检测到 Region 迁移,将转换为 insertion 以避免数据丢失。"; + public static final String LOG_START_LOADING_ARG_TSFILES_BDB79D79 = "开始加载 {} 个 TsFile"; + public static final String LOG_LOAD_TSFILE_ARG_SPLIT_PATH_ARG_A9F2E219 = + "加载 TsFile {},拆分路径={}"; + public static final String LOG_LOADING_TSFILE_ARG_ARG_ARG_F29B2E19 = "正在加载 TsFile {}({}/{})"; + public static final String + LOG_LOADING_TSFILE_ARG_ARG_ARG_WITH_STRATEGY_ARG_CURRENTLY_LOADING_ARG_FEA695E6 = + "正在加载 TsFile {}({}/{}),策略 {},当前正在加载的文件:{}"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_START_FILESIZE_ARG_D7E5D5F4 = + "加载 TsFile {},dispatch 阶段开始,fileSize={}"; + public static final String + LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_DISPATCHEDBYTES_ARG_FILESIZE_ARG_PROGRESS_ARG_668540F9 = + "加载 TsFile {},dispatch 阶段,dispatchedBytes={},fileSize={},进度={}%"; + public static final String + LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_END_PIECES_ARG_BYTES_ARG_ELAPSEDMS_ARG_DB1D69B6 = + "加载 TsFile {},dispatch 阶段结束,pieces={},bytes={},elapsedMs={}"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_START_B225907D = + "加载 TsFile {},阶段 {} 开始"; + public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_END_SUCCESS_ARG_ELAPSEDMS_ARG_6126B0D0 = + "加载 TsFile {},阶段 {} 结束,success={},elapsedMs={}"; public static final String LOAD_TSFILE_ARG_SUCCESSFULLY_LOAD_PROCESS_ARG_ARG = "成功加载 TsFile {},加载进度 [{}/{}]"; public static final String @@ -5684,4 +5705,10 @@ private DataNodeQueryMessages() {} public static final String LOG_QUERYFAILED_QUERYID_ARG_SQL_ARG_249264E9 = "[QueryFailed] queryId: {}, SQL:{}"; + public static final String EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC = + "Whole-file load 未解析出唯一的 region:%s。"; + + public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT = + "加载 - 分析阶段:已分析 {}/{} 个 TsFile,进度:{}%"; + } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index a2429b5438cd3..057231d10e731 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1386,6 +1386,15 @@ private StorageEngineMessages() {} "无法转换树模型语句 {} 的数据类型。"; public static final String STORAGE_LOG_LOAD_INSERTING_TABLET_TO_CASTING_TYPE_FROM_TO_AE808A8B = "Load:正在向 {}.{} 插入 tablet。将类型从 {} 转换为 {}。"; + public static final String + LOG_SPLIT_TSFILE_ARG_START_FILESIZE_ARG_CHUNKMETADATA_ARG_FORCEDECODE_ARG_243A182F = + "开始拆分 TsFile {},fileSize={},chunkMetadata={},forceDecode={}"; + public static final String + LOG_SPLIT_TSFILE_ARG_END_COMPLETED_ARG_DEVICES_ARG_ENTIRECHUNKS_ARG_DECODEDCHUNKS_ARG_SKIPPEDCHUNKS_ARG_TIMEPARTITIONS_ARG_DELETIONS_ARG_ELAPSEDMS_ARG_063A3789 = + "结束拆分 TsFile {},completed={},devices={},entireChunks={},decodedChunks={}," + + "skippedChunks={},timePartitions={},deletions={},elapsedMs={}"; + public static final String LOG_SKIP_CHUNK_WITHOUT_METADATA_IN_TSFILE_ARG_OFFSET_ARG_DATASIZE_ARG_41F9ED2B = + "跳过 TsFile {} 中缺少 ChunkMetadata 的 chunk,offset={},dataSize={}"; public static final String STORAGE_LOG_TRY_TO_LOAD_TSFILE_V3_INTO_CURRENT_VERSION_V4_FILE_PATH_B8D38E22 = "尝试将 TsFile V3 加载到当前版本 (V4),文件路径:{}"; public static final String STORAGE_LOG_THE_FILE_S_VERSION_NUMBER_IS_HIGHER_THAN_CURRENT_FILE_PATH_6D17349F = @@ -1410,6 +1419,9 @@ private StorageEngineMessages() {} "拒绝自动加载 TsFile {} (isGeneratedByPipe = {}),原因:当前查询已超时,稍后重试。"; public static final String STORAGE_LOG_SUCCESSFULLY_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_ADB5FEC9 = "成功自动加载 TsFile {} (isGeneratedByPipe = {})"; + public static final String + LOG_START_AUTO_LOADING_TSFILE_ARG_ISGENERATEDBYPIPE_ARG_PENDING_ARG_LOADING_ARG_7B71F0D6 = + "开始自动加载 TsFile {} (isGeneratedByPipe = {}),pending={},loading={}"; public static final String STORAGE_LOG_ERROR_OCCURRED_DURING_CREATING_FAIL_DIRECTORY_FOR_ACTIVE_7D3BEB38 = "为 active load 创建失败目录 {} 时发生错误。"; public static final String STORAGE_LOG_FAILED_TO_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_STATUS_FILE_F43E9EF7 = @@ -1808,4 +1820,45 @@ private StorageEngineMessages() {} public static final String EXCEPTION_SOURCE_FILES_ARE_CLEANED_UP_83638D68 = "源文件已清理"; public static final String EXCEPTION_TASK_FINISHED_A075ACB1 = "任务已完成"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878 = + "Whole-file 切片内容不能为空。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_RANGE_IS_INVALID_OFFSET_D_LENGTH_D_SOURCE_LENGTH_D_402A1599 = + "Whole-file 切片范围非法:offset %d,length %d,source length %d。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24 = + "Whole-file 切片缺少时间分区。"; + + + public static final String EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1 = + "Whole-file 切片的 payload 长度非法:%d。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_ENDED_AFTER_D_OF_D_PAYLOAD_BYTES_39E892F4 = + "Whole-file 切片在读取到 %d/%d 字节时提前结束。"; + + public static final String EXCEPTION_A_WHOLE_FILE_LOAD_PIECE_CANNOT_MIX_WHOLE_FILE_SLICES_AND_CHUNK_DATA_B1297688 = + "Whole-file LOAD piece 不能同时包含 whole-file 切片和 chunk 数据。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_CHECKSUM_MISMATCH_AT_OFFSET_D_EXPECTED_D_ACTUAL_D_48238B7E = + "Whole-file 切片在 offset %d 处校验和不一致:期望 %d,实际 %d。"; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_5F80C60D = + "Whole-file staging 文件 %s 已存在,但无法恢复。"; + + public static final String EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB = + "Whole-file 切片(source length %d,time partition %d)与已 staging 的任务(source length %d)不匹配。"; + + + + + public static final String EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5 = + "Whole-file 切片 offset %d 超过已 staging 的长度 %d(任务 %s)。"; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C = + "Whole-file staging 缺少任务 %s 的 TsFile。"; + + public static final String EXCEPTION_WHOLE_FILE_STAGING_LENGTH_D_DOES_NOT_MATCH_THE_EXPECTED_LENGTH_D_FOR_TASK_S_89AC32B6 = + "Whole-file staging 长度 %d 与期望长度 %d 不匹配(任务 %s)。"; + } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index c71d98f72504f..aae2e95101c14 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -68,6 +68,9 @@ public class LoadSingleTsFileNode extends WritePlanNode { private TRegionReplicaSet localRegionReplicaSet; + /** The one time partition of a whole-file load, or null when the file has to be split. */ + private TTimePartitionSlot wholeFileTimePartitionSlot; + public LoadSingleTsFileNode( final PlanNodeId id, final TsFileResource resource, @@ -119,19 +122,7 @@ public boolean needDecodeTsFile( return true; } - List> slotList = - new ArrayList<>(resource.getDevices().size() << 1); - for (final IDeviceID device : resource.getDevices()) { - // iterating the index, must present - final TTimePartitionSlot startSlot = - TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(device).get()); - final TTimePartitionSlot endSlot = - TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(device).get()); - slotList.add(new Pair<>(device, startSlot)); - if (!startSlot.equals(endSlot)) { - slotList.add(new Pair<>(device, endSlot)); - } - } + final List> slotList = buildSlotList(); if (slotList.isEmpty()) { throw new IllegalStateException( @@ -153,6 +144,83 @@ public boolean needDecodeTsFile( return needDecodeTsFile; } + /** + * Resolves how this file has to be loaded. A file whose devices all fall into one time slot and + * one data region does not need to be split by chunks even when the region has several replicas: + * the consensus LOAD transaction can carry it as whole-file slices. + */ + public LoadMode resolveLoadMode( + final Function>, List> + partitionFetcher) { + if (needDecodeTsFile) { + return LoadMode.CONSENSUS_SPLIT; + } + + final List> slotList = buildSlotList(); + if (slotList.isEmpty()) { + return LoadMode.CONSENSUS_SPLIT; + } + + // A whole-file load needs exactly one time slot and exactly one target region. Several device + // slots are fine when the partition placement puts all of them into that one region. + final Set timePartitionSlots = new HashSet<>(); + for (final Pair slot : slotList) { + timePartitionSlots.add(slot.right); + } + if (timePartitionSlots.size() != 1) { + return LoadMode.CONSENSUS_SPLIT; + } + + final TTimePartitionSlot firstSlot = timePartitionSlots.iterator().next(); + final Set replicaSets = new HashSet<>(partitionFetcher.apply(slotList)); + if (replicaSets.size() != 1) { + return LoadMode.CONSENSUS_SPLIT; + } + final TRegionReplicaSet replicaSet = replicaSets.iterator().next(); + localRegionReplicaSet = replicaSet; + wholeFileTimePartitionSlot = firstSlot; + + if (replicaSet.getDataNodeLocations().size() == 1) { + return isDispatchedToLocal(replicaSet.getDataNodeLocations().get(0).getInternalEndPoint()) + ? LoadMode.LOCAL_WHOLE_FILE + : LoadMode.CONSENSUS_WHOLE_FILE; + } + return LoadMode.CONSENSUS_WHOLE_FILE; + } + + private List> buildSlotList() { + final List> slotList = + new ArrayList<>(resource.getDevices().size() << 1); + for (final IDeviceID device : resource.getDevices()) { + // iterating the index, must present + final TTimePartitionSlot startSlot = + TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(device).get()); + final TTimePartitionSlot endSlot = + TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(device).get()); + slotList.add(new Pair<>(device, startSlot)); + if (!startSlot.equals(endSlot)) { + slotList.add(new Pair<>(device, endSlot)); + } + } + return slotList; + } + + public TTimePartitionSlot getWholeFileTimePartitionSlot() { + return wholeFileTimePartitionSlot; + } + + /** The load paths a source TsFile can take, resolved before the strategy is created. */ + public enum LoadMode { + /** One local replica and one region: import the local file directly, as before. */ + LOCAL_WHOLE_FILE, + /** + * One region with several replicas: copy the whole file through the consensus LOAD protocol. + */ + CONSENSUS_WHOLE_FILE, + /** More than one region or a file that has to be decoded: split into chunk pieces. */ + CONSENSUS_SPLIT + } + private boolean isDispatchedToLocal(Set replicaSets) { if (replicaSets.size() > 1) { return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileConsensusNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileConsensusNode.java index 44e016bc8804a..b002f6925dd87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileConsensusNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileConsensusNode.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.load.splitter.SerializedSizes; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; +import org.apache.iotdb.db.storageengine.load.splitter.TsFileDataType; import org.apache.tsfile.exception.NotImplementedException; import org.apache.tsfile.exception.write.PageException; @@ -373,6 +374,16 @@ public boolean hasChunkData() { return !tsFileDataList.isEmpty(); } + /** Whether this PIECE carries slices of a whole source file instead of staged TsFileData. */ + public boolean hasWholeFileData() { + for (final TsFileData data : tsFileDataList) { + if (data.getType() == TsFileDataType.WHOLE_FILE) { + return true; + } + } + return false; + } + public Map getTimePartition2ProgressIndex() { return timePartition2ProgressIndex; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java index 745643678bae6..b0a6842635a3b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -53,6 +53,13 @@ public class LoadTsFilePieceNode extends WritePlanNode { private File tsFile; private long pieceIndex; + /** + * Whether this piece carries slices of the complete source TsFile instead of decoded chunk data. + * The receiver recognises the same mode from the {@code WholeFileData} it receives; the flag + * makes the coordinator side explicit and keeps logs/diagnostics unambiguous. + */ + private boolean wholeFile; + private long dataSize; private List tsFileDataList; @@ -76,6 +83,21 @@ public long getPieceIndex() { return pieceIndex; } + public boolean isWholeFilePiece() { + return wholeFile; + } + + public LoadTsFilePieceNode markAsWholeFilePiece() { + this.wholeFile = true; + return this; + } + + /** A piece of a complete source TsFile, carried as whole-file slices instead of chunks. */ + public static LoadTsFilePieceNode wholeFilePiece( + final PlanNodeId id, final File tsFile, final long pieceIndex) { + return new LoadTsFilePieceNode(id, tsFile, pieceIndex).markAsWholeFilePiece(); + } + public long getDataSize() { return dataSize; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index 24c78410dd5a5..bfd4c002eef4a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -278,6 +278,14 @@ public class LoadTsFileScheduler implements IScheduler { /** Global thread-safe set tracking file paths actively being loaded across all schedulers. */ private static final Set ACTIVE_LOADING_FILES = ConcurrentHashMap.newKeySet(); + private static final String LOCAL_SPLIT_PATH = "local"; + + private static final String DISTRIBUTED_SPLIT_PATH = "distributed"; + + private static final String TABLET_REWRITE_SPLIT_PATH = "tablet-rewrite"; + + private static final String WHOLE_FILE_SPLIT_PATH = "whole-file"; + private final MPPQueryContext queryContext; private final QueryStateMachine stateMachine; private final LoadTsFileDispatcherImpl dispatcher; @@ -364,6 +372,7 @@ public void start() { boolean allFilesSucceeded = true; final int totalFiles = tsFileNodeList.size(); + logLoadProgress(DataNodeQueryMessages.LOG_START_LOADING_ARG_TSFILES_BDB79D79, totalFiles); for (int i = 0; i < totalFiles; ++i) { if (stopped) { return; @@ -438,6 +447,7 @@ private boolean processSingleNode( } final TsFileLoadStrategy strategy = resolveStrategy(node, userName); + logLoadFileStart(filePath, index + 1, totalFiles, strategy.getClass().getSimpleName()); final boolean loadSuccess = strategy.execute(node); if (isRegionMigrating(startTimeMs)) { @@ -445,12 +455,12 @@ private boolean processSingleNode( DataNodeQueryMessages .LOADTSFILESCHEDULER_REGION_MIGRATION_WAS_DETECTED_DURING_LOADING_TSFILE_ARG_WILL_CONVERT, filePath); - logCannotLoad(node, index, totalFiles); + logCannotLoad(node, index + 1, totalFiles); return false; } if (!loadSuccess) { - logCannotLoad(node, index, totalFiles); + logCannotLoad(node, index + 1, totalFiles); return false; } @@ -471,20 +481,48 @@ private boolean processSingleNode( */ private TsFileLoadStrategy resolveStrategy(final LoadSingleTsFileNode node, final String userName) throws Exception { + final String filePath = node.getTsFileResource().getTsFilePath(); if (EncryptDBUtils.requiresFileRewrite()) { + logSplitPath(filePath, TABLET_REWRITE_SPLIT_PATH); return new TabletRewriteLoadStrategy(queryContext, isGeneratedByPipe); } // The judgement resolves the partitions of the devices anyway, so the routing reuses them. final DataPartitionResolver partitionResolver = new DataPartitionResolver(partitionFetcher, userName); - final boolean needDecode = node.needDecodeTsFile(partitionResolver::resolve); + final LoadSingleTsFileNode.LoadMode loadMode = node.resolveLoadMode(partitionResolver::resolve); + + switch (loadMode) { + case LOCAL_WHOLE_FILE: + logSplitPath(filePath, LOCAL_SPLIT_PATH); + return new LocalLoadStrategy(queryContext, fragmentId, dispatcher); + case CONSENSUS_WHOLE_FILE: + logSplitPath(filePath, WHOLE_FILE_SPLIT_PATH); + return new TwoPhaseConsensusLoadStrategy( + dispatcher, + partitionResolver, + memoryBlock, + consensusSubmitter, + isGeneratedByPipe, + true); + case CONSENSUS_SPLIT: + default: + logSplitPath(filePath, DISTRIBUTED_SPLIT_PATH); + return new TwoPhaseConsensusLoadStrategy( + dispatcher, + partitionResolver, + memoryBlock, + consensusSubmitter, + isGeneratedByPipe, + false); + } + } - if (!needDecode) { - return new LocalLoadStrategy(queryContext, fragmentId, dispatcher); - } else { - return new TwoPhaseConsensusLoadStrategy( - dispatcher, partitionResolver, memoryBlock, consensusSubmitter, isGeneratedByPipe); + private void logSplitPath(final String filePath, final String splitPath) { + if (!LOGGER.isDebugEnabled()) { + return; } + LOGGER.debug( + DataNodeQueryMessages.LOG_LOAD_TSFILE_ARG_SPLIT_PATH_ARG_A9F2E219, filePath, splitPath); } private boolean isRegionMigrating(final long taskStartTimeMs) { @@ -561,6 +599,31 @@ private void teardown() { } } + private void logLoadFileStart( + final String filePath, final int current, final int total, final String strategy) { + logLoadProgress( + DataNodeQueryMessages.LOG_LOADING_TSFILE_ARG_ARG_ARG_F29B2E19, filePath, current, total); + if (!LOGGER.isDebugEnabled()) { + return; + } + LOGGER.debug( + DataNodeQueryMessages + .LOG_LOADING_TSFILE_ARG_ARG_ARG_WITH_STRATEGY_ARG_CURRENTLY_LOADING_ARG_FEA695E6, + filePath, + current, + total, + strategy, + String.join(", ", ACTIVE_LOADING_FILES)); + } + + private void logLoadProgress(final String message, final Object... args) { + if (isGeneratedByPipe) { + LOGGER.debug(message, args); + } else { + LOGGER.info(message, args); + } + } + private void logLoadSuccess(final String filePath, final int current, final int total) { if (isGeneratedByPipe) { LOGGER.debug( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TsFileLoadStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TsFileLoadStrategy.java index 97b89f8208af1..cb0c9a4b8b7f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TsFileLoadStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TsFileLoadStrategy.java @@ -32,7 +32,8 @@ *
    *
  • {@link LocalLoadStrategy} - no decode needed, dispatch the file to the local region; *
  • {@link TwoPhaseConsensusLoadStrategy} - decode, stream PIECE batches through consensus, - * then PREPARE+COMMIT or ABORT. + * then PREPARE+COMMIT or ABORT. When the whole file belongs to one data region it is copied + * as whole-file pieces instead of being split into chunks. *
*/ public interface TsFileLoadStrategy { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java index aa0ec86cddd07..80488837d7d0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/TwoPhaseConsensusLoadStrategy.java @@ -32,23 +32,35 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.load.memory.LoadTsFileDataCacheMemoryBlock; import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet; import org.apache.iotdb.db.storageengine.load.splitter.ChunkData; +import org.apache.iotdb.db.storageengine.load.splitter.DeletionData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileSplitter; +import org.apache.iotdb.db.storageengine.load.splitter.WholeFileData; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.zip.CRC32C; /** * Two-phase consensus LOAD strategy for files that need decoding. @@ -88,6 +100,15 @@ public class TwoPhaseConsensusLoadStrategy implements TsFileLoadStrategy { private static final long LOAD_CONSENSUS_SUBMIT_RETRY_BACKOFF_MS = 100L; + /** Progress of one file is logged on the first piece, each further 10%, and at least every 5s. */ + static final long LOAD_DISPATCH_PROGRESS_LOG_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(5); + + static final int LOAD_DISPATCH_PROGRESS_BUCKETS = 10; + + private static final String PREPARE_COMMIT_PHASE = "prepare-commit"; + + private static final String ABORT_PHASE = "abort"; + private final LoadTsFileDispatcherImpl dispatcher; /** Resolves the data partitions of the file, reusing the ones the judgement already resolved. */ @@ -97,6 +118,9 @@ public class TwoPhaseConsensusLoadStrategy implements TsFileLoadStrategy { private final LoadConsensusSubmitter consensusSubmitter; private final boolean isGeneratedByPipe; + /** When true the source file is copied as whole-file slices instead of being split by chunks. */ + private final boolean wholeFile; + /** The source file being loaded, kept for the phase-two commands. */ private LoadSingleTsFileNode currentNode; @@ -129,17 +153,40 @@ public class TwoPhaseConsensusLoadStrategy implements TsFileLoadStrategy { private final Map timePartitionSlotToProgressIndex = new HashMap<>(); + private String sourceFilePath = ""; + + private long sourceFileLength; + + private long dispatchedPayloadBytes; + + private long dispatchedPieceCount; + + private int lastLoggedProgressBucket = -1; + + private long lastProgressLogNanos; + public TwoPhaseConsensusLoadStrategy( LoadTsFileDispatcherImpl dispatcher, DataPartitionResolver partitionResolver, LoadTsFileDataCacheMemoryBlock block, LoadConsensusSubmitter consensusSubmitter, boolean isGeneratedByPipe) { + this(dispatcher, partitionResolver, block, consensusSubmitter, isGeneratedByPipe, false); + } + + public TwoPhaseConsensusLoadStrategy( + LoadTsFileDispatcherImpl dispatcher, + DataPartitionResolver partitionResolver, + LoadTsFileDataCacheMemoryBlock block, + LoadConsensusSubmitter consensusSubmitter, + boolean isGeneratedByPipe, + boolean wholeFile) { this.dispatcher = dispatcher; this.partitionResolver = partitionResolver; this.block = block; this.consensusSubmitter = consensusSubmitter; this.isGeneratedByPipe = isGeneratedByPipe; + this.wholeFile = wholeFile; } @Override @@ -152,29 +199,130 @@ public boolean execute(LoadSingleTsFileNode node) { regionTotalBytes.clear(); regionExpectedDataEnd.clear(); timePartitionSlotToProgressIndex.clear(); + dispatchedPayloadBytes = 0L; + dispatchedPieceCount = 0L; + lastLoggedProgressBucket = -1; + lastProgressLogNanos = 0L; + final File sourceFile = node.getTsFileResource().getTsFile(); + sourceFilePath = sourceFile.getAbsolutePath(); + sourceFileLength = sourceFile.length(); + + logLoadProgress( + DataNodeQueryMessages.LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_START_FILESIZE_ARG_D7E5D5F4, + sourceFilePath, + sourceFileLength); long startTime = System.nanoTime(); final boolean isFirstPhaseSuccess; try { isFirstPhaseSuccess = firstPhase(node); } finally { + final long elapsedNanos = System.nanoTime() - startTime; LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost( - LoadTsFileCostMetricsSet.FIRST_PHASE, System.nanoTime() - startTime); + LoadTsFileCostMetricsSet.FIRST_PHASE, elapsedNanos); + logLoadProgress( + DataNodeQueryMessages + .LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_END_PIECES_ARG_BYTES_ARG_ELAPSEDMS_ARG_DB1D69B6, + sourceFilePath, + dispatchedPieceCount, + dispatchedPayloadBytes, + TimeUnit.NANOSECONDS.toMillis(elapsedNanos)); } + final String secondPhaseName = isFirstPhaseSuccess ? PREPARE_COMMIT_PHASE : ABORT_PHASE; + logLoadProgress( + DataNodeQueryMessages.LOG_LOAD_TSFILE_ARG_PHASE_ARG_START_B225907D, + sourceFilePath, + secondPhaseName); startTime = System.nanoTime(); - final boolean isSecondPhaseSuccess; + boolean isSecondPhaseSuccess = false; try { isSecondPhaseSuccess = secondPhase(isFirstPhaseSuccess); + return isFirstPhaseSuccess && isSecondPhaseSuccess; } finally { + final long elapsedNanos = System.nanoTime() - startTime; LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost( - LoadTsFileCostMetricsSet.SECOND_PHASE, System.nanoTime() - startTime); + LoadTsFileCostMetricsSet.SECOND_PHASE, elapsedNanos); + logLoadProgress( + DataNodeQueryMessages + .LOG_LOAD_TSFILE_ARG_PHASE_ARG_END_SUCCESS_ARG_ELAPSEDMS_ARG_6126B0D0, + sourceFilePath, + secondPhaseName, + isSecondPhaseSuccess, + TimeUnit.NANOSECONDS.toMillis(elapsedNanos)); } + } - return isFirstPhaseSuccess && isSecondPhaseSuccess; + private void logDispatchProgress(final long pieceBytes) { + dispatchedPayloadBytes += pieceBytes; + if (!isLoadProgressLogEnabled()) { + return; + } + final long nowNanos = System.nanoTime(); + if (!shouldLogDispatchProgress( + sourceFileLength, + dispatchedPayloadBytes, + lastLoggedProgressBucket, + lastProgressLogNanos, + nowNanos)) { + return; + } + lastLoggedProgressBucket = progressBucket(sourceFileLength, dispatchedPayloadBytes); + lastProgressLogNanos = nowNanos; + logLoadProgress( + DataNodeQueryMessages + .LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_DISPATCHEDBYTES_ARG_FILESIZE_ARG_PROGRESS_ARG_668540F9, + sourceFilePath, + dispatchedPayloadBytes, + sourceFileLength, + progressPercent(dispatchedPayloadBytes, sourceFileLength)); + } + + private void logLoadProgress(final String message, final Object... args) { + if (!LOGGER.isDebugEnabled()) { + return; + } + LOGGER.debug(message, args); + } + + private boolean isLoadProgressLogEnabled() { + return LOGGER.isDebugEnabled(); + } + + static boolean shouldLogDispatchProgress( + final long fileSize, + final long dispatchedBytes, + final int lastBucket, + final long lastLogNanos, + final long nowNanos) { + if (lastLogNanos == 0L) { + return true; + } + if (nowNanos - lastLogNanos >= LOAD_DISPATCH_PROGRESS_LOG_INTERVAL_NANOS) { + return true; + } + return progressBucket(fileSize, dispatchedBytes) > lastBucket; + } + + static int progressBucket(final long fileSize, final long dispatchedBytes) { + if (fileSize <= 0L || dispatchedBytes <= 0L) { + return 0; + } + if (dispatchedBytes >= fileSize) { + return LOAD_DISPATCH_PROGRESS_BUCKETS; + } + return (int) (dispatchedBytes * LOAD_DISPATCH_PROGRESS_BUCKETS / fileSize); + } + + private static String progressPercent(final long dispatchedBytes, final long fileSize) { + final double percent = fileSize <= 0L ? 0.0 : dispatchedBytes * 100.0 / fileSize; + return String.format("%.3f", percent); } private boolean firstPhase(LoadSingleTsFileNode node) { + if (wholeFile) { + return dispatchWholeFile(node); + } final TsFileSplitConsumer pipeline = new TsFileSplitConsumer( node, @@ -205,6 +353,156 @@ private boolean firstPhase(LoadSingleTsFileNode node) { } } + /** How many source bytes one whole-file PIECE carries. Kept well below the RPC frame limit. */ + private static final int WHOLE_FILE_SLICE_SIZE = 4 * 1024 * 1024; + + /** + * Copies one source file through the consensus LOAD transaction: every slice is a PIECE, so the + * consensus layer replicates it and PREPARE/COMMIT still decide the transaction atomically. + */ + private boolean dispatchWholeFile(final LoadSingleTsFileNode node) { + final TRegionReplicaSet replicaSet = node.getLocalRegionReplicaSet(); + final TTimePartitionSlot timePartitionSlot = node.getWholeFileTimePartitionSlot(); + final File sourceFile = node.getTsFileResource().getTsFile(); + if (replicaSet == null || timePartitionSlot == null || sourceFile == null) { + LOGGER.warn( + String.format( + DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, + sourceFile == null ? node.getTsFileResource().getTsFilePath() : sourceFile), + new IllegalStateException( + String.format( + DataNodeQueryMessages + .EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC, + node.getTsFileResource().getTsFilePath()))); + return false; + } + + final TConsensusGroupId regionId = replicaSet.getRegionId(); + final String loadId = + regionLoadIds.computeIfAbsent(regionId, ignored -> UUID.randomUUID().toString()); + regionRoutes.putIfAbsent(regionId, replicaSet); + computeTimePartitionSlotToProgressIndexIfAbsent(timePartitionSlot); + + final long sourceLength = sourceFile.length(); + if (sourceLength <= 0L) { + LOGGER.warn( + String.format( + DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, sourceFile.getAbsolutePath())); + return false; + } + regionExpectedDataEnd + .computeIfAbsent(regionId, ignored -> new HashMap<>()) + .put(timePartitionSlot, sourceLength); + + return dispatchWholeFileSlices(sourceFile, sourceLength, timePartitionSlot, replicaSet, node); + } + + /** How many deletion bytes one PIECE carries; the mods themselves are usually small. */ + private static final int WHOLE_FILE_DELETION_PIECE_SIZE = 1024 * 1024; + + /** + * Sends the deletion sidecar of the source TsFile as the existing {@code DeletionData} records. + * They are part of the same consensus LOAD transaction, so the receiver stages them next to the + * whole-file bytes and {@code loadNewTsFile} applies them on COMMIT. + */ + private boolean dispatchWholeFileDeletions( + final File sourceFile, + final TTimePartitionSlot timePartitionSlot, + final TRegionReplicaSet replicaSet, + final LoadSingleTsFileNode node) { + final List deletions; + try { + deletions = ModificationFile.readAllModifications(sourceFile, true); + } catch (final IOException e) { + LOGGER.warn( + String.format( + DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, sourceFile.getAbsolutePath()), + e); + return false; + } + if (deletions.isEmpty()) { + return true; + } + + long pieceIndex = regionPieceCounts.getOrDefault(replicaSet.getRegionId(), 0L); + LoadTsFilePieceNode pieceNode = null; + long pieceBytes = 0L; + for (final ModEntry deletion : deletions) { + final DeletionData data = new DeletionData(deletion); + if (pieceNode == null) { + pieceNode = + LoadTsFilePieceNode.wholeFilePiece(node.getPlanNodeId(), sourceFile, pieceIndex++); + } else if (pieceBytes + data.getDataSize() > WHOLE_FILE_DELETION_PIECE_SIZE) { + if (!dispatchOnePieceNode(pieceNode, replicaSet)) { + return false; + } + pieceNode = + LoadTsFilePieceNode.wholeFilePiece(node.getPlanNodeId(), sourceFile, pieceIndex++); + pieceBytes = 0L; + } + pieceNode.addTsFileData(data); + pieceBytes += data.getDataSize(); + } + return pieceNode == null || dispatchOnePieceNode(pieceNode, replicaSet); + } + + private boolean dispatchWholeFileSlices( + final File file, + final long sourceLength, + final TTimePartitionSlot timePartitionSlot, + final TRegionReplicaSet replicaSet, + final LoadSingleTsFileNode node) { + long offset = 0L; + long pieceIndex = regionPieceCounts.getOrDefault(replicaSet.getRegionId(), 0L); + try (final FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { + while (offset < sourceLength) { + final int expected = (int) Math.min(WHOLE_FILE_SLICE_SIZE, sourceLength - offset); + final ByteBuffer buffer = ByteBuffer.allocate(expected); + int read = 0; + while (read < expected) { + final int current = channel.read(buffer, offset + read); + if (current < 0) { + break; + } + read += current; + } + if (read <= 0) { + LOGGER.warn( + String.format( + DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, file.getAbsolutePath())); + return false; + } + final byte[] payload = Arrays.copyOf(buffer.array(), read); + final LoadTsFilePieceNode pieceNode = + LoadTsFilePieceNode.wholeFilePiece(node.getPlanNodeId(), file, pieceIndex++); + pieceNode.addTsFileData( + new WholeFileData( + offset, + sourceLength, + timePartitionSlot.getStartTime(), + isGeneratedByPipe, + crc32c(payload), + payload)); + if (!dispatchOnePieceNode(pieceNode, replicaSet)) { + return false; + } + offset += read; + } + } catch (final IOException e) { + LOGGER.warn( + String.format(DataNodeQueryMessages.PARSE_OR_SEND_TSFILE_S_ERROR, file.getAbsolutePath()), + e); + return false; + } + return offset == sourceLength; + } + + private static long crc32c(final byte[] payload) { + final CRC32C crc32c = new CRC32C(); + crc32c.update(payload, 0, payload.length); + return crc32c.getValue(); + } + private boolean dispatchOnePieceNode( LoadTsFilePieceNode pieceNode, TRegionReplicaSet replicaSet) { // The first piece of a region pins the route the splitter resolved; a route that a re-resolve @@ -381,6 +679,8 @@ private boolean dispatchConsensusPiece( piece.getDataSize()); regionPieceCounts.merge(regionId, 1L, Long::sum); regionTotalBytes.merge(regionId, piece.getDataSize(), Long::sum); + dispatchedPieceCount++; + logDispatchProgress(piece.getDataSize()); final Map expectedDataEnd = regionExpectedDataEnd.computeIfAbsent(regionId, id -> new HashMap<>()); for (final TsFileData data : pieceNode.getAllTsFileData()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java index 93a537c1e4aca..bafc0c8584dd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java @@ -326,6 +326,11 @@ static long lastRecordedSearchIndex(final File taskDir) throws IOException { /** Verifies that every staged TsFile in the directory has no missing chunks. */ private static boolean isComplete(final File taskDir) { + // A whole-file task is complete once its marker exists: its bytes were validated by PREPARE, + // and it has no chunk records to check. + if (new File(taskDir, TsFileWriterManager.WHOLE_FILE_META_NAME).isFile()) { + return true; + } final File[] files = taskDir.listFiles(LoadTsFileProgress.PROGRESS_FILE_FILTER); if (files == null) { return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 7e373d774668c..ab5f5bd29f76a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -338,23 +338,25 @@ public List writePiece(final LoadTsFileConsens // of // a staged file can be read back command by command. final List refs; + final boolean wholeFileTask; if (stopped) { throw newStoppedIOException(); } final Lock readLock = writersLock.readLock(); readLock.lock(); try { - refs = - getOrCreateWriterManager(node.getLoadId()) - .writePiece(node.getTsFileDataList(), node.getSearchIndex()); + final TsFileWriterManager writerManager = getOrCreateWriterManager(node.getLoadId()); + wholeFileTask = writerManager.isWholeFileTask(); + refs = writerManager.writePiece(node.getTsFileDataList(), node.getSearchIndex()); } finally { readLock.unlock(); } - if (node.getPieceRefs().isEmpty()) { + if (!wholeFileTask && !node.hasWholeFileData() && node.getPieceRefs().isEmpty()) { // The chunks of this piece have just been written into the staged file, which recorded where // the payload of each of them landed. The WAL entry keeps everything the leader received - // device, headers, statistics and layout - and only replaces the payload with those - // references, so a replica can be given the very same piece later on + // references, so a replica can be given the very same piece later on. A whole-file slice has + // no reference form and is already in the WAL, so it must not be logged a second time. node.setPieceRefs(refs); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java index 64e3e44a524fc..dedb766ce87fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java @@ -167,6 +167,10 @@ private static void copyTask(final File taskDir, final File targetDir) throws IO } private static void discardUnrecordedStagedFiles(final File snapshotTaskDir) throws IOException { + // A whole-file task has no chunk records by design: its marker and its bytes are the state. + if (new File(snapshotTaskDir, TsFileWriterManager.WHOLE_FILE_META_NAME).isFile()) { + return; + } for (final File stagedFile : listFiles(snapshotTaskDir)) { if (!stagedFile.isFile() || !stagedFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java index ca30a78b6df23..ee4674256177a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java @@ -39,11 +39,14 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; import org.apache.iotdb.db.storageengine.load.metrics.LoadPointCountMetrics; import org.apache.iotdb.db.storageengine.load.splitter.ChunkData; import org.apache.iotdb.db.storageengine.load.splitter.ChunkPayloadRef; import org.apache.iotdb.db.storageengine.load.splitter.DeletionData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; +import org.apache.iotdb.db.storageengine.load.splitter.TsFileDataType; +import org.apache.iotdb.db.storageengine.load.splitter.WholeFileData; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.tsfile.common.conf.TSFileConfig; @@ -60,19 +63,28 @@ import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.TsFilePrecalculatedChunkWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -84,6 +96,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import java.util.zip.CRC32C; /** * Manages the physical writing, crash recovery, and consensus transitions (PIECE, PREPARE, COMMIT, @@ -115,6 +128,24 @@ final class TsFileWriterManager { private final Map timePartition2ModificationFile = new ConcurrentHashMap<>(); + private static final String WHOLE_FILE_MAGIC = "LTWHOLE1"; + private static final int WHOLE_FILE_META_VERSION = 1; + static final String WHOLE_FILE_TSFILE_NAME = "whole-file.tsfile"; + static final String WHOLE_FILE_META_NAME = "whole-file.meta"; + + /** Whole-file staging: the source TsFile is copied byte for byte, not rebuilt from chunks. */ + private File wholeFile; + + private FileChannel wholeFileChannel; + private long wholeFileSourceLength = -1L; + private long wholeFileSealedLength = -1L; + private long wholeFileTimePartitionStart = Long.MIN_VALUE; + private boolean wholeFileGeneratedByPipe; + private ModificationFile wholeFileModificationFile; + private TsFileResource wholeFileResource; + private LoadTsFileProgress wholeFileProgress; + private TTimePartitionSlot wholeFileTimePartition; + private volatile long currentSearchIndex = -1L; private volatile boolean isClosed; @@ -137,12 +168,20 @@ File getTaskDir() { return taskDir; } + /** Whether this task stages a complete source TsFile instead of per-partition chunks. */ + boolean isWholeFileTask() { + return wholeFile != null; + } + boolean hasPendingTsFiles() { - return !timePartition2Resource.isEmpty(); + return wholeFile != null || wholeFileChannel != null || !timePartition2Resource.isEmpty(); } boolean hasStagedData() { - return !timePartition2Writer.isEmpty() || !timePartition2ModificationFile.isEmpty(); + return wholeFileChannel != null + || wholeFileResource != null + || !timePartition2Writer.isEmpty() + || !timePartition2ModificationFile.isEmpty(); } // ------------------------------------------------------------------------- @@ -183,6 +222,11 @@ List writePiece(final List tsFileD .EXCEPTION_PRECALCULATED_CHUNK_STAGING_IS_NOT_SUPPORTED_BY_THE_FILE_ENCRYPTION_POLICY_13DA8C32); } + if (containsWholeFileData(tsFileDataList)) { + writeWholeFilePiece(tsFileDataList); + return Collections.emptyList(); + } + for (final TsFileData tsFileData : tsFileDataList) { switch (tsFileData.getType()) { case CHUNK: @@ -190,7 +234,11 @@ List writePiece(final List tsFileD write(chunkData.getTimePartitionSlot(), chunkData); break; case DELETION: - writeDeletion((DeletionData) tsFileData); + if (wholeFile != null) { + writeWholeFileDeletion((DeletionData) tsFileData); + } else { + writeDeletion((DeletionData) tsFileData); + } break; default: throw new IOException( @@ -205,6 +253,182 @@ List writePiece(final List tsFileD return publishPieceRefs(); } + // ------------------------------------------------------------------------- + // Whole-file staging + // ------------------------------------------------------------------------- + + private static boolean containsWholeFileData(final List tsFileDataList) { + for (final TsFileData data : tsFileDataList) { + if (data.getType() == TsFileDataType.WHOLE_FILE) { + return true; + } + } + return false; + } + + private void writeWholeFilePiece(final List tsFileDataList) throws IOException { + for (final TsFileData data : tsFileDataList) { + if (data.getType() != TsFileDataType.WHOLE_FILE) { + throw new IOException( + StorageEngineMessages + .EXCEPTION_A_WHOLE_FILE_LOAD_PIECE_CANNOT_MIX_WHOLE_FILE_SLICES_AND_CHUNK_DATA_B1297688); + } + writeWholeFileData((WholeFileData) data); + } + if (wholeFileChannel != null) { + wholeFileChannel.force(false); + } + } + + private void writeWholeFileData(final WholeFileData data) throws IOException { + final CRC32C crc32c = new CRC32C(); + crc32c.update(data.getPayload(), 0, data.getPayload().length); + if (crc32c.getValue() != data.getChecksum()) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_CHECKSUM_MISMATCH_AT_OFFSET_D_EXPECTED_D_ACTUAL_D_48238B7E, + data.getOffset(), + data.getChecksum(), + crc32c.getValue())); + } + + ensureWholeFile(data); + appendWholeFile(wholeFile, wholeFileChannel, data); + } + + /** + * Writes one deletion of a whole-file task to the sidecar of the staged TsFile. {@code + * loadNewTsFile} picks that sidecar up on COMMIT, exactly as it does for a chunk-staged file. + */ + private void writeWholeFileDeletion(final DeletionData deletionData) throws IOException { + if (wholeFile == null) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C, + taskDir.getAbsolutePath())); + } + if (wholeFileModificationFile == null) { + final File modsFile = ModificationFile.getExclusiveMods(wholeFile); + if (!modsFile.exists() && !modsFile.createNewFile()) { + throw new IOException( + StorageEngineMessages.EXCEPTION_FAILED_TO_CREATE_MODIFICATION_FILE_0D020A0C); + } + wholeFileModificationFile = new ModificationFile(modsFile, false); + } + deletionData.writeToModificationFile(wholeFileModificationFile); + } + + private void ensureWholeFile(final WholeFileData data) throws IOException { + if (wholeFileChannel != null) { + validateWholeFile(data); + return; + } + if (wholeFile != null) { + // Recovered from disk: only the metadata still has to match the slices that arrive next. + validateWholeFile(data); + if (wholeFileChannel == null) { + wholeFileChannel = + FileChannel.open(wholeFile.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); + } + return; + } + wholeFile = new File(taskDir, WHOLE_FILE_TSFILE_NAME); + if (!wholeFile.createNewFile()) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_5F80C60D, + wholeFile.getAbsolutePath())); + } + wholeFileChannel = + FileChannel.open(wholeFile.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); + wholeFileSourceLength = data.getSourceLength(); + wholeFileTimePartitionStart = data.getTimePartitionStart(); + wholeFileTimePartition = new TTimePartitionSlot(wholeFileTimePartitionStart); + wholeFileGeneratedByPipe = data.isGeneratedByPipe(); + wholeFileProgress = new LoadTsFileProgress(wholeFile); + writeWholeFileMeta(); + } + + private void validateWholeFile(final WholeFileData data) throws IOException { + if (wholeFileSourceLength != data.getSourceLength() + || wholeFileTimePartitionStart != data.getTimePartitionStart() + || wholeFileGeneratedByPipe != data.isGeneratedByPipe()) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB, + data.getSourceLength(), + data.getTimePartitionStart(), + wholeFileSourceLength)); + } + } + + private void appendWholeFile(final File file, final FileChannel channel, final WholeFileData data) + throws IOException { + final long length = file.length(); + if (data.getOffset() > length) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5, + data.getOffset(), + length, + taskDir.getAbsolutePath())); + } + final long alreadyStagedLong = Math.max(0L, length - data.getOffset()); + if (alreadyStagedLong >= data.getPayload().length) { + return; + } + final int alreadyStaged = (int) alreadyStagedLong; + final ByteBuffer buffer = + ByteBuffer.wrap(data.getPayload(), alreadyStaged, data.getPayload().length - alreadyStaged); + channel.position(length); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.force(false); + } + + private void writeWholeFileMeta() throws IOException { + final File meta = new File(taskDir, WHOLE_FILE_META_NAME); + try (final FileOutputStream fos = new FileOutputStream(meta, false); + final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(fos))) { + ReadWriteIOUtils.write(WHOLE_FILE_MAGIC, out); + ReadWriteIOUtils.write(WHOLE_FILE_META_VERSION, out); + ReadWriteIOUtils.write(wholeFileSourceLength, out); + ReadWriteIOUtils.write(wholeFileTimePartitionStart, out); + ReadWriteIOUtils.write(wholeFileGeneratedByPipe, out); + ReadWriteIOUtils.write(wholeFile.getName(), out); + out.flush(); + fos.getFD().sync(); + } + } + + private static WholeFileMeta readWholeFileMeta(final File meta) throws IOException { + if (!meta.isFile()) { + return null; + } + try (final FileInputStream fis = new FileInputStream(meta); + final DataInputStream in = new DataInputStream(new BufferedInputStream(fis))) { + final String magic = ReadWriteIOUtils.readString(in); + if (!WHOLE_FILE_MAGIC.equals(magic)) { + return null; + } + final int version = ReadWriteIOUtils.readInt(in); + if (version != WHOLE_FILE_META_VERSION) { + return null; + } + return new WholeFileMeta( + ReadWriteIOUtils.readLong(in), + ReadWriteIOUtils.readLong(in), + ReadWriteIOUtils.readBool(in), + ReadWriteIOUtils.readString(in)); + } + } + /** One fsync per staged file of the piece, after every chunk of that piece is written. */ private void forceStagedFiles() throws IOException { for (final TsFilePrecalculatedChunkWriter writer : timePartition2Writer.values()) { @@ -228,6 +452,24 @@ void recordTerminal( final long searchIndex, final Map timePartition2ExpectedDataEnd) throws IOException { + if (wholeFile != null) { + final long expectedDataEnd = + expectedDataEndOf(timePartition2ExpectedDataEnd, wholeFileTimePartition); + if (wholeFileProgress == null + || !canMarkWholeFileTerminal( + terminalOp, + wholeFileSealedLength >= 0L ? wholeFileSealedLength : wholeFile.length(), + expectedDataEnd)) { + LOGGER.warn( + StorageEngineMessages + .LOG_STAGED_FILE_ARG_OF_THE_LOAD_TASK_IS_NOT_CONTINUOUS_SO_IT_CANNOT_BE_MARKED_ARG_AND_THE_TERMINAL_RECORD_IS_WITHHELD_7B21F365, + wholeFile.getAbsolutePath(), + terminalOp); + return; + } + wholeFileProgress.recordTerminal(terminalOp, searchIndex); + return; + } for (final Map.Entry entry : timePartition2Progress.entrySet()) { final TsFilePrecalculatedChunkWriter writer = timePartition2Writer.get(entry.getKey()); @@ -279,6 +521,17 @@ static boolean canMarkTerminal( : progress.getFirstHoleOffset() < 0L && fileLength >= progress.getTotalLength(); } + /** Whether a whole-file staging has the bytes the terminal command of the task may describe. */ + static boolean canMarkWholeFileTerminal( + final LoadTsFileConsensusOp terminalOp, final long fileLength, final long expectedDataEnd) { + if (expectedDataEnd >= 0L && fileLength < expectedDataEnd) { + return false; + } + return terminalOp != LoadTsFileConsensusOp.COMMIT + || expectedDataEnd < 0L + || fileLength == expectedDataEnd; + } + /** Writes a chunk into the target staged file or skips if the byte range was already recorded. */ private void write(final TTimePartitionSlot timePartitionSlot, final ChunkData chunkData) throws IOException, PageException { @@ -519,6 +772,12 @@ void prepare( checkNotClosed(); touch(); + if (wholeFile != null) { + prepareWholeFile( + isGeneratedByPipe, timePartitionProgressIndexMap, timePartition2ExpectedDataEnd); + return; + } + // Validate every announced partition before sealing any file. A missing partition or a file // without durable chunk records must not turn into a successful partial import. validateStagedFiles(timePartition2ExpectedDataEnd); @@ -560,9 +819,82 @@ void prepare( } } + /** + * Seals a whole-file staging: the copied TsFile is validated and its resource is rebuilt. Nothing + * is imported here; COMMIT does that through {@link #loadAll}. + */ + private void prepareWholeFile( + final boolean isGeneratedByPipe, + final Map timePartitionProgressIndexMap, + final Map timePartition2ExpectedDataEnd) + throws IOException, LoadFileException { + if (wholeFile == null || wholeFileProgress == null) { + throw new LoadFileException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C, + taskDir.getAbsolutePath())); + } + + if (wholeFileChannel != null) { + wholeFileChannel.force(false); + } + final long fileLength = wholeFile.length(); + wholeFileSealedLength = fileLength; + final long expectedDataEnd = + expectedDataEndOf(timePartition2ExpectedDataEnd, wholeFileTimePartition); + if ((wholeFileSourceLength >= 0L && fileLength != wholeFileSourceLength) + || (expectedDataEnd >= 0L && fileLength != expectedDataEnd)) { + throw new LoadFileException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_LENGTH_D_DOES_NOT_MATCH_THE_EXPECTED_LENGTH_D_FOR_TASK_S_89AC32B6, + fileLength, + expectedDataEnd >= 0L ? expectedDataEnd : wholeFileSourceLength, + taskDir.getAbsolutePath())); + } + + if (wholeFileChannel != null) { + wholeFileChannel.close(); + wholeFileChannel = null; + } + if (wholeFileModificationFile != null) { + wholeFileModificationFile.close(); + wholeFileModificationFile = null; + } + + final TsFileResource resource = new TsFileResource(wholeFile); + try (final TsFileSequenceReader reader = + new TsFileSequenceReader(wholeFile.getAbsolutePath())) { + resource.updatePlanIndexes(reader.getMinPlanIndex()); + resource.updatePlanIndexes(reader.getMaxPlanIndex()); + TsFileResourceUtils.updateTsFileResource(reader, resource); + } + resource.setStatus(TsFileResourceStatus.NORMAL); + resource.setGeneratedByPipe(isGeneratedByPipe); + resource.setProgressIndex( + timePartitionProgressIndexMap.getOrDefault( + wholeFileTimePartition, MinimumProgressIndex.INSTANCE)); + resource.serialize(); + + wholeFileResource = resource; + timePartition2Resource.put(wholeFileTimePartition, resource); + timePartition2Progress.put(wholeFileTimePartition, wholeFileProgress); + } + /** Formally imports sealed staged TsFiles into the target storage engine data regions. */ void loadAll(final boolean isGeneratedByPipe, final boolean deleteStagedSource) throws IOException, LoadFileException { + if (wholeFile != null && wholeFileSealedLength < 0L && wholeFile.isFile()) { + wholeFileSealedLength = wholeFile.length(); + } + if (wholeFile != null && wholeFileResource == null) { + throw new LoadFileException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C, + taskDir.getAbsolutePath())); + } for (final Map.Entry entry : timePartition2Resource.entrySet()) { final DataRegion targetDataRegion = dataRegion; @@ -708,6 +1040,11 @@ private long getTsFileWritePointCount(final TsFilePrecalculatedChunkWriter write * writers. */ TsFileWriterManager recoverFromDisk() throws IOException { + final File wholeFileMeta = new File(taskDir, WHOLE_FILE_META_NAME); + if (wholeFileMeta.isFile()) { + return recoverWholeFileFromDisk(wholeFileMeta); + } + final File[] files = taskDir.listFiles(LoadTsFileProgress.PROGRESS_FILE_FILTER); if (files == null) { return this; @@ -830,6 +1167,40 @@ TsFileWriterManager recoverFromDisk() throws IOException { return this; } + private TsFileWriterManager recoverWholeFileFromDisk(final File metaFile) throws IOException { + final WholeFileMeta meta = readWholeFileMeta(metaFile); + if (meta == null) { + return this; + } + wholeFile = new File(taskDir, meta.tsFileName); + if (!wholeFile.isFile()) { + return this; + } + wholeFileChannel = + FileChannel.open(wholeFile.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); + wholeFileSourceLength = meta.sourceLength; + wholeFileSealedLength = -1L; + wholeFileTimePartitionStart = meta.timePartitionStart; + wholeFileTimePartition = new TTimePartitionSlot(wholeFileTimePartitionStart); + wholeFileGeneratedByPipe = meta.generatedByPipe; + wholeFileProgress = new LoadTsFileProgress(wholeFile); + final File modsFile = ModificationFile.getExclusiveMods(wholeFile); + if (modsFile.isFile()) { + wholeFileModificationFile = new ModificationFile(modsFile, false); + } + + if (new File(wholeFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).isFile()) { + wholeFileSealedLength = wholeFile.length(); + final TsFileResource resource = new TsFileResource(wholeFile); + resource.deserialize(); + resource.setStatus(TsFileResourceStatus.NORMAL); + wholeFileResource = resource; + timePartition2Resource.put(wholeFileTimePartition, resource); + timePartition2Progress.put(wholeFileTimePartition, wholeFileProgress); + } + return this; + } + /** * Replays serialized chunk headers and statistics to restore writer metadata state without * physical pages. @@ -884,6 +1255,57 @@ void close(final boolean retainStagedFiles) { } isClosed = true; + if (wholeFileChannel != null) { + try { + wholeFileChannel.close(); + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.CLOSE_TSFILE_IO_WRITER_ERROR, + wholeFile == null ? taskDir.getPath() : wholeFile.getPath(), + e); + } + wholeFileChannel = null; + } + if (wholeFileModificationFile != null) { + try { + wholeFileModificationFile.close(); + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.CLOSE_MODIFICATION_FILE_ERROR, + wholeFileModificationFile.getFile(), + e); + } + wholeFileModificationFile = null; + } + if (!retainStagedFiles) { + for (final File staged : + new File[] { + wholeFile, + wholeFile == null ? null : ModificationFile.getExclusiveMods(wholeFile), + new File(taskDir, WHOLE_FILE_META_NAME) + }) { + if (staged == null) { + continue; + } + try { + deleteIfExistsWithRetry(staged.toPath()); + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.LOG_FAILED_TO_DELETE_ARG_3A7BD6FD, staged.getAbsolutePath(), e); + } + } + if (wholeFileProgress != null) { + try { + deleteIfExistsWithRetry(wholeFileProgress.getProgressFile().toPath()); + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.LOG_FAILED_TO_DELETE_ARG_3A7BD6FD, + wholeFileProgress.getProgressFile().getAbsolutePath(), + e); + } + } + } + for (final TsFilePrecalculatedChunkWriter writer : timePartition2Writer.values()) { try { writer.getOutput().close(); @@ -938,6 +1360,11 @@ void close(final boolean retainStagedFiles) { timePartition2Resource.clear(); timePartition2Progress.clear(); timePartition2ModificationFile.clear(); + wholeFile = null; + wholeFileModificationFile = null; + wholeFileResource = null; + wholeFileProgress = null; + wholeFileTimePartition = null; } // ------------------------------------------------------------------------- @@ -995,6 +1422,24 @@ private long parseTimePartitionStart(final String tsFileName) { tsFileName.length() - TsFileConstant.TSFILE_SUFFIX.length())); } + private static final class WholeFileMeta { + private final long sourceLength; + private final long timePartitionStart; + private final boolean generatedByPipe; + private final String tsFileName; + + private WholeFileMeta( + final long sourceLength, + final long timePartitionStart, + final boolean generatedByPipe, + final String tsFileName) { + this.sourceLength = sourceLength; + this.timePartitionStart = timePartitionStart; + this.generatedByPipe = generatedByPipe; + this.tsFileName = tsFileName; + } + } + /** The bytes this piece added to each staged file, as the refs a replica reads them back with. */ private List publishPieceRefs() { final List refs = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java index 19bc95756b01a..abe0316fd790c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java @@ -70,6 +70,8 @@ static TsFileData deserialize(InputStream stream) return ChunkData.deserialize(stream); case DELETION: return DeletionData.deserialize(stream); + case WHOLE_FILE: + return WholeFileData.deserialize(stream); default: throw new UnsupportedOperationException( StorageEngineMessages.UNKNOWN_TSFILE_DATA_TYPE + type); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileDataType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileDataType.java index 9e3f73f724dd5..1e3e81e0eff3b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileDataType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileDataType.java @@ -21,5 +21,6 @@ public enum TsFileDataType { CHUNK, - DELETION + DELETION, + WHOLE_FILE } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java new file mode 100644 index 0000000000000..c0abcf29becf2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileData.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.load.splitter; + +import org.apache.iotdb.db.i18n.StorageEngineMessages; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Objects; + +/** + * One slice of a complete source TsFile that a LOAD task transfers byte for byte. + * + *

The deletion sidecar of a TsFile is not part of this record. A file that has modifications is + * not loaded through the whole-file path at all: it falls back to the chunk-splitting path, which + * already transfers its deletions. The whole-file path therefore only ever carries the {@code + * .tsfile} bytes, and a receiver can rebuild the exact file without any extra artifact. + * + *

Slices are identified by {@code offset} and are applied in order, so a retried slice is + * skipped instead of being appended twice. The source length, time partition and pipe flag are + * repeated in every slice: the receiver creates the staging state on the first slice and needs them + * to recognise the task and to rebuild its resource on PREPARE. + */ +public class WholeFileData implements TsFileData { + + private static final int TYPE_ORDINAL_BYTES = Integer.BYTES; + private static final int LONG_BYTES = Long.BYTES; + private static final int BOOLEAN_BYTES = Byte.BYTES; + private static final int PAYLOAD_LENGTH_BYTES = Integer.BYTES; + private static final int METADATA_BYTES = + TYPE_ORDINAL_BYTES + 4 * LONG_BYTES + BOOLEAN_BYTES + PAYLOAD_LENGTH_BYTES; + + private final long offset; + private final long sourceLength; + private final long timePartitionStart; + private final boolean generatedByPipe; + private final long checksum; + private final byte[] payload; + + public WholeFileData( + final long offset, + final long sourceLength, + final long timePartitionStart, + final boolean generatedByPipe, + final long checksum, + final byte[] payload) { + if (payload == null || payload.length == 0) { + throw new IllegalArgumentException( + StorageEngineMessages.EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878); + } + if (offset < 0 || sourceLength < 0 || offset > sourceLength - payload.length) { + throw new IllegalArgumentException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_RANGE_IS_INVALID_OFFSET_D_LENGTH_D_SOURCE_LENGTH_D_402A1599, + offset, + payload.length, + sourceLength)); + } + if (generatedByPipe && timePartitionStart == Long.MIN_VALUE) { + throw new IllegalArgumentException( + StorageEngineMessages.EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24); + } + this.offset = offset; + this.sourceLength = sourceLength; + this.timePartitionStart = timePartitionStart; + this.generatedByPipe = generatedByPipe; + this.checksum = checksum; + this.payload = payload; + } + + public long getOffset() { + return offset; + } + + public long getSourceLength() { + return sourceLength; + } + + public long getTimePartitionStart() { + return timePartitionStart; + } + + public boolean isGeneratedByPipe() { + return generatedByPipe; + } + + public long getChecksum() { + return checksum; + } + + public byte[] getPayload() { + return payload; + } + + @Override + public long getDataSize() { + return payload.length; + } + + @Override + public TsFileDataType getType() { + return TsFileDataType.WHOLE_FILE; + } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + serialize(stream, true); + } + + @Override + public void serialize(final DataOutputStream stream, final boolean includeContent) + throws IOException { + // A whole-file slice has no reference form: its bytes are the payload, so they are always + // written. A caller that asks for references therefore still gets a self-contained slice. + ReadWriteIOUtils.write(getType().ordinal(), stream); + ReadWriteIOUtils.write(offset, stream); + ReadWriteIOUtils.write(sourceLength, stream); + ReadWriteIOUtils.write(timePartitionStart, stream); + ReadWriteIOUtils.write(generatedByPipe, stream); + ReadWriteIOUtils.write(checksum, stream); + ReadWriteIOUtils.write(payload.length, stream); + stream.write(payload); + } + + @Override + public int serializedSize(final boolean includeContent) { + return Math.toIntExact(METADATA_BYTES + (long) payload.length); + } + + public static WholeFileData deserialize(final InputStream stream) throws IOException { + final long offset = ReadWriteIOUtils.readLong(stream); + final long sourceLength = ReadWriteIOUtils.readLong(stream); + final long timePartitionStart = ReadWriteIOUtils.readLong(stream); + final boolean generatedByPipe = ReadWriteIOUtils.readBool(stream); + final long checksum = ReadWriteIOUtils.readLong(stream); + final int payloadLength = ReadWriteIOUtils.readInt(stream); + if (payloadLength <= 0) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1, + payloadLength)); + } + final byte[] payload = new byte[payloadLength]; + int read = 0; + while (read < payloadLength) { + final int current = stream.read(payload, read, payloadLength - read); + if (current < 0) { + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_WHOLE_FILE_SLICE_ENDED_AFTER_D_OF_D_PAYLOAD_BYTES_39E892F4, + read, + payloadLength)); + } + read += current; + } + return new WholeFileData( + offset, sourceLength, timePartitionStart, generatedByPipe, checksum, payload); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WholeFileData)) { + return false; + } + final WholeFileData that = (WholeFileData) o; + return offset == that.offset + && sourceLength == that.sourceLength + && timePartitionStart == that.timePartitionStart + && generatedByPipe == that.generatedByPipe + && checksum == that.checksum + && Arrays.equals(payload, that.payload); + } + + @Override + public int hashCode() { + return Objects.hash(offset, sourceLength, timePartitionStart, generatedByPipe, checksum); + } + + @Override + public String toString() { + return "WholeFileData{" + + "offset=" + + offset + + ", length=" + + payload.length + + ", sourceLength=" + + sourceLength + + '}'; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java new file mode 100644 index 0000000000000..0333e118e3970 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadSingleTsFileNodeLoadModeTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.node.load; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadSingleTsFileNode; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; + +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; + +public class LoadSingleTsFileNodeLoadModeTest { + + @Test + public void singleDeviceSlotAndTimeSlotUsesWholeFileConsensus() { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final LoadSingleTsFileNode node = nodeWithDevices(0L, 1L, false, device("root.sg.d1")); + + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_WHOLE_FILE, mode); + Assert.assertEquals(region, node.getLocalRegionReplicaSet()); + Assert.assertNotNull(node.getWholeFileTimePartitionSlot()); + } + + @Test + public void severalDeviceSlotsInOneRegionUseWholeFileConsensus() { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final StringArrayDeviceID first = device("root.sg.d1"); + final SeriesPartitionExecutor executor = + SeriesPartitionExecutor.getSeriesPartitionExecutor( + IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionExecutorClass(), + IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionSlotNum()); + + StringArrayDeviceID second = null; + for (int i = 2; i < 10_000 && second == null; i++) { + final StringArrayDeviceID candidate = device("root.sg.d" + i); + if (!executor + .getSeriesPartitionSlot(first) + .equals(executor.getSeriesPartitionSlot(candidate))) { + second = candidate; + } + } + Assert.assertNotNull( + "the test needs two device ids that map to different series slots", second); + + final LoadSingleTsFileNode node = nodeWithDevices(0L, 100L, false, first, second); + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_WHOLE_FILE, mode); + } + + @Test + public void severalTimeSlotsKeepTheSplitPath() { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final long interval = TimePartitionUtils.getTimePartitionInterval(); + final LoadSingleTsFileNode node = + nodeWithDevices(1L, 2 * interval, false, device("root.sg.d1")); + + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_SPLIT, mode); + } + + @Test + public void deletionSidecarStillUsesWholeFileConsensus() throws Exception { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final File tsFile = Files.createTempFile("load-mode", ".tsfile").toFile(); + final File mods = ModificationFile.getExclusiveMods(tsFile); + Assert.assertTrue(mods.createNewFile()); + try { + final LoadSingleTsFileNode node = nodeWithFile(tsFile, 0L, 1L, false, device("root.sg.d1")); + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_WHOLE_FILE, mode); + } finally { + Files.deleteIfExists(mods.toPath()); + Files.deleteIfExists(tsFile.toPath()); + } + } + + @Test + public void severalRegionsKeepTheSplitPath() { + final TRegionReplicaSet first = region(1, 1, 2); + final TRegionReplicaSet second = region(2, 3, 4); + final LoadSingleTsFileNode node = nodeWithDevices(0L, 100L, false, device("root.sg.d1")); + + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Arrays.asList(first, second)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_SPLIT, mode); + } + + @Test + public void timeColumnConversionKeepsTheSplitPath() { + final TRegionReplicaSet region = region(1, 1, 2, 3); + final LoadSingleTsFileNode node = nodeWithDevices(0L, 100L, true, device("root.sg.d1")); + + final LoadSingleTsFileNode.LoadMode mode = + node.resolveLoadMode(ignored -> Collections.singletonList(region)); + Assert.assertEquals(LoadSingleTsFileNode.LoadMode.CONSENSUS_SPLIT, mode); + } + + private static StringArrayDeviceID device(final String deviceId) { + return new StringArrayDeviceID(deviceId); + } + + private static LoadSingleTsFileNode nodeWithDevices( + final long startTime, + final long endTime, + final boolean needDecodeTsFile, + final StringArrayDeviceID... devices) { + return nodeWithFile(new File("1.tsfile"), startTime, endTime, needDecodeTsFile, devices); + } + + private static LoadSingleTsFileNode nodeWithFile( + final File tsFile, + final long startTime, + final long endTime, + final boolean needDecodeTsFile, + final StringArrayDeviceID... devices) { + final TsFileResource resource = new TsFileResource(tsFile); + for (final StringArrayDeviceID device : devices) { + resource.updateStartTime(device, startTime); + resource.updateEndTime(device, endTime); + } + return new LoadSingleTsFileNode( + new PlanNodeId(""), resource, false, "root.sg", false, 1L, needDecodeTsFile); + } + + private static TRegionReplicaSet region(final int regionId, final int... dataNodeIds) { + final TDataNodeLocation[] locations = new TDataNodeLocation[dataNodeIds.length]; + for (int i = 0; i < dataNodeIds.length; i++) { + locations[i] = + new TDataNodeLocation() + .setDataNodeId(dataNodeIds[i]) + .setInternalEndPoint(new TEndPoint("127.0.0.1", 6667 + dataNodeIds[i])); + } + return new TRegionReplicaSet( + new TConsensusGroupId(TConsensusGroupType.DataRegion, regionId), Arrays.asList(locations)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java new file mode 100644 index 0000000000000..7181108a266a3 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManagerWholeFileTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.load; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.load.splitter.WholeFileData; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Collections; +import java.util.List; +import java.util.zip.CRC32C; + +public class TsFileWriterManagerWholeFileTest { + + private File taskDir; + private TsFileWriterManager writerManager; + + @Before + public void setUp() throws Exception { + taskDir = Files.createTempDirectory("whole-file-staging-test").toFile(); + final DataRegion dataRegion = Mockito.mock(DataRegion.class); + writerManager = new TsFileWriterManager(dataRegion, taskDir); + } + + @After + public void tearDown() throws Exception { + if (writerManager != null) { + writerManager.close(); + } + deleteRecursively(taskDir); + } + + @Test + public void testSlicesAreAppendedInOrderAndDuplicatesAreSkipped() throws Exception { + final byte[] source = new byte[] {0, 1, 2, 3, 4, 5, 6, 7}; + final List firstRefs = + writerManager.writePiece(Collections.singletonList(slice(0, source, 0, 3)), 1L); + Assert.assertTrue(firstRefs.isEmpty()); + final List secondRefs = + writerManager.writePiece(Collections.singletonList(slice(3, source, 3, 5)), 2L); + Assert.assertTrue(secondRefs.isEmpty()); + + // A retried first slice must not append its bytes a second time. + writerManager.writePiece(Collections.singletonList(slice(0, source, 0, 3)), 3L); + + final File staged = new File(taskDir, "whole-file.tsfile"); + Assert.assertTrue(staged.isFile()); + Assert.assertArrayEquals(source, Files.readAllBytes(staged.toPath())); + Assert.assertTrue(writerManager.hasStagedData()); + } + + @Test + public void testHoleIsRejected() throws Exception { + final byte[] source = new byte[] {0, 1, 2, 3}; + try { + writerManager.writePiece(Collections.singletonList(slice(2, source, 2, 2)), 1L); + Assert.fail("a slice that leaves a hole must be rejected"); + } catch (final IOException expected) { + // expected: the receiver cannot append a slice whose predecessor never arrived + } + } + + private static WholeFileData slice( + final int offset, final byte[] source, final int from, final int length) { + final byte[] payload = new byte[length]; + System.arraycopy(source, from, payload, 0, length); + final CRC32C crc32c = new CRC32C(); + crc32c.update(payload, 0, payload.length); + return new WholeFileData(offset, source.length, 0L, false, crc32c.getValue(), payload); + } + + private static void deleteRecursively(final File file) { + final File[] children = file.listFiles(); + if (children != null) { + for (final File child : children) { + deleteRecursively(child); + } + } + file.delete(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java new file mode 100644 index 0000000000000..cc1c70d0dfb6b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/WholeFileDataTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.load.splitter; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.util.zip.CRC32C; + +public class WholeFileDataTest { + + @Test + public void testRoundTripKeepsEveryField() throws Exception { + final byte[] payload = new byte[] {1, 2, 3, 4, 5}; + final CRC32C crc32c = new CRC32C(); + crc32c.update(payload, 0, payload.length); + final WholeFileData data = + new WholeFileData(10L, 100L, 1_000L, true, crc32c.getValue(), payload); + + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (final DataOutputStream out = new DataOutputStream(baos)) { + data.serialize(out); + } + Assert.assertEquals(data.serializedSize(true), baos.size()); + + final TsFileData deserialized = + TsFileData.deserialize(new ByteArrayInputStream(baos.toByteArray())); + Assert.assertTrue(deserialized instanceof WholeFileData); + final WholeFileData wholeFileData = (WholeFileData) deserialized; + Assert.assertEquals(data, wholeFileData); + Assert.assertArrayEquals(payload, wholeFileData.getPayload()); + Assert.assertEquals(10L, wholeFileData.getOffset()); + Assert.assertEquals(100L, wholeFileData.getSourceLength()); + Assert.assertEquals(1_000L, wholeFileData.getTimePartitionStart()); + Assert.assertTrue(wholeFileData.isGeneratedByPipe()); + } + + @Test + public void testRejectsSliceOutsideSourceLength() { + try { + new WholeFileData(95L, 100L, 0L, false, 0L, new byte[10]); + Assert.fail(); + } catch (final IllegalArgumentException expected) { + // expected: the slice must not extend past the declared source length + } + } +}