Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,38 @@ public void testLoadTsFileInCluster() throws Exception {
}
}

/**
* A TsFile whose devices all fall into one time partition and one data region is loaded through
* the whole-file consensus path: three replicas, no chunk splitting, and every replica must end
* up with the same data.
*/
@Test
public void testLoadWholeTsFileIntoSingleRegion() throws Exception {
final long writtenPointCount;
try (final TsFileGenerator generator =
new TsFileGenerator(new File(tmpDir, "load-cluster-whole-file.tsfile"))) {
final List<IMeasurementSchema> schemas =
Arrays.asList(
new MeasurementSchema(MEASUREMENTS.get(0), TSDataType.INT64, TSEncoding.PLAIN),
new MeasurementSchema(MEASUREMENTS.get(1), TSDataType.INT64, TSEncoding.PLAIN),
new MeasurementSchema(MEASUREMENTS.get(2), TSDataType.INT64, TSEncoding.PLAIN),
new MeasurementSchema(MEASUREMENTS.get(3), TSDataType.INT64, TSEncoding.PLAIN));
generator.registerTimeseries(DEVICES.get(0), schemas);
generator.generateData(DEVICES.get(0), 1_000, 1L, false);
writtenPointCount = generator.getTotalNumber();
}

try (final Connection connection = EnvFactory.getEnv().getConnection();
final Statement statement = connection.createStatement()) {
createDatabaseAndSchema(statement);
statement.execute("load \"" + tmpDir.getAbsolutePath() + "\"");
Assert.assertEquals(writtenPointCount, queryPointCount(statement));
}

// The whole-file transaction must reach all three replicas, not only the coordinator.
awaitAllReplicasHaveData();
}

/** Four concurrent LOADs on separate time partitions must expose their combined point count. */
@Test
public void testLoadConcurrently() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2773,8 +2773,11 @@ public final class DataNodeQueryMessages {
"Device {} is not in the tsFileDevice2IsAligned cache {}.";
public static final String LOADTSFILEANALYZER_CURRENT_DATANODE_IS_READ_ONLY_WILL_TRY_TO_CONVERT_TO_TABLETS_AND =
"LoadTsFileAnalyzer: Current datanode is read only, will try to convert to tablets and insert later.";
public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT =
"Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%";
public static final String LOG_LOAD_ANALYSIS_STAGE_ANALYZING_TSFILE_ARG_ARG_ARG_31359828 =
"Load - Analysis Stage: analyzing TsFile {} ({}/{})";
public static final String
LOG_LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_FILE_ARG_3339452F =
"Load - Analysis Stage: {}/{} TsFiles have been analyzed, progress: {}%, file: {}";
public static final String THE_FILE_ARG_IS_NOT_A_VALID_TSFILE_PLEASE_CHECK_THE_INPUT_FILE =
"The file {} is not a valid tsfile. Please check the input file.";
public static final String TSFILE_ARG_IS_A_ARG_MODEL_FILE =
Expand Down Expand Up @@ -2878,6 +2881,25 @@ public final class DataNodeQueryMessages {
"The prefix of sourceKey is not 'source.'. Please check the parameters passed in: {}";
public static final String LOADTSFILESCHEDULER_REGION_MIGRATION_WAS_DETECTED_DURING_LOADING_TSFILE_ARG_WILL_CONVERT =
"LoadTsFileScheduler: Region migration was detected during loading TsFile {}, will convert to insertion to avoid data loss";
public static final String LOG_START_LOADING_ARG_TSFILES_BDB79D79 = "Start loading {} TsFiles";
public static final String LOG_LOAD_TSFILE_ARG_SPLIT_PATH_ARG_A9F2E219 =
"Load TsFile {}, split path={}";
public static final String LOG_LOADING_TSFILE_ARG_ARG_ARG_F29B2E19 = "Loading TsFile {} ({}/{})";
public static final String
LOG_LOADING_TSFILE_ARG_ARG_ARG_WITH_STRATEGY_ARG_CURRENTLY_LOADING_ARG_FEA695E6 =
"Loading TsFile {} ({}/{}) with strategy {}, currently loading: {}";
public static final String LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_START_FILESIZE_ARG_D7E5D5F4 =
"Load TsFile {}, phase=dispatch start, fileSize={}";
public static final String
LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_DISPATCHEDBYTES_ARG_FILESIZE_ARG_PROGRESS_ARG_668540F9 =
"Load TsFile {}, phase=dispatch, dispatchedBytes={}, fileSize={}, progress={}%";
public static final String
LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_END_PIECES_ARG_BYTES_ARG_ELAPSEDMS_ARG_DB1D69B6 =
"Load TsFile {}, phase=dispatch end, pieces={}, bytes={}, elapsedMs={}";
public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_START_B225907D =
"Load TsFile {}, phase={} start";
public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_END_SUCCESS_ARG_ELAPSEDMS_ARG_6126B0D0 =
"Load TsFile {}, phase={} end, success={}, elapsedMs={}";
public static final String LOAD_TSFILE_ARG_SUCCESSFULLY_LOAD_PROCESS_ARG_ARG =
"Load TsFile {} Successfully, load process [{}/{}]";
public static final String
Expand Down Expand Up @@ -4983,4 +5005,10 @@ private DataNodeQueryMessages() {}
public static final String LOG_QUERYFAILED_QUERYID_ARG_SQL_ARG_249264E9 =
"[QueryFailed] queryId: {}, sql: {}";

public static final String EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC =
"Whole-file load has no single resolved region for %s.";

public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT =
"Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%";

}
Original file line number Diff line number Diff line change
Expand Up @@ -1452,6 +1452,15 @@ private StorageEngineMessages() {}
"Failed to convert data types for tree model statement {}.";
public static final String STORAGE_LOG_LOAD_INSERTING_TABLET_TO_CASTING_TYPE_FROM_TO_AE808A8B =
"Load: Inserting tablet to {}.{}. Casting type from {} to {}.";
public static final String
LOG_SPLIT_TSFILE_ARG_START_FILESIZE_ARG_CHUNKMETADATA_ARG_FORCEDECODE_ARG_243A182F =
"Split TsFile {} start, fileSize={}, chunkMetadata={}, forceDecode={}";
public static final String
LOG_SPLIT_TSFILE_ARG_END_COMPLETED_ARG_DEVICES_ARG_ENTIRECHUNKS_ARG_DECODEDCHUNKS_ARG_SKIPPEDCHUNKS_ARG_TIMEPARTITIONS_ARG_DELETIONS_ARG_ELAPSEDMS_ARG_063A3789 =
"Split TsFile {} end, completed={}, devices={}, entireChunks={}, decodedChunks={}, "
+ "skippedChunks={}, timePartitions={}, deletions={}, elapsedMs={}";
public static final String LOG_SKIP_CHUNK_WITHOUT_METADATA_IN_TSFILE_ARG_OFFSET_ARG_DATASIZE_ARG_41F9ED2B =
"Skip chunk without metadata in TsFile {}, offset={}, dataSize={}";
public static final String STORAGE_LOG_TRY_TO_LOAD_TSFILE_V3_INTO_CURRENT_VERSION_V4_FILE_PATH_B8D38E22 =
"try to load TsFile V3 into current version (V4), file path: {}";
public static final String STORAGE_LOG_THE_FILE_S_VERSION_NUMBER_IS_HIGHER_THAN_CURRENT_FILE_PATH_6D17349F =
Expand Down Expand Up @@ -1482,6 +1491,9 @@ private StorageEngineMessages() {}
+ "will retry later.";
public static final String STORAGE_LOG_SUCCESSFULLY_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_ADB5FEC9 =
"Successfully auto load tsfile {} (isGeneratedByPipe = {})";
public static final String
LOG_START_AUTO_LOADING_TSFILE_ARG_ISGENERATEDBYPIPE_ARG_PENDING_ARG_LOADING_ARG_7B71F0D6 =
"Start auto loading TsFile {} (isGeneratedByPipe = {}), pending={}, loading={}";
public static final String STORAGE_LOG_ERROR_OCCURRED_DURING_CREATING_FAIL_DIRECTORY_FOR_ACTIVE_7D3BEB38 =
"Error occurred during creating fail directory {} for active load.";
public static final String STORAGE_LOG_FAILED_TO_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_STATUS_FILE_F43E9EF7 =
Expand Down Expand Up @@ -1906,4 +1918,45 @@ private StorageEngineMessages() {}
"source files are cleaned up";

public static final String EXCEPTION_TASK_FINISHED_A075ACB1 = "task finished";

public static final String EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878 =
"Whole-file slice payload cannot be empty.";

public static final String EXCEPTION_WHOLE_FILE_SLICE_RANGE_IS_INVALID_OFFSET_D_LENGTH_D_SOURCE_LENGTH_D_402A1599 =
"Whole-file slice range is invalid: offset %d, length %d, source length %d.";

public static final String EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24 =
"Whole-file slice time partition is missing.";


public static final String EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1 =
"Whole-file slice has invalid payload length %d.";

public static final String EXCEPTION_WHOLE_FILE_SLICE_ENDED_AFTER_D_OF_D_PAYLOAD_BYTES_39E892F4 =
"Whole-file slice ended after %d of %d payload bytes.";

public static final String EXCEPTION_A_WHOLE_FILE_LOAD_PIECE_CANNOT_MIX_WHOLE_FILE_SLICES_AND_CHUNK_DATA_B1297688 =
"A whole-file LOAD piece cannot mix whole-file slices and chunk data.";

public static final String EXCEPTION_WHOLE_FILE_SLICE_CHECKSUM_MISMATCH_AT_OFFSET_D_EXPECTED_D_ACTUAL_D_48238B7E =
"Whole-file slice checksum mismatch at offset %d: expected %d, actual %d.";

public static final String EXCEPTION_WHOLE_FILE_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_5F80C60D =
"Whole-file staging %s already exists but cannot be resumed.";

public static final String EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB =
"Whole-file slice (source length %d, time partition %d) does not match the staged task (source length %d).";




public static final String EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5 =
"Whole-file slice offset %d exceeds the staged length %d for task %s.";

public static final String EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C =
"Whole-file staging is missing the TsFile of task %s.";

public static final String EXCEPTION_WHOLE_FILE_STAGING_LENGTH_D_DOES_NOT_MATCH_THE_EXPECTED_LENGTH_D_FOR_TASK_S_89AC32B6 =
"Whole-file staging length %d does not match the expected length %d for task %s.";

}
Original file line number Diff line number Diff line change
Expand Up @@ -3196,9 +3196,11 @@ public final class DataNodeQueryMessages {
public static final String LOADTSFILEANALYZER_CURRENT_DATANODE_IS_READ_ONLY_WILL_TRY_TO_CONVERT_TO_TABLETS_AND =

"LoadTsFileAnalyzer: 当前 DataNode 为只读,将尝试转换为 tablet 后再插入。";
public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT =

"加载 - 分析阶段:已分析 {}/{} 个 TsFile,进度:{}%";
public static final String LOG_LOAD_ANALYSIS_STAGE_ANALYZING_TSFILE_ARG_ARG_ARG_31359828 =
"加载 - 分析阶段:正在分析 TsFile {}({}/{})";
public static final String
LOG_LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_FILE_ARG_3339452F =
"加载 - 分析阶段:已分析 {}/{} 个 TsFile,进度:{}%,文件:{}";
public static final String THE_FILE_ARG_IS_NOT_A_VALID_TSFILE_PLEASE_CHECK_THE_INPUT_FILE =

"文件 {} 不是有效的 TsFile,请检查输入文件。";
Expand Down Expand Up @@ -3354,6 +3356,25 @@ public final class DataNodeQueryMessages {


"LoadTsFileScheduler:加载 TsFile {} 期间检测到 Region 迁移,将转换为 insertion 以避免数据丢失。";
public static final String LOG_START_LOADING_ARG_TSFILES_BDB79D79 = "开始加载 {} 个 TsFile";
public static final String LOG_LOAD_TSFILE_ARG_SPLIT_PATH_ARG_A9F2E219 =
"加载 TsFile {},拆分路径={}";
public static final String LOG_LOADING_TSFILE_ARG_ARG_ARG_F29B2E19 = "正在加载 TsFile {}({}/{})";
public static final String
LOG_LOADING_TSFILE_ARG_ARG_ARG_WITH_STRATEGY_ARG_CURRENTLY_LOADING_ARG_FEA695E6 =
"正在加载 TsFile {}({}/{}),策略 {},当前正在加载的文件:{}";
public static final String LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_START_FILESIZE_ARG_D7E5D5F4 =
"加载 TsFile {},dispatch 阶段开始,fileSize={}";
public static final String
LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_DISPATCHEDBYTES_ARG_FILESIZE_ARG_PROGRESS_ARG_668540F9 =
"加载 TsFile {},dispatch 阶段,dispatchedBytes={},fileSize={},进度={}%";
public static final String
LOG_LOAD_TSFILE_ARG_PHASE_DISPATCH_END_PIECES_ARG_BYTES_ARG_ELAPSEDMS_ARG_DB1D69B6 =
"加载 TsFile {},dispatch 阶段结束,pieces={},bytes={},elapsedMs={}";
public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_START_B225907D =
"加载 TsFile {},阶段 {} 开始";
public static final String LOG_LOAD_TSFILE_ARG_PHASE_ARG_END_SUCCESS_ARG_ELAPSEDMS_ARG_6126B0D0 =
"加载 TsFile {},阶段 {} 结束,success={},elapsedMs={}";
public static final String LOAD_TSFILE_ARG_SUCCESSFULLY_LOAD_PROCESS_ARG_ARG =
"成功加载 TsFile {},加载进度 [{}/{}]";
public static final String
Expand Down Expand Up @@ -5684,4 +5705,10 @@ private DataNodeQueryMessages() {}
public static final String LOG_QUERYFAILED_QUERYID_ARG_SQL_ARG_249264E9 =
"[QueryFailed] queryId: {}, SQL:{}";

public static final String EXCEPTION_WHOLE_FILE_LOAD_HAS_NO_SINGLE_RESOLVED_REGION_FOR_S_DAC09CDC =
"Whole-file load 未解析出唯一的 region:%s。";

public static final String LOAD_ANALYSIS_STAGE_ARG_ARG_TSFILES_HAVE_BEEN_ANALYZED_PROGRESS_ARG_PERCENT =
"加载 - 分析阶段:已分析 {}/{} 个 TsFile,进度:{}%";

}
Original file line number Diff line number Diff line change
Expand Up @@ -1386,6 +1386,15 @@ private StorageEngineMessages() {}
"无法转换树模型语句 {} 的数据类型。";
public static final String STORAGE_LOG_LOAD_INSERTING_TABLET_TO_CASTING_TYPE_FROM_TO_AE808A8B =
"Load:正在向 {}.{} 插入 tablet。将类型从 {} 转换为 {}。";
public static final String
LOG_SPLIT_TSFILE_ARG_START_FILESIZE_ARG_CHUNKMETADATA_ARG_FORCEDECODE_ARG_243A182F =
"开始拆分 TsFile {},fileSize={},chunkMetadata={},forceDecode={}";
public static final String
LOG_SPLIT_TSFILE_ARG_END_COMPLETED_ARG_DEVICES_ARG_ENTIRECHUNKS_ARG_DECODEDCHUNKS_ARG_SKIPPEDCHUNKS_ARG_TIMEPARTITIONS_ARG_DELETIONS_ARG_ELAPSEDMS_ARG_063A3789 =
"结束拆分 TsFile {},completed={},devices={},entireChunks={},decodedChunks={},"
+ "skippedChunks={},timePartitions={},deletions={},elapsedMs={}";
public static final String LOG_SKIP_CHUNK_WITHOUT_METADATA_IN_TSFILE_ARG_OFFSET_ARG_DATASIZE_ARG_41F9ED2B =
"跳过 TsFile {} 中缺少 ChunkMetadata 的 chunk,offset={},dataSize={}";
public static final String STORAGE_LOG_TRY_TO_LOAD_TSFILE_V3_INTO_CURRENT_VERSION_V4_FILE_PATH_B8D38E22 =
"尝试将 TsFile V3 加载到当前版本 (V4),文件路径:{}";
public static final String STORAGE_LOG_THE_FILE_S_VERSION_NUMBER_IS_HIGHER_THAN_CURRENT_FILE_PATH_6D17349F =
Expand All @@ -1410,6 +1419,9 @@ private StorageEngineMessages() {}
"拒绝自动加载 TsFile {} (isGeneratedByPipe = {}),原因:当前查询已超时,稍后重试。";
public static final String STORAGE_LOG_SUCCESSFULLY_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_ADB5FEC9 =
"成功自动加载 TsFile {} (isGeneratedByPipe = {})";
public static final String
LOG_START_AUTO_LOADING_TSFILE_ARG_ISGENERATEDBYPIPE_ARG_PENDING_ARG_LOADING_ARG_7B71F0D6 =
"开始自动加载 TsFile {} (isGeneratedByPipe = {}),pending={},loading={}";
public static final String STORAGE_LOG_ERROR_OCCURRED_DURING_CREATING_FAIL_DIRECTORY_FOR_ACTIVE_7D3BEB38 =
"为 active load 创建失败目录 {} 时发生错误。";
public static final String STORAGE_LOG_FAILED_TO_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_STATUS_FILE_F43E9EF7 =
Expand Down Expand Up @@ -1808,4 +1820,45 @@ private StorageEngineMessages() {}
public static final String EXCEPTION_SOURCE_FILES_ARE_CLEANED_UP_83638D68 = "源文件已清理";

public static final String EXCEPTION_TASK_FINISHED_A075ACB1 = "任务已完成";

public static final String EXCEPTION_WHOLE_FILE_SLICE_PAYLOAD_CANNOT_BE_EMPTY_C14A7878 =
"Whole-file 切片内容不能为空。";

public static final String EXCEPTION_WHOLE_FILE_SLICE_RANGE_IS_INVALID_OFFSET_D_LENGTH_D_SOURCE_LENGTH_D_402A1599 =
"Whole-file 切片范围非法:offset %d,length %d,source length %d。";

public static final String EXCEPTION_WHOLE_FILE_SLICE_TIME_PARTITION_IS_MISSING_4EE30E24 =
"Whole-file 切片缺少时间分区。";


public static final String EXCEPTION_WHOLE_FILE_SLICE_HAS_INVALID_PAYLOAD_LENGTH_D_0D04C9C1 =
"Whole-file 切片的 payload 长度非法:%d。";

public static final String EXCEPTION_WHOLE_FILE_SLICE_ENDED_AFTER_D_OF_D_PAYLOAD_BYTES_39E892F4 =
"Whole-file 切片在读取到 %d/%d 字节时提前结束。";

public static final String EXCEPTION_A_WHOLE_FILE_LOAD_PIECE_CANNOT_MIX_WHOLE_FILE_SLICES_AND_CHUNK_DATA_B1297688 =
"Whole-file LOAD piece 不能同时包含 whole-file 切片和 chunk 数据。";

public static final String EXCEPTION_WHOLE_FILE_SLICE_CHECKSUM_MISMATCH_AT_OFFSET_D_EXPECTED_D_ACTUAL_D_48238B7E =
"Whole-file 切片在 offset %d 处校验和不一致:期望 %d,实际 %d。";

public static final String EXCEPTION_WHOLE_FILE_STAGING_S_ALREADY_EXISTS_BUT_CANNOT_BE_RESUMED_5F80C60D =
"Whole-file staging 文件 %s 已存在,但无法恢复。";

public static final String EXCEPTION_WHOLE_FILE_SLICE_SOURCE_LENGTH_D_TIME_PARTITION_D_DOES_NOT_MATCH_THE_STAGED_TASK_SOURCE_LENGTH_D_FCD90EBB =
"Whole-file 切片(source length %d,time partition %d)与已 staging 的任务(source length %d)不匹配。";




public static final String EXCEPTION_WHOLE_FILE_SLICE_OFFSET_D_EXCEEDS_THE_STAGED_LENGTH_D_FOR_TASK_S_8E5B27C5 =
"Whole-file 切片 offset %d 超过已 staging 的长度 %d(任务 %s)。";

public static final String EXCEPTION_WHOLE_FILE_STAGING_IS_MISSING_THE_TSFILE_OF_TASK_S_AC2FAD2C =
"Whole-file staging 缺少任务 %s 的 TsFile。";

public static final String EXCEPTION_WHOLE_FILE_STAGING_LENGTH_D_DOES_NOT_MATCH_THE_EXPECTED_LENGTH_D_FOR_TASK_S_89AC32B6 =
"Whole-file staging 长度 %d 与期望长度 %d 不匹配(任务 %s)。";

}
Loading