From 260a0088c9bb4396d642f00246d276d385befccd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:08:05 +0800 Subject: [PATCH 1/2] [Pipe] Fix aggregate window statistics --- .../aggregate/AggregateProcessor.java | 118 +++- .../aggregate/TimeSeriesRuntimeState.java | 53 +- .../AggregatedResultOperator.java | 8 + .../standardstatistics/AverageOperator.java | 2 +- .../ClearanceFactorOperator.java | 16 +- .../CrestFactorOperator.java | 2 +- .../FormFactorOperator.java | 24 +- .../standardstatistics/KurtosisOperator.java | 2 +- .../standardstatistics/PeakOperator.java | 20 +- .../PulseFactorOperator.java | 16 +- .../RootMeanSquareOperator.java | 2 +- .../standardstatistics/SkewnessOperator.java | 2 +- .../standardstatistics/VarianceOperator.java | 4 +- ...CustomizedReadableIntermediateResults.java | 9 +- .../IntermediateResultOperator.java | 8 + .../sametype/numeric/AbsoluteMaxOperator.java | 80 ++- .../numeric/IntegralPoweredSumOperator.java | 50 +- .../FractionPoweredSumOperator.java | 32 +- .../integertype/CountOperator.java | 15 +- .../StandardStatisticsOperatorProcessor.java | 2 + .../datastructure/TimeSeriesWindow.java | 67 ++- .../aggregate/AggregateProcessorTest.java | 535 ++++++++++++++++++ .../TumblingWindowingProcessorTest.java | 31 + 23 files changed, 1000 insertions(+), 98 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessorTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java index d451a57ae6ce0..768e3ec9a0422 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java @@ -77,11 +77,13 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; @@ -151,6 +153,15 @@ public void validate(final PipeParameterValidator validator) throws Exception { PROCESSOR_OPERATORS_KEY), parameters.getStringOrDefault( PROCESSOR_OPERATORS_KEY, PROCESSOR_OPERATORS_DEFAULT_VALUE)) + .validate( + arg -> areUniqueOperatorNames((String) arg), + String.format( + DataNodePipeMessages + .PIPE_EXCEPTION_THE_AGGREGATOR_AND_OUTPUT_NAME_S_IS_INVALID_BC22CF92, + parameters.getStringOrDefault( + PROCESSOR_OPERATORS_KEY, PROCESSOR_OPERATORS_DEFAULT_VALUE)), + parameters.getStringOrDefault( + PROCESSOR_OPERATORS_KEY, PROCESSOR_OPERATORS_DEFAULT_VALUE)) .validate( arg -> !((String) arg).isEmpty(), String.format( @@ -168,9 +179,10 @@ public void validate(final PipeParameterValidator validator) throws Exception { PROCESSOR_OUTPUT_DATABASE_KEY, PROCESSOR_OUTPUT_DATABASE_DEFAULT_VALUE)) .validate( arg -> - ((String) arg).isEmpty() - || Arrays.stream(((String) arg).replace(" ", "").split(",")) - .allMatch(this::isLegalMeasurement), + areLegalOutputMeasurements( + (String) arg, + parameters.getStringOrDefault( + PROCESSOR_OPERATORS_KEY, PROCESSOR_OPERATORS_DEFAULT_VALUE)), String.format( DataNodePipeMessages .MESSAGE_THE_OUTPUT_MEASUREMENTS_ARG_CONTAINS_ILLEGAL_MEASUREMENTS_THE_MEASUREMENTS_MUST_BE_THE_LAST_LEVEL_OF_A_LEGAL_PATH_793FA8AE, @@ -181,6 +193,44 @@ public void validate(final PipeParameterValidator validator) throws Exception { PROCESSOR_OUTPUT_MEASUREMENTS_KEY, PROCESSOR_OUTPUT_MEASUREMENTS_DEFAULT_VALUE)); } + private boolean areUniqueOperatorNames(final String operatorString) { + final String[] operatorNames = operatorString.replace(" ", "").split(",", -1); + final Set uniqueOperatorNames = new HashSet<>(); + return Arrays.stream(operatorNames) + .allMatch( + operatorName -> + !operatorName.isEmpty() + && uniqueOperatorNames.add(operatorName.toLowerCase(Locale.ROOT))); + } + + private boolean areLegalOutputMeasurements( + final String outputMeasurementString, final String operatorString) { + if (outputMeasurementString.isEmpty()) { + return true; + } + + final String[] outputMeasurementNames = + Arrays.stream(outputMeasurementString.replace(" ", "").split(",", -1)) + .toArray(String[]::new); + final String[] operatorNames = operatorString.replace(" ", "").split(",", -1); + if (outputMeasurementNames.length > operatorNames.length) { + return false; + } + + final Set uniqueOutputMeasurementNames = new HashSet<>(); + for (int index = 0; index < operatorNames.length; ++index) { + final String outputMeasurementName = + index < outputMeasurementNames.length + ? outputMeasurementNames[index] + : operatorNames[index]; + if ((index < outputMeasurementNames.length && !isLegalMeasurement(outputMeasurementName)) + || !uniqueOutputMeasurementNames.add(outputMeasurementName)) { + return false; + } + } + return true; + } + private boolean isLegalMeasurement(final String measurement) { try { PathUtils.isLegalPath("root." + measurement); @@ -226,12 +276,14 @@ public void customize( // The output max delay milliseconds must be set to at least 1 // to guarantee the correctness of the CAS in last receive time outputMaxDelayMilliseconds = - outputMaxDelaySeconds < 0 ? Long.MAX_VALUE : Math.max(outputMaxDelaySeconds * 1000, 1); + outputMaxDelaySeconds < 0 + ? Long.MAX_VALUE + : Math.max(secondsToMilliseconds(outputMaxDelaySeconds), 1); outputMinReportIntervalMilliseconds = - parameters.getLongOrDefault( + secondsToMilliseconds( + parameters.getLongOrDefault( PROCESSOR_OUTPUT_MIN_REPORT_INTERVAL_SECONDS_KEY, - PROCESSOR_OUTPUT_MIN_REPORT_INTERVAL_SECONDS_DEFAULT_VALUE) - * 1000; + PROCESSOR_OUTPUT_MIN_REPORT_INTERVAL_SECONDS_DEFAULT_VALUE)); outputDatabase = parameters.getStringOrDefault( PROCESSOR_OUTPUT_DATABASE_KEY, PROCESSOR_OUTPUT_DATABASE_DEFAULT_VALUE); @@ -260,10 +312,10 @@ public void customize( for (int i = 0; i < operatorNameList.size(); ++i) { if (i < outputMeasurementNameList.size()) { aggregatorName2OutputNameMap.put( - operatorNameList.get(i).toLowerCase(), outputMeasurementNameList.get(i)); + operatorNameList.get(i).toLowerCase(Locale.ROOT), outputMeasurementNameList.get(i)); } else { aggregatorName2OutputNameMap.put( - operatorNameList.get(i).toLowerCase(), operatorNameList.get(i)); + operatorNameList.get(i).toLowerCase(Locale.ROOT), operatorNameList.get(i)); } } @@ -281,11 +333,13 @@ public void customize( operatorProcessor.getAggregatorOperatorSet().stream() .filter( operator -> - aggregatorName2OutputNameMap.containsKey(operator.getName().toLowerCase())) + aggregatorName2OutputNameMap.containsKey( + operator.getName().toLowerCase(Locale.ROOT))) .forEach( operator -> { outputName2OperatorMap.put( - aggregatorName2OutputNameMap.get(operator.getName().toLowerCase()), operator); + aggregatorName2OutputNameMap.get(operator.getName().toLowerCase(Locale.ROOT)), + operator); declaredIntermediateResultSet.addAll(operator.getDeclaredIntermediateValueNames()); }); @@ -453,7 +507,7 @@ private Map> processRow( final String timeSeries = (outputDatabaseWithPathSeparator.isEmpty() ? row.getDeviceId() - : row.getDeviceId().replaceFirst(databaseWithPathSeparator, "")) + : stripDatabasePrefix(row.getDeviceId(), databaseWithPathSeparator)) + TsFileConstant.PATH_SEPARATOR + row.getColumnName(index); @@ -585,6 +639,16 @@ private void resetOutputDatabaseForGeneratedEvent(final RowCollector rowCollecto } } + static String stripDatabasePrefix(final String deviceId, final String databaseWithPathSeparator) { + return deviceId.startsWith(databaseWithPathSeparator) + ? deviceId.substring(databaseWithPathSeparator.length()) + : deviceId; + } + + static long secondsToMilliseconds(final long seconds) { + return TimeUnit.SECONDS.toMillis(seconds); + } + /** * Collect {@link WindowOutput}s of a single timeSeries in one turn. The {@link TSDataType}s shall * be the same because the {@link AggregatedResultOperator}s shall return the same value for the @@ -600,19 +664,7 @@ public void collectWindowOutputs( if (Objects.isNull(outputs) || outputs.isEmpty()) { return; } - // Sort and same timestamps removal - outputs.sort(Comparator.comparingLong(WindowOutput::getTimestamp)); - - final AtomicLong lastValue = new AtomicLong(Long.MIN_VALUE); - final List distinctOutputs = new ArrayList<>(); - outputs.forEach( - output -> { - final long timeStamp = output.getTimestamp(); - if (timeStamp != lastValue.get()) { - lastValue.set(timeStamp); - distinctOutputs.add(output); - } - }); + final List distinctOutputs = sortAndDeduplicateOutputs(outputs); final MeasurementSchema[] measurementSchemaList = new MeasurementSchema[columnNameStringList.length]; @@ -756,6 +808,22 @@ private static Type getOutputTabletType(final TSDataType dataType) { } } + static List sortAndDeduplicateOutputs(final List outputs) { + outputs.sort(Comparator.comparingLong(WindowOutput::getTimestamp)); + + final List distinctOutputs = new ArrayList<>(outputs.size()); + long lastTimestamp = 0; + boolean hasLastTimestamp = false; + for (final WindowOutput output : outputs) { + if (!hasLastTimestamp || output.getTimestamp() != lastTimestamp) { + distinctOutputs.add(output); + lastTimestamp = output.getTimestamp(); + hasLastTimestamp = true; + } + } + return distinctOutputs; + } + @Override public void close() throws Exception { boolean isLastReference = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java index 2c5d114e2dd1a..ac796fa2ea2c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java @@ -61,6 +61,7 @@ public class TimeSeriesRuntimeState { // Outer set values private long lastReportTimeStamp = Long.MIN_VALUE; + private boolean hasReportedTimeStamp; private final List currentOpeningWindows = new ArrayList<>(); // Variables to avoid "new" operation @@ -103,7 +104,7 @@ public Pair, Pair> updateWindows( throws IOException { Pair, Pair> output = null; - if (timestamp <= lastReportTimeStamp) { + if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) { return null; } final Set addedWindows = @@ -127,8 +128,7 @@ public Pair, Pair> updateWindows( if (stateWindowOutputPair.getLeft().isEmit() && Objects.nonNull(stateWindowOutputPair.getRight())) { outputList.add(stateWindowOutputPair.getRight()); - lastReportTimeStamp = - Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime()); } if (stateWindowOutputPair.getLeft().isPurge()) { windowIterator.remove(); @@ -149,7 +149,7 @@ public Pair, Pair> updateWindows( throws IOException { Pair, Pair> output = null; - if (timestamp <= lastReportTimeStamp) { + if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) { return null; } final Set addedWindows = @@ -173,8 +173,7 @@ public Pair, Pair> updateWindows( if (stateWindowOutputPair.getLeft().isEmit() && Objects.nonNull(stateWindowOutputPair.getRight())) { outputList.add(stateWindowOutputPair.getRight()); - lastReportTimeStamp = - Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime()); } if (stateWindowOutputPair.getLeft().isPurge()) { windowIterator.remove(); @@ -195,7 +194,7 @@ public Pair, Pair> updateWindows( throws IOException { Pair, Pair> output = null; - if (timestamp <= lastReportTimeStamp) { + if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) { return null; } final Set addedWindows = @@ -219,8 +218,7 @@ public Pair, Pair> updateWindows( if (stateWindowOutputPair.getLeft().isEmit() && Objects.nonNull(stateWindowOutputPair.getRight())) { outputList.add(stateWindowOutputPair.getRight()); - lastReportTimeStamp = - Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime()); } if (stateWindowOutputPair.getLeft().isPurge()) { windowIterator.remove(); @@ -241,7 +239,7 @@ public Pair, Pair> updateWindows( throws IOException { Pair, Pair> output = null; - if (timestamp <= lastReportTimeStamp) { + if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) { return null; } final Set addedWindows = @@ -265,8 +263,7 @@ public Pair, Pair> updateWindows( if (stateWindowOutputPair.getLeft().isEmit() && Objects.nonNull(stateWindowOutputPair.getRight())) { outputList.add(stateWindowOutputPair.getRight()); - lastReportTimeStamp = - Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime()); } if (stateWindowOutputPair.getLeft().isPurge()) { windowIterator.remove(); @@ -287,7 +284,7 @@ public Pair, Pair> updateWindows( throws IOException { Pair, Pair> output = null; - if (timestamp <= lastReportTimeStamp) { + if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) { return null; } final Set addedWindows = @@ -311,8 +308,7 @@ public Pair, Pair> updateWindows( if (stateWindowOutputPair.getLeft().isEmit() && Objects.nonNull(stateWindowOutputPair.getRight())) { outputList.add(stateWindowOutputPair.getRight()); - lastReportTimeStamp = - Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime()); } if (stateWindowOutputPair.getLeft().isPurge()) { windowIterator.remove(); @@ -333,7 +329,7 @@ public Pair, Pair> updateWindows( throws IOException { Pair, Pair> output = null; - if (timestamp <= lastReportTimeStamp) { + if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) { return null; } final Set addedWindows = @@ -357,8 +353,7 @@ public Pair, Pair> updateWindows( if (stateWindowOutputPair.getLeft().isEmit() && Objects.nonNull(stateWindowOutputPair.getRight())) { outputList.add(stateWindowOutputPair.getRight()); - lastReportTimeStamp = - Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime()); } if (stateWindowOutputPair.getLeft().isPurge()) { windowIterator.remove(); @@ -379,7 +374,7 @@ public Pair, Pair> updateWindows( throws IOException { Pair, Pair> output = null; - if (timestamp <= lastReportTimeStamp) { + if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) { return null; } final Set addedWindows = @@ -403,8 +398,7 @@ public Pair, Pair> updateWindows( if (stateWindowOutputPair.getLeft().isEmit() && Objects.nonNull(stateWindowOutputPair.getRight())) { outputList.add(stateWindowOutputPair.getRight()); - lastReportTimeStamp = - Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime()); } if (stateWindowOutputPair.getLeft().isPurge()) { windowIterator.remove(); @@ -425,7 +419,7 @@ public Pair, Pair> updateWindows( throws IOException { Pair, Pair> output = null; - if (timestamp <= lastReportTimeStamp) { + if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) { return null; } final Set addedWindows = @@ -449,8 +443,7 @@ public Pair, Pair> updateWindows( if (stateWindowOutputPair.getLeft().isEmit() && Objects.nonNull(stateWindowOutputPair.getRight())) { outputList.add(stateWindowOutputPair.getRight()); - lastReportTimeStamp = - Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime()); } if (stateWindowOutputPair.getLeft().isPurge()) { windowIterator.remove(); @@ -472,6 +465,11 @@ public List forceOutput() { .collect(Collectors.toList()); } + private void updateLastReportTimeStamp(final long timestamp) { + lastReportTimeStamp = Math.max(lastReportTimeStamp, timestamp); + hasReportedTimeStamp = true; + } + // A runtime state with window buffer shall be reported after at least output min report // interval time to avoid frequent serialization // Return null if this should not report @@ -499,13 +497,18 @@ private Pair getTimestampWindowBufferPair( public void restoreTimestampAndWindows(final Pair timestampWindowBufferPair) throws IOException { - if (timestampWindowBufferPair.getLeft() <= lastReportTimeStamp) { + if (hasReportedTimeStamp && timestampWindowBufferPair.getLeft() <= lastReportTimeStamp) { // The runtime state may be initialized by different processorSubtasks on one DataNode // Only the subtask with the largest timestamp wins return; } this.lastReportTimeStamp = timestampWindowBufferPair.getLeft(); + hasReportedTimeStamp = true; + currentOpeningWindows.clear(); final ByteBuffer buffer = timestampWindowBufferPair.getRight(); + if (buffer == null) { + return; + } final int size = ReadWriteIOUtils.readInt(buffer); for (int i = 0; i < size; ++i) { // The runtime value will be deserialized if exists diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/AggregatedResultOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/AggregatedResultOperator.java index a792bc0d3935d..751372bd59143 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/AggregatedResultOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/AggregatedResultOperator.java @@ -54,6 +54,14 @@ public interface AggregatedResultOperator { /** Get the needed intermediate value names of this aggregate result. */ Set getDeclaredIntermediateValueNames(); + default boolean supportsLegacyIntermediateResults() { + return false; + } + + default Set getLegacyIntermediateValueNames() { + return getDeclaredIntermediateValueNames(); + } + /** * terminateWindow receives a Map to get intermediate results by its names, and produce a * characteristic value of this window. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/AverageOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/AverageOperator.java index 226893acaec97..7dcfa2c1ff48e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/AverageOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/AverageOperator.java @@ -52,6 +52,6 @@ public Pair terminateWindow( TSDataType measurementDataType, CustomizedReadableIntermediateResults intermediateResults) { return new Pair<>( TSDataType.DOUBLE, - intermediateResults.getDouble("sum_x1") / intermediateResults.getInt("count")); + intermediateResults.getDouble("sum_x1") / intermediateResults.getLong("count")); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/ClearanceFactorOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/ClearanceFactorOperator.java index d77e49bf3f24e..309d0e0c00e14 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/ClearanceFactorOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/ClearanceFactorOperator.java @@ -44,6 +44,17 @@ public void configureSystemParameters(Map systemParams) { @Override public Set getDeclaredIntermediateValueNames() { + return Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("abs_max", "sum_x0.5", "sum_abs_x0.5", "count"))); + } + + @Override + public boolean supportsLegacyIntermediateResults() { + return true; + } + + @Override + public Set getLegacyIntermediateValueNames() { return Collections.unmodifiableSet( new HashSet<>(Arrays.asList("abs_max", "sum_x0.5", "count"))); } @@ -51,11 +62,14 @@ public Set getDeclaredIntermediateValueNames() { @Override public Pair terminateWindow( TSDataType measurementDataType, CustomizedReadableIntermediateResults intermediateResults) { + final String averageSquareRootAbsoluteValueName = + intermediateResults.isLegacyStatisticsFormula() ? "sum_x0.5" : "sum_abs_x0.5"; return new Pair<>( TSDataType.DOUBLE, intermediateResults.getDouble("abs_max") / Math.pow( - intermediateResults.getDouble("sum_x0.5") / intermediateResults.getInt("count"), + intermediateResults.getDouble(averageSquareRootAbsoluteValueName) + / intermediateResults.getLong("count"), 2)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/CrestFactorOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/CrestFactorOperator.java index 53710e1e3263c..f0ee250a00cf3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/CrestFactorOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/CrestFactorOperator.java @@ -54,7 +54,7 @@ public Pair terminateWindow( TSDataType.DOUBLE, intermediateResults.getDouble("abs_max") / Math.pow( - intermediateResults.getDouble("sum_x2") / intermediateResults.getInt("count"), + intermediateResults.getDouble("sum_x2") / intermediateResults.getLong("count"), 0.5)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/FormFactorOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/FormFactorOperator.java index a0e8cac3b9e9c..b834e954be57a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/FormFactorOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/FormFactorOperator.java @@ -44,15 +44,35 @@ public void configureSystemParameters(Map systemParams) { @Override public Set getDeclaredIntermediateValueNames() { + return Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("sum_x2", "sum_x1", "sum_abs_x1", "count"))); + } + + @Override + public boolean supportsLegacyIntermediateResults() { + return true; + } + + @Override + public Set getLegacyIntermediateValueNames() { return Collections.unmodifiableSet(new HashSet<>(Arrays.asList("sum_x2", "sum_x1", "count"))); } @Override public Pair terminateWindow( TSDataType measurementDataType, CustomizedReadableIntermediateResults intermediateResults) { + if (intermediateResults.isLegacyStatisticsFormula()) { + return new Pair<>( + TSDataType.DOUBLE, + Math.pow( + intermediateResults.getDouble("sum_x2") / intermediateResults.getLong("count"), + 0.5) + / (intermediateResults.getDouble("sum_x1") / intermediateResults.getLong("count"))); + } return new Pair<>( TSDataType.DOUBLE, - Math.pow(intermediateResults.getDouble("sum_x2") / intermediateResults.getInt("count"), 0.5) - / (intermediateResults.getDouble("sum_x1") / intermediateResults.getInt("count"))); + Math.pow( + intermediateResults.getDouble("sum_x2") / intermediateResults.getLong("count"), 0.5) + / (intermediateResults.getDouble("sum_abs_x1") / intermediateResults.getLong("count"))); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/KurtosisOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/KurtosisOperator.java index efd3025cf5ccf..452d72fae135c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/KurtosisOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/KurtosisOperator.java @@ -55,7 +55,7 @@ public Pair terminateWindow( double sumX3 = intermediateResults.getDouble("sum_x3"); double sumX2 = intermediateResults.getDouble("sum_x2"); double sumX1 = intermediateResults.getDouble("sum_x1"); - int count = intermediateResults.getInt("count"); + long count = intermediateResults.getLong("count"); return new Pair<>( TSDataType.DOUBLE, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PeakOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PeakOperator.java index 55f60523e0dc2..d7449ed413378 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PeakOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PeakOperator.java @@ -48,6 +48,24 @@ public Set getDeclaredIntermediateValueNames() { @Override public Pair terminateWindow( TSDataType measurementDataType, CustomizedReadableIntermediateResults intermediateResults) { - return new Pair<>(measurementDataType, intermediateResults.getObject("abs_max")); + final Object absoluteMaximum; + switch (measurementDataType) { + case INT32: + absoluteMaximum = intermediateResults.getInt("abs_max"); + break; + case INT64: + absoluteMaximum = intermediateResults.getLong("abs_max"); + break; + case FLOAT: + absoluteMaximum = intermediateResults.getFloat("abs_max"); + break; + case DOUBLE: + absoluteMaximum = intermediateResults.getDouble("abs_max"); + break; + default: + absoluteMaximum = intermediateResults.getObject("abs_max"); + break; + } + return new Pair<>(measurementDataType, absoluteMaximum); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PulseFactorOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PulseFactorOperator.java index aae68777728ef..aab040b054777 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PulseFactorOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PulseFactorOperator.java @@ -44,15 +44,29 @@ public void configureSystemParameters(Map systemParams) { @Override public Set getDeclaredIntermediateValueNames() { + return Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("abs_max", "sum_x1", "sum_abs_x1", "count"))); + } + + @Override + public boolean supportsLegacyIntermediateResults() { + return true; + } + + @Override + public Set getLegacyIntermediateValueNames() { return Collections.unmodifiableSet(new HashSet<>(Arrays.asList("abs_max", "sum_x1", "count"))); } @Override public Pair terminateWindow( TSDataType measurementDataType, CustomizedReadableIntermediateResults intermediateResults) { + final String averageAbsoluteValueName = + intermediateResults.isLegacyStatisticsFormula() ? "sum_x1" : "sum_abs_x1"; return new Pair<>( TSDataType.DOUBLE, intermediateResults.getDouble("abs_max") - / (intermediateResults.getDouble("sum_x1") / intermediateResults.getInt("count"))); + / (intermediateResults.getDouble(averageAbsoluteValueName) + / intermediateResults.getLong("count"))); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/RootMeanSquareOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/RootMeanSquareOperator.java index 2937072946c3e..9f89052a632f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/RootMeanSquareOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/RootMeanSquareOperator.java @@ -53,6 +53,6 @@ public Pair terminateWindow( return new Pair<>( TSDataType.DOUBLE, Math.pow( - intermediateResults.getDouble("sum_x2") / intermediateResults.getInt("count"), 0.5)); + intermediateResults.getDouble("sum_x2") / intermediateResults.getLong("count"), 0.5)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/SkewnessOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/SkewnessOperator.java index 1fd61727e5f54..823d4010ef1a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/SkewnessOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/SkewnessOperator.java @@ -54,7 +54,7 @@ public Pair terminateWindow( double sumX3 = intermediateResults.getDouble("sum_x3"); double sumX2 = intermediateResults.getDouble("sum_x2"); double sumX1 = intermediateResults.getDouble("sum_x1"); - int count = intermediateResults.getInt("count"); + long count = intermediateResults.getLong("count"); return new Pair<>( TSDataType.DOUBLE, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/VarianceOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/VarianceOperator.java index 386618df635f8..c3edb6bd2f4f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/VarianceOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/VarianceOperator.java @@ -53,8 +53,8 @@ public Pair terminateWindow( TSDataType measurementDataType, CustomizedReadableIntermediateResults intermediateResults) { return new Pair<>( TSDataType.DOUBLE, - intermediateResults.getDouble("sum_x2") / intermediateResults.getInt("count") + intermediateResults.getDouble("sum_x2") / intermediateResults.getLong("count") - Math.pow( - intermediateResults.getDouble("sum_x1") / intermediateResults.getInt("count"), 2)); + intermediateResults.getDouble("sum_x1") / intermediateResults.getLong("count"), 2)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java index 548b03022fdab..ade61d12b89b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java @@ -35,10 +35,17 @@ public class CustomizedReadableIntermediateResults { private final Map> intermediateResults; + private final boolean legacyStatisticsFormula; public CustomizedReadableIntermediateResults( - final Map> intermediateResults) { + final Map> intermediateResults, + final boolean legacyStatisticsFormula) { this.intermediateResults = intermediateResults; + this.legacyStatisticsFormula = legacyStatisticsFormula; + } + + public boolean isLegacyStatisticsFormula() { + return legacyStatisticsFormula; } public boolean getBoolean(final String key) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java index 95e1a9cb0d1e6..e5043bb061527 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java @@ -43,6 +43,9 @@ */ public interface IntermediateResultOperator { + int LEGACY_STATE_FORMAT_VERSION = 0; + int CURRENT_STATE_FORMAT_VERSION = 1; + /** * Return the name of the operator, the name shall be in lower case * @@ -121,4 +124,9 @@ public interface IntermediateResultOperator { /** Deserialize the object from byteBuffer to allow shutdown restart */ void deserialize(final ByteBuffer byteBuffer) throws IOException; + + default void deserialize(final ByteBuffer byteBuffer, final int stateFormatVersion) + throws IOException { + deserialize(byteBuffer); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbsoluteMaxOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbsoluteMaxOperator.java index b121e4c99513f..410eba1f02bd6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbsoluteMaxOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbsoluteMaxOperator.java @@ -19,6 +19,11 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.sametype.numeric; +import org.apache.tsfile.enums.TSDataType; + +import java.io.IOException; +import java.nio.ByteBuffer; + public class AbsoluteMaxOperator extends AbstractSameTypeNumericOperator { @Override public String getName() { @@ -27,36 +32,64 @@ public String getName() { @Override public boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp) { - intValue = Math.abs(initialInput); - return super.initAndGetIsSupport(initialInput, initialTimestamp); + if (initialInput == Integer.MIN_VALUE) { + doubleValue = -(double) initialInput; + outPutDataType = TSDataType.DOUBLE; + } else { + intValue = Math.abs(initialInput); + outPutDataType = TSDataType.INT32; + } + return true; } @Override public boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp) { - longValue = Math.abs(initialInput); - return super.initAndGetIsSupport(initialInput, initialTimestamp); + if (initialInput == Long.MIN_VALUE) { + doubleValue = -(double) initialInput; + outPutDataType = TSDataType.DOUBLE; + } else { + longValue = Math.abs(initialInput); + outPutDataType = TSDataType.INT64; + } + return true; } @Override public boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp) { floatValue = Math.abs(initialInput); - return super.initAndGetIsSupport(initialInput, initialTimestamp); + outPutDataType = TSDataType.FLOAT; + return true; } @Override public boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp) { doubleValue = Math.abs(initialInput); - return super.initAndGetIsSupport(initialInput, initialTimestamp); + outPutDataType = TSDataType.DOUBLE; + return true; } @Override public void updateValue(final int input, final long timestamp) { - intValue = Math.max(intValue, Math.abs(input)); + if (outPutDataType == TSDataType.DOUBLE) { + doubleValue = Math.max(doubleValue, Math.abs((double) input)); + } else if (input == Integer.MIN_VALUE) { + doubleValue = -(double) input; + outPutDataType = TSDataType.DOUBLE; + } else { + intValue = Math.max(intValue, Math.abs(input)); + } } @Override public void updateValue(final long input, final long timestamp) { - longValue = Math.max(longValue, Math.abs(input)); + if (outPutDataType == TSDataType.DOUBLE) { + doubleValue = Math.max(doubleValue, Math.abs((double) input)); + } else if (input == Long.MIN_VALUE) { + doubleValue = -(double) input; + outPutDataType = TSDataType.DOUBLE; + } else { + longValue = Math.max(longValue, Math.abs(input)); + } } @Override @@ -68,4 +101,35 @@ public void updateValue(final float input, final long timestamp) { public void updateValue(final double input, final long timestamp) { doubleValue = Math.max(doubleValue, Math.abs(input)); } + + @Override + public void deserialize(final ByteBuffer byteBuffer) throws IOException { + super.deserialize(byteBuffer); + switch (outPutDataType) { + case INT32: + if (intValue == Integer.MIN_VALUE) { + doubleValue = -(double) intValue; + outPutDataType = TSDataType.DOUBLE; + } else { + intValue = Math.abs(intValue); + } + break; + case INT64: + if (longValue == Long.MIN_VALUE) { + doubleValue = -(double) longValue; + outPutDataType = TSDataType.DOUBLE; + } else { + longValue = Math.abs(longValue); + } + break; + case FLOAT: + floatValue = Math.abs(floatValue); + break; + case DOUBLE: + doubleValue = Math.abs(doubleValue); + break; + default: + throw new IOException(); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/IntegralPoweredSumOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/IntegralPoweredSumOperator.java index f5b532319c1e2..00fe119d1e4e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/IntegralPoweredSumOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/IntegralPoweredSumOperator.java @@ -19,6 +19,11 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.sametype.numeric; +import org.apache.tsfile.enums.TSDataType; + +import java.io.IOException; +import java.nio.ByteBuffer; + public class IntegralPoweredSumOperator extends AbstractSameTypeNumericOperator { private final transient int power; @@ -33,45 +38,70 @@ public String getName() { @Override public boolean initAndGetIsSupport(int initialInput, long initialTimestamp) { - intValue = (int) Math.pow(initialInput, power); - return super.initAndGetIsSupport(initialInput, initialTimestamp); + doubleValue = Math.pow(initialInput, power); + outPutDataType = TSDataType.DOUBLE; + return true; } @Override public boolean initAndGetIsSupport(long initialInput, long initialTimestamp) { - longValue = (long) Math.pow(initialInput, power); - return super.initAndGetIsSupport(initialInput, initialTimestamp); + doubleValue = Math.pow(initialInput, power); + outPutDataType = TSDataType.DOUBLE; + return true; } @Override public boolean initAndGetIsSupport(float initialInput, long initialTimestamp) { - floatValue = (float) Math.pow(initialInput, power); - return super.initAndGetIsSupport(initialInput, initialTimestamp); + doubleValue = Math.pow(initialInput, power); + outPutDataType = TSDataType.DOUBLE; + return true; } @Override public boolean initAndGetIsSupport(double initialInput, long initialTimestamp) { doubleValue = Math.pow(initialInput, power); - return super.initAndGetIsSupport(initialInput, initialTimestamp); + outPutDataType = TSDataType.DOUBLE; + return true; } @Override public void updateValue(int input, long timestamp) { - intValue += (int) Math.pow(input, power); + doubleValue += Math.pow(input, power); } @Override public void updateValue(long input, long timestamp) { - longValue += (long) Math.pow(input, power); + doubleValue += Math.pow(input, power); } @Override public void updateValue(float input, long timestamp) { - floatValue += (float) Math.pow(input, power); + doubleValue += Math.pow(input, power); } @Override public void updateValue(double input, long timestamp) { doubleValue += Math.pow(input, power); } + + @Override + public void deserialize(final ByteBuffer byteBuffer) throws IOException { + super.deserialize(byteBuffer); + switch (outPutDataType) { + case INT32: + doubleValue = intValue; + break; + case INT64: + doubleValue = longValue; + break; + case FLOAT: + doubleValue = floatValue; + break; + case DOUBLE: + break; + default: + throw new IOException(); + } + outPutDataType = TSDataType.DOUBLE; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java index 0e84372fa3865..36414ae4edc32 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java @@ -36,6 +36,7 @@ public class FractionPoweredSumOperator implements IntermediateResultOperator { private double sum; private final transient double power; + private final transient boolean absolute; /** * Warning: Do not use fraction values like "1/3" to initiate the operator, unless you know what @@ -45,12 +46,25 @@ public class FractionPoweredSumOperator implements IntermediateResultOperator { * @param power the power to use */ public FractionPoweredSumOperator(final double power) { + this(power, false); + } + + public FractionPoweredSumOperator(final double power, final boolean absolute) { this.power = power; + this.absolute = absolute; } @Override public String getName() { - return "sum_x" + power; + return "sum_" + (absolute ? "abs_" : "") + "x" + formatPower(power); + } + + private static String formatPower(final double power) { + return power == (int) power ? Integer.toString((int) power) : Double.toString(power); + } + + private double poweredValue(final double value) { + return Math.pow(absolute ? Math.abs(value) : value, power); } @Override @@ -65,7 +79,7 @@ public boolean initAndGetIsSupport(final boolean initialInput, final long initia @Override public boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp) { - sum = Math.pow(initialInput, power); + sum = poweredValue(initialInput); return true; } @@ -76,19 +90,19 @@ public boolean initAndGetIsSupport(final LocalDate initialInput, final long init @Override public boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp) { - sum = Math.pow(initialInput, power); + sum = poweredValue(initialInput); return true; } @Override public boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp) { - sum = Math.pow(initialInput, power); + sum = poweredValue(initialInput); return true; } @Override public boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp) { - sum = Math.pow(initialInput, power); + sum = poweredValue(initialInput); return true; } @@ -110,7 +124,7 @@ public void updateValue(final boolean input, final long timestamp) { @Override public void updateValue(final int input, final long timestamp) { - sum += Math.pow(input, power); + sum += poweredValue(input); } @Override @@ -121,17 +135,17 @@ public void updateValue(final LocalDate input, final long timestamp) { @Override public void updateValue(final long input, final long timestamp) { - sum += Math.pow(input, power); + sum += poweredValue(input); } @Override public void updateValue(final float input, final long timestamp) { - sum += Math.pow(input, power); + sum += poweredValue(input); } @Override public void updateValue(final double input, final long timestamp) { - sum += Math.pow(input, power); + sum += poweredValue(input); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java index a42dfa179425a..e895921c5176f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java @@ -33,7 +33,7 @@ import java.util.Map; public class CountOperator implements IntermediateResultOperator { - private int count; + private long count; @Override public String getName() { @@ -135,7 +135,7 @@ public void updateValue(Binary input, long timestamp) { @Override public Pair getResult() { - return new Pair<>(TSDataType.INT32, count); + return new Pair<>(TSDataType.INT64, count); } @Override @@ -145,6 +145,15 @@ public void serialize(final DataOutputStream outputStream) throws IOException { @Override public void deserialize(final ByteBuffer byteBuffer) throws IOException { - count = ReadWriteIOUtils.readInt(byteBuffer); + count = ReadWriteIOUtils.readLong(byteBuffer); + } + + @Override + public void deserialize(final ByteBuffer byteBuffer, final int stateFormatVersion) + throws IOException { + count = + stateFormatVersion == LEGACY_STATE_FORMAT_VERSION + ? ReadWriteIOUtils.readInt(byteBuffer) + : ReadWriteIOUtils.readLong(byteBuffer); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/processor/StandardStatisticsOperatorProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/processor/StandardStatisticsOperatorProcessor.java index 414c0f38e51b3..f8dd6f3355a76 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/processor/StandardStatisticsOperatorProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/processor/StandardStatisticsOperatorProcessor.java @@ -75,6 +75,8 @@ public Set> getIntermediateResultOperatorSu Arrays.asList( AbsoluteMaxOperator::new, CountOperator::new, + () -> new FractionPoweredSumOperator(1, true), + () -> new FractionPoweredSumOperator(0.5, true), () -> new FractionPoweredSumOperator(0.5), () -> new IntegralPoweredSumOperator(1), () -> new IntegralPoweredSumOperator(2), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java index ca636fd789889..f85faafb06c9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java @@ -38,18 +38,24 @@ import java.time.LocalDate; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; public class TimeSeriesWindow { private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesWindow.class); + private static final int STATE_FORMAT_MARKER = -1; + private static final int PREVIOUS_STATE_FORMAT_VERSION = 1; + private static final int CURRENT_STATE_FORMAT_VERSION = 2; // A window is typically with a timestamp. We define it here to avoid // boxing/unboxing and simplify the logics. private long timestamp = 0; + private boolean legacyStatisticsFormula; private Map aggregatedOutputName2OperatorMap; private final Map> intermediateResultName2tsTypeAndOperatorMap = new HashMap<>(); @@ -567,8 +573,20 @@ private Map> getAggregatedResults(final TSDataT intermediateResultName2tsTypeAndOperatorMap.entrySet().stream() .collect( Collectors.toMap( - Map.Entry::getKey, entry -> entry.getValue().getRight().getResult()))); + Map.Entry::getKey, entry -> entry.getValue().getRight().getResult())), + legacyStatisticsFormula); return aggregatedOutputName2OperatorMap.entrySet().stream() + .filter( + entry -> { + final AggregatedResultOperator operator = entry.getValue(); + final Set requiredIntermediateResultNames = + legacyStatisticsFormula && operator.supportsLegacyIntermediateResults() + ? operator.getLegacyIntermediateValueNames() + : operator.getDeclaredIntermediateValueNames(); + return intermediateResultName2tsTypeAndOperatorMap + .keySet() + .containsAll(requiredIntermediateResultNames); + }) .collect( Collectors.toMap( Map.Entry::getKey, @@ -579,6 +597,9 @@ private Map> getAggregatedResults(final TSDataT public void serialize(final DataOutputStream outputStream) throws IOException { ReadWriteIOUtils.write(timestamp, outputStream); + ReadWriteIOUtils.write(STATE_FORMAT_MARKER, outputStream); + ReadWriteIOUtils.write(CURRENT_STATE_FORMAT_VERSION, outputStream); + ReadWriteIOUtils.write(legacyStatisticsFormula ? 1 : 0, outputStream); ReadWriteIOUtils.write(intermediateResultName2tsTypeAndOperatorMap.size(), outputStream); for (final Map.Entry> entry : intermediateResultName2tsTypeAndOperatorMap.entrySet()) { @@ -597,15 +618,51 @@ public void serialize(final DataOutputStream outputStream) throws IOException { // processor, only adding intermediate values is permitted. public void deserialize(final ByteBuffer byteBuffer) throws IOException { timestamp = ReadWriteIOUtils.readLong(byteBuffer); - final int size = ReadWriteIOUtils.readInt(byteBuffer); + final int sizeOrFormatMarker = ReadWriteIOUtils.readInt(byteBuffer); + final int stateFormatVersion; + final int size; + if (sizeOrFormatMarker == STATE_FORMAT_MARKER) { + final int windowStateFormatVersion = ReadWriteIOUtils.readInt(byteBuffer); + if (windowStateFormatVersion == PREVIOUS_STATE_FORMAT_VERSION) { + legacyStatisticsFormula = true; + stateFormatVersion = IntermediateResultOperator.CURRENT_STATE_FORMAT_VERSION; + size = ReadWriteIOUtils.readInt(byteBuffer); + } else if (windowStateFormatVersion == CURRENT_STATE_FORMAT_VERSION) { + final int legacyStatisticsFormulaFlag = ReadWriteIOUtils.readInt(byteBuffer); + if (legacyStatisticsFormulaFlag != 0 && legacyStatisticsFormulaFlag != 1) { + throw new IOException(); + } + legacyStatisticsFormula = legacyStatisticsFormulaFlag == 1; + stateFormatVersion = IntermediateResultOperator.CURRENT_STATE_FORMAT_VERSION; + size = ReadWriteIOUtils.readInt(byteBuffer); + } else { + throw new IOException(); + } + } else if (sizeOrFormatMarker >= 0) { + legacyStatisticsFormula = true; + stateFormatVersion = IntermediateResultOperator.LEGACY_STATE_FORMAT_VERSION; + size = sizeOrFormatMarker; + } else { + throw new IOException(); + } + final Set restoredIntermediateResultNames = new HashSet<>(); for (int i = 0; i < size; i++) { final String intermediateResultName = ReadWriteIOUtils.readString(byteBuffer); final Pair initializedAndOperatorPair = intermediateResultName2tsTypeAndOperatorMap.get(intermediateResultName); - if (Objects.nonNull(initializedAndOperatorPair)) { - initializedAndOperatorPair.setLeft(TSDataType.deserializeFrom(byteBuffer)); - initializedAndOperatorPair.getRight().deserialize(byteBuffer); + if (Objects.isNull(initializedAndOperatorPair)) { + throw new IOException(); } + restoredIntermediateResultNames.add(intermediateResultName); + initializedAndOperatorPair.setLeft(TSDataType.deserializeFrom(byteBuffer)); + initializedAndOperatorPair.getRight().deserialize(byteBuffer, stateFormatVersion); + } + if (legacyStatisticsFormula) { + intermediateResultName2tsTypeAndOperatorMap + .keySet() + .removeIf( + intermediateResultName -> + !restoredIntermediateResultNames.contains(intermediateResultName)); } processor.deserializeCustomizedAttributes(this, byteBuffer); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessorTest.java new file mode 100644 index 0000000000000..ecd9cecfa427e --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessorTest.java @@ -0,0 +1,535 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.pipe.processor.aggregate; + +import org.apache.iotdb.db.pipe.processor.aggregate.operator.aggregatedresult.AggregatedResultOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.aggregatedresult.standardstatistics.AverageOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.aggregatedresult.standardstatistics.ClearanceFactorOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.aggregatedresult.standardstatistics.FormFactorOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.aggregatedresult.standardstatistics.PeakOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.aggregatedresult.standardstatistics.PulseFactorOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.CustomizedReadableIntermediateResults; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.sametype.numeric.AbsoluteMaxOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.sametype.numeric.IntegralPoweredSumOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.specifictype.doubletype.FractionPoweredSumOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.specifictype.integertype.CountOperator; +import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.TimeSeriesWindow; +import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput; +import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.TumblingWindowingProcessor; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_OPERATORS_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_OUTPUT_MEASUREMENTS_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_SLIDING_BOUNDARY_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_SLIDING_SECONDS_KEY; + +public class AggregateProcessorTest { + + @Test(expected = PipeParameterNotValidException.class) + public void testRejectsDuplicateOperatorNamesIgnoringCase() throws Exception { + validate("avg,AVG", ""); + } + + @Test(expected = PipeParameterNotValidException.class) + public void testRejectsEmptyOperatorNames() throws Exception { + validate("avg,", ""); + } + + @Test(expected = PipeParameterNotValidException.class) + public void testRejectsDuplicateOutputMeasurements() throws Exception { + validate("avg,max", "result,result"); + } + + @Test(expected = PipeParameterNotValidException.class) + public void testRejectsOutputMeasurementCollidingWithDefaultName() throws Exception { + validate("avg,max", "max"); + } + + @Test(expected = PipeParameterNotValidException.class) + public void testRejectsEmptyOutputMeasurements() throws Exception { + validate("avg,max", "result,"); + } + + @Test(expected = PipeParameterNotValidException.class) + public void testRejectsMoreOutputMeasurementsThanOperators() throws Exception { + validate("avg,max", "avg,max,extra"); + } + + @Test + public void testStripDatabasePrefixTreatsRegexCharactersLiterally() { + Assert.assertEquals( + "device", AggregateProcessor.stripDatabasePrefix("root.db$.device", "root.db$.")); + Assert.assertEquals( + "root.dbx.device", AggregateProcessor.stripDatabasePrefix("root.dbx.device", "root.db$.")); + } + + @Test + public void testStripDatabasePrefixOnlyAtTheBeginning() { + Assert.assertEquals( + "device.root.db.device", + AggregateProcessor.stripDatabasePrefix("device.root.db.device", "root.db.")); + } + + @Test + public void testSecondsToMillisecondsSaturatesOnOverflow() { + Assert.assertEquals(Long.MAX_VALUE, AggregateProcessor.secondsToMilliseconds(Long.MAX_VALUE)); + Assert.assertEquals(Long.MIN_VALUE, AggregateProcessor.secondsToMilliseconds(Long.MIN_VALUE)); + } + + @Test + public void testDeduplicatesMinimumTimestamp() { + final List outputs = + new ArrayList<>( + Arrays.asList( + new WindowOutput().setTimestamp(Long.MIN_VALUE), + new WindowOutput().setTimestamp(Long.MIN_VALUE), + new WindowOutput().setTimestamp(0))); + + final List distinctOutputs = + AggregateProcessor.sortAndDeduplicateOutputs(outputs); + + Assert.assertEquals(2, distinctOutputs.size()); + Assert.assertEquals(Long.MIN_VALUE, distinctOutputs.get(0).getTimestamp()); + Assert.assertEquals(0L, distinctOutputs.get(1).getTimestamp()); + } + + @Test + public void testMinimumTimestampCreatesFirstWindow() throws Exception { + final TumblingWindowingProcessor windowingProcessor = + createWindowingProcessor(Long.MIN_VALUE, 1); + final TimeSeriesRuntimeState state = + new TimeSeriesRuntimeState( + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + windowingProcessor); + + state.updateWindows(Long.MIN_VALUE, 1, 0); + + final List outputs = state.forceOutput(); + Assert.assertEquals(1, outputs.size()); + Assert.assertEquals(Long.MIN_VALUE, outputs.get(0).getTimestamp()); + } + + @Test + public void testLegacyWindowStatePromotesCountAndMomentSums() throws Exception { + final TumblingWindowingProcessor windowingProcessor = createWindowingProcessor(0, 1); + final Map> intermediateOperators = new HashMap<>(); + intermediateOperators.put("count", CountOperator::new); + intermediateOperators.put("sum_x1", () -> new IntegralPoweredSumOperator(1)); + final Map aggregatedOperators = + Collections.singletonMap("avg", new AverageOperator()); + final TimeSeriesWindow window = new TimeSeriesWindow(windowingProcessor, null); + window.initWindow(intermediateOperators, aggregatedOperators, Collections.emptyMap()); + final ByteBuffer legacyState = serializeLegacyWindow(0, Integer.MAX_VALUE, Integer.MAX_VALUE); + final ByteBuffer legacyStateReader = legacyState.duplicate(); + Assert.assertEquals(0L, ReadWriteIOUtils.readLong(legacyStateReader)); + Assert.assertEquals(2, ReadWriteIOUtils.readInt(legacyStateReader)); + Assert.assertEquals("count", ReadWriteIOUtils.readString(legacyStateReader)); + Assert.assertEquals(TSDataType.INT32, TSDataType.deserializeFrom(legacyStateReader)); + final CountOperator countOperator = new CountOperator(); + countOperator.deserialize( + legacyStateReader, IntermediateResultOperator.LEGACY_STATE_FORMAT_VERSION); + Assert.assertEquals( + Integer.MAX_VALUE, ((Number) countOperator.getResult().getRight()).longValue()); + Assert.assertEquals("sum_x1", ReadWriteIOUtils.readString(legacyStateReader)); + Assert.assertEquals(TSDataType.INT32, TSDataType.deserializeFrom(legacyStateReader)); + final IntegralPoweredSumOperator sumOperator = new IntegralPoweredSumOperator(1); + sumOperator.deserialize(legacyStateReader); + Assert.assertEquals( + (double) Integer.MAX_VALUE, (double) sumOperator.getResult().getRight(), 0.0); + window.deserialize(legacyState); + + window.updateIntermediateResult(1, 1); + Assert.assertEquals( + 1.0, (double) window.forceOutput().getAggregatedResults().get("avg").getRight(), 0.0); + + final ByteBuffer currentState; + try (final PublicBAOS output = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(output)) { + window.serialize(outputStream); + currentState = ByteBuffer.wrap(output.getBuf(), 0, output.size()); + } + final TimeSeriesWindow restoredWindow = new TimeSeriesWindow(windowingProcessor, null); + restoredWindow.initWindow(intermediateOperators, aggregatedOperators, Collections.emptyMap()); + restoredWindow.deserialize(currentState); + Assert.assertEquals( + 1.0, + (double) restoredWindow.forceOutput().getAggregatedResults().get("avg").getRight(), + 0.0); + } + + @Test + public void testMagnitudeFactorsHandleNegativeValues() throws Exception { + final TumblingWindowingProcessor windowingProcessor = createWindowingProcessor(0, 1); + final TimeSeriesWindow window = new TimeSeriesWindow(windowingProcessor, null); + window.initWindow( + createFactorIntermediateOperators(), + createFactorAggregatedOperators(), + Collections.emptyMap()); + + window.updateIntermediateResult(0, -2); + window.updateIntermediateResult(1, 4); + + final Map> results = + window.forceOutput().getAggregatedResults(); + Assert.assertEquals(Math.sqrt(10) / 3, (double) results.get("ff").getRight(), 1e-12); + Assert.assertEquals(4.0 / 3, (double) results.get("pf").getRight(), 1e-12); + Assert.assertEquals( + 4.0 / Math.pow((Math.sqrt(2) + 2) / 2, 2), (double) results.get("ce").getRight(), 1e-12); + } + + @Test + public void testLegacyFactorWindowKeepsFormulaAcrossRestore() throws Exception { + final TumblingWindowingProcessor windowingProcessor = createWindowingProcessor(0, 1); + final Map> intermediateOperators = + createFactorIntermediateOperators(); + final Map aggregatedOperators = + createFactorAggregatedOperators(); + final List requiredIntermediateResultNames = + aggregatedOperators.values().stream() + .flatMap(operator -> operator.getDeclaredIntermediateValueNames().stream()) + .collect(Collectors.toList()); + intermediateOperators.keySet().retainAll(requiredIntermediateResultNames); + final TimeSeriesWindow window = new TimeSeriesWindow(windowingProcessor, null); + window.initWindow(intermediateOperators, aggregatedOperators, Collections.emptyMap()); + window.deserialize(serializeLegacyFactorWindow()); + Assert.assertTrue( + "Expected the legacy clearance factor to be NaN immediately after restore, got " + + window.forceOutput().getAggregatedResults().get("ce").getRight(), + Double.isNaN((double) window.forceOutput().getAggregatedResults().get("ce").getRight())); + + window.updateIntermediateResult(2, 5); + final Map> results = + window.forceOutput().getAggregatedResults(); + Assert.assertEquals(Math.sqrt(15) / (7.0 / 3), (double) results.get("ff").getRight(), 1e-12); + Assert.assertEquals(15.0 / 7, (double) results.get("pf").getRight(), 1e-12); + Assert.assertTrue( + "Expected the legacy clearance factor to remain NaN, got " + results.get("ce").getRight(), + Double.isNaN((double) results.get("ce").getRight())); + + final ByteBuffer currentState; + try (final PublicBAOS output = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(output)) { + window.serialize(outputStream); + currentState = ByteBuffer.wrap(output.getBuf(), 0, output.size()); + } + final TimeSeriesWindow restoredWindow = new TimeSeriesWindow(windowingProcessor, null); + restoredWindow.initWindow(intermediateOperators, aggregatedOperators, Collections.emptyMap()); + restoredWindow.deserialize(currentState); + Assert.assertEquals( + 15.0 / 7, + (double) restoredWindow.forceOutput().getAggregatedResults().get("pf").getRight(), + 1e-12); + } + + @Test + public void testLegacyFactorWindowSkipsAggregatorsWithoutRestoredInputs() throws Exception { + final TumblingWindowingProcessor windowingProcessor = createWindowingProcessor(0, 1); + final TimeSeriesWindow window = new TimeSeriesWindow(windowingProcessor, null); + window.initWindow( + createFactorIntermediateOperators(), + createFactorAggregatedOperators(), + Collections.emptyMap()); + window.deserialize(serializeLegacyCountWindow()); + + Assert.assertTrue(window.forceOutput().getAggregatedResults().isEmpty()); + } + + @Test(expected = IOException.class) + public void testLegacyWindowStateRejectsRemovedIntermediateResult() throws Exception { + final TumblingWindowingProcessor windowingProcessor = createWindowingProcessor(0, 1); + final TimeSeriesWindow window = new TimeSeriesWindow(windowingProcessor, null); + window.initWindow( + Collections.singletonMap("count", CountOperator::new), + Collections.emptyMap(), + Collections.emptyMap()); + + window.deserialize(serializeLegacyWindow(0, 1, 1)); + } + + @Test + public void testIntegralPoweredSumAvoidsIntegerOverflow() { + final IntegralPoweredSumOperator sumOfSquares = new IntegralPoweredSumOperator(2); + Assert.assertTrue(sumOfSquares.initAndGetIsSupport(50_000, 0)); + sumOfSquares.updateValue(50_000, 1); + + Assert.assertEquals(5_000_000_000.0, (double) sumOfSquares.getResult().getRight(), 0.0); + } + + @Test + public void testAbsoluteMaxHandlesMinimumIntegerAndLegacyState() throws IOException { + final AbsoluteMaxOperator absoluteMaxOperator = new AbsoluteMaxOperator(); + Assert.assertTrue(absoluteMaxOperator.initAndGetIsSupport(Integer.MIN_VALUE, 0)); + Assert.assertEquals(TSDataType.DOUBLE, absoluteMaxOperator.getResult().getLeft()); + Assert.assertEquals(2_147_483_648.0, (double) absoluteMaxOperator.getResult().getRight(), 0.0); + + final ByteBuffer legacyState; + try (final PublicBAOS output = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(output)) { + TSDataType.INT32.serializeTo(outputStream); + ReadWriteIOUtils.write(Integer.MIN_VALUE, outputStream); + legacyState = ByteBuffer.wrap(output.getBuf(), 0, output.size()); + } + absoluteMaxOperator.deserialize(legacyState); + Assert.assertEquals(2_147_483_648.0, (double) absoluteMaxOperator.getResult().getRight(), 0.0); + } + + @Test + public void testPeakPreservesInputTypeAndIntegerPrecision() throws IOException { + final AbsoluteMaxOperator intAbsoluteMax = new AbsoluteMaxOperator(); + Assert.assertTrue(intAbsoluteMax.initAndGetIsSupport(-17, 0)); + Assert.assertEquals(17, intAbsoluteMax.getResult().getRight()); + final Pair intPeak = terminatePeak(TSDataType.INT32, intAbsoluteMax); + Assert.assertEquals(TSDataType.INT32, intPeak.getLeft()); + Assert.assertEquals(17, intPeak.getRight()); + + final AbsoluteMaxOperator minimumIntAbsoluteMax = new AbsoluteMaxOperator(); + Assert.assertTrue(minimumIntAbsoluteMax.initAndGetIsSupport(Integer.MIN_VALUE, 0)); + final Pair minimumIntPeak = + terminatePeak(TSDataType.INT32, minimumIntAbsoluteMax); + Assert.assertEquals(TSDataType.INT32, minimumIntPeak.getLeft()); + Assert.assertEquals(Integer.MAX_VALUE, minimumIntPeak.getRight()); + + final long largeLongValue = Long.MAX_VALUE - 1024; + final AbsoluteMaxOperator longAbsoluteMax = new AbsoluteMaxOperator(); + Assert.assertTrue(longAbsoluteMax.initAndGetIsSupport(-largeLongValue, 0)); + Assert.assertEquals(TSDataType.INT64, longAbsoluteMax.getResult().getLeft()); + + final ByteBuffer serializedState; + try (final PublicBAOS output = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(output)) { + longAbsoluteMax.serialize(outputStream); + serializedState = ByteBuffer.wrap(output.getBuf(), 0, output.size()); + } + final AbsoluteMaxOperator restoredLongAbsoluteMax = new AbsoluteMaxOperator(); + restoredLongAbsoluteMax.deserialize(serializedState); + final Pair longPeak = + terminatePeak(TSDataType.INT64, restoredLongAbsoluteMax); + Assert.assertEquals(TSDataType.INT64, longPeak.getLeft()); + Assert.assertEquals(largeLongValue, longPeak.getRight()); + } + + @Test + public void testPeakSaturatesLongMinimumAfterCheckpointRestore() throws IOException { + final AbsoluteMaxOperator absoluteMaxOperator = new AbsoluteMaxOperator(); + Assert.assertTrue(absoluteMaxOperator.initAndGetIsSupport(Long.MIN_VALUE, 0)); + Assert.assertEquals( + 9_223_372_036_854_775_808.0, (double) absoluteMaxOperator.getResult().getRight(), 0.0); + + final ByteBuffer serializedState; + try (final PublicBAOS output = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(output)) { + absoluteMaxOperator.serialize(outputStream); + serializedState = ByteBuffer.wrap(output.getBuf(), 0, output.size()); + } + final AbsoluteMaxOperator restoredAbsoluteMaxOperator = new AbsoluteMaxOperator(); + restoredAbsoluteMaxOperator.deserialize(serializedState); + Assert.assertEquals(TSDataType.DOUBLE, restoredAbsoluteMaxOperator.getResult().getLeft()); + Assert.assertEquals( + 9_223_372_036_854_775_808.0, + (double) restoredAbsoluteMaxOperator.getResult().getRight(), + 0.0); + restoredAbsoluteMaxOperator.updateValue(Long.MAX_VALUE, 1); + + final Pair peak = + terminatePeak(TSDataType.INT64, restoredAbsoluteMaxOperator); + Assert.assertEquals(TSDataType.INT64, peak.getLeft()); + Assert.assertEquals(Long.MAX_VALUE, peak.getRight()); + } + + @Test + public void testNewerRestoredStateReplacesOlderWindows() throws IOException { + final TumblingWindowingProcessor windowingProcessor = new TumblingWindowingProcessor(); + final TimeSeriesRuntimeState state = + new TimeSeriesRuntimeState( + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + windowingProcessor); + + state.restoreTimestampAndWindows(new Pair<>(10L, serializeWindow(windowingProcessor, 100L))); + state.restoreTimestampAndWindows(new Pair<>(20L, serializeWindow(windowingProcessor, 200L))); + + final List outputs = state.forceOutput(); + Assert.assertEquals(1, outputs.size()); + Assert.assertEquals(200L, outputs.get(0).getTimestamp()); + } + + @Test + public void testNewerRestoredStateWithoutBufferClearsOlderWindows() throws IOException { + final TumblingWindowingProcessor windowingProcessor = new TumblingWindowingProcessor(); + final TimeSeriesRuntimeState state = + new TimeSeriesRuntimeState( + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + windowingProcessor); + + state.restoreTimestampAndWindows(new Pair<>(10L, serializeWindow(windowingProcessor, 100L))); + state.restoreTimestampAndWindows(new Pair<>(20L, null)); + + Assert.assertTrue(state.forceOutput().isEmpty()); + } + + private static ByteBuffer serializeWindow( + final TumblingWindowingProcessor windowingProcessor, final long timestamp) + throws IOException { + try (final PublicBAOS output = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(output)) { + ReadWriteIOUtils.write(1, outputStream); + final TimeSeriesWindow window = new TimeSeriesWindow(windowingProcessor, null); + window.setTimestamp(timestamp); + window.serialize(outputStream); + return ByteBuffer.wrap(output.getBuf(), 0, output.size()); + } + } + + private static ByteBuffer serializeLegacyWindow( + final long timestamp, final int count, final int sum) throws IOException { + try (final PublicBAOS output = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(output)) { + ReadWriteIOUtils.write(timestamp, outputStream); + ReadWriteIOUtils.write(2, outputStream); + ReadWriteIOUtils.write("count", outputStream); + TSDataType.INT32.serializeTo(outputStream); + ReadWriteIOUtils.write(count, outputStream); + ReadWriteIOUtils.write("sum_x1", outputStream); + TSDataType.INT32.serializeTo(outputStream); + TSDataType.INT32.serializeTo(outputStream); + ReadWriteIOUtils.write(sum, outputStream); + return ByteBuffer.wrap(output.getBuf(), 0, output.size()); + } + } + + private static ByteBuffer serializeLegacyFactorWindow() throws IOException { + try (final PublicBAOS output = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(output)) { + ReadWriteIOUtils.write(0L, outputStream); + ReadWriteIOUtils.write(5, outputStream); + writeLegacyIntermediate(outputStream, "abs_max", TSDataType.INT32, 4, true); + writeLegacyIntermediate(outputStream, "count", TSDataType.INT32, 2, false); + writeLegacyIntermediate(outputStream, "sum_x1", TSDataType.INT32, 2, true); + writeLegacyIntermediate(outputStream, "sum_x2", TSDataType.INT32, 20, true); + ReadWriteIOUtils.write("sum_x0.5", outputStream); + TSDataType.INT32.serializeTo(outputStream); + ReadWriteIOUtils.write(Double.NaN, outputStream); + return ByteBuffer.wrap(output.getBuf(), 0, output.size()); + } + } + + private static ByteBuffer serializeLegacyCountWindow() throws IOException { + try (final PublicBAOS output = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(output)) { + ReadWriteIOUtils.write(0L, outputStream); + ReadWriteIOUtils.write(1, outputStream); + ReadWriteIOUtils.write("count", outputStream); + TSDataType.INT32.serializeTo(outputStream); + ReadWriteIOUtils.write(2, outputStream); + return ByteBuffer.wrap(output.getBuf(), 0, output.size()); + } + } + + private static void writeLegacyIntermediate( + final DataOutputStream outputStream, + final String name, + final TSDataType measurementDataType, + final int value, + final boolean operatorStateHasDataType) + throws IOException { + ReadWriteIOUtils.write(name, outputStream); + measurementDataType.serializeTo(outputStream); + if (operatorStateHasDataType) { + measurementDataType.serializeTo(outputStream); + } + ReadWriteIOUtils.write(value, outputStream); + } + + private static Map> + createFactorIntermediateOperators() { + final Map> intermediateOperators = new HashMap<>(); + intermediateOperators.put("abs_max", AbsoluteMaxOperator::new); + intermediateOperators.put("count", CountOperator::new); + intermediateOperators.put("sum_abs_x0.5", () -> new FractionPoweredSumOperator(0.5, true)); + intermediateOperators.put("sum_abs_x1", () -> new FractionPoweredSumOperator(1, true)); + intermediateOperators.put("sum_x0.5", () -> new FractionPoweredSumOperator(0.5)); + intermediateOperators.put("sum_x1", () -> new IntegralPoweredSumOperator(1)); + intermediateOperators.put("sum_x2", () -> new IntegralPoweredSumOperator(2)); + return intermediateOperators; + } + + private static Map createFactorAggregatedOperators() { + final Map aggregatedOperators = new HashMap<>(); + aggregatedOperators.put("ff", new FormFactorOperator()); + aggregatedOperators.put("pf", new PulseFactorOperator()); + aggregatedOperators.put("ce", new ClearanceFactorOperator()); + return aggregatedOperators; + } + + private static Pair terminatePeak( + final TSDataType measurementDataType, final AbsoluteMaxOperator absoluteMaxOperator) { + return new PeakOperator() + .terminateWindow( + measurementDataType, + new CustomizedReadableIntermediateResults( + Collections.singletonMap("abs_max", absoluteMaxOperator.getResult()), false)); + } + + private static TumblingWindowingProcessor createWindowingProcessor( + final long boundaryTime, final long slidingSeconds) throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PROCESSOR_SLIDING_BOUNDARY_TIME_KEY, Long.toString(boundaryTime)); + attributes.put(PROCESSOR_SLIDING_SECONDS_KEY, Long.toString(slidingSeconds)); + final PipeParameters parameters = new PipeParameters(attributes); + final TumblingWindowingProcessor processor = new TumblingWindowingProcessor(); + processor.validate(new PipeParameterValidator(parameters)); + processor.customize(parameters, () -> null); + return processor; + } + + private static void validate(final String operators, final String outputMeasurements) + throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PROCESSOR_OPERATORS_KEY, operators); + attributes.put(PROCESSOR_OUTPUT_MEASUREMENTS_KEY, outputMeasurements); + new AggregateProcessor().validate(new PipeParameterValidator(new PipeParameters(attributes))); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java index a9c78e7d73cf8..990f3d6d6d5a7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java @@ -42,6 +42,37 @@ public class TumblingWindowingProcessorTest { + @Test + public void testWindowUsesHalfOpenTimeRange() throws Exception { + final TumblingWindowingProcessor processor = createProcessor(0, 1); + final long interval = TimestampPrecisionUtils.convertToCurrPrecision(1, TimeUnit.SECONDS); + final TimeSeriesWindow window = new TimeSeriesWindow(processor, null); + window.setTimestamp(0); + + final Pair beforeEnd = + processor.updateAndMaySetWindowState(window, interval - 1); + final Pair atEnd = + processor.updateAndMaySetWindowState(window, interval); + + Assert.assertEquals(WindowState.COMPUTE, beforeEnd.getLeft()); + Assert.assertNull(beforeEnd.getRight()); + Assert.assertEquals(WindowState.EMIT_AND_PURGE_WITHOUT_COMPUTE, atEnd.getLeft()); + Assert.assertEquals(0L, atEnd.getRight().getTimestamp()); + Assert.assertEquals(interval, atEnd.getRight().getProgressTime()); + } + + @Test + public void testNegativeTimestampAlignsFromNegativeBoundary() throws Exception { + final long interval = TimestampPrecisionUtils.convertToCurrPrecision(1, TimeUnit.SECONDS); + final long boundary = -(interval + interval / 2); + final long timestamp = -(interval * 2 / 5); + final TumblingWindowingProcessor processor = createProcessor(boundary, 1); + final List windows = new ArrayList<>(); + + Assert.assertEquals(1, processor.mayAddWindow(windows, timestamp).size()); + Assert.assertEquals(boundary + interval, windows.get(0).getTimestamp()); + } + @Test public void testMayAddWindowAlignsExtremeRangeWithoutOverflow() throws Exception { final TumblingWindowingProcessor processor = createProcessor(Long.MIN_VALUE, 1); From 1b9ea5af45e01aa167cb055a1d880da44320a25b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:07:52 +0800 Subject: [PATCH 2/2] [Pipe] Route aggregate type dispatch through TypeService --- .../standardstatistics/PeakOperator.java | 36 +++++------ .../sametype/numeric/AbsoluteMaxOperator.java | 63 +++++++++++-------- .../numeric/IntegralPoweredSumOperator.java | 36 ++++++----- 3 files changed, 76 insertions(+), 59 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PeakOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PeakOperator.java index d7449ed413378..8e48eac32234a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PeakOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PeakOperator.java @@ -23,13 +23,27 @@ import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.CustomizedReadableIntermediateResults; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Pair; import java.util.Collections; import java.util.Map; import java.util.Set; +import java.util.function.Function; public class PeakOperator implements AggregatedResultOperator { + private static final TypeService> + ABSOLUTE_MAXIMUM_GETTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> intermediateResults -> intermediateResults.getInt("abs_max"); + case INT64 -> intermediateResults -> intermediateResults.getLong("abs_max"); + case FLOAT -> intermediateResults -> intermediateResults.getFloat("abs_max"); + case DOUBLE -> intermediateResults -> intermediateResults.getDouble("abs_max"); + default -> intermediateResults -> intermediateResults.getObject("abs_max"); + }; + @Override public String getName() { return "peak"; @@ -48,24 +62,10 @@ public Set getDeclaredIntermediateValueNames() { @Override public Pair terminateWindow( TSDataType measurementDataType, CustomizedReadableIntermediateResults intermediateResults) { - final Object absoluteMaximum; - switch (measurementDataType) { - case INT32: - absoluteMaximum = intermediateResults.getInt("abs_max"); - break; - case INT64: - absoluteMaximum = intermediateResults.getLong("abs_max"); - break; - case FLOAT: - absoluteMaximum = intermediateResults.getFloat("abs_max"); - break; - case DOUBLE: - absoluteMaximum = intermediateResults.getDouble("abs_max"); - break; - default: - absoluteMaximum = intermediateResults.getObject("abs_max"); - break; - } + final Object absoluteMaximum = + ABSOLUTE_MAXIMUM_GETTER_SERVICE + .call(Type.fromTsDataType(measurementDataType)) + .apply(intermediateResults); return new Pair<>(measurementDataType, absoluteMaximum); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbsoluteMaxOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbsoluteMaxOperator.java index 410eba1f02bd6..2694a7f88272e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbsoluteMaxOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbsoluteMaxOperator.java @@ -20,11 +20,47 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.sametype.numeric; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import java.io.IOException; import java.nio.ByteBuffer; public class AbsoluteMaxOperator extends AbstractSameTypeNumericOperator { + private static final TypeService DESERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + operator -> { + if (operator.intValue == Integer.MIN_VALUE) { + operator.doubleValue = -(double) operator.intValue; + operator.outPutDataType = TSDataType.DOUBLE; + } else { + operator.intValue = Math.abs(operator.intValue); + } + }; + case INT64 -> + operator -> { + if (operator.longValue == Long.MIN_VALUE) { + operator.doubleValue = -(double) operator.longValue; + operator.outPutDataType = TSDataType.DOUBLE; + } else { + operator.longValue = Math.abs(operator.longValue); + } + }; + case FLOAT -> operator -> operator.floatValue = Math.abs(operator.floatValue); + case DOUBLE -> operator -> operator.doubleValue = Math.abs(operator.doubleValue); + default -> + operator -> { + throw new IOException(); + }; + }; + + @FunctionalInterface + private interface Deserializer { + void deserialize(AbsoluteMaxOperator operator) throws IOException; + } + @Override public String getName() { return "abs_max"; @@ -105,31 +141,6 @@ public void updateValue(final double input, final long timestamp) { @Override public void deserialize(final ByteBuffer byteBuffer) throws IOException { super.deserialize(byteBuffer); - switch (outPutDataType) { - case INT32: - if (intValue == Integer.MIN_VALUE) { - doubleValue = -(double) intValue; - outPutDataType = TSDataType.DOUBLE; - } else { - intValue = Math.abs(intValue); - } - break; - case INT64: - if (longValue == Long.MIN_VALUE) { - doubleValue = -(double) longValue; - outPutDataType = TSDataType.DOUBLE; - } else { - longValue = Math.abs(longValue); - } - break; - case FLOAT: - floatValue = Math.abs(floatValue); - break; - case DOUBLE: - doubleValue = Math.abs(doubleValue); - break; - default: - throw new IOException(); - } + DESERIALIZER_SERVICE.call(Type.fromTsDataType(outPutDataType)).deserialize(this); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/IntegralPoweredSumOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/IntegralPoweredSumOperator.java index 00fe119d1e4e0..5feaaf97f53a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/IntegralPoweredSumOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/IntegralPoweredSumOperator.java @@ -20,6 +20,8 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.sametype.numeric; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import java.io.IOException; import java.nio.ByteBuffer; @@ -27,6 +29,24 @@ public class IntegralPoweredSumOperator extends AbstractSameTypeNumericOperator { private final transient int power; + private static final TypeService DESERIALIZER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> operator -> operator.doubleValue = operator.intValue; + case INT64 -> operator -> operator.doubleValue = operator.longValue; + case FLOAT -> operator -> operator.doubleValue = operator.floatValue; + case DOUBLE -> operator -> {}; + default -> + operator -> { + throw new IOException(); + }; + }; + + @FunctionalInterface + private interface Deserializer { + void deserialize(IntegralPoweredSumOperator operator) throws IOException; + } + public IntegralPoweredSumOperator(int power) { this.power = power; } @@ -87,21 +107,7 @@ public void updateValue(double input, long timestamp) { @Override public void deserialize(final ByteBuffer byteBuffer) throws IOException { super.deserialize(byteBuffer); - switch (outPutDataType) { - case INT32: - doubleValue = intValue; - break; - case INT64: - doubleValue = longValue; - break; - case FLOAT: - doubleValue = floatValue; - break; - case DOUBLE: - break; - default: - throw new IOException(); - } + DESERIALIZER_SERVICE.call(Type.fromTsDataType(outPutDataType)).deserialize(this); outPutDataType = TSDataType.DOUBLE; } }