Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,13 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
Expand Down Expand Up @@ -151,6 +153,15 @@ public void validate(final PipeParameterValidator validator) throws Exception {
PROCESSOR_OPERATORS_KEY),
parameters.getStringOrDefault(
PROCESSOR_OPERATORS_KEY, PROCESSOR_OPERATORS_DEFAULT_VALUE))
.validate(
arg -> areUniqueOperatorNames((String) arg),
String.format(
DataNodePipeMessages
.PIPE_EXCEPTION_THE_AGGREGATOR_AND_OUTPUT_NAME_S_IS_INVALID_BC22CF92,
parameters.getStringOrDefault(
PROCESSOR_OPERATORS_KEY, PROCESSOR_OPERATORS_DEFAULT_VALUE)),
parameters.getStringOrDefault(
PROCESSOR_OPERATORS_KEY, PROCESSOR_OPERATORS_DEFAULT_VALUE))
.validate(
arg -> !((String) arg).isEmpty(),
String.format(
Expand All @@ -168,9 +179,10 @@ public void validate(final PipeParameterValidator validator) throws Exception {
PROCESSOR_OUTPUT_DATABASE_KEY, PROCESSOR_OUTPUT_DATABASE_DEFAULT_VALUE))
.validate(
arg ->
((String) arg).isEmpty()
|| Arrays.stream(((String) arg).replace(" ", "").split(","))
.allMatch(this::isLegalMeasurement),
areLegalOutputMeasurements(
(String) arg,
parameters.getStringOrDefault(
PROCESSOR_OPERATORS_KEY, PROCESSOR_OPERATORS_DEFAULT_VALUE)),
String.format(
DataNodePipeMessages
.MESSAGE_THE_OUTPUT_MEASUREMENTS_ARG_CONTAINS_ILLEGAL_MEASUREMENTS_THE_MEASUREMENTS_MUST_BE_THE_LAST_LEVEL_OF_A_LEGAL_PATH_793FA8AE,
Expand All @@ -181,6 +193,44 @@ public void validate(final PipeParameterValidator validator) throws Exception {
PROCESSOR_OUTPUT_MEASUREMENTS_KEY, PROCESSOR_OUTPUT_MEASUREMENTS_DEFAULT_VALUE));
}

private boolean areUniqueOperatorNames(final String operatorString) {
final String[] operatorNames = operatorString.replace(" ", "").split(",", -1);
final Set<String> uniqueOperatorNames = new HashSet<>();
return Arrays.stream(operatorNames)
.allMatch(
operatorName ->
!operatorName.isEmpty()
&& uniqueOperatorNames.add(operatorName.toLowerCase(Locale.ROOT)));
}

private boolean areLegalOutputMeasurements(
final String outputMeasurementString, final String operatorString) {
if (outputMeasurementString.isEmpty()) {
return true;
}

final String[] outputMeasurementNames =
Arrays.stream(outputMeasurementString.replace(" ", "").split(",", -1))
.toArray(String[]::new);
final String[] operatorNames = operatorString.replace(" ", "").split(",", -1);
if (outputMeasurementNames.length > operatorNames.length) {
return false;
}

final Set<String> uniqueOutputMeasurementNames = new HashSet<>();
for (int index = 0; index < operatorNames.length; ++index) {
final String outputMeasurementName =
index < outputMeasurementNames.length
? outputMeasurementNames[index]
: operatorNames[index];
if ((index < outputMeasurementNames.length && !isLegalMeasurement(outputMeasurementName))
|| !uniqueOutputMeasurementNames.add(outputMeasurementName)) {
return false;
}
}
return true;
}

private boolean isLegalMeasurement(final String measurement) {
try {
PathUtils.isLegalPath("root." + measurement);
Expand Down Expand Up @@ -226,12 +276,14 @@ public void customize(
// The output max delay milliseconds must be set to at least 1
// to guarantee the correctness of the CAS in last receive time
outputMaxDelayMilliseconds =
outputMaxDelaySeconds < 0 ? Long.MAX_VALUE : Math.max(outputMaxDelaySeconds * 1000, 1);
outputMaxDelaySeconds < 0
? Long.MAX_VALUE
: Math.max(secondsToMilliseconds(outputMaxDelaySeconds), 1);
outputMinReportIntervalMilliseconds =
parameters.getLongOrDefault(
secondsToMilliseconds(
parameters.getLongOrDefault(
PROCESSOR_OUTPUT_MIN_REPORT_INTERVAL_SECONDS_KEY,
PROCESSOR_OUTPUT_MIN_REPORT_INTERVAL_SECONDS_DEFAULT_VALUE)
* 1000;
PROCESSOR_OUTPUT_MIN_REPORT_INTERVAL_SECONDS_DEFAULT_VALUE));
outputDatabase =
parameters.getStringOrDefault(
PROCESSOR_OUTPUT_DATABASE_KEY, PROCESSOR_OUTPUT_DATABASE_DEFAULT_VALUE);
Expand Down Expand Up @@ -260,10 +312,10 @@ public void customize(
for (int i = 0; i < operatorNameList.size(); ++i) {
if (i < outputMeasurementNameList.size()) {
aggregatorName2OutputNameMap.put(
operatorNameList.get(i).toLowerCase(), outputMeasurementNameList.get(i));
operatorNameList.get(i).toLowerCase(Locale.ROOT), outputMeasurementNameList.get(i));
} else {
aggregatorName2OutputNameMap.put(
operatorNameList.get(i).toLowerCase(), operatorNameList.get(i));
operatorNameList.get(i).toLowerCase(Locale.ROOT), operatorNameList.get(i));
}
}

Expand All @@ -281,11 +333,13 @@ public void customize(
operatorProcessor.getAggregatorOperatorSet().stream()
.filter(
operator ->
aggregatorName2OutputNameMap.containsKey(operator.getName().toLowerCase()))
aggregatorName2OutputNameMap.containsKey(
operator.getName().toLowerCase(Locale.ROOT)))
.forEach(
operator -> {
outputName2OperatorMap.put(
aggregatorName2OutputNameMap.get(operator.getName().toLowerCase()), operator);
aggregatorName2OutputNameMap.get(operator.getName().toLowerCase(Locale.ROOT)),
operator);
declaredIntermediateResultSet.addAll(operator.getDeclaredIntermediateValueNames());
});

Expand Down Expand Up @@ -453,7 +507,7 @@ private Map<String, Pair<Long, ByteBuffer>> processRow(
final String timeSeries =
(outputDatabaseWithPathSeparator.isEmpty()
? row.getDeviceId()
: row.getDeviceId().replaceFirst(databaseWithPathSeparator, ""))
: stripDatabasePrefix(row.getDeviceId(), databaseWithPathSeparator))
+ TsFileConstant.PATH_SEPARATOR
+ row.getColumnName(index);

Expand Down Expand Up @@ -585,6 +639,16 @@ private void resetOutputDatabaseForGeneratedEvent(final RowCollector rowCollecto
}
}

static String stripDatabasePrefix(final String deviceId, final String databaseWithPathSeparator) {
return deviceId.startsWith(databaseWithPathSeparator)
? deviceId.substring(databaseWithPathSeparator.length())
: deviceId;
}

static long secondsToMilliseconds(final long seconds) {
return TimeUnit.SECONDS.toMillis(seconds);
}

/**
* Collect {@link WindowOutput}s of a single timeSeries in one turn. The {@link TSDataType}s shall
* be the same because the {@link AggregatedResultOperator}s shall return the same value for the
Expand All @@ -600,19 +664,7 @@ public void collectWindowOutputs(
if (Objects.isNull(outputs) || outputs.isEmpty()) {
return;
}
// Sort and same timestamps removal
outputs.sort(Comparator.comparingLong(WindowOutput::getTimestamp));

final AtomicLong lastValue = new AtomicLong(Long.MIN_VALUE);
final List<WindowOutput> distinctOutputs = new ArrayList<>();
outputs.forEach(
output -> {
final long timeStamp = output.getTimestamp();
if (timeStamp != lastValue.get()) {
lastValue.set(timeStamp);
distinctOutputs.add(output);
}
});
final List<WindowOutput> distinctOutputs = sortAndDeduplicateOutputs(outputs);

final MeasurementSchema[] measurementSchemaList =
new MeasurementSchema[columnNameStringList.length];
Expand Down Expand Up @@ -756,6 +808,22 @@ private static Type getOutputTabletType(final TSDataType dataType) {
}
}

static List<WindowOutput> sortAndDeduplicateOutputs(final List<WindowOutput> outputs) {
outputs.sort(Comparator.comparingLong(WindowOutput::getTimestamp));

final List<WindowOutput> distinctOutputs = new ArrayList<>(outputs.size());
long lastTimestamp = 0;
boolean hasLastTimestamp = false;
for (final WindowOutput output : outputs) {
if (!hasLastTimestamp || output.getTimestamp() != lastTimestamp) {
distinctOutputs.add(output);
lastTimestamp = output.getTimestamp();
hasLastTimestamp = true;
}
}
return distinctOutputs;
}

@Override
public void close() throws Exception {
boolean isLastReference = false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ public class TimeSeriesRuntimeState {

// Outer set values
private long lastReportTimeStamp = Long.MIN_VALUE;
private boolean hasReportedTimeStamp;
private final List<TimeSeriesWindow> currentOpeningWindows = new ArrayList<>();

// Variables to avoid "new" operation
Expand Down Expand Up @@ -103,7 +104,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
throws IOException {
Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> output = null;

if (timestamp <= lastReportTimeStamp) {
if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) {
return null;
}
final Set<TimeSeriesWindow> addedWindows =
Expand All @@ -127,8 +128,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
if (stateWindowOutputPair.getLeft().isEmit()
&& Objects.nonNull(stateWindowOutputPair.getRight())) {
outputList.add(stateWindowOutputPair.getRight());
lastReportTimeStamp =
Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime());
}
if (stateWindowOutputPair.getLeft().isPurge()) {
windowIterator.remove();
Expand All @@ -149,7 +149,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
throws IOException {
Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> output = null;

if (timestamp <= lastReportTimeStamp) {
if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) {
return null;
}
final Set<TimeSeriesWindow> addedWindows =
Expand All @@ -173,8 +173,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
if (stateWindowOutputPair.getLeft().isEmit()
&& Objects.nonNull(stateWindowOutputPair.getRight())) {
outputList.add(stateWindowOutputPair.getRight());
lastReportTimeStamp =
Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime());
}
if (stateWindowOutputPair.getLeft().isPurge()) {
windowIterator.remove();
Expand All @@ -195,7 +194,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
throws IOException {
Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> output = null;

if (timestamp <= lastReportTimeStamp) {
if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) {
return null;
}
final Set<TimeSeriesWindow> addedWindows =
Expand All @@ -219,8 +218,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
if (stateWindowOutputPair.getLeft().isEmit()
&& Objects.nonNull(stateWindowOutputPair.getRight())) {
outputList.add(stateWindowOutputPair.getRight());
lastReportTimeStamp =
Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime());
}
if (stateWindowOutputPair.getLeft().isPurge()) {
windowIterator.remove();
Expand All @@ -241,7 +239,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
throws IOException {
Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> output = null;

if (timestamp <= lastReportTimeStamp) {
if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) {
return null;
}
final Set<TimeSeriesWindow> addedWindows =
Expand All @@ -265,8 +263,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
if (stateWindowOutputPair.getLeft().isEmit()
&& Objects.nonNull(stateWindowOutputPair.getRight())) {
outputList.add(stateWindowOutputPair.getRight());
lastReportTimeStamp =
Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime());
}
if (stateWindowOutputPair.getLeft().isPurge()) {
windowIterator.remove();
Expand All @@ -287,7 +284,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
throws IOException {
Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> output = null;

if (timestamp <= lastReportTimeStamp) {
if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) {
return null;
}
final Set<TimeSeriesWindow> addedWindows =
Expand All @@ -311,8 +308,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
if (stateWindowOutputPair.getLeft().isEmit()
&& Objects.nonNull(stateWindowOutputPair.getRight())) {
outputList.add(stateWindowOutputPair.getRight());
lastReportTimeStamp =
Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime());
}
if (stateWindowOutputPair.getLeft().isPurge()) {
windowIterator.remove();
Expand All @@ -333,7 +329,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
throws IOException {
Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> output = null;

if (timestamp <= lastReportTimeStamp) {
if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) {
return null;
}
final Set<TimeSeriesWindow> addedWindows =
Expand All @@ -357,8 +353,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
if (stateWindowOutputPair.getLeft().isEmit()
&& Objects.nonNull(stateWindowOutputPair.getRight())) {
outputList.add(stateWindowOutputPair.getRight());
lastReportTimeStamp =
Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime());
}
if (stateWindowOutputPair.getLeft().isPurge()) {
windowIterator.remove();
Expand All @@ -379,7 +374,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
throws IOException {
Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> output = null;

if (timestamp <= lastReportTimeStamp) {
if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) {
return null;
}
final Set<TimeSeriesWindow> addedWindows =
Expand All @@ -403,8 +398,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
if (stateWindowOutputPair.getLeft().isEmit()
&& Objects.nonNull(stateWindowOutputPair.getRight())) {
outputList.add(stateWindowOutputPair.getRight());
lastReportTimeStamp =
Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime());
}
if (stateWindowOutputPair.getLeft().isPurge()) {
windowIterator.remove();
Expand All @@ -425,7 +419,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
throws IOException {
Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> output = null;

if (timestamp <= lastReportTimeStamp) {
if (hasReportedTimeStamp && timestamp <= lastReportTimeStamp) {
return null;
}
final Set<TimeSeriesWindow> addedWindows =
Expand All @@ -449,8 +443,7 @@ public Pair<List<WindowOutput>, Pair<Long, ByteBuffer>> updateWindows(
if (stateWindowOutputPair.getLeft().isEmit()
&& Objects.nonNull(stateWindowOutputPair.getRight())) {
outputList.add(stateWindowOutputPair.getRight());
lastReportTimeStamp =
Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
updateLastReportTimeStamp(stateWindowOutputPair.getRight().getProgressTime());
}
if (stateWindowOutputPair.getLeft().isPurge()) {
windowIterator.remove();
Expand All @@ -472,6 +465,11 @@ public List<WindowOutput> forceOutput() {
.collect(Collectors.toList());
}

private void updateLastReportTimeStamp(final long timestamp) {
lastReportTimeStamp = Math.max(lastReportTimeStamp, timestamp);
hasReportedTimeStamp = true;
}

// A runtime state with window buffer shall be reported after at least output min report
// interval time to avoid frequent serialization
// Return null if this should not report
Expand Down Expand Up @@ -499,13 +497,18 @@ private Pair<Long, ByteBuffer> getTimestampWindowBufferPair(

public void restoreTimestampAndWindows(final Pair<Long, ByteBuffer> timestampWindowBufferPair)
throws IOException {
if (timestampWindowBufferPair.getLeft() <= lastReportTimeStamp) {
if (hasReportedTimeStamp && timestampWindowBufferPair.getLeft() <= lastReportTimeStamp) {
// The runtime state may be initialized by different processorSubtasks on one DataNode
// Only the subtask with the largest timestamp wins
return;
}
this.lastReportTimeStamp = timestampWindowBufferPair.getLeft();
hasReportedTimeStamp = true;
currentOpeningWindows.clear();
final ByteBuffer buffer = timestampWindowBufferPair.getRight();
if (buffer == null) {
return;
}
final int size = ReadWriteIOUtils.readInt(buffer);
for (int i = 0; i < size; ++i) {
// The runtime value will be deserialized if exists
Expand Down
Loading
Loading