diff --git a/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookupConcurrencyTest.java b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookupConcurrencyTest.java new file mode 100644 index 000000000..b3a638238 --- /dev/null +++ b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookupConcurrencyTest.java @@ -0,0 +1,314 @@ +/******************************************************************************* + * Copyright (c) 2026 Avaloq Group AG and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Avaloq Group AG - initial API and implementation + *******************************************************************************/ +package com.avaloq.tools.ddk.xtext.naming; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntConsumer; + +import org.eclipse.xtext.naming.QualifiedName; +import org.junit.jupiter.api.Test; + + +/** + * Tests lookups in a {@link QualifiedNameSegmentTreeLookup} while another thread inserts into nodes with many children. + *

+ * A lookup during an insertion is not exact in general: it searches a child and then gets it by its index, and a child inserted in between shifts the + * children after it. The tests assert what holds nevertheless: no lookup fails, and a name which sorts before all names inserted later, and so keeps its + * index, is found with its own values. + */ +@SuppressWarnings("nls") +public class QualifiedNameSegmentTreeLookupConcurrencyTest { + + private static final long TIMEOUT_SECONDS = 60; + private static final int READERS = 4; + + /** A node keeps 4096 or more children in chunks; the prefilled nodes have a few children less. */ + private static final int PREFILLED = 4088; + /** The prefilled name after which the first test inserts. */ + private static final int INSERTION_POINT = 2000; + private static final int ROUNDS = 10; + private static final int INSERTS = 20_000; + /** Readers look up one of this many names inserted last, which are in the chunks being split. */ + private static final int RECENT = 1100; + + /** The size of a chunk; the nodes of a lookup read from its serialized form have full chunks, which the first insertion splits. */ + private static final int CHUNK_SIZE = 1024; + private static final int CHUNKS = 64; + private static final int SPLIT_ROUNDS = 20; + + /** More children than the smallest large node has, so that the nodes are large before the readers start. */ + private static final int LARGE = 4200; + private static final int TRAVERSAL_ROUNDS = 5; + /** A lookup throws if a clear lands in the short window between searching a child and getting it; few clears with long reads in between keep that rare. */ + private static final int CLEARS = 2; + private static final int CLEARED = 20_000; + + private static final String CHILD = "c"; + private static final String SHARED = "shared"; + private static final String NESTED = "big"; + private static final String CLEARED_PREFIX = "k"; + + /** + * A read done repeatedly by every reader thread. + */ + @FunctionalInterface + private interface Read { + /** + * Reads from the lookup under test. + * + * @param random + * the random numbers of the reader thread, never {@code null} + * @param inserted + * the number of insertions done so far; the next one may be under way + */ + void read(Random random, int inserted); + } + + /** + * The root and a node below it grow from just below the size of a large node while readers look up, exactly and by pattern, the names inserted last, which + * have no child {@value #CHILD} and so no values, and, exactly and by recursive pattern, the prefilled names before the insertion point. + */ + @Test + public void testLookupsWhileNodesBecomeLarge() throws InterruptedException { + String[] inserted = insertedNames(); + for (int round = 0; round < ROUNDS; round++) { + QualifiedNameLookup lookup = new QualifiedNameSegmentTreeLookup<>(String.class, round % 2 == 0); + for (int i = 0; i < PREFILLED; i++) { + put(lookup, QualifiedName.create(prefilledName(i), CHILD)); + put(lookup, QualifiedName.create(NESTED, prefilledName(i), CHILD)); + } + readDuring(INSERTS, (random, done) -> { + String recent = inserted[Math.max(0, done - random.nextInt(RECENT))]; + switch (random.nextInt(5)) { + case 0 -> lookup.get(QualifiedName.create(recent, CHILD)); + case 1 -> lookup.get(QualifiedName.create(NESTED, recent, CHILD)); + case 2 -> lookup.get(QualifiedNamePattern.create(recent.substring(0, recent.length() - 1) + '*'), false); + case 3 -> { + // before the insertion point, so no inserted name matches + String kept = prefilledName(random.nextInt(INSERTION_POINT)); + assertEquals(List.of(kept + '.' + CHILD), lookup.get(QualifiedNamePattern.create(kept + "**"), false), "values matching a recursive pattern"); + assertEquals(List.of(NESTED + '.' + kept + '.' + CHILD), lookup.get(QualifiedNamePattern.create(NESTED, kept + "**"), false), + "values matching a nested recursive pattern"); + } + default -> { + String kept = prefilledName(random.nextInt(INSERTION_POINT + 1)); + assertFound(lookup, QualifiedName.create(kept, CHILD)); + assertFound(lookup, QualifiedName.create(NESTED, kept, CHILD)); + } + } + }, i -> { + put(lookup, QualifiedName.create(inserted[i])); + put(lookup, QualifiedName.create(NESTED, inserted[i])); + }); + + for (int i = 0; i < PREFILLED; i++) { + assertFound(lookup, QualifiedName.create(prefilledName(i), CHILD)); + assertFound(lookup, QualifiedName.create(NESTED, prefilledName(i), CHILD)); + } + for (String name : inserted) { + assertFound(lookup, QualifiedName.create(name)); + assertFound(lookup, QualifiedName.create(NESTED, name)); + } + } + } + + /** + * Every chunk of a large root is split once, in ascending order, while readers look up, exactly and by recursive pattern, names of the upper half of the + * chunk being split which sort before the inserted name. The assertions do not depend on the size of a chunk, only how likely the readers are to search a chunk during its split. + */ + @Test + public void testExactLookupsWhileChunksSplit() throws InterruptedException, IOException, ClassNotFoundException { + QualifiedName[] names = new QualifiedName[CHUNKS * CHUNK_SIZE]; + QualifiedNameLookup source = new QualifiedNameSegmentTreeLookup<>(String.class, false); + for (int i = 0; i < names.length; i++) { + names[i] = QualifiedName.create(String.format("s%07d", i)); + put(source, names[i]); + } + ByteArrayOutputStream serialized = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(serialized)) { + out.writeObject(source); + } + for (int round = 0; round < SPLIT_ROUNDS; round++) { + QualifiedNameLookup lookup = read(serialized.toByteArray()); + readDuring(CHUNKS, (random, done) -> { + QualifiedName name = names[done * CHUNK_SIZE + CHUNK_SIZE / 2 + random.nextInt(CHUNK_SIZE / 4)]; + assertFound(lookup, name); + // reads the children from the name on in one go + assertEquals(List.of(name.toString()), lookup.get(QualifiedNamePattern.create(name.getFirstSegment() + "**"), false), "values matching a pattern"); + }, + // after the names which the readers look up + i -> put(lookup, QualifiedName.create(names[i * CHUNK_SIZE + CHUNK_SIZE * 3 / 4].getFirstSegment() + '_'))); + + for (QualifiedName name : names) { + assertFound(lookup, name); + } + } + } + + /** + * The root and a node below it, both large from the start, grow while readers visit all their children: to find the names mapped to a value, and to match + * a recursive pattern. Such a traversal works on the children which a large node has at one moment, so it does not fail, finds every name which was + * there before, and may or may not find a name inserted meanwhile. (Iterating over the children of a small node during an insertion fails.) + */ + @Test + public void testTraversalsWhileLargeNodesGrow() throws InterruptedException { + String[] inserted = insertedNames(); + for (int round = 0; round < TRAVERSAL_ROUNDS; round++) { + QualifiedNameLookup lookup = new QualifiedNameSegmentTreeLookup<>(String.class, round % 2 == 0); + List nested = new ArrayList<>(); + for (int i = 0; i < LARGE; i++) { + put(lookup, QualifiedName.create(prefilledName(i), CHILD)); + QualifiedName name = QualifiedName.create(NESTED, prefilledName(i), CHILD); + put(lookup, name); + nested.add(name.toString()); + } + // before and after the insertion point + Set shared = Set.of(QualifiedName.create(prefilledName(1), CHILD), QualifiedName.create(prefilledName(LARGE - 1), CHILD), QualifiedName.create(NESTED, prefilledName(2), CHILD), QualifiedName.create(NESTED, prefilledName(LARGE - 2), CHILD)); + for (QualifiedName name : shared) { + lookup.put(name, SHARED); + } + // matches the node "big" and all below it + QualifiedNamePattern pattern = QualifiedNamePattern.create(NESTED.substring(0, 2) + "**"); + readDuring(INSERTS, (random, done) -> { + if (random.nextBoolean()) { + assertEquals(shared, Set.copyOf(lookup.getMappings(SHARED)), "names mapped to a value"); + } else { + Collection values = lookup.get(pattern, true); + assertTrue(values.containsAll(nested), "values of the names which were there before"); + } + }, i -> { + put(lookup, QualifiedName.create(inserted[i])); + put(lookup, QualifiedName.create(NESTED, inserted[i])); + }); + + assertEquals(shared, Set.copyOf(lookup.getMappings(SHARED)), "names mapped to a value after the insertions"); + assertEquals(nested.size() + INSERTS + 1, lookup.get(pattern, true).size(), "values below the nested node"); + } + } + + /** + * A large root is cleared and refilled while readers match a pattern which reads all its children. A read across a clear may return any part of the names, + * but it does not fail. + */ + @Test + public void testPatternLookupsWhileLargeNodeIsCleared() throws InterruptedException { + QualifiedNameLookup lookup = new QualifiedNameSegmentTreeLookup<>(String.class, false); + IntConsumer fill = i -> { + for (int n = 0; n < CLEARED; n++) { + put(lookup, QualifiedName.create(String.format("%s%05d", CLEARED_PREFIX, n))); + } + }; + // a large node stays large when it is cleared + fill.accept(0); + QualifiedNamePattern pattern = QualifiedNamePattern.create(CLEARED_PREFIX + "**"); + readDuring(CLEARS, (random, done) -> lookup.get(pattern, false), i -> { + lookup.clear(); + fill.accept(i); + }); + + assertEquals(CLEARED, lookup.get(pattern, false).size(), "values matching the pattern after the last refill"); + } + + /** + * Runs the given read in {@value #READERS} reader threads while this thread does the given insertions, and fails if a read failed. + * + * @param inserts + * the number of insertions + * @param read + * the read which the readers repeat until the last insertion is done, must not be {@code null} + * @param insert + * does the insertion with the given index, must not be {@code null} + */ + private static void readDuring(final int inserts, final Read read, final IntConsumer insert) throws InterruptedException { + AtomicInteger inserted = new AtomicInteger(); + CountDownLatch reading = new CountDownLatch(READERS); + List failures = Collections.synchronizedList(new ArrayList<>()); + List readers = new ArrayList<>(); + for (int r = 0; r < READERS; r++) { + Random random = new Random(r); + Thread reader = new Thread(() -> { + reading.countDown(); + for (int done = inserted.get(); done < inserts; done = inserted.get()) { + read.read(random, done); + } + }, "reader-" + r); + reader.setDaemon(true); + reader.setUncaughtExceptionHandler((thread, failure) -> failures.add(failure)); + readers.add(reader); + reader.start(); + } + try { + assertTrue(reading.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "readers started"); + for (int i = 0; i < inserts; i++) { + insert.accept(i); + inserted.set(i + 1); + } + } finally { + // stops the readers + inserted.set(inserts); + } + for (Thread reader : readers) { + reader.join(TimeUnit.SECONDS.toMillis(TIMEOUT_SECONDS)); + assertFalse(reader.isAlive(), "reader finished"); + } + if (!failures.isEmpty()) { + fail(failures.size() + " of " + READERS + " readers failed during the insertions, the first with " + failures.get(0), failures.get(0)); + } + } + + /** The names to insert: all between two prefilled names, in ascending order. */ + private static String[] insertedNames() { + String[] inserted = new String[INSERTS]; + for (int i = 0; i < INSERTS; i++) { + inserted[i] = String.format("%s_%07d", prefilledName(INSERTION_POINT), i); + } + return inserted; + } + + private static String prefilledName(final int i) { + return String.format("m%04d", i); + } + + /** Maps the given name to itself, as a string. */ + private static void put(final QualifiedNameLookup lookup, final QualifiedName name) { + lookup.put(name, name.toString()); + } + + private static void assertFound(final QualifiedNameLookup lookup, final QualifiedName name) { + assertEquals(List.of(name.toString()), lookup.get(name), "values of a name which keeps its index"); + } + + @SuppressWarnings("unchecked") + private static QualifiedNameLookup read(final byte[] serialized) throws IOException, ClassNotFoundException { + try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(serialized))) { + return (QualifiedNameLookup) in.readObject(); + } + } + +} diff --git a/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookupTest.java b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookupTest.java index c42cf67eb..e41b2690a 100644 --- a/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookupTest.java +++ b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookupTest.java @@ -18,14 +18,18 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.ObjectInputFilter; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Random; import java.util.Set; import org.apache.logging.log4j.LogManager; @@ -43,6 +47,8 @@ public class QualifiedNameSegmentTreeLookupTest { private static final Logger LOGGER = LogManager.getLogger(QualifiedNameSegmentTreeLookupTest.class); + private static final URI SHARED = URI.createURI("scheme:/shared"); + private final QualifiedNameSegmentTreeLookup lookup = new QualifiedNameSegmentTreeLookup(URI.class, true); @Test @@ -255,6 +261,161 @@ public void testPatternMatchesWithSharedValues() { } } + @Test + public void testLargeNodes() { + for (boolean shareValues : new boolean[] {true, false}) { + QualifiedNameSegmentTreeLookup tree = new QualifiedNameSegmentTreeLookup(URI.class, shareValues); + Map> expected = new HashMap<>(); + putLargeNodes(tree, expected, 1); + assertLookup(tree, expected); + + QualifiedNameSegmentTreeLookup copy = new QualifiedNameSegmentTreeLookup(URI.class, shareValues); + copy.initializeFrom(tree); + assertLookup(copy, expected); + + Set shared = new HashSet<>(); + expected.forEach((name, values) -> { + if (values.contains(SHARED)) { + shared.add(name); + } + }); + assertEquals(shared, new HashSet<>(tree.getMappings(SHARED))); + + QualifiedName removed = shared.iterator().next(); + tree.remove(removed, SHARED); + expected.get(removed).remove(SHARED); + shared.remove(removed); + assertLookup(tree, expected); + assertEquals(shared, new HashSet<>(tree.getMappings(SHARED))); + + tree.removeMappings(SHARED); + expected.values().forEach(values -> values.remove(SHARED)); + assertLookup(tree, expected); + assertEquals(0, tree.getMappings(SHARED).size()); + } + } + + @Test + public void testLargeNodesLoadStore() throws IOException, ClassNotFoundException { + QualifiedNameSegmentTreeLookup tree = new QualifiedNameSegmentTreeLookup(URI.class, true); + Map> expected = new HashMap<>(); + putLargeNodes(tree, expected, 1); + + QualifiedNameLookup readBack = writeAndRead(tree); + assertLookup(readBack, expected); + + // insert into the large nodes read back, and store them again + putLargeNodes(readBack, expected, 2); + assertLookup(readBack, expected); + assertLookup(writeAndRead(readBack), expected); + } + + /** + * Serializes and deserializes the given lookup, accepting no other lists than {@link ArrayList} in the serialized form. + * + * @param tree + * lookup to serialize, must not be {@code null} + * @return the deserialized lookup, never {@code null} + */ + @SuppressWarnings("unchecked") + private QualifiedNameLookup writeAndRead(final QualifiedNameLookup tree) throws IOException, ClassNotFoundException { + ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); + try (ObjectOutputStream objectOutStream = new ObjectOutputStream(byteOutStream)) { + objectOutStream.writeObject(tree); + } + try (ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(byteOutStream.toByteArray()))) { + inStream.setObjectInputFilter(info -> info.serialClass() != null && List.class.isAssignableFrom(info.serialClass()) + && info.serialClass() != ArrayList.class ? ObjectInputFilter.Status.REJECTED : ObjectInputFilter.Status.UNDECIDED); + return (QualifiedNameLookup) inStream.readObject(); + } + } + + /** + * Puts names in random order which give the root and the node {@code big} several times more children than a node keeps in an array list. Every seventh + * name is also mapped to {@link #SHARED}. + * + * @param tree + * lookup to fill, must not be {@code null} + * @param expected + * expected mappings to update, must not be {@code null} + * @param seed + * seed of the random names and order + */ + private void putLargeNodes(final QualifiedNameLookup tree, final Map> expected, final long seed) { + Random random = new Random(seed); + List names = new ArrayList<>(); + for (int i = 0; i < 20_000; i++) { + QualifiedName top = name("n" + token(random)); + names.add(top); + if (i % 10 == 0) { + names.add(top.append("c" + token(random))); + } + } + for (int i = 0; i < 10_000; i++) { + names.add(name("big." + token(random))); + } + for (int i = 0; i < 100; i++) { + names.add(name("z" + token(random))); + } + Collections.shuffle(names, random); + for (int i = 0; i < names.size(); i++) { + QualifiedName qn = names.get(i); + tree.put(qn, uri(qn)); + expected.computeIfAbsent(qn, k -> new HashSet<>()).add(uri(qn)); + if (i % 7 == 0) { + tree.put(qn, SHARED); + expected.get(qn).add(SHARED); + } + } + } + + private static String token(final Random random) { + return Integer.toString(random.nextInt(1_000_000_000), Character.MAX_RADIX); + } + + /** + * Asserts that exact and pattern lookups find the expected values; the expected pattern matches are computed with {@link QualifiedNamePattern#matches}. + * + * @param tree + * lookup to query, must not be {@code null} + * @param expected + * expected mappings, must not be {@code null} + */ + private void assertLookup(final QualifiedNameLookup tree, final Map> expected) { + for (Map.Entry> entry : expected.entrySet()) { + assertEquals(entry.getValue(), new HashSet<>(tree.get(entry.getKey())), entry.getKey().toString()); + } + for (String missing : List.of("big", "n", "y", "zzzzzzzzzz", "big.zzzzzzzzzz", "big.0.0")) { + assertNull(tree.get(name(missing)), missing); + } + + // z* and z** end at the upper bound node, y* and zzzzzzzzzz* match nothing + List patterns = new ArrayList<>(); + for (String pattern : List.of("n*", "n**", "big.*", "big.**", "z*", "z**", "y*", "zzzzzzzzzz*", "big.zzzzzzzzzz*")) { + patterns.add(pattern(pattern)); + } + for (char c : "05afkpuz".toCharArray()) { + for (String pattern : List.of("n" + c + "*", "n" + c + "**", "big." + c + "*", "big." + c + "**")) { + patterns.add(pattern(pattern)); + } + } + expected.keySet().stream().limit(10).forEach(name -> { + patterns.add(pattern(name)); + patterns.add(pattern(name.toString() + "*")); + patterns.add(pattern(name.toString() + "**")); + }); + for (QualifiedNamePattern pattern : patterns) { + Set matches = new HashSet<>(); + expected.forEach((name, values) -> { + if (pattern.matches(name)) { + matches.addAll(values); + } + }); + assertEquals(matches, tree.get(pattern, true), pattern.toString()); + assertEquals(matches, new HashSet<>(tree.get(pattern, false)), pattern.toString()); + } + } + // when sharing values, foo, foo.bar, foo.bar.baz and foo.baz share one value array private QualifiedNameSegmentTreeLookup createTreeWithSharedValues(final boolean shareValues, final URI v1, final URI v2, final URI v3) { QualifiedNameSegmentTreeLookup tree = new QualifiedNameSegmentTreeLookup(URI.class, shareValues); diff --git a/com.avaloq.tools.ddk.xtext/META-INF/MANIFEST.MF b/com.avaloq.tools.ddk.xtext/META-INF/MANIFEST.MF index fb7c23e82..3bb6da660 100644 --- a/com.avaloq.tools.ddk.xtext/META-INF/MANIFEST.MF +++ b/com.avaloq.tools.ddk.xtext/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: com.avaloq.tools.ddk.xtext Bundle-SymbolicName: com.avaloq.tools.ddk.xtext;singleton:=true -Bundle-Version: 17.4.7.qualifier +Bundle-Version: 17.4.8.qualifier Bundle-Vendor: Avaloq Group AG Bundle-RequiredExecutionEnvironment: JavaSE-21 Bundle-ActivationPolicy: lazy diff --git a/com.avaloq.tools.ddk.xtext/pom.xml b/com.avaloq.tools.ddk.xtext/pom.xml index bb5dd06de..da2a73f77 100644 --- a/com.avaloq.tools.ddk.xtext/pom.xml +++ b/com.avaloq.tools.ddk.xtext/pom.xml @@ -6,7 +6,7 @@ 18.0.1-SNAPSHOT ../ddk-parent - 17.4.7-SNAPSHOT + 17.4.8-SNAPSHOT com.avaloq.tools.ddk com.avaloq.tools.ddk.xtext eclipse-plugin diff --git a/com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/ChunkedList.java b/com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/ChunkedList.java new file mode 100644 index 000000000..b5764a3dd --- /dev/null +++ b/com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/ChunkedList.java @@ -0,0 +1,383 @@ +/******************************************************************************* + * Copyright (c) 2026 Avaloq Group AG and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Avaloq Group AG - initial API and implementation + *******************************************************************************/ +package com.avaloq.tools.ddk.xtext.naming; + +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Objects; +import java.util.RandomAccess; +import java.util.concurrent.locks.StampedLock; +import java.util.function.ToIntFunction; + + +/** + * A list which stores its elements in chunks of at most B = {@value #CHUNK_CAPACITY} elements. Inserting an element shifts the following elements of its + * chunk and the start indexes of the following chunks, instead of all following elements as {@link ArrayList#add(int, Object)} does. Inserting into a list of + * {@code n} elements thus takes O(B + n/B) instead of O(n) time, and building a large sorted list by binary search and insertion is no longer quadratic. + * {@link #get(int)} takes O(log(n/B)) time. + *

+ * Supports {@link #get(int)}, {@link #add(int, Object)}, {@link #size()}, {@link #clear()}, {@link #toArray()}, iteration, a + * {@link #binarySearch(ToIntFunction) binary search} which is faster than one with {@link #get(int)} and {@link #copy(Object[], int) copying a range}, which + * is faster than getting its elements one by one. Other modifications are not supported. + *

+ * The list must be modified by one thread at a time. {@link #get(int)}, {@link #binarySearch(ToIntFunction)}, {@link #toArray()} and + * {@link #copy(Object[], int)} may run concurrently with a modification. They then see the list before or after it, never a half-done modification, so that + * they do not fail where reads of an {@link ArrayList} in practice do not fail either. + * + * @param + * element type + */ +final class ChunkedList extends AbstractList implements RandomAccess { + + /** Maximum number of elements of a chunk. */ + static final int CHUNK_CAPACITY = 1024; + + private static final int HALF_CHUNK = CHUNK_CAPACITY / 2; + + /** Number of chunks there is room for beyond the initial ones. */ + private static final int SPARE_CHUNKS = 8; + + /** Number of reads without a lock before a read which a concurrent modification keeps invalidating takes the read lock and so blocks the writer. */ + private static final int OPTIMISTIC_READ_ATTEMPTS = 16; + + private final StampedLock lock = new StampedLock(); + + /** The chunks; only the first {@link #chunkCount} are used. Each holds at least one element unless the list is empty. */ + private Object[][] chunks; + + /** The index in the list of the first element of each chunk. */ + private int[] starts; + + private int chunkCount; + + private int elementCount; + + /** + * Creates a list with the elements of the given collection, in the order of its iterator. + * + * @param source + * the elements, must not be {@code null} + */ + ChunkedList(final Collection source) { + Object[] elements = source.toArray(); + int count = Math.max(1, (elements.length + CHUNK_CAPACITY - 1) / CHUNK_CAPACITY); + // the list may be published without synchronization; since the lock is final, a reader sees it released and so the fields initialized + long stamp = lock.writeLock(); + try { + chunks = new Object[count + SPARE_CHUNKS][]; + starts = new int[count + SPARE_CHUNKS]; + for (int c = 0; c < count; c++) { + int from = c * CHUNK_CAPACITY; + chunks[c] = Arrays.copyOfRange(elements, from, Math.min(from + CHUNK_CAPACITY, elements.length), Object[].class); + starts[c] = from; + } + chunkCount = count; + elementCount = elements.length; + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public int size() { + return elementCount; + } + + @Override + @SuppressWarnings("unchecked") + public E get(final int index) { + for (int attempt = 0; attempt < OPTIMISTIC_READ_ATTEMPTS; attempt++) { + long stamp = lock.tryOptimisticRead(); + int currentSize = elementCount; + Object element = find(index); + if (lock.validate(stamp)) { + Objects.checkIndex(index, currentSize); + return (E) element; + } + Thread.onSpinWait(); + } + long stamp = lock.readLock(); + try { + Objects.checkIndex(index, elementCount); + return (E) find(index); + } finally { + lock.unlockRead(stamp); + } + } + + /** + * Returns the element at the given index. Since {@link #get(int)} calls this without a lock first, it must not fail on a state which a concurrent + * modification left inconsistent. + * + * @param index + * the index of the element + * @return the element, or any value if the state is inconsistent or the index out of range + */ + private Object find(final int index) { + Object[][] currentChunks = chunks; + int[] currentStarts = starts; + int high = Math.min(chunkCount, Math.min(currentChunks.length, currentStarts.length)) - 1; + if (high < 0) { + return null; + } + int low = 0; + while (low < high) { + int mid = (low + high + 1) >>> 1; + if (currentStarts[mid] <= index) { + low = mid; + } else { + high = mid - 1; + } + } + Object[] chunk = currentChunks[low]; + int offset = index - currentStarts[low]; + return chunk != null && offset >= 0 && offset < chunk.length ? chunk[offset] : null; + } + + /** + * Searches this list, which must be sorted consistently with the given comparison, in O(log(n)) time like + * {@link java.util.Collections#binarySearch(java.util.List, Object, java.util.Comparator)}. Unlike a search with {@link #get(int)}, it searches the chunks + * first and then one chunk, instead of the chunks again for every element it compares. + * + * @param comparison + * compares an element with the searched key: negative if the element is smaller, zero if equal and positive if greater, must not be {@code null} + * @return index of an element equal to the key, or {@code (-(insertion point) - 1)} + */ + int binarySearch(final ToIntFunction comparison) { + for (int attempt = 0; attempt < OPTIMISTIC_READ_ATTEMPTS; attempt++) { + long stamp = lock.tryOptimisticRead(); + int result = search(comparison, stamp); + if (lock.validate(stamp)) { + return result; + } + Thread.onSpinWait(); + } + long stamp = lock.readLock(); + try { + return search(comparison, stamp); + } finally { + lock.unlockRead(stamp); + } + } + + /** + * Implements {@link #binarySearch(ToIntFunction)}. Since that calls this without a lock first, it must not fail on a state which a concurrent modification + * left inconsistent. In a consistent state, no element is {@code null}. It validates the stamp before comparing an element, so that the comparison sees + * the element as it was added, and not an element which a concurrent modification is adding. + * + * @param comparison + * compares an element with the searched key, must not be {@code null} + * @param stamp + * the stamp of the optimistic read or the read lock + * @return index of an element equal to the key, or {@code (-(insertion point) - 1)}, or any value if the state is inconsistent + */ + @SuppressWarnings("unchecked") + private int search(final ToIntFunction comparison, final long stamp) { + Object[][] currentChunks = chunks; + int[] currentStarts = starts; + int count = Math.min(chunkCount, Math.min(currentChunks.length, currentStarts.length)); + if (count == 0) { + return -1; + } + // the last chunk whose first element is not greater than the key + int low = 0; + int high = count - 1; + while (low < high) { + int mid = (low + high + 1) >>> 1; + Object[] chunk = currentChunks[mid]; + Object first = chunk != null && chunk.length > 0 ? chunk[0] : null; + if (first == null || !lock.validate(stamp)) { + return -1; + } + int cmp = comparison.applyAsInt((E) first); + if (cmp < 0) { + low = mid; + } else if (cmp > 0) { + high = mid - 1; + } else { + return currentStarts[mid]; + } + } + Object[] chunk = currentChunks[low]; + int start = currentStarts[low]; + int chunkSize = (low + 1 < count ? currentStarts[low + 1] : elementCount) - start; + if (chunk == null || chunkSize < 0 || chunkSize > chunk.length) { + return -1; + } + int lowIdx = 0; + int highIdx = chunkSize - 1; + while (lowIdx <= highIdx) { + int mid = (lowIdx + highIdx) >>> 1; + Object element = chunk[mid]; + if (element == null || !lock.validate(stamp)) { + return -1; + } + int cmp = comparison.applyAsInt((E) element); + if (cmp < 0) { + lowIdx = mid + 1; + } else if (cmp > 0) { + highIdx = mid - 1; + } else { + return start + mid; + } + } + return -(start + lowIdx + 1); + } + + @Override + public void add(final int index, final E element) { + if (index < 0 || index > elementCount) { + throw new IndexOutOfBoundsException("Index: " + index + ", Size: " + elementCount); //$NON-NLS-1$ //$NON-NLS-2$ + } + long stamp = lock.writeLock(); + try { + int c = index == elementCount ? chunkCount - 1 : chunkIndex(index); + int offset = index - starts[c]; + int chunkSize = chunkSize(c); + if (chunkSize == chunks[c].length) { + if (chunkSize < CHUNK_CAPACITY) { + chunks[c] = Arrays.copyOf(chunks[c], CHUNK_CAPACITY); + } else { + split(c); + if (offset > HALF_CHUNK) { + c++; + offset -= HALF_CHUNK; + } + chunkSize = chunkSize(c); + } + } + Object[] chunk = chunks[c]; + System.arraycopy(chunk, offset, chunk, offset + 1, chunkSize - offset); + chunk[offset] = element; + for (int i = c + 1; i < chunkCount; i++) { + starts[i]++; + } + elementCount++; + modCount++; + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void clear() { + long stamp = lock.writeLock(); + try { + chunks = new Object[1 + SPARE_CHUNKS][]; + chunks[0] = new Object[0]; + starts = new int[1 + SPARE_CHUNKS]; + chunkCount = 1; + elementCount = 0; + modCount++; + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public Object[] toArray() { + long stamp = lock.readLock(); + try { + Object[] result = new Object[elementCount]; + for (int c = 0; c < chunkCount; c++) { + System.arraycopy(chunks[c], 0, result, starts[c], chunkSize(c)); + } + return result; + } finally { + lock.unlockRead(stamp); + } + } + + /** + * Copies the elements from the given index on into the given array, as many as it holds and there are. Copies nothing if the index is not below the size, + * so that a reader which advances its index across a {@link #clear()} ends there. + * + * @param target + * the array to copy into, from its start, must not be {@code null} + * @param from + * the index of the first element to copy, must not be negative + * @return the number of elements copied + * @throws IndexOutOfBoundsException + * if the index is negative + */ + int copy(final Object[] target, final int from) { + if (from < 0) { + throw new IndexOutOfBoundsException(from); + } + long stamp = lock.readLock(); + try { + int count = Math.max(0, Math.min(target.length, elementCount - from)); + int copied = 0; + int c = count > 0 ? chunkIndex(from) : 0; + while (copied < count) { + int offset = from + copied - starts[c]; + int length = Math.min(chunkSize(c) - offset, count - copied); + System.arraycopy(chunks[c], offset, target, copied, length); + copied += length; + c++; + } + return count; + } finally { + lock.unlockRead(stamp); + } + } + + /** + * Returns the index of the chunk holding the element at the given index. + * + * @param index + * the index of an element, {@code 0 <= index < size()} + * @return the index of the chunk + */ + private int chunkIndex(final int index) { + int low = 0; + int high = chunkCount - 1; + while (low < high) { + int mid = (low + high + 1) >>> 1; + if (starts[mid] <= index) { + low = mid; + } else { + high = mid - 1; + } + } + return low; + } + + private int chunkSize(final int c) { + return (c + 1 < chunkCount ? starts[c + 1] : elementCount) - starts[c]; + } + + /** + * Splits the given full chunk into two chunks of half its size. + * + * @param c + * the index of a chunk holding {@value #CHUNK_CAPACITY} elements + */ + private void split(final int c) { + if (chunkCount == chunks.length) { + int newLength = chunkCount + (chunkCount >> 1) + 1; + chunks = Arrays.copyOf(chunks, newLength); + starts = Arrays.copyOf(starts, newLength); + } + System.arraycopy(chunks, c + 1, chunks, c + 2, chunkCount - c - 1); + System.arraycopy(starts, c + 1, starts, c + 2, chunkCount - c - 1); + Object[] left = chunks[c]; + Object[] right = new Object[CHUNK_CAPACITY]; + System.arraycopy(left, HALF_CHUNK, right, 0, CHUNK_CAPACITY - HALF_CHUNK); + Arrays.fill(left, HALF_CHUNK, CHUNK_CAPACITY, null); + chunks[c + 1] = right; + starts[c + 1] = starts[c] + HALF_CHUNK; + chunkCount++; + } + +} diff --git a/com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookup.java b/com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookup.java index fdd60ee48..5e192b72c 100644 --- a/com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookup.java +++ b/com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookup.java @@ -64,6 +64,12 @@ private static class SegmentNode implements Externalizable { protected static final int DEFAULT_CHILD_CAPACITY = 4; + /** + * Number of children from which a node keeps them in a {@link ChunkedList}. Inserting into an {@link ArrayList} shifts all following children, which makes + * filling nodes with several 100'000 children quadratic. + */ + protected static final int LARGE_CHILD_COUNT = 4096; + private String segment; protected Object[] values; protected List children; @@ -187,22 +193,22 @@ protected boolean collectMatches(final QualifiedName lower, int lowerIdx, final return true; } if (lower.getSegmentCount() == lowerIdx) { - for (; idx < children.size(); idx++) { - SegmentNode child = children.get(idx); + ChildReader reader = new ChildReader(children, idx); + for (SegmentNode child = reader.next(); child != null; child = reader.next()) { if (child == upper) { return false; } - visitor.visit(children.get(idx)); + visitor.visit(child); if (recursive) { - children.get(idx).visitChildren(visitor, upper); + child.visitChildren(visitor, upper); } } return true; } else { boolean collect = children.get(idx++).collectMatches(lower, lowerIdx, upper, recursive, visitor); if (collect) { - for (; idx < children.size(); idx++) { - SegmentNode child = children.get(idx); + ChildReader reader = new ChildReader(children, idx); + for (SegmentNode child = reader.next(); child != null; child = reader.next()) { if (child == upper || !child.visitChildren(visitor, upper)) { return false; } @@ -226,7 +232,7 @@ private boolean visitChildren(final Visitor visitor, final SegmentNode stopOn) { if (children == null) { return true; } - for (SegmentNode child : children) { + for (SegmentNode child : childrenToVisit()) { if (stopOn.equals(child)) { return false; } @@ -260,7 +266,7 @@ void merge(final QualifiedName name, int segIdx, final Object[] newValues) { // if (idx < 0) { SegmentNode child = new SegmentNode(seg); idx = -(idx + 1); - children.add(idx, child); + addChild(idx, child); if (name.getSegmentCount() == segIdx) { child.values = newValues; return; @@ -274,6 +280,33 @@ void merge(final QualifiedName name, int segIdx, final Object[] newValues) { // } } + /** + * Inserts the given child, keeping the children in a {@link ChunkedList} once there are {@value #LARGE_CHILD_COUNT} of them. + * + * @param idx + * index at which to insert the child + * @param child + * child to insert, must not be {@code null} + */ + protected void addChild(final int idx, final SegmentNode child) { + if (children.size() >= LARGE_CHILD_COUNT && !(children instanceof ChunkedList)) { + children = new ChunkedList<>(children); + } + children.add(idx, child); + } + + /** + * Returns the children for visiting all of them. Those of a large node are a copy, since getting them one by one searches the chunks for each child. The + * copy is not affected by a concurrent insertion, while iterating over the children of a small node then fails. + * + * @return the children, must not be modified; {@link #children} must not be {@code null} + */ + @SuppressWarnings({"unchecked", "PMD.LooseCoupling"}) // only a ChunkedList copies safely during an insertion + private List childrenToVisit() { + List list = children; + return list instanceof ChunkedList ? (List) (List) Arrays.asList(list.toArray()) : list; + } + /** * Accepts the given visitor and uses it to visit all children in depth-first search order. * @@ -283,7 +316,7 @@ void merge(final QualifiedName name, int segIdx, final Object[] newValues) { // void accept(final Visitor visitor) { visitor.visit(this); if (children != null) { - for (SegmentNode child : children) { + for (SegmentNode child : childrenToVisit()) { child.accept(visitor); } } @@ -292,7 +325,7 @@ void accept(final Visitor visitor) { List> getMappings(final Object value) { List> result = new ArrayList<>(); if (children != null) { - for (SegmentNode node : children) { + for (SegmentNode node : childrenToVisit()) { for (List mapping : node.getMappings(value)) { if (!segment.isBlank()) { mapping.add(0, segment); @@ -319,7 +352,11 @@ List> getMappings(final Object value) { * key, corresponding to {@link SegmentNode#segment}, to search for * @return index of found node or (-(insertion point) - 1) */ + @SuppressWarnings("PMD.LooseCoupling") // a ChunkedList searches faster itself than through get protected int binarySearch(final List list, final String key) { + if (list instanceof ChunkedList chunkedList) { + return chunkedList.binarySearch(node -> node.segment.compareTo(key)); + } int low = 0; int high = list.size() - 1; @@ -361,7 +398,8 @@ public void writeExternal(final ObjectOutput out) throws IOException { } } } - out.writeObject(children); + // the serialized form keeps a plain ArrayList + out.writeObject(children instanceof ChunkedList ? new ArrayList<>(children) : children); } @SuppressWarnings("unchecked") @@ -382,7 +420,8 @@ public void readExternal(final ObjectInput in) throws IOException, ClassNotFound } } } - children = (List) in.readObject(); + List list = (List) in.readObject(); + children = list != null && list.size() >= LARGE_CHILD_COUNT ? new ChunkedList<>(list) : list; } } @@ -435,7 +474,7 @@ void merge(final QualifiedName name, int segIdx, final Object[] newValues) { // if (idx < 0) { SegmentNode child = new ValueSharingSegmentNode(seg); idx = -(idx + 1); - children.add(idx, child); + addChild(idx, child); if (name.getSegmentCount() == segIdx) { child.values = Arrays.equals(values, newValues) ? values : newValues; return; @@ -455,6 +494,58 @@ void merge(final QualifiedName name, int segIdx, final Object[] newValues) { // // CHECKSTYLE:CHECK-ON ParameterAssignmentCheck // CHECKSTYLE:CHECK-ON FinalParametersCheck + /** + * Reads the children of a node in order, from a given index on. Those of a large node are copied in batches with {@link ChunkedList#copy(Object[], int)}, + * since getting them one by one searches the chunks for each; the batches grow, as a lookup often stops after a few children. + */ + private static final class ChildReader { + private static final int FIRST_BATCH = 32; + + private final List list; + @SuppressWarnings("PMD.LooseCoupling") // only a ChunkedList copies a range + private final ChunkedList chunkedList; + private Object[] batch; + private int index; + private int position; + private int count; + + /** + * Creates a reader of the given children. + * + * @param list + * the children, must not be {@code null} + * @param from + * the index of the first child to read + */ + @SuppressWarnings("unchecked") + ChildReader(final List list, final int from) { + this.list = list; + this.chunkedList = list instanceof ChunkedList ? (ChunkedList) list : null; + this.index = from; + } + + /** + * Returns the next child. + * + * @return the next child, or {@code null} after the last one + */ + SegmentNode next() { + if (chunkedList == null) { + return index < list.size() ? list.get(index++) : null; + } + if (position == count) { + batch = new Object[batch == null ? FIRST_BATCH : Math.min(batch.length * 2, ChunkedList.CHUNK_CAPACITY)]; + count = chunkedList.copy(batch, index); + index += count; + position = 0; + if (count == 0) { + return null; + } + } + return (SegmentNode) batch[position++]; + } + } + /** * A visitor to visit the nodes of the tree in a {@link SegmentNode#accept(Visitor) depth-first order}. */