diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java index c703d87..dcea786 100644 --- a/native/src/main/java/org/humemai/nativelib/BulkOps.java +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -26,8 +26,11 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.IntConsumer; final class BulkOps { @@ -85,6 +88,17 @@ static String insertColumnsReply(final Database db, final String type, final Str return "{\"ok\":true,\"n\":" + insertColumns(db, type, h, spec, NativeExtras.FailureToken.of(fid.longValue()).onError) + "}"; } + /** + * The records of a frame for the async writers, each with its own callbacks (the "create_records" operation of arcadedb_n_async_bin): + * as the "parallel" form above, but record i goes to {@link NativeExtras.RecordHooks#submit} instead of one shared error callback. The + * header is the one of arcadedb_n_insert_colp ("n", "commit", "cols"); the frame is checked before the first record is handed over. + * A failure after that is thrown as an IllegalStateException "after handing k of n rows", and hooks.handed is k. + */ + static long handColumns(final Database db, final String type, final Map header, final long[] spec, + final NativeExtras.RecordHooks hooks) { + return insertColumns(db, type, header, spec, hooks); + } + /** * header: {"n":rows,"commit":every,"cols":[[name,kind,dim],..]}. Kinds: q (int64), d (float64), b (bool byte), i (int32), e (float32), * f (float32 matrix, dim floats per row), s (UTF-8 string: int32 offsets[n+1] in the data slot, the bytes they index in the aux slot, as @@ -103,73 +117,9 @@ static String insertColumnsReply(final Database db, final String type, final Str @SuppressWarnings("unchecked") private static long insertColumns(final Database db, final String type, final Map h, final long[] spec, final ErrorCallback onError) { - final long nRows = ((Number) h.get("n")).longValue(); - if (nRows < 0 || nRows > Integer.MAX_VALUE) - throw new IllegalArgumentException("row count out of range: " + nRows); - final int n = (int) nRows; final int commitEvery = ((Number) h.get("commit")).intValue(); - final List cols = (List) h.get("cols"); - final int k = cols.size(); - if (spec.length != SLOTS * k) - throw new IllegalArgumentException("address table has " + spec.length + " entries for " + k + " columns, expected " + SLOTS * k); - final String[] names = new String[k]; - final char[] kinds = new char[k]; - final int[] dims = new int[k]; - final long[] data = new long[k]; // addresses: a Word value cannot live in an array or be compared with null - final long[] nulls = new long[k]; // 0 = no nulls - final ByteBuffer[] blobs = new ByteBuffer[k]; - int longest = 0; - for (int c = 0; c < k; c++) { - final List col = (List) cols.get(c); - names[c] = (String) col.get(0); - kinds[c] = ((String) col.get(1)).charAt(0); - dims[c] = ((Number) col.get(2)).intValue(); - final long dataAddr = spec[SLOTS * c], dataBytes = spec[SLOTS * c + 1]; - final long auxAddr = spec[SLOTS * c + 2], auxBytes = spec[SLOTS * c + 3]; - final long nullAddr = spec[SLOTS * c + 4], nullBytes = spec[SLOTS * c + 5]; - final long need = switch (kinds[c]) { - case 'q', 'd' -> 8L * n; - case 'b' -> n; - case 'i', 'e', 'D' -> 4L * n; - case 'T' -> { - if (dims[c] != 1 && dims[c] != 1_000 && dims[c] != 1_000_000 && dims[c] != 1_000_000_000) - throw new IllegalArgumentException("column '" + names[c] + "': a date-time column has 1, 1000, 10^6 or 10^9 ticks per second, not " + dims[c]); - yield 8L * n; - } - case 'f' -> { - if (dims[c] <= 0) - throw new IllegalArgumentException("column '" + names[c] + "': a vector column needs a dimension above 0"); - yield 4L * n * dims[c]; - } - case 's', 'n', 'x' -> 4L * (n + 1L); - default -> throw new IllegalArgumentException("unknown column kind " + kinds[c]); - }; - if (dataBytes < need || (need > 0 && dataAddr == 0)) - throw new IllegalArgumentException("column '" + names[c] + "' has " + dataBytes + " bytes, " + need + " needed"); - data[c] = dataAddr; - if (nullAddr != 0) { - if (nullBytes < (n + 7L) / 8) - throw new IllegalArgumentException("column '" + names[c] + "' has a null bitmap of " + nullBytes + " bytes, " + (n + 7L) / 8 + " needed"); - nulls[c] = nullAddr; - } - if (kinds[c] == 's' || kinds[c] == 'n' || kinds[c] == 'x') { - if (auxBytes < 0 || auxBytes > Integer.MAX_VALUE || (auxBytes > 0 && auxAddr == 0)) - throw new IllegalArgumentException("column '" + names[c] + "': the string bytes are missing or beyond 2 GiB"); - blobs[c] = auxBytes == 0 ? ByteBuffer.allocate(0) : CTypeConversion.asByteBuffer(WordFactory.pointer(auxAddr), (int) auxBytes); - final Pointer offsets = WordFactory.pointer(data[c]); - int prev = offsets.readInt(WordFactory.signed(0)); - if (prev < 0) - throw new IllegalArgumentException("column '" + names[c] + "': negative string offset"); - for (int i = 1; i <= n; i++) { - final int off = offsets.readInt(WordFactory.signed(4L * i)); - if (off < prev || off > auxBytes) - throw new IllegalArgumentException("column '" + names[c] + "': string offsets are not increasing within the " + auxBytes + " bytes (row " + (i - 1) + ")"); - longest = Math.max(longest, off - prev); - prev = off; - } - } - } - final byte[] scratch = new byte[longest]; + final Table table = Table.of(h, spec); + final int n = table.n; final Object schemaType = db.getSchema().getType(type); final boolean vertex = schemaType instanceof VertexType; if (onError != null) { @@ -181,8 +131,11 @@ private static long insertColumns(final Database db, final String type, final Ma try { for (; handed < n; handed++) { final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); - setRow(doc, handed, names, kinds, dims, data, nulls, blobs, scratch); - async.createRecord(doc, null, onError); + setRow(doc, handed, table); + if (onError instanceof NativeExtras.RecordHooks hooks) + hooks.submit(async, doc, handed); // an async create_record batch: each record has its own callbacks + else + async.createRecord(doc, null, onError); } } catch (final Throwable t) { // the records handed over before the failure are the writers' now, whatever this call does @@ -192,7 +145,7 @@ private static long insertColumns(final Database db, final String type, final Ma } writeChunks(db, n, commitEvery, i -> { final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); - setRow(doc, i, names, kinds, dims, data, nulls, blobs, scratch); + setRow(doc, i, table); doc.save(); }); return n; @@ -266,58 +219,193 @@ private static void backoffBeforeRetry(final int attempt) { } } - /** Sets the properties of row i on doc from the columns: the one place that reads a value out of a buffer. */ - private static void setRow(final MutableDocument doc, final int i, final String[] names, final char[] kinds, final int[] dims, - final long[] data, final long[] nulls, final ByteBuffer[] blobs, final byte[] scratch) { - final int k = names.length; - for (int c = 0; c < k; c++) { - final Pointer base = WordFactory.pointer(data[c]); - if (nulls[c] != 0 && ((WordFactory.pointer(nulls[c]).readByte(WordFactory.signed(i >> 3)) >> (i & 7)) & 1) == 0) { - doc.set(names[c], null); - continue; + /** Sets the properties of row i on doc from the columns. */ + private static void setRow(final MutableDocument doc, final int i, final Table t) { + for (int c = 0; c < t.k; c++) + doc.set(t.names[c], t.cell(c, i)); + } + + /** + * The rows of a column table as GraphBatch.createVertices takes them: per row {name, value, name, value, ...}. The names are in the + * iteration order of a java.util.HashMap that took them in column order: the order in which the JSON rows of the same call set their + * properties (Json.rows reads every row, which has the keys of the first row in the same order, into a fresh HashMap), so a vertex is + * stored with the property order it has always had. The values are the objects the JSON rows give (Table.cell). The columns that + * {@code last} names (float32 array vectors: the old route took them out of the rows and appended them, in column order) follow the others. + */ + static Object[][] vertexRows(final Table t, final Object last) { + final Set tail = new HashSet<>(); + if (last instanceof List names) + for (final Object name : names) + tail.add((String) name); + final HashMap byName = new HashMap<>(); + for (int c = 0; c < t.k; c++) + if (!tail.contains(t.names[c])) + byName.put(t.names[c], c); + final int[] order = new int[t.k]; + int at = 0; + for (final Integer c : byName.values()) + order[at++] = c; + int tailFound = 0; + for (int c = 0; c < t.k; c++) + if (tail.contains(t.names[c])) { + order[at++] = c; + tailFound++; + } + if (at != t.k || tailFound != tail.size()) + throw new IllegalArgumentException("two columns have the same name, or \"last\" names a column that is not there"); + final Object[][] rows = new Object[t.n][]; + for (int i = 0; i < t.n; i++) { + final Object[] row = new Object[2 * t.k]; + for (int p = 0; p < t.k; p++) { + row[2 * p] = t.names[order[p]]; + row[2 * p + 1] = t.cell(order[p], i); } + rows[i] = row; + } + return rows; + } + + /** + * The columns of one call: the header ("n" rows, "cols" [name, kind, dim]) and the address table (SLOTS longs per column), validated + * before anything is built, and the one place that reads a value out of a buffer ({@link #cell}). The memory belongs to the caller for + * the duration of the call. + */ + static final class Table { + final int n, k; + final String[] names; + final char[] kinds; + final int[] dims; + final long[] data; // addresses: a Word value cannot live in an array or be compared with null + final long[] nulls; // 0 = no nulls + final ByteBuffer[] blobs; + final byte[] scratch; + + @SuppressWarnings("unchecked") + static Table of(final Map h, final long[] spec) { + final long nRows = ((Number) h.get("n")).longValue(); + if (nRows < 0 || nRows > Integer.MAX_VALUE) + throw new IllegalArgumentException("row count out of range: " + nRows); + final List cols = (List) h.get("cols"); + if (spec.length != SLOTS * cols.size()) + throw new IllegalArgumentException("address table has " + spec.length + " entries for " + cols.size() + " columns, expected " + SLOTS * cols.size()); + return new Table((int) nRows, cols, spec); + } + + @SuppressWarnings("unchecked") + private Table(final int n, final List cols, final long[] spec) { + this.n = n; + k = cols.size(); + names = new String[k]; + kinds = new char[k]; + dims = new int[k]; + data = new long[k]; + nulls = new long[k]; + blobs = new ByteBuffer[k]; + int longest = 0; + for (int c = 0; c < k; c++) { + final List col = (List) cols.get(c); + names[c] = (String) col.get(0); + kinds[c] = ((String) col.get(1)).charAt(0); + dims[c] = ((Number) col.get(2)).intValue(); + final long dataAddr = spec[SLOTS * c], dataBytes = spec[SLOTS * c + 1]; + final long auxAddr = spec[SLOTS * c + 2], auxBytes = spec[SLOTS * c + 3]; + final long nullAddr = spec[SLOTS * c + 4], nullBytes = spec[SLOTS * c + 5]; + final long need = switch (kinds[c]) { + case 'q', 'd' -> 8L * n; + case 'b' -> n; + case 'i', 'e', 'D' -> 4L * n; + case 'T' -> { + if (dims[c] != 1 && dims[c] != 1_000 && dims[c] != 1_000_000 && dims[c] != 1_000_000_000) + throw new IllegalArgumentException("column '" + names[c] + "': a date-time column has 1, 1000, 10^6 or 10^9 ticks per second, not " + dims[c]); + yield 8L * n; + } + case 'f' -> { + if (dims[c] <= 0) + throw new IllegalArgumentException("column '" + names[c] + "': a vector column needs a dimension above 0"); + yield 4L * n * dims[c]; + } + case 's', 'n', 'x' -> 4L * (n + 1L); + default -> throw new IllegalArgumentException("unknown column kind " + kinds[c]); + }; + if (dataBytes < need || (need > 0 && dataAddr == 0)) + throw new IllegalArgumentException("column '" + names[c] + "' has " + dataBytes + " bytes, " + need + " needed"); + data[c] = dataAddr; + if (nullAddr != 0) { + if (nullBytes < (n + 7L) / 8) + throw new IllegalArgumentException("column '" + names[c] + "' has a null bitmap of " + nullBytes + " bytes, " + (n + 7L) / 8 + " needed"); + nulls[c] = nullAddr; + } + if (kinds[c] == 's' || kinds[c] == 'n' || kinds[c] == 'x') { + if (auxBytes < 0 || auxBytes > Integer.MAX_VALUE || (auxBytes > 0 && auxAddr == 0)) + throw new IllegalArgumentException("column '" + names[c] + "': the string bytes are missing or beyond 2 GiB"); + blobs[c] = auxBytes == 0 ? ByteBuffer.allocate(0) : CTypeConversion.asByteBuffer(WordFactory.pointer(auxAddr), (int) auxBytes); + final Pointer offsets = WordFactory.pointer(data[c]); + int prev = offsets.readInt(WordFactory.signed(0)); + if (prev < 0) + throw new IllegalArgumentException("column '" + names[c] + "': negative string offset"); + for (int i = 1; i <= n; i++) { + final int off = offsets.readInt(WordFactory.signed(4L * i)); + if (off < prev || off > auxBytes) + throw new IllegalArgumentException("column '" + names[c] + "': string offsets are not increasing within the " + auxBytes + " bytes (row " + (i - 1) + ")"); + longest = Math.max(longest, off - prev); + prev = off; + } + } + } + scratch = new byte[longest]; + } + + /** The value of column c in row i (null for a null), as the JSON rows give it: Long, Double, Boolean, String, LocalDateTime, LocalDate, BigDecimal, byte[], float[]. */ + Object cell(final int c, final int i) { + if (nulls[c] != 0 && ((WordFactory.pointer(nulls[c]).readByte(WordFactory.signed(i >> 3)) >> (i & 7)) & 1) == 0) + return null; + final Pointer base = WordFactory.pointer(data[c]); switch (kinds[c]) { - case 'q' -> doc.set(names[c], base.readLong(WordFactory.signed(8L * i))); - case 'd' -> doc.set(names[c], base.readDouble(WordFactory.signed(8L * i))); - case 'b' -> doc.set(names[c], base.readByte(WordFactory.signed(i)) != 0); - case 'i' -> doc.set(names[c], base.readInt(WordFactory.signed(4L * i))); - case 'e' -> doc.set(names[c], base.readFloat(WordFactory.signed(4L * i))); - case 'T' -> { + case 'q': + return Long.valueOf(base.readLong(WordFactory.signed(8L * i))); + case 'd': + return Double.valueOf(base.readDouble(WordFactory.signed(8L * i))); + case 'b': + return Boolean.valueOf(base.readByte(WordFactory.signed(i)) != 0); + case 'i': + return Integer.valueOf(base.readInt(WordFactory.signed(4L * i))); + case 'e': + return Float.valueOf(base.readFloat(WordFactory.signed(4L * i))); + case 'T': { final long ticks = base.readLong(WordFactory.signed(8L * i)); final long perSecond = dims[c]; - doc.set(names[c], LocalDateTime.ofEpochSecond(Math.floorDiv(ticks, perSecond), - (int) (Math.floorMod(ticks, perSecond) * (1_000_000_000L / perSecond)), ZoneOffset.UTC)); + return LocalDateTime.ofEpochSecond(Math.floorDiv(ticks, perSecond), + (int) (Math.floorMod(ticks, perSecond) * (1_000_000_000L / perSecond)), ZoneOffset.UTC); } - case 'D' -> doc.set(names[c], LocalDate.ofEpochDay(base.readInt(WordFactory.signed(4L * i)))); - case 'n' -> { + case 'D': + return LocalDate.ofEpochDay(base.readInt(WordFactory.signed(4L * i))); + case 'n': { final int from = base.readInt(WordFactory.signed(4L * i)); final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; blobs[c].get(from, scratch, 0, len); - doc.set(names[c], new BigDecimal(new String(scratch, 0, len, StandardCharsets.US_ASCII))); + return new BigDecimal(new String(scratch, 0, len, StandardCharsets.US_ASCII)); } - case 'x' -> { + case 'x': { final int from = base.readInt(WordFactory.signed(4L * i)); final byte[] bytes = new byte[base.readInt(WordFactory.signed(4L * (i + 1L))) - from]; blobs[c].get(from, bytes, 0, bytes.length); - doc.set(names[c], bytes); + return bytes; } - case 's' -> { + case 's': { final int from = base.readInt(WordFactory.signed(4L * i)); final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; if (len == 0) - doc.set(names[c], ""); - else { - blobs[c].get(from, scratch, 0, len); - doc.set(names[c], new String(scratch, 0, len, StandardCharsets.UTF_8)); - } + return ""; + blobs[c].get(from, scratch, 0, len); + return new String(scratch, 0, len, StandardCharsets.UTF_8); } - default -> { + default: { final int dim = dims[c]; final float[] v = new float[dim]; final long at = 4L * i * dim; for (int j = 0; j < dim; j++) v[j] = base.readFloat(WordFactory.signed(at + 4L * j)); - doc.set(names[c], v); + return v; } } } diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index d230919..6245f6d 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -501,6 +501,42 @@ public static CCharPointer gbBin(final IsolateThread thread, final long h, final return gbImpl(h, op, argsJson, buf.rawValue(), bufFloats); } + /** + * create_vertices from columns, by pointer, as arcadedb_n_insert_colp takes them (see BulkOps.Table): header JSON {"n":rows,"cols":[[name, + * kind, dim],..],"last":[names]} (the columns stored after the others, see BulkOps.vertexRows) and an address table of nspec longs (BulkOps.SLOTS per column). The rows are built from the columns (BulkOps.vertexRows: + * the objects, and the property order, that the JSON rows of arcadedb_n_gb give) and go to the same GraphBatch.createVertices(type, + * Object[][]), so the transaction, the retries and the commit are the engine's and nothing differs from the JSON route but the transport. + * The RIDs do not come back as text: rids points at ridLongs int64 slots of the caller's (at least 2 per row), and row i gets bucket id + * at slot 2i and position at slot 2i + 1. Returns {"ok":true,"n":rows}. The caller keeps every buffer alive until the call returns. + */ + @CEntryPoint(name = "arcadedb_n_gb_colp") + public static CCharPointer gbColp(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer header, + final CCharPointer spec, final int nspec, final CCharPointer rids, final long ridLongs) { + try { + final BatchEntry entry = BATCHES.get(h); + if (entry == null) + throw new IllegalArgumentException("unknown GraphBatch " + h); + if (nspec < 0 || nspec % BulkOps.SLOTS != 0) + throw new IllegalArgumentException("address table of " + nspec + " entries is not a multiple of " + BulkOps.SLOTS); + final long[] table = new long[nspec]; + for (int i = 0; i < nspec; i++) + table[i] = ((Pointer) spec).readLong(WordFactory.signed(8L * i)); + final Map head = args(header); + final BulkOps.Table columns = BulkOps.Table.of(head, table); + if (rids.isNull() || ridLongs < 2L * columns.n) + throw new IllegalArgumentException("the RID buffer holds " + ridLongs + " int64 slots, " + 2L * columns.n + " needed"); + final RID[] created = entry.batch().createVertices(ArcadeNative.str(type), BulkOps.vertexRows(columns, head.get("last"))); + final Pointer out = (Pointer) rids; + for (int i = 0; i < created.length; i++) { + out.writeLong(WordFactory.signed(16L * i), created[i].getBucketId()); + out.writeLong(WordFactory.signed(16L * i + 8L), created[i].getPosition()); + } + return ArcadeNative.out("{\"ok\":true,\"n\":" + created.length + "}"); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + @SuppressWarnings("unchecked") private static CCharPointer gbImpl(final long h, final CCharPointer op, final CCharPointer argsJson, final long buf, final long bufFloats) { @@ -614,6 +650,78 @@ void drop() { } } + /** + * The callback of a record that an async writer stored: Python gets the stored record (its RID included) under the callback id, or + * nothing when the record has no callback. + */ + private static com.arcadedb.database.async.NewRecordCallback recordCallback(final long cb) { + if (cb == 0) + return null; + return rec -> { + try { + final StringBuilder sb = new StringBuilder("\"rec\":"); + NativeCodec.append(sb, rec); + fire("record callback", cb, event("record", sb.toString())); + } catch (final Throwable e) { + swallowed("record callback", e); // the writer goes on + } + }; + } + + /** + * The callbacks of the records of one "create_records" batch (callback id and error callback id per record, 0 for none). It is the + * error callback that BulkOps gets for the frame; BulkOps hands record i to {@link #submit} when it is one of these, so that every + * record has its own callbacks, as a "create_record" call has. + */ + static final class RecordHooks implements ErrorCallback { + private final long[] cbs; + private final long[] errs; + /** The records handed to the writers so far: after a failure, the number of records that the writers hold. */ + int handed; + + RecordHooks(final long[] cbs, final long[] errs) { + this.cbs = cbs; + this.errs = errs; + } + + /** Not used: every record is handed over with its own error callback. */ + @Override + public void call(final Throwable t) { + swallowed("create_records error callback", t); + } + + void submit(final DatabaseAsyncExecutor ex, final MutableDocument doc, final int i) { + final com.arcadedb.database.async.NewRecordCallback ok = recordCallback(cbs[i]); + if (errs[i] == 0) + ex.createRecord(doc, ok); + else + ex.createRecord(doc, ok, errorCallback(errs[i])); + handed = i + 1; + } + } + + /** The address table of arcadedb_n_async_bin as a long[] of BulkOps.SLOTS entries per column (the layout of arcadedb_n_insert_colp). */ + private static long[] columnTable(final long table, final int nTable) { + if (table == 0 || nTable <= 0 || nTable % BulkOps.SLOTS != 0) + throw new IllegalArgumentException("address table of " + nTable + " entries is not a multiple of " + BulkOps.SLOTS); + final Pointer t = WordFactory.pointer(table); + final long[] spec = new long[nTable]; + for (int i = 0; i < nTable; i++) + spec[i] = t.readLong(WordFactory.signed(8L * i)); + return spec; + } + + @SuppressWarnings("unchecked") + private static long[] callbackIds(final Object list, final int n, final String what) { + final List in = (List) list; + if (in == null || in.size() != n) + throw new IllegalArgumentException("\"" + what + "\" needs one entry per record (" + n + ")"); + final long[] out = new long[n]; + for (int i = 0; i < n; i++) + out[i] = ((Number) in.get(i)).longValue(); + return out; + } + private static String replyRows(final ResultSet rs) { final StringBuilder sb = new StringBuilder("{\"ok\":true,\"rs\":0,"); NativeCodec.appendBatch(sb, rs, Integer.MAX_VALUE); @@ -673,8 +781,8 @@ private static boolean isVertexType(final Database db, final String typeName) { } /** - * ops: get | set {key,value} | wait {timeout_ms} | kill | create_record {type, kind, props, cb, err} | command / query {lang, text, params, - * cb, err} | on_ok {cb} | on_error {cb} | append_samples {type, ts, cols} | insert_parallel {type, rows} | failures {fid}. + * ops: get | set {key,value} | wait {timeout_ms} | kill | create_record {type, kind, props, cb, err} | create_records {type, kind, n, commit, + * cols, cbs, errs} (arcadedb_n_async_bin only: a frame) | command / query {lang, text, params, cb, err} | on_ok {cb} | on_error {cb} | append_samples {type, ts, cols} | insert_parallel {type, rows} | failures {fid}. */ @CEntryPoint(name = "arcadedb_n_async") public static CCharPointer async(final IsolateThread thread, final long dbh, final CCharPointer op, final CCharPointer argsJson) { @@ -781,20 +889,37 @@ private static CCharPointer asyncImpl(final long dbh, final CCharPointer op, fin final MutableDocument doc = "v".equals(a.get("kind")) ? db.newVertex(type) : db.newDocument(type); for (final Map.Entry e : ((Map) a.get("props")).entrySet()) doc.set(e.getKey(), e.getValue()); - final com.arcadedb.database.async.NewRecordCallback ok = cb == 0 ? null : rec -> { - try { - final StringBuilder sb = new StringBuilder("\"rec\":"); - NativeCodec.append(sb, rec); - fire("record callback", cb, event("record", sb.toString())); - } catch (final Throwable e) { - swallowed("record callback", e); // the writer goes on - } - }; + final com.arcadedb.database.async.NewRecordCallback ok = recordCallback(cb); if (errCb == 0) ex.createRecord(doc, ok); else ex.createRecord(doc, ok, errorCallback(errCb)); } + case "create_records" -> { + // a frame of records of one type and kind, each with its own callbacks (cbs, errs: callback ids, 0 for none), through BulkOps' column + // reader. The frame and the type are checked before the first record is handed over (an exception then: nothing was queued). A + // record that fails while the batch is handed over ends the call with {"ok":true,"n":k,"err":..}: k records are the writers' + // now and the record k is refused with that error, so the caller can give it to its error callback and send the rest again. + final String type = (String) a.get("type"); + final boolean vertex = "v".equals(a.get("kind")); + final Object schemaType = db.getSchema().getType(type); + if (schemaType instanceof EdgeType) + throw new IllegalArgumentException("Cannot bulk-insert into '" + type + + "' because it is an edge type; load the vertices, then create the edges (graph_batch().new_edges() or CREATE EDGE)"); + if (vertex != (schemaType instanceof VertexType)) + throw new IllegalArgumentException("'" + type + "' is " + (vertex ? "not a vertex type" : "a vertex type") + + ", but its records were created as " + (vertex ? "vertices" : "documents")); + final int n = intOf(a.get("n"), -1); + final RecordHooks hooks = new RecordHooks(callbackIds(a.get("cbs"), n, "cbs"), callbackIds(a.get("errs"), n, "errs")); + try { + final long handed = BulkOps.handColumns(db, type, a, columnTable(table, nTable), hooks); + return ArcadeNative.out("{\"ok\":true,\"n\":" + handed + "}"); + } catch (final IllegalStateException e) { + if (e.getMessage() == null || !e.getMessage().startsWith("after handing ") || e.getCause() == null) + throw e; + return ArcadeNative.out("{\"ok\":true,\"n\":" + hooks.handed + "," + errorBody(e.getCause()) + "}"); + } + } case "command", "query" -> { final boolean command = "command".equals(ArcadeNative.str(op)); final com.arcadedb.database.async.AsyncResultsetCallback callback = cb == 0 && errCb == 0 ? null : new com.arcadedb.database.async.AsyncResultsetCallback() { diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py index c16d225..b372344 100644 --- a/src/arcadedb/_native/bulk.py +++ b/src/arcadedb/_native/bulk.py @@ -95,3 +95,58 @@ def insert_rows_columnar(db, type_name, chunk): rows, cols = packed _send(db, type_name, cols, rows, 0, f"Failed to bulk-insert into '{type_name}'") return True + + +def rids_of(out): + """The RID texts (``#bucket:position``, the ``RID.toString`` text) of an int64 array of ``(bucket, position)`` pairs, in row order.""" + pairs = out.reshape(-1, 2) + return list(map("#{}:{}".format, pairs[:, 0].tolist(), pairs[:, 1].tolist())) + + +def create_vertices_columns(gb, type_name, rows, declared, prefix): + """One ``GraphBatch.create_vertices`` chunk (a list of dicts) through ``arcadedb_n_gb_colp``: the rows cross as columns by address and + the RIDs come back in an int64 buffer, one crossing and one transaction, as the JSON route has it. Returns the list of RID texts, or None + when the chunk needs the JSON route (nothing was sent then): rows that do not all have the same keys in the same order, a value no column + holds (a nested list or map, a mixed-type column, a set, a NaT, a numpy scalar, an integer past 64 bits, a lone surrogate), an empty chunk or a + chunk of fewer rows than ``lib.GB_COLP_MIN_ROWS`` (64 by default: a handful of rows does not repay the fixed cost of the columnar entry). + + ``declared``: a :class:`columnar.Declared` of the type, read once by the caller for all chunks of a call. + """ + if not (lib.GB_COLP and _usable()) or len(rows) < lib.GB_COLP_MIN_ROWS: + return None # a chunk below the floor keeps the JSON route of post3 (lib.GB_COLP_MIN_ROWS), as does a chunk the columns cannot carry + columns = columnar.rows_to_columns(rows, same_order=True) + if columns is None: + return None + packed = columnar.pack( + columns, declared, exact_nanos=True + ) # the JSON route keeps the nanoseconds of a Timestamp + if packed is None: + return None + count, cols = packed + # a column of float32 arrays is stored after the other properties of a row: the old route took those vectors out of the JSON rows and + # appended them (NativeExtras.withVectors); the other columns keep the order of a HashMap of their names, as the JSON rows do + last = [ + c.name + for c in cols + if c.kind == "f" and isinstance(columns[c.name][0], columnar._np.ndarray) + ] + spec = columnar.spec_of(cols) + out = columnar._np.empty(2 * count, dtype=columnar._np.int64) + from .database import _call + + reply = _call( + prefix, + "arcadedb_n_gb_colp", + gb._gb, + lib.b(type_name), + lib.b(columnar.header_of(count, 0, cols, last=last)), + spec, + len(spec), + out.ctypes.data, + out.size, + ) # `cols` and `out` hold the memory the addresses point into until the call returns + if reply["n"] != count: + raise ArcadeDBError( + f"{prefix}: the library created {reply['n']} vertices for {count} rows" + ) + return rids_of(out) diff --git a/src/arcadedb/_native/columnar.py b/src/arcadedb/_native/columnar.py index d001b3d..dad4d67 100644 --- a/src/arcadedb/_native/columnar.py +++ b/src/arcadedb/_native/columnar.py @@ -215,7 +215,7 @@ def _epoch_micros(value): return seconds * 1_000_000 + value.microsecond -def _list_column(name, values, declared): +def _list_column(name, values, declared, exact_nanos=False): """A list or tuple of Python values as a column: one type (None allowed) or a list of equal-length float arrays; else None.""" n = len(values) types = set(map(type, values)) @@ -255,6 +255,12 @@ def _list_column(name, values, declared): if issubclass(only, datetime): if only.__name__ == "NaTType": return None # the JSON path stores it as a null + if ( + exact_nanos + and only is not datetime + and any(getattr(v, "nanosecond", 0) for v in values if v is not None) + ): + return None # a pandas Timestamp with nanoseconds, for a caller whose JSON route keeps them (insert_many's keeps microseconds) try: ticks = [0 if v is None else _epoch_micros(v) for v in values] except (OverflowError, ValueError): @@ -407,7 +413,7 @@ def _arrow_column(name, arr, declared): return None -def pack(columns, declared): +def pack(columns, declared, exact_nanos=False): """``(rows, [Packed])`` for a dict of name -> column, or None when any column cannot be laid out or the columns differ in length. A column is a numpy array (1-D ints, floats and bools, or a 2-D float32 matrix), a list or tuple of Python values of one type (None @@ -425,14 +431,14 @@ def pack(columns, declared): elif isinstance(values, _np.ndarray): if values.dtype.kind in "OUS": col = ( - _list_column(name, values.tolist(), declared) + _list_column(name, values.tolist(), declared, exact_nanos) if values.ndim == 1 else None ) else: col = _numpy_column(name, values, declared) elif isinstance(values, (list, tuple)) and values: - col = _list_column(name, values, declared) + col = _list_column(name, values, declared, exact_nanos) else: col = None # a pandas Series, a generator, an empty list: the row path handles them if col is None: @@ -454,7 +460,7 @@ def spec_of(packed): return (ctypes.c_int64 * len(flat))(*flat) -def header_of(rows, commit_every, packed, fid=None): +def header_of(rows, commit_every, packed, fid=None, last=()): """The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column; with a failure token ``fid`` it also says ``"parallel": true``, which hands the records to the async executor (the image then ignores the commit interval and counts the failed records on that token). @@ -466,14 +472,22 @@ def header_of(rows, commit_every, packed, fid=None): "commit": min(int(commit_every), 2**31 - 1), "cols": [[c.name, c.kind, c.dim] for c in packed], } + if ( + last + ): # create_vertices: the columns the old route appended to the end of each row (float32 array vectors) + header["last"] = list(last) if fid is not None: header["parallel"] = True header["fid"] = fid return json.dumps(header) -def rows_to_columns(rows): - """A list of dicts with the same keys as a dict of name -> list of values, or None (no rows, different keys, a non-str key).""" +def rows_to_columns(rows, same_order=False): + """A list of dicts with the same keys as a dict of name -> list of values, or None (no rows, different keys, a non-str key). + + ``same_order``: every row must also list its keys in the order of the first row (a row with the same keys in another order gives + None), for a caller whose stored property order follows the key order of its rows. + """ if not rows: return None try: @@ -482,6 +496,8 @@ def rows_to_columns(rows): return None if set(map(len, rows)) != {len(keys)}: return None + if same_order and not all(map(keys.__eq__, map(list, rows))): + return None return {key: [row[key] for row in rows] for key in keys} except ( KeyError, diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index 1b106ed..715b092 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -7,10 +7,13 @@ ctypes takes the GIL for the call, and the reply is JSON (see the header of ``NativeExtras.java`` for GIL and exception rules). """ +import copy import ctypes import itertools import json +import logging import threading +import time import weakref from collections.abc import Mapping from typing import Callable, Optional @@ -20,7 +23,7 @@ from ..exceptions import ArcadeDBError from ..graph_batch import GraphBatch as _BaseGraphBatch from ..type_conversion import take_vector_columns -from . import interrupts, lib +from . import bulk, columnar, interrupts, lib from .codec import ( _default, _np, @@ -190,13 +193,32 @@ def _op(self, prefix: str, op: str, **args) -> dict: return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(text)) return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(_enc(args))) - def _create_chunk(self, prefix: str, type_name: str, rows: list) -> list: + def _create_chunk( + self, prefix: str, type_name: str, rows: list, declared=None + ) -> list: """One crossing and one transaction for `rows` (dicts): the RIDs of the new vertices. + Rows with the same keys in the same order, whose values are scalars, strings, dates, decimals, bytes or float32 vectors of one type + per property, cross as columns by address and the RIDs come back in an int64 buffer (``bulk.create_vertices_columns``, no JSON text in + either direction). Any other chunk goes as JSON text, as follows. + A property whose value in every row is a float32 numpy array of one length crosses as one float32 matrix in the side buffer (``take_vector_columns``, ``encode_vertex_args``), with no tag, hook call or Python code per row; every other value is written as :func:`encode_gb_args` writes it. """ + rids = bulk.create_vertices_columns( + self, + type_name, + rows, + ( + declared + if declared is not None + else columnar.Declared(self._db, type_name) + ), + prefix, + ) + if rids is not None: + return rids taken = take_vector_columns(rows, (), undeclared=True) if lib.GB_BIN else None if taken is None: return self._op(prefix, "create_vertices", type=type_name, rows=rows)[ @@ -307,12 +329,15 @@ def create_vertices(self, type_name: str, count_or_properties) -> list: # chunked, as the JPype wrapper does: one crossing, one JSON text and one reply per chunk, so a 1M-vertex load # never holds the text, the parsed rows and the reply of all of it at once (the Serial GC copies what lives) rids = [] + declared = columnar.Declared( + db, type_name + ) # the schema is read at most once for the chunks of this call has_none = type(None) in kinds for start in range(0, len(rows), chunk): part = rows[start : start + chunk] if has_none: part = [r or {} for r in part] - rids.extend(self._create_chunk(prefix, type_name, part)) + rids.extend(self._create_chunk(prefix, type_name, part, declared)) if not rows: rids = self._op(prefix, "create_vertices", type=type_name, rows=[])[ "rids" @@ -446,6 +471,35 @@ def get_deferred_incoming_edge_count(self) -> int: # -------------------------------------------------------------------------------------------------------- async +# records per crossing of create_record (the same size as the commit interval of insert_many on this build) +_ASYNC_BATCH = 1000 +# seconds a collected record may wait for the rest of its batch before a thread of the executor hands it over: a caller that creates +# records and waits for a callback, or reads after a pause, with no other call on the executor, is never kept waiting longer +_ASYNC_LINGER = 0.05 +# the first records of an executor go one op each, exactly as in 26.11.1.post3, before it starts to collect: no copy, no buffer, no linger thread, no frame. Collecting has a fixed cost +# per executor (the linger thread that is started for it and the locks) and a frame has its own fixed cost on top (see _ASYNC_FRAME_MIN), so a small load repays neither. In an untimed +# micro of one executor call (a new executor, N records, wait_completion, native, cores 14 and 15) the first head of this change, which collected from the first record, read 1.6x slower +# than post3 for 1 record and 1.2x for 16, and a direct phase of 32 with the frame floor still read 1.2x to 1.3x slower for 33 to 64 records. An executor of up to _ASYNC_DIRECT records is +# therefore exactly post3. Beyond it the records collect, and a frame repays itself only from _ASYNC_DIRECT + 64 records, 192: an executor of 129 to 191 records collects a few records, +# hands them over one op each and pays the collection cost without a frame, about 0.2 ms more than post3 whatever its size (10% to 15% of its time in the micro). The cost is near +# constant per executor, so the relative slowdown falls as this number rises; 128 was chosen after a direct phase of 96 put the band on the 100-record case, and it is not tuned further. +# A linger thread shared by all executors of the process would remove the cost (ArcadeData/arcadedb-python#406); 0 turns the direct phase off. +_ASYNC_DIRECT = 128 +# a hand-over of fewer collected records than this goes one op each, as in post3: a columnar frame has a fixed cost per crossing (the untimed micro put it near 270 us on this build, +# against about 12 us per record for the per-record op and about 3 us per record inside a frame, so a frame repays itself from about 30 records), and without this floor an executor of +# 33 to 64 records (with a direct phase of 32) read 1.2x to 1.7x slower than post3 in the same micro. 64 is the threshold that was decided; 0 turns the floor off. +_ASYNC_FRAME_MIN = 64 +# the values of a property that cannot change after create_record returns; any other value is copied at the call +_ATOMIC = frozenset((int, float, str, bool, bytes, type(None))) + + +def _snapshot(props: dict) -> dict: + return { + key: value if type(value) in _ATOMIC else copy.deepcopy(value) + for key, value in props.items() + } + + class NativeAsyncExecutor: """The database's async executor (one per database) with the JPype wrapper's names; callbacks run on the engine's writer threads and take the GIL, as the JPype proxies do.""" @@ -462,6 +516,20 @@ def __init__(self, database): self._cb_lock = ( threading.Lock() ) # guards the three above; never held across a call into the library + # the records of create_record that are not handed to the library yet: (properties, callback id, error callback id, the + # caller's error callback), all of one type and kind (_buf_key). _buf_lock is held while the buffer is handed over, so a + # flush point that returns has seen every create_record that returned before it. + self._buf: list = [] + self._buf_key = None + self._buf_lock = threading.RLock() + # the linger thread (started with the first collected record) hands a batch over when its first record has waited + # _ASYNC_LINGER; a failure it meets is kept and raised by the next call that hands records over + self._linger = threading.Condition(self._buf_lock) + self._linger_thread = None + self._linger_due = 0.0 + self._linger_stop = False + self._linger_error = None + self._direct = 0 # the records that went one op each (see _ASYNC_DIRECT) def __enter__(self): return self @@ -473,6 +541,10 @@ def is_closed(self) -> bool: return self._closed def _op(self, op: str, **args) -> dict: + self._flush() # every call on the executor goes after the records that were handed to create_record before it + return self._op_raw(op, **args) + + def _op_raw(self, op: str, **args) -> dict: return _call( "Async executor failed", "arcadedb_n_async", @@ -573,16 +645,204 @@ def _error_callback(self, callback): return _register(lambda message: callback(_error_from(message))) def create_record(self, document, callback=None, error_callback=None): + """Queue ``document`` for an async writer; ``callback(record)`` runs on the writer once it is stored, ``error_callback(error)`` + when the engine refuses it. + + The first ``_ASYNC_DIRECT`` (128) records of an executor go one op each, exactly as in 26.11.1.post3 (an executor that creates a few + records pays nothing for collecting). From the next record on the records of one type and kind are collected (a copy of their properties, so a document that the caller changes or reuses + afterwards is stored as it was at this call) and cross the library boundary ``_ASYNC_BATCH`` at a time, as one columnar frame with the callback ids of each record (not one JSON op per record) when at least ``_ASYNC_FRAME_MIN`` (64) + of them are handed over together, else one op each as in post3. Every other call on the executor (``wait_completion``, + ``is_processing``, ``query``, ``command``, ``kill``, ``close``, ...) hands the collected records over first, so the engine + queue keeps the order of the calls and a record is never held back from a ``wait_completion`` that follows its call. A thread of + the executor hands a batch over when its first record has waited ``_ASYNC_LINGER`` (50 ms), so a caller that waits for a + callback or reads after a pause, with no other call, is not kept waiting; a failure that thread meets is raised by the next + call that hands records over. Records + whose shape the frame cannot carry (different keys, nested values, numpy scalars, mixed-type columns, integers beyond 64 bits) + and a batch of one cross as the per-record op they always did. A record that the library refuses before it is queued (the frame is checked + before the first record is handed over; a record that cannot be built) reaches its own ``error_callback`` when one was given, + else it is raised by the call that hands the batch over, after the records around it were handed over. + """ kind = "v" if hasattr(document, "get_out_edges") else "d" + if self._closed or self._direct < _ASYNC_DIRECT: + self._direct += 1 + self._queue_record( + document._type, kind, document._props, callback, error_callback + ) + return + cb = self._record_callback(callback, kind) if callback is not None else 0 + err = self._error_callback(error_callback) if error_callback is not None else 0 + entry = (_snapshot(document._props), cb, err, error_callback) + key = (document._type, kind) + with self._buf_lock: + if self._buf and self._buf_key != key: + self._flush() + self._buf_key = key + self._buf.append(entry) + if len(self._buf) >= _ASYNC_BATCH: + self._flush() + elif ( + len(self._buf) == 1 + ): # the first record of a batch: its hand-over is due after the linger time + self._start_linger() + + def _start_linger(self): + """Under ``_buf_lock``: the batch that has just got its first record is handed over at the latest after ``_ASYNC_LINGER``.""" + self._linger_due = time.monotonic() + _ASYNC_LINGER + if self._linger_thread is None: + self._linger_thread = threading.Thread( + target=self._linger_loop, name="arcadedb-async-linger", daemon=True + ) + self._linger_thread.start() + else: + self._linger.notify() + + def _linger_loop(self): + with self._buf_lock: + while not self._linger_stop: + if not self._buf: + self._linger.wait() + continue + left = self._linger_due - time.monotonic() + if left > 0: + self._linger.wait(left) + continue + try: + self._flush_buffer() + except ( + BaseException + ) as e: # no caller to raise it to: the next call that hands records over raises it + self._linger_error = e + + def _stop_linger(self): + with self._buf_lock: + self._linger_stop = True + self._linger.notify_all() + + def _queue_record(self, type_name, kind, props, callback, error_callback): cb = self._record_callback(callback, kind) if callback is not None else 0 err = self._error_callback(error_callback) if error_callback is not None else 0 self._queue( - "create_record", - type=document._type, - kind=kind, - props=document._props, - cb=cb, - err=err, + "create_record", type=type_name, kind=kind, props=props, cb=cb, err=err + ) + + def _flush(self): + """Hand the collected ``create_record`` calls to the library, in order, and raise what the linger thread's hand-over met.""" + if not self._buf and self._linger_error is None: + return + with self._buf_lock: + self._flush_buffer() + error, self._linger_error = self._linger_error, None + if error is not None: + raise error + + def _flush_buffer(self): + """Hand the collected ``create_record`` calls to the library, in order; see :meth:`create_record`.""" + if not self._buf: + return + with self._buf_lock: + batch, self._buf = self._buf, [] + if not batch: + return + type_name, kind = self._buf_key + problems = [] + state = [0] # the records of the batch that are handed over or failed + try: + while state[0] < len(batch): + error = self._hand(type_name, kind, batch, state) + if error is None: + continue + failed = batch[state[0]] + state[0] += 1 + _forget( + failed[1] + ) # no callback of a record that was not queued will ever run + _forget(failed[2]) + if failed[3] is None: + problems.append(error) + continue + try: + failed[3](error) + except ( + Exception + ): # the batch goes on, as an async writer goes on after a failing handler + logging.getLogger(__name__).exception( + "the error callback of an async create_record failed" + ) + except BaseException: + for entry in batch[state[0] :]: + _forget(entry[1]) + _forget(entry[2]) + raise + if problems: + raise problems[0] + + def _remember(self, entries): + with self._cb_lock: + for entry in entries: + self._cb_ids.extend(i for i in (entry[1], entry[2]) if i) + + def _hand(self, type_name, kind, batch, state): + """Hand ``batch[state[0]:]`` over until one record is refused; returns that record's error, or None when all went. + + ``state[0]`` is the index of the first record that is neither handed over nor failed, updated as records go, so that a + caller that sees an exception (an interrupt) knows which records the engine holds. + """ + part = batch[state[0] :] + if len(part) > 1 and len(part) >= _ASYNC_FRAME_MIN and bulk._usable(): + reply = self._send_frame(type_name, kind, part) + if reply is not None: + handed = int(reply["n"]) + self._remember(part[:handed]) + state[0] += handed + return None if reply.get("err") is None else _error_from(reply) + # one op per record: fewer than _ASYNC_FRAME_MIN records (a frame of one is not cheaper, a small one does not repay its fixed cost) or a shape the frame cannot carry + for entry in part: + try: + self._op_raw( + "create_record", + type=type_name, + kind=kind, + props=entry[0], + cb=entry[1], + err=entry[2], + ) + except ArcadeDBError as e: + return e + self._remember([entry]) + state[0] += 1 + return None + + def _send_frame(self, type_name, kind, part): + """The records of ``part`` as one columnar frame (``arcadedb_n_async_bin``, op ``create_records``), or None when a column + cannot be sent that way. The reply says how many records the library handed over and, when it stopped, the error of the next one. + """ + columns = columnar.rows_to_columns([entry[0] for entry in part]) + if columns is None: + return None + packed = columnar.pack(columns, columnar.Declared(self._db, type_name)) + if packed is None: + return None + rows, cols = packed + if rows != len(part): + return None + spec = columnar.spec_of(cols) + header = { + "type": type_name, + "kind": kind, + "n": rows, + "commit": 0, + "cols": [[c.name, c.kind, c.dim] for c in cols], + "cbs": [entry[1] for entry in part], + "errs": [entry[2] for entry in part], + } + return _call( + "Async executor failed", + "arcadedb_n_async_bin", + self._db._h, + b"create_records", + lib.b(_enc(header)), + spec, # `cols` holds the arrays that the addresses point into until the call returns + len(spec), ) def _statement(self, op, language, text, callback, args, params, error_callback): @@ -653,6 +913,8 @@ def append_samples( 0.370 (95% interval 0.362 to 0.390, median of 11 clean paired runs). The figure is for that shape. """ + self._flush() # the address-table call below does not go through _op + def plain(values): kind = getattr(getattr(values, "dtype", None), "kind", None) if ( @@ -874,6 +1136,7 @@ def _release_callbacks(self, ids=None): def wait_completion(self, timeout_ms: Optional[int] = None): if timeout_ms is not None and timeout_ms < 0: raise ValueError(f"timeout_ms must be None or >= 0, got {timeout_ms}") + self._flush() # first, so that the ids of the collected records are among the ones this call releases # the calls queued so far: a call that another thread queues while this one waits keeps its callbacks with self._cb_lock: finished, self._cb_ids = self._cb_ids, [] @@ -916,10 +1179,18 @@ def on_error(self, callback: Callable[[Exception], None]): def close(self): if self._closed: return + pending = None try: - self._op("close") + try: + self._flush() + except BaseException as e: # the engine's executor is closed all the same + pending = e + self._op_raw("close") + if pending is not None: + raise pending finally: self._closed = True + self._stop_linger() self._release_callbacks() owner, self._owner = self._owner, None if owner is not None: diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index 9f7474d..2f17659 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -36,6 +36,15 @@ # ARCADEDB_NATIVE_GB_BIN: the float32 vectors of a GraphBatch op (create_vertices rows, new_edges props) go through arcadedb_n_gb_bin in a float32 side buffer, not as number text GB_BIN = os.environ.get("ARCADEDB_NATIVE_GB_BIN", "1") != "0" +# ARCADEDB_NATIVE_GB_COLP: the rows of a GraphBatch create_vertices chunk cross as columns by address (arcadedb_n_gb_colp, the layout of +# arcadedb_n_insert_colp) and the RIDs come back in an int64 buffer, not as JSON text both ways; a chunk the columns cannot carry keeps the JSON route +GB_COLP = os.environ.get("ARCADEDB_NATIVE_GB_COLP", "1") != "0" +# ARCADEDB_NATIVE_GB_COLP_MIN_ROWS: a create_vertices chunk of fewer rows than this takes the JSON route of 26.11.1.post3, whatever the columns could carry (0: no floor). +# The columnar entry has a fixed cost (a header, the column specs, the buffers) that a handful of rows does not repay: the laptop A/B of this change measured a single row +# 1.63 times slower than post3 and a batch of 100 rows faster. An untimed micro of create_vertices calls in one open graph batch (this change against post3, native, cores 14 and 15) +# measured slower for 1, 8, 16 and 32 rows (1.7x, 1.4x, 1.2x, 1.03x) and faster for 64 and 100 rows (0.90, 0.86): the crossover is near 40 rows, so 64 is above it and a chunk below +# 64 rows takes the post3 path and cannot be slower. 64 to 99 rows were measured in the micro only (not in the pre-registered A/B, which has sizes 1 and 100). +GB_COLP_MIN_ROWS = int(os.environ.get("ARCADEDB_NATIVE_GB_COLP_MIN_ROWS", "64")) # the directory the engine log, logging.properties and the crash files go to (None until the library is loaded) LOG_DIR_USED = None @@ -252,6 +261,7 @@ class _Params(ctypes.Structure): "arcadedb_n_gb_create": [P, L, S], "arcadedb_n_gb": [P, L, S, S], "arcadedb_n_gb_bin": [P, L, S, S, S, L], + "arcadedb_n_gb_colp": [P, L, S, S, P, N, P, L], "arcadedb_n_async": [P, L, S, S], "arcadedb_n_async_bin": [P, L, S, S, P, N], "arcadedb_n_vindex": [P, L, S, S], @@ -280,7 +290,7 @@ class _Params(ctypes.Structure): ) else ctypes.c_longlong if name in returns_long else ctypes.c_void_p ) - global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY, NEXT_BIN, GB_BIN, TS_BIN + global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY, NEXT_BIN, GB_BIN, GB_COLP, TS_BIN if not hasattr( lib, "arcadedb_n_exec_bin" ): # a library built before the binary path @@ -301,6 +311,10 @@ class _Params(ctypes.Structure): lib, "arcadedb_n_gb_bin" ): # a library built before the side buffer of GraphBatch ops GB_BIN = False + if not hasattr( + lib, "arcadedb_n_gb_colp" + ): # a library built before the columnar vertex load of GraphBatch + GB_COLP = False if not hasattr( lib, "arcadedb_n_async_bin" ): # a library built before the address table of append_samples diff --git a/src/arcadedb/graph.py b/src/arcadedb/graph.py index 4e3cff6..608790a 100644 --- a/src/arcadedb/graph.py +++ b/src/arcadedb/graph.py @@ -220,6 +220,9 @@ def to_dict(self, convert_types: bool = True) -> Dict[str, Any]: def get_rid(self) -> str: """Get Record ID.""" + # Two JPype operations (``getIdentity()``, then ``str()`` of the RID). A bridge call would be two as well (the static call and the + # conversion of the String it returns) plus the lookup of the bridge class, so there is none here: the timed A/B of a one-call + # version read 1.6 times slower per call (ArcadeData/arcadedb-python#395, figure ``j_rid_docobj_100k``). return str(self._java_document.getIdentity()) def get_java_document(self): diff --git a/src/arcadedb/results.py b/src/arcadedb/results.py index bbdd31c..d8a0f83 100644 --- a/src/arcadedb/results.py +++ b/src/arcadedb/results.py @@ -1212,6 +1212,16 @@ def get_raw(self, name: str) -> Any: return None return self._java_result.getProperty(name) + def _element_or_none(self, bridge_method: str, java_getter: str): + """The Java vertex, edge or record of the row, or None when the row is not one: one + crossing through ``ReaderCalls``, or the three calls (``getX()``, ``isPresent()``, + ``get()``) of the plain path when the bridge class is unavailable.""" + reader = _bridge_class("ReaderCalls") + if reader is not None: + return getattr(reader, bridge_method)(self._java_result) + optional = getattr(self._java_result, java_getter)() + return optional.get() if optional.isPresent() else None + def get_rid(self) -> Optional[str]: """ Get the Record ID (RID) if available. @@ -1224,10 +1234,14 @@ def get_rid(self) -> Optional[str]: >>> print(result.get_rid()) #10:5 """ - identity = self._java_result.getIdentity() - if identity.isPresent(): - return str(identity.get().toString()) - return None + reader = _bridge_class("ReaderCalls") + if reader is None: # no bridge jar: the four calls of the plain path + identity = self._java_result.getIdentity() + if identity.isPresent(): + return str(identity.get().toString()) + return None + rid = reader.ridOf(self._java_result) + return None if rid is None else str(rid) def get_vertex(self) -> Optional[Vertex]: """ @@ -1237,10 +1251,8 @@ def get_vertex(self) -> Optional[Vertex]: Vertex object or None """ self._check_open() - vertex = self._java_result.getVertex() - if vertex.isPresent(): - return Vertex(vertex.get(), self._database) - return None + vertex = self._element_or_none("vertexOrNull", "getVertex") + return None if vertex is None else Vertex(vertex, self._database) def get_edge(self) -> Optional[Edge]: """ @@ -1250,10 +1262,8 @@ def get_edge(self) -> Optional[Edge]: Edge object or None """ self._check_open() - edge = self._java_result.getEdge() - if edge.isPresent(): - return Edge(edge.get(), self._database) - return None + edge = self._element_or_none("edgeOrNull", "getEdge") + return None if edge is None else Edge(edge, self._database) def get_element(self) -> Optional[Document]: """ @@ -1263,10 +1273,8 @@ def get_element(self) -> Optional[Document]: Document, Vertex, or Edge object or None """ self._check_open() - element = self._java_result.getElement() - if element.isPresent(): - return Document.wrap(element.get(), self._database) - return None + element = self._element_or_none("elementOrNull", "getElement") + return None if element is None else Document.wrap(element, self._database) def get_property_names(self) -> List[str]: """ diff --git a/src/java/com/arcadedb/python/ReaderCalls.java b/src/java/com/arcadedb/python/ReaderCalls.java index dd93799..f9461dc 100644 --- a/src/java/com/arcadedb/python/ReaderCalls.java +++ b/src/java/com/arcadedb/python/ReaderCalls.java @@ -82,6 +82,29 @@ public static Object[] documentJson(final Document doc) { return new Object[] { sb.toString(), side.toArray() }; } + /** + * The record id of a row as text, or null when the row is not a record: the {@code getIdentity()}, {@code isPresent()}, + * {@code get()} and {@code toString()} of {@code Result.get_rid()} in one crossing. + */ + public static String ridOf(final Result row) { + return row.getIdentity().map(Object::toString).orElse(null); + } + + /** The vertex of a row, or null when the row is not a vertex: {@code getVertex()}, {@code isPresent()} and {@code get()} in one crossing. */ + public static Object vertexOrNull(final Result row) { + return row.getVertex().orElse(null); + } + + /** The edge of a row, or null when the row is not an edge: {@code getEdge()}, {@code isPresent()} and {@code get()} in one crossing. */ + public static Object edgeOrNull(final Result row) { + return row.getEdge().orElse(null); + } + + /** The record of a row, or null when the row is not a record: {@code getElement()}, {@code isPresent()} and {@code get()} in one crossing. */ + public static Object elementOrNull(final Result row) { + return row.getElement().orElse(null); + } + /** The property names of a row as one JSON array of strings, in the order {@code getPropertyNames()} gives them. */ public static String rowNames(final Result row) { return namesJson(row.getPropertyNames()); diff --git a/tests/conftest.py b/tests/conftest.py index 2920c28..3d9802c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -71,6 +71,11 @@ collect_ignore.append("test_native_mixed_chunk.py") # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library collect_ignore.append("test_native_ingest_defaults.py") + # create_vertices on the native library: the columnar route (arcadedb_n_gb_colp) stores what the JSON route stores + collect_ignore.append("test_native_graph_batch_frame.py") + # create_record on the native async executor hands records over as columnar frames with per-record callbacks (and its recorded-library twin) + collect_ignore.append("test_native_async_create_batch.py") + collect_ignore.append("test_native_async_create_batch_unit.py") # the float32 side buffer of the native GraphBatch entry (arcadedb_n_gb_bin), observed on the native library collect_ignore.append("test_native_graph_batch_binary_vectors.py") # the native library's start-up output (#477) can only be checked with the native library loaded @@ -158,6 +163,7 @@ "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", + "test_reader_identity_crossings.py": "crossing counts of get_rid, get_vertex, get_edge and get_element on the JVM build (JPype proxies of the Java row and the bridge class)", "test_typed_batch_guard.py": "the object_hook and the side array of the JVM build's TypedRows batches (JPype arrays, the bridge class, a spy on json.loads)", "test_jvm_ingest_integrity_faults.py": "rows-dropped faults put into the JVM build's column conversion and bulk vertex path (core._column_to_java, GraphBatch._create_vertices_json_bulk)", } diff --git a/tests/test_native_async_create_batch.py b/tests/test_native_async_create_batch.py new file mode 100644 index 0000000..c18abfd --- /dev/null +++ b/tests/test_native_async_create_batch.py @@ -0,0 +1,298 @@ +"""``create_record`` on the native async executor stores what the per-record op stored, and every record keeps its own callbacks. + +Records of one type go to the library as one columnar frame (``arcadedb_n_async_bin``, op ``create_records``) with the callback ids of +each record. Here the real library runs: the stored values of a batch are compared with the ones the per-record JSON op stores (a batch +of one, so one ``create_record`` op per record), the callbacks are the records' own and in the order of the calls, and a record that an +async writer refuses in the middle of a batch reaches its own error callback while the others are stored. The counts are crossings. + +Needs the native library, so conftest.py does not collect this file on the JPype backend; it runs in the native parity run. The +Python side with a recorded library is in test_native_async_create_batch_unit.py. +""" + +import datetime +import decimal + +import numpy as np +import pytest + +from arcadedb._native import extras, lib + + +@pytest.fixture(autouse=True) +def _collect_from_the_first_record(monkeypatch): + """The tests below pin the collecting route (a frame for a batch of 12); the direct phase of the first 128 records has its own tests at the end.""" + monkeypatch.setattr(extras, "_ASYNC_DIRECT", 0) + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 0) + + +class _Crossings: + """Records every library call whose entry point is the async executor's: ``(entry point, op)`` in order.""" + + def __init__(self, monkeypatch): + self.calls = [] + real = lib.call + + def counting(name, *args): + if name in ("arcadedb_n_async", "arcadedb_n_async_bin"): + self.calls.append( + (name, args[1].decode() if isinstance(args[1], bytes) else args[1]) + ) + return real(name, *args) + + monkeypatch.setattr(lib, "call", counting) + + def ops(self, op): + return [c for c in self.calls if c[1] == op] + + +class _Stub: + """A record of a type that does not exist (the database refuses ``new_document`` for it, the executor is asked all the same).""" + + def __init__(self, type_name, props): + self._type = type_name + self._props = props + + +def _executor(db): + executor = db.async_executor() + executor.set_parallel_level( + 1 + ) # one writer: the callbacks run in the order of the calls + executor.set_commit_every( + 1 + ) # one record per transaction: a refused record takes no other with it + return executor + + +def _store(db, type_name, rows, one_by_one): + """Create the rows through the executor. ``one_by_one``: wait after each (a batch of one: the per-record op).""" + executor = _executor(db) + for row in rows: + executor.create_record(_document(db, type_name, row)) + if one_by_one: + executor.wait_completion() + # not closed: close() shuts the database's async executor down for good, and the next _store of the test needs it + executor.wait_completion() + + +def _document(db, type_name, props): + doc = db.new_document(type_name) + for key, value in props.items(): + doc.set(key, value) + return doc + + +_DUMPS = { + "Old": "SELECT FROM Old ORDER BY i", + "New": "SELECT FROM New ORDER BY i", +} + + +def _dump(db, type_name): + rows = db.query("sql", _DUMPS[type_name]).to_list() + rows = [ + dict(sorted((k, v) for k, v in row.items() if not k.startswith("@"))) + for row in rows + ] # the rid and the type name differ by design; the keys are sorted: the per-record op parses the properties into a hash map, so + # it stores them in hash order and the batch in the order of the call, and the order of the properties of a document is no promise + return repr(rows) # repr: a NaN equals a NaN, a None differs from a missing key + + +_N = 12 + +SHAPES = { + "ints": lambda i: {"i": i, "v": i * 7}, + "big_ints": lambda i: {"i": i, "v": 2**62 + i}, + "floats": lambda i: {"i": i, "v": i / 3}, + "nan_inf": lambda i: { + "i": i, + "v": [float("nan"), float("inf"), -float("inf")][i % 3], + }, + "strings": lambda i: {"i": i, "v": f"s{i}é\U0001f600"}, + "empty_strings": lambda i: {"i": i, "v": ""}, + "bools": lambda i: {"i": i, "v": bool(i % 2)}, + "bytes": lambda i: {"i": i, "v": bytes([i, 0, 255])}, + "datetimes": lambda i: {"i": i, "v": datetime.datetime(2020, 1, 1, 0, 0, i, 1)}, + "dates": lambda i: {"i": i, "v": datetime.date(2020, 1, 1 + i)}, + "decimals": lambda i: {"i": i, "v": decimal.Decimal(i) / decimal.Decimal(7)}, + "nulls": lambda i: {"i": i, "v": None}, + "some_nulls": lambda i: {"i": i, "v": None if i % 2 else i}, + "numpy_scalars": lambda i: {"i": i, "v": np.int64(i), "w": np.float64(i) / 3}, + "nested": lambda i: {"i": i, "v": [i, [i, {"k": i}]], "m": {"a": [i]}}, + "unsorted_keys": lambda i: {"z": 1, "i": i, "a": 2}, + "keys_differ": lambda i: {"i": i, "v": i} if i % 2 else {"i": i}, +} + + +@pytest.mark.parametrize("shape", sorted(SHAPES)) +def test_a_batch_stores_what_the_per_record_op_stores(temp_db, monkeypatch, shape): + temp_db.command("sql", "CREATE DOCUMENT TYPE Old") + temp_db.command("sql", "CREATE DOCUMENT TYPE New") + rows = [SHAPES[shape](i) for i in range(_N)] + crossings = _Crossings(monkeypatch) + _store(temp_db, "Old", rows, one_by_one=True) + old = crossings.calls[:] + crossings.calls.clear() + _store(temp_db, "New", rows, one_by_one=False) + + assert _dump(temp_db, "New") == _dump(temp_db, "Old") + assert ( + len([c for c in old if c[1] == "create_record"]) == _N + ) # the control: one op per record + batched = crossings.ops("create_records") + if shape in ( + "nested", + "keys_differ", + "numpy_scalars", + ): # no frame for these (the column reader takes Python scalars only): the per-record op, as before + assert batched == [] and len(crossings.ops("create_record")) == _N + else: + assert len(batched) == 1 and crossings.ops("create_record") == [] + + +def test_empty_and_single_batches(temp_db, monkeypatch): + temp_db.command("sql", "CREATE DOCUMENT TYPE One") + crossings = _Crossings(monkeypatch) + executor = _executor(temp_db) + executor.wait_completion() # nothing was created: nothing is handed over + assert ( + crossings.ops("create_records") == [] and crossings.ops("create_record") == [] + ) + executor.create_record(_document(temp_db, "One", {"i": 1, "v": "x"})) + executor.wait_completion() + executor.close() + assert ( + crossings.ops("create_records") == [] + and len(crossings.ops("create_record")) == 1 + ) + assert temp_db.query("sql", "SELECT FROM One").to_list()[0]["v"] == "x" + + +def test_every_record_gets_its_own_callback_in_the_order_of_the_calls( + temp_db, monkeypatch +): + temp_db.command("sql", "CREATE DOCUMENT TYPE Item") + temp_db.command("sql", "CREATE VERTEX TYPE Node") + crossings = _Crossings(monkeypatch) + executor = _executor(temp_db) + seen = [] + for type_name, make in ( + ("Item", temp_db.new_document), + ("Node", temp_db.new_vertex), + ): + for i in range(50): + doc = make(type_name) + doc.set("i", i) + executor.create_record( + doc, + callback=lambda record, i=i, t=type_name: seen.append( + (t, i, record.get("i"), record.get_rid()) + ), + error_callback=lambda e, i=i: seen.append(("error", i, str(e))), + ) + executor.wait_completion() + executor.close() + + assert len(crossings.ops("create_records")) == 2 # one frame per type, not 100 ops + assert [(t, i, v) for t, i, v, _ in seen] == [ + (t, i, i) for t in ("Item", "Node") for i in range(50) + ] + assert all( + str(rid).startswith("#") for _, _, _, rid in seen + ) # the stored record: it has its RID + assert temp_db.count_type("Item") == 50 and temp_db.count_type("Node") == 50 + + +def test_a_record_the_writer_refuses_in_the_middle_reaches_its_own_error_callback( + temp_db, monkeypatch +): + temp_db.command("sql", "CREATE DOCUMENT TYPE Uniq") + temp_db.command("sql", "CREATE PROPERTY Uniq.k INTEGER") + temp_db.command("sql", "CREATE INDEX ON Uniq (k) UNIQUE") + keys = [ + 1, + 2, + 3, + 4, + 2, + 5, + 6, + 3, + 7, + ] # records 4 and 7 repeat an earlier key: the writer refuses them + results = {} + for batched in (False, True): + with temp_db.transaction(): + temp_db.command("sql", "DELETE FROM Uniq") + seen = {"ok": [], "err": []} + executor = _executor(temp_db) + for index, key in enumerate(keys): + executor.create_record( + _document(temp_db, "Uniq", {"k": key, "n": index}), + callback=lambda record, index=index: seen["ok"].append(index), + error_callback=lambda e, index=index: seen["err"].append( + (index, type(e).__name__) + ), + ) + if not batched: + executor.wait_completion() + executor.wait_completion() # not closed: the database's async executor is shut down for good by close() + stored = sorted( + (r["k"], r["n"]) for r in temp_db.query("sql", "SELECT FROM Uniq").to_list() + ) + results[batched] = (sorted(seen["ok"]), sorted(seen["err"]), stored) + assert results[True] == results[False] + ok, err, stored = results[True] + assert [index for index, _ in err] == [ + 4, + 7, + ] # exactly the refused records, each on its own callback + # the writer reports a record stored once it is in the transaction, and a refusal at the commit comes after that: the records 4 and 7 + # reach their callback and then their error callback (on both routes, as the equal results say), so what is stored is the proof + assert {0, 1, 2, 3, 5, 6, 8} <= set(ok) + assert stored == [(1, 0), (2, 1), (3, 2), (4, 3), (5, 5), (6, 6), (7, 8)] + + +def test_the_executor_keeps_working_after_a_refused_batch(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE Item") + executor = _executor(temp_db) + errors = [] + for i in range(5): + executor.create_record( + _Stub("NoSuchType", {"i": i}), error_callback=errors.append + ) + with pytest.raises(Exception, match="NoSuchType"): + executor.wait_completion() # the batch of a type that does not exist is refused before any record is handed over + for i in range(5): + executor.create_record(_document(temp_db, "Item", {"i": i})) + executor.wait_completion() + executor.close() + assert temp_db.count_type("Item") == 5 + + +@pytest.mark.parametrize("n", [1, 127, 128, 129, 191, 192, 256]) +def test_the_direct_phase_stores_what_the_collecting_route_stores( + temp_db, monkeypatch, n +): + """#393: the first 128 records the post3 op per record, the records after them one frame when 64 or more are collected and the post3 op each otherwise; the stored data are the same either way.""" + temp_db.command("sql", "CREATE DOCUMENT TYPE Old") + temp_db.command("sql", "CREATE DOCUMENT TYPE New") + rows = [SHAPES["ints"](i) for i in range(n)] + crossings = _Crossings(monkeypatch) + monkeypatch.setattr(extras, "_ASYNC_DIRECT", 128) + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 64) + _store(temp_db, "Old", rows, one_by_one=False) + direct_default = crossings.calls[:] + crossings.calls.clear() + monkeypatch.setattr(extras, "_ASYNC_DIRECT", 0) + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 0) + _store(temp_db, "New", rows, one_by_one=False) + + assert _dump(temp_db, "Old") == _dump(temp_db, "New") + ops = [c for c in direct_default if c[1] == "create_record"] + frames = [c for c in direct_default if c[1] == "create_records"] + collected = max( + n - 128, 0 + ) # the records after the direct phase: a frame when 64 or more of them, else one op each + assert len(ops) == min(n, 128) + (collected if collected < 64 else 0) + assert len(frames) == (1 if collected >= 64 else 0) diff --git a/tests/test_native_async_create_batch_unit.py b/tests/test_native_async_create_batch_unit.py new file mode 100644 index 0000000..a2945f6 --- /dev/null +++ b/tests/test_native_async_create_batch_unit.py @@ -0,0 +1,576 @@ +"""The native async executor hands ``create_record`` calls to the library in batches, and every record keeps its own callbacks. + +``NativeAsyncExecutor.create_record`` used to send one JSON op (``{type, kind, props, cb, err}``) per record: one crossing, one +``json.dumps`` and one JSON parse in the image per record. It now buffers the records of one type and kind and hands a batch over +as one columnar frame (``arcadedb_n_async_bin`` with the ``create_records`` op, the layout of ``arcadedb_n_insert_colp``) plus the +callback ids of each record. These tests pin what must not change: nothing is ever lost or reordered (every other call on the +executor, ``wait_completion``, ``close`` and ``kill`` hand the buffer over first), a record changed after ``create_record`` returned +is stored as it was at the call, a shape the frame cannot carry keeps the per-record op, and a record that the image refuses in the +middle of a batch reaches its own error callback while the records around it still go in, in order. + +No JVM and no native library: the library call is replaced by a recorder, so the counts are crossings, not seconds. The stored +values and the callbacks of the real library are in test_native_async_create_batch.py. +""" + +import json +import time + +import pytest + +from arcadedb import ArcadeDBError +from arcadedb._native import bulk, columnar, extras + +BATCH = getattr(extras, "_ASYNC_BATCH", 1000) +REFUSAL = {"cls": "DuplicatedKeyException", "error": "duplicate", "chain": []} + + +class _Doc: + """What the executor needs of a document: its type and its properties.""" + + def __init__(self, type_name, props): + self._type = type_name + self._props = props + + +class _Db: + _h = 7 + + def _discard_async_executor(self, executor): + pass + + def query( + self, *args, **kwargs + ): # the schema read of columnar.Declared: no engine here, so no declared types + raise RuntimeError("no engine") + + +class _Library: + """Stands for ``extras._call``: records every crossing in order and answers like the image.""" + + def __init__(self): + self.log = [] # (kind, op, args or header, n_spec) + self.refusals = ( + [] + ) # replies for the next create_records calls, None for a plain success + self.raise_on_batch = None + + def __call__(self, prefix, name, handle, op, text, *rest): + args = json.loads(text) + op = op.decode() if isinstance(op, bytes) else op + if name == "arcadedb_n_async_bin": + self.log.append(("bin", op, args, rest[1])) + if self.raise_on_batch is not None: + raise self.raise_on_batch + reply = self.refusals.pop(0) if self.refusals else None + return {"ok": True, "n": args["n"]} if reply is None else reply + self.log.append(("op", op, args, 0)) + return { + "ok": True, + "parallel": 1, + "back_pressure": 0, + "commit_every": 1, + "wal": True, + "sync": "NO", + "threads": 1, + "processing": False, + "done": True, + } + + def batches(self): + return [entry for entry in self.log if entry[0] == "bin"] + + def per_record(self): + return [entry for entry in self.log if entry[1] == "create_record"] + + def order(self): + return [(entry[1], entry[2].get("n")) for entry in self.log] + + +@pytest.fixture +def library(monkeypatch): + fake = _Library() + # the linger time (how long a collected record waits for the rest of its batch) is out of the way, so "still buffered" is a fact + monkeypatch.setattr(extras, "_ASYNC_LINGER", 3600.0, raising=False) + # the tests below pin the collecting route from the first record; the direct phase of the first records has its own tests at the end + monkeypatch.setattr(extras, "_ASYNC_DIRECT", 0) + # and a frame from two records on: the floor below which a hand-over goes one op each has its own tests at the end + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 0) + monkeypatch.setattr(extras, "_call", fake) + monkeypatch.setattr(extras, "_ensure_upcall", lambda: None) + monkeypatch.setattr(bulk, "_usable", lambda: True) + return fake + + +@pytest.fixture +def executor(library): + return extras.NativeAsyncExecutor(_Db()) + + +def _size(): + with extras._callbacks_lock: + return len(extras._callbacks) + + +def _rows(n, type_name="Item"): + return [_Doc(type_name, {"id": i, "name": f"n{i}"}) for i in range(n)] + + +def test_records_of_one_type_cross_in_one_batch(library, executor): + seen = [] + for doc in _rows(250): + executor.create_record( + doc, callback=seen.append, error_callback=lambda e: seen.append(e) + ) + executor.wait_completion() + + assert len(library.batches()) == 1 + assert library.per_record() == [] + _, op, header, n_spec = library.batches()[0] + assert op == "create_records" + assert header["type"] == "Item" and header["kind"] == "d" and header["n"] == 250 + assert [c[0] for c in header["cols"]] == ["id", "name"] + assert len(header["cbs"]) == 250 and len(header["errs"]) == 250 + assert all(header["cbs"]) and all(header["errs"]) + assert ( + len(set(header["cbs"])) == 250 + ) # one callback id per record: each record keeps its own + assert n_spec == 6 * len(header["cols"]) + + +def test_a_record_without_callbacks_sends_zero_ids(library, executor): + for doc in _rows(5): + executor.create_record(doc) + executor.wait_completion() + ((_, _, header, _),) = library.batches() + assert header["cbs"] == [0] * 5 and header["errs"] == [0] * 5 + + +def test_the_buffer_is_bounded(library, executor): + for doc in _rows(2 * BATCH + 10): + executor.create_record(doc) + # the full batches went over while the records were being added, the rest waits for the next flush point + assert [b[2]["n"] for b in library.batches()] == [BATCH] * 2 + executor.wait_completion() + assert [b[2]["n"] for b in library.batches()] == [BATCH] * 2 + [10] + + +@pytest.mark.parametrize( + "later", + [ + lambda e: e.command("sql", "SELECT 1", lambda row: None), + lambda e: e.wait_completion(), + lambda e: e.is_processing(), + lambda e: e.close(), + lambda e: e.kill(), + lambda e: e.get_parallel_level(), + ], + ids=["command", "wait_completion", "is_processing", "close", "kill", "a_getter"], +) +def test_every_other_call_hands_the_buffer_over_first(library, executor, later): + for doc in _rows(3): + executor.create_record(doc) + assert library.log == [] # still buffered + later(executor) + ops = library.order() + assert ops[0] == ( + "create_records", + 3, + ) # first, so the engine queue keeps the order of the calls + assert len(ops) >= 2 + + +def test_records_after_another_call_form_their_own_batch(library, executor): + for doc in _rows(3): + executor.create_record(doc) + executor.command("sql", "SELECT 1") + for doc in _rows(2): + executor.create_record(doc) + executor.wait_completion() + assert library.order()[:3] == [ + ("create_records", 3), + ("command", None), + ("create_records", 2), + ] + + +def test_a_change_of_type_or_kind_hands_the_buffer_over(library, executor): + for doc in _rows(2, "A") + _rows(2, "B"): + executor.create_record(doc) + vertex = _Doc("B", {"id": 9, "name": "v"}) + vertex.get_out_edges = lambda: None # what a vertex has and a document has not + executor.create_record(vertex) + executor.create_record(_Doc("B", {"id": 10, "name": "x"})) + executor.wait_completion() + batches = [(b[2]["type"], b[2]["kind"], b[2]["n"]) for b in library.batches()] + # a batch of one goes as the per-record op (a frame of one is not cheaper), so only the two pairs are frames + assert batches == [("A", "d", 2), ("B", "d", 2)] + singles = [(e[2]["type"], e[2]["kind"]) for e in library.per_record()] + assert singles == [("B", "v"), ("B", "d")] + order = [e[1] for e in library.log if e[1] in ("create_records", "create_record")] + assert order == [ + "create_records", + "create_records", + "create_record", + "create_record", + ] + + +def test_a_batch_of_one_keeps_the_per_record_op(library, executor): + executor.create_record(_Doc("Item", {"id": 1}), callback=lambda r: None) + executor.wait_completion() + assert library.batches() == [] + ((_, op, args, _),) = library.per_record() + assert args["type"] == "Item" and args["props"] == {"id": 1} and args["cb"] + + +@pytest.mark.parametrize( + "shape", + [ + [{"a": 1}, {"b": 2}, {"a": 3}], # different keys + [ + {"a": [1, 2]}, + {"a": [3]}, + {"a": []}, + ], # a nested value the frame has no column for + [{"a": 1}, {"a": "x"}, {"a": 2}], # a column of mixed types + [{"a": 2**70}, {"a": 1}, {"a": 2}], # beyond 64 bits + ], + ids=["keys", "lists", "mixed", "big"], +) +def test_a_shape_the_frame_cannot_carry_keeps_the_per_record_op( + library, executor, shape +): + for props in shape: + executor.create_record(_Doc("Item", dict(props))) + executor.wait_completion() + assert library.batches() == [] + assert [ + e[2]["props"] for e in library.per_record() + ] == shape # every record, in the order given + + +def test_a_record_changed_after_the_call_is_stored_as_it_was( + library, executor, monkeypatch +): + captured = [] + real_pack = columnar.pack + + def spying_pack(columns, declared): + captured.append({k: list(v) for k, v in columns.items()}) + return real_pack(columns, declared) + + monkeypatch.setattr(columnar, "pack", spying_pack) + doc = _Doc("Item", {"id": 0, "name": "zero"}) + for i in range(4): + doc._props["id"] = i # the caller reuses one document object + doc._props["name"] = f"n{i}" + executor.create_record(doc) + executor.wait_completion() + assert captured == [{"id": [0, 1, 2, 3], "name": ["n0", "n1", "n2", "n3"]}] + + # a list that the caller mutates later is also copied at the call (the per-record JSON op copied it by encoding it) + library.log.clear() + tags = ["a"] + doc = _Doc("Item", {"id": 1, "tags": tags}) + executor.create_record(doc) + tags.append("b") + executor.create_record(_Doc("Item", {"id": 2, "tags": ["c"]})) + executor.wait_completion() + assert [e[2]["props"]["tags"] for e in library.per_record()] == [["a"], ["c"]] + + +def test_a_record_the_image_refuses_in_the_middle_reaches_its_own_callback( + library, executor +): + baseline = _size() + seen = [] + docs = _rows(6) + for i, doc in enumerate(docs): + executor.create_record( + doc, + callback=lambda r, i=i: seen.append(("ok", i)), + error_callback=lambda e, i=i: seen.append(("err", i, str(e))), + ) + # the image handed records 0 and 1 to the writers and refused record 2 + library.refusals.append({"ok": True, "n": 2, "err": REFUSAL}) + executor.wait_completion() + + assert [b[2]["n"] for b in library.batches()] == [ + 6, + 3, + ] # the rest (3, 4, 5) went on in a second batch + first, second = (b[2] for b in library.batches()) + assert second["cbs"] == first["cbs"][3:] and second["errs"] == first["errs"][3:] + assert [s[:2] for s in seen] == [ + ("err", 2) + ] # exactly the refused record's own error callback ran, once + assert "DuplicatedKeyException" in seen[0][2] and "duplicate" in seen[0][2] + # the refused record's ids are given back, the handed records' ids are released by wait_completion + assert all( + cid not in extras._callbacks for cid in (first["cbs"][2], first["errs"][2]) + ) + assert _size() == baseline + assert executor._cb_ids == [] + + +def test_a_record_the_image_refuses_without_an_error_callback_raises_after_the_rest_is_handed( + library, executor +): + for doc in _rows(5): + executor.create_record(doc) + library.refusals.append({"ok": True, "n": 1, "err": REFUSAL}) + with pytest.raises(ArcadeDBError, match="DuplicatedKeyException: duplicate"): + executor.wait_completion() + # the records around the refused one were still handed over, in order + assert [b[2]["n"] for b in library.batches()] == [5, 3] + # and the executor works on: a later flush point does not raise the same error again + executor.wait_completion() + + +def test_a_batch_the_library_refuses_gives_back_its_ids(library, executor): + baseline = _size() + library.raise_on_batch = ArcadeDBError("Async executor failed: type Nope not found") + for doc in _rows(4, "Nope"): + executor.create_record( + doc, callback=lambda r: None, error_callback=lambda e: None + ) + assert _size() > baseline # the buffered records hold their ids + with pytest.raises(ArcadeDBError, match="Nope"): + executor.wait_completion() + assert _size() == baseline + assert executor._cb_ids == [] + + +def test_wait_completion_releases_the_ids_of_the_batch(library, executor): + baseline = _size() + for doc in _rows(30): + executor.create_record( + doc, callback=lambda r: None, error_callback=lambda e: None + ) + assert _size() == baseline + 60 + executor.wait_completion() + assert _size() == baseline + + +def test_an_empty_executor_hands_nothing_over(library, executor): + executor.wait_completion() + executor.close() + assert library.batches() == [] and library.per_record() == [] + + +def _wait_for(condition, seconds=10.0): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if condition(): + return True + time.sleep(0.005) + return condition() + + +def test_a_collected_record_is_handed_over_after_the_linger_time_with_no_other_call( + library, executor, monkeypatch +): + # an existing async test creates one record and waits for its callback, with no other call on the executor: a record must + # not sit in Python until the caller happens to make one + monkeypatch.setattr(extras, "_ASYNC_LINGER", 0.02, raising=False) + for doc in _rows(3): + executor.create_record(doc) + assert _wait_for( + lambda: library.log + ), "the collected records were never handed over" + assert library.order() == [("create_records", 3)] + # a second round: the linger thread waits for the next first record of a batch + library.log.clear() + for doc in _rows(1): + executor.create_record(doc) + assert _wait_for(lambda: library.log), "the second batch was never handed over" + assert [(e[1], e[2].get("type")) for e in library.log] == [ + ("create_record", "Item") + ] + + +def test_a_failure_of_the_linger_flush_is_raised_by_the_next_call( + library, executor, monkeypatch +): + monkeypatch.setattr(extras, "_ASYNC_LINGER", 0.02, raising=False) + library.raise_on_batch = ArcadeDBError("Async executor failed: type Nope not found") + for doc in _rows(3, "Nope"): + executor.create_record(doc) + assert _wait_for(lambda: library.batches() and executor._linger_error is not None) + library.raise_on_batch = None + with pytest.raises(ArcadeDBError, match="Nope"): + executor.wait_completion() + executor.wait_completion() # raised once, not again + + +def test_close_stops_the_linger_thread(library, executor, monkeypatch): + monkeypatch.setattr(extras, "_ASYNC_LINGER", 3600.0, raising=False) + executor.create_record(_Doc("Item", {"id": 1})) + thread = executor._linger_thread + assert thread is not None and thread.is_alive() + executor.close() + assert _wait_for( + lambda: not thread.is_alive() + ), "the linger thread outlived close()" + assert library.order()[0] == ( + "create_record", + None, + ) # the record was handed over by close(), before the engine closed + + +# the direct phase (ArcadeData/arcadedb-python #388, #393): the first _ASYNC_DIRECT (128) records of an executor go one op each, exactly as in post3. +# The laptop A/B measured one record 1.57 times slower than post3; the cost is collecting (a linger thread, locks, a copy), which a small load does not repay. + + +@pytest.mark.parametrize("n", [1, 2, 127, 128]) +def test_an_executor_of_up_to_128_records_sends_one_op_each_and_collects_nothing( + library, monkeypatch, n +): + monkeypatch.setattr(extras, "_ASYNC_DIRECT", 128) + executor = extras.NativeAsyncExecutor(_Db()) + seen = [] + for doc in _rows(n): + executor.create_record( + doc, callback=seen.append, error_callback=lambda e: seen.append(e) + ) + assert executor._buf == [] and executor._linger_thread is None + assert library.batches() == [] + ops = library.per_record() + assert len(ops) == n + assert [o[2]["props"]["id"] for o in ops] == list( + range(n) + ) # in the order of the calls + assert all( + o[2]["cb"] and o[2]["err"] for o in ops + ) # every record keeps its own ids + executor.wait_completion() + assert library.batches() == [] and len(library.per_record()) == n + + +@pytest.mark.parametrize("n", [129, 130, 192, 256]) +def test_from_the_129th_record_on_the_executor_collects_and_nothing_is_lost_or_reordered( + library, monkeypatch, n +): + monkeypatch.setattr(extras, "_ASYNC_DIRECT", 128) + executor = extras.NativeAsyncExecutor(_Db()) + for doc in _rows(n): + executor.create_record(doc, callback=lambda r: None) + executor.wait_completion() + assert len(library.per_record()) == 128 + ( + 1 if n - 128 == 1 else 0 + ) # a collected batch of one is the per-record op + batches = library.batches() + assert sum(b[2]["n"] for b in batches) == n - len(library.per_record()) + # the order of every crossing is the order of the calls: the 128 direct ops, then the collected rest + first = [e for e in library.log if e[1] == "create_record"][0] + assert library.log.index(first) == 0 + ids = [o[2]["props"]["id"] for o in library.per_record()[:128]] + assert ids == list(range(128)) + + +def test_zero_turns_the_direct_phase_off(library, monkeypatch): + monkeypatch.setattr(extras, "_ASYNC_DIRECT", 0) + executor = extras.NativeAsyncExecutor(_Db()) + for doc in _rows(5): + executor.create_record(doc) + assert len(executor._buf) == 5 and library.log == [] + executor.wait_completion() + assert len(library.batches()) == 1 and library.per_record() == [] + + +def test_the_default_is_128(): + assert extras._ASYNC_DIRECT == 128 + + +# the frame floor (ArcadeData/arcadedb-python #388, #393): a hand-over of fewer than _ASYNC_FRAME_MIN (64) collected records goes one op each, as in post3. +# A frame has a fixed cost per crossing that a small batch does not repay (an untimed micro: with a direct phase of 32 and no floor, executors of 33 to 64 records read 1.2x to 1.7x slower than post3). + + +@pytest.mark.parametrize("n", [2, 3, 63]) +def test_fewer_than_64_collected_records_go_one_op_each(library, monkeypatch, n): + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 64) + executor = extras.NativeAsyncExecutor(_Db()) + for doc in _rows(n): + executor.create_record( + doc, callback=lambda r: None, error_callback=lambda e: None + ) + assert len(executor._buf) == n # collected, then handed over at the flush point + executor.wait_completion() + assert library.batches() == [] + ops = library.per_record() + assert [o[2]["props"]["id"] for o in ops] == list(range(n)) + assert all(o[2]["cb"] and o[2]["err"] for o in ops) # every record keeps its ids + + +@pytest.mark.parametrize("n", [64, 65, 100]) +def test_64_or_more_collected_records_cross_as_one_frame(library, monkeypatch, n): + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 64) + executor = extras.NativeAsyncExecutor(_Db()) + for doc in _rows(n): + executor.create_record(doc, callback=lambda r: None) + executor.wait_completion() + assert [b[2]["n"] for b in library.batches()] == [n] + assert library.per_record() == [] + + +@pytest.mark.parametrize( + "n, direct, frame", + [ + (1, 1, 0), + (127, 127, 0), + (128, 128, 0), + (129, 129, 0), + (191, 191, 0), + (192, 128, 64), + (256, 128, 128), + ], +) +def test_the_defaults_of_an_executor_of_n_records( + library, monkeypatch, n, direct, frame +): + """The production values: the first 128 records one op each, the rest a frame only when 64 or more of them are collected.""" + monkeypatch.setattr(extras, "_ASYNC_DIRECT", 128) + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 64) + executor = extras.NativeAsyncExecutor(_Db()) + for doc in _rows(n): + executor.create_record(doc, callback=lambda r: None) + executor.wait_completion() + assert len(library.per_record()) == direct + assert [b[2]["n"] for b in library.batches()] == ([frame] if frame else []) + # the order of the calls is the order of the crossings, frame last + ids = [o[2]["props"]["id"] for o in library.per_record()] + assert ids == list(range(direct)) + + +def test_a_remainder_below_the_floor_after_a_refused_record_goes_one_op_each( + library, monkeypatch +): + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 64) + executor = extras.NativeAsyncExecutor(_Db()) + seen = [] + for doc in _rows(100): + executor.create_record( + doc, + callback=lambda r: None, + error_callback=lambda e: seen.append(str(e)), + ) + # the image handed records 0 to 39 to the writers and refused record 40: the 59 that are left are fewer than the floor + library.refusals.append({"ok": True, "n": 40, "err": REFUSAL}) + executor.wait_completion() + assert [b[2]["n"] for b in library.batches()] == [100] + ops = library.per_record() + assert [o[2]["props"]["id"] for o in ops] == list(range(41, 100)) + assert len(seen) == 1 and "duplicate" in seen[0] + + +def test_a_floor_of_zero_turns_the_floor_off(library, monkeypatch): + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 0) + executor = extras.NativeAsyncExecutor(_Db()) + for doc in _rows(2): + executor.create_record(doc) + executor.wait_completion() + assert [b[2]["n"] for b in library.batches()] == [2] + + +def test_the_frame_floor_default_is_64(): + assert extras._ASYNC_FRAME_MIN == 64 diff --git a/tests/test_native_graph_batch_binary_vectors.py b/tests/test_native_graph_batch_binary_vectors.py index c554310..99dd913 100644 --- a/tests/test_native_graph_batch_binary_vectors.py +++ b/tests/test_native_graph_batch_binary_vectors.py @@ -22,7 +22,10 @@ @pytest.fixture -def db(temp_db): +def db(temp_db, monkeypatch): + # this file pins the side-buffer route of arcadedb_n_gb_bin, which the chunks that the columnar entry cannot carry still take; a plain + # vector load takes arcadedb_n_gb_colp now (test_native_graph_batch_frame*.py), so the columnar route is off here + monkeypatch.setattr(lib, "GB_COLP", False) temp_db.command("sql", "CREATE VERTEX TYPE Vx") temp_db.command("sql", "CREATE PROPERTY Vx.id LONG") temp_db.command("sql", "CREATE PROPERTY Vx.embedding ARRAY_OF_FLOATS") diff --git a/tests/test_native_graph_batch_frame.py b/tests/test_native_graph_batch_frame.py new file mode 100644 index 0000000..f75cfc5 --- /dev/null +++ b/tests/test_native_graph_batch_frame.py @@ -0,0 +1,321 @@ +"""Native GraphBatch.create_vertices: the vertices that the columnar route stores are the ones the JSON route stores (ArcadeData/arcadedb-python #388, rank 7). + +The same rows are loaded into two vertex types of one schema, once with ``lib.GB_COLP`` on (``arcadedb_n_gb_colp``: columns by address, the +RIDs back in an int64 buffer) and once with it off (the JSON text of ``arcadedb_n_gb``), and every stored record is compared: the property +order, the type and the value of each property (``None`` and ``NaN`` included), and the RIDs the call returned (a ``list`` of ``str`` in row +order). A shape the columns cannot carry must come out the same because it takes the JSON route; the entry that each shape crossed is checked +too, so a shape cannot move from one route to the other unseen. Needs the native library, so conftest.py collects this file on the native +backend only; the routing, the header and the RID decode are in test_native_graph_batch_frame_pack.py (both backends, no library). +""" + +import re +from datetime import date, datetime, timedelta, timezone +from decimal import Decimal + +import numpy as np +import pandas as pd +import pytest + +from arcadedb._native import lib + +COLP = "arcadedb_n_gb_colp" + + +@pytest.fixture +def db(temp_db): + for name in ( + "A", + "B", + "C", + ): # the same schema three times: A is loaded with the columnar route, B with the JSON route, C with the columns forced + temp_db.command("sql", f"CREATE VERTEX TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.i LONG") + temp_db.command("sql", f"CREATE PROPERTY {name}.n INTEGER") + temp_db.command("sql", f"CREATE PROPERTY {name}.f FLOAT") + temp_db.command("sql", f"CREATE PROPERTY {name}.v ARRAY_OF_FLOATS") + return temp_db + + +def _vectors(n, dim=6): + x = np.random.default_rng(5).standard_normal((n, dim)).astype(np.float32) + x[0, 0] = np.float32(-0.0) + return x + + +def _rows(n, make): + return [{"i": i, **make(i)} for i in range(n)] + + +SHAPES = { + "scalars": (lambda i: {"s": f"né{i}", "d": i * 0.5, "b": i % 2 == 0}, True), + "big integers": ( + lambda i: {"big": [2**53 + 1, -(2**63), 2**63 - 1, 7][i % 4]}, + True, + ), + "nan inf and negative zero": ( + lambda i: {"x": [float("nan"), float("inf"), float("-inf"), -0.0, 1.5][i % 5]}, + True, + ), + "empty and odd strings": ( + lambda i: {"s": ["", "a\u0000b", "\U0001f600", "x" * 300][i % 4]}, + True, + ), + "nulls in a column": ( + lambda i: {"s": None if i % 3 == 0 else f"s{i}", "d": None if i % 2 else 1.0}, + True, + ), + "only nulls": (lambda i: {"s": None}, True), + "dates and datetimes": ( + lambda i: { + "day": date(2020, 1, 1) + timedelta(days=i), + "at": datetime(2020, 1, 2, 3, 4, 5, 123456 + i), + "utc": datetime(2020, 1, 2, 3, 4, 5, tzinfo=timezone(timedelta(hours=9))), + }, + True, + ), + "decimals and bytes": ( + lambda i: {"dec": Decimal(f"{i}.50"), "raw": bytes([i, 0, 255])}, + True, + ), + "declared integer and float": (lambda i: {"n": i, "f": i * 0.1}, True), + "float32 vectors": (lambda i: {"emb": _vectors(40)[i]}, True), + "declared vectors from lists": (lambda i: {"v": [i * 0.5, i + 0.25, 3.0]}, True), + "declared vectors from float32 arrays": (lambda i: {"v": _vectors(40)[i]}, True), + "float64 vectors": (lambda i: {"emb": _vectors(40)[i].astype(np.float64)}, False), + "nested lists and maps": ( + lambda i: {"l": [i, "x", [1, 2]], "m": {"k": i, "z": None}}, + False, + ), + "numpy scalars": ( + lambda i: {"a": np.int64(i), "b": np.float64(i) / 3, "c": np.bool_(i % 2)}, + False, + ), + "mixed int and float": (lambda i: {"a": i if i % 2 else i + 0.5}, False), + "a set and a tuple": (lambda i: {"s": {i}, "t": (i, 1)}, False), + "a pandas timestamp with nanoseconds": ( + lambda i: { + "t": pd.Timestamp("2020-01-01 00:00:00.000000123") + pd.Timedelta(i, "ns") + }, + False, + ), +} + + +def _dump(db, type_name): + """Every stored record as (property, type, repr) in its stored order, sorted by the row index.""" + out = {} + sql = f"SELECT FROM {type_name}" # nosec B608 - test-owned name + for row in db.query("sql", sql).to_list(): + props = [ + (k, type(v).__name__, repr(v)) + for k, v in row.items() + if not str(k).startswith("@") + ] + out[row["i"]] = props + return [out[i] for i in sorted(out)] + + +def _load(db, monkeypatch, type_name, rows, columnar, floor=None): + calls = [] + real = lib.call + + def spy(name, *args): + calls.append(name) + return real(name, *args) + + monkeypatch.setattr(lib, "call", spy) + monkeypatch.setattr(lib, "GB_COLP", columnar) + monkeypatch.setattr( + lib, "GB_COLP_MIN_ROWS", 0 if floor is None else floor + ) # the route tests force the columns; floor=64 is the default + try: + with db.graph_batch() as gb: + rids = gb.create_vertices(type_name, rows) + finally: + monkeypatch.undo() + return rids, calls + + +@pytest.mark.parametrize("shape", sorted(SHAPES)) +@pytest.mark.parametrize("n", [1, 40]) +def test_the_columnar_route_stores_what_the_json_route_stores( + db, monkeypatch, shape, n +): + make, carried = SHAPES[shape] + if shape == "mixed int and float" and n == 1: + carried = True # one row holds one type: a column of one float + rows = _rows(n, make) + rids_on, calls_on = _load(db, monkeypatch, "A", rows, True) + rids_off, calls_off = _load(db, monkeypatch, "B", rows, False) + assert (COLP in calls_on) is carried, (shape, calls_on) + assert COLP not in calls_off + assert _dump(db, "A") == _dump(db, "B") + for rids in (rids_on, rids_off): + assert type(rids) is list and len(rids) == n + assert all(type(r) is str and re.fullmatch(r"#\d+:\d+", r) for r in rids) + + +def test_the_rids_are_the_vertices_in_row_order_across_chunks(db, monkeypatch): + rows = [ + {"i": i, "s": f"p{i}"} for i in range(2500) + ] # the default chunk is 1000 rows: three crossings + rids, calls = _load(db, monkeypatch, "A", rows, True) + assert calls.count(COLP) == 3 + by_rid = { + str(r["rid"]): r["i"] + for r in db.query("sql", "SELECT i, @rid AS rid FROM A").to_list() + } + assert [by_rid[r] for r in rids] == list(range(2500)) + assert len(set(rids)) == 2500 + + +@pytest.mark.parametrize( + "rows", + [ + [ + {"i": 0, "a": 1, "b": 2}, + {"i": 1, "b": 4, "a": 3}, + ], # the same keys in another order + [{"i": 0, "a": 1, "b": 2}, {"i": 1, "a": 3}], # a missing key + [{"i": 0, "a": None}, {"i": 1}], # None against a missing key + ], +) +def test_rows_with_other_keys_keep_the_json_route_and_the_stored_order( + db, monkeypatch, rows +): + _, calls_on = _load(db, monkeypatch, "A", rows, True) + _load(db, monkeypatch, "B", rows, False) + assert COLP not in calls_on + assert _dump(db, "A") == _dump(db, "B") + + +def test_a_none_row_among_dict_rows_is_an_empty_vertex_on_both_routes(db, monkeypatch): + rows = [{"i": 0, "a": 1}, None, {"i": 2, "a": 3}] + rids_on, calls_on = _load(db, monkeypatch, "A", rows, True) + rids_off, _ = _load(db, monkeypatch, "B", rows, False) + assert COLP not in calls_on and len(rids_on) == len(rids_off) == 3 + for type_name in ("A", "B"): + sql = f"SELECT count(*) AS c FROM {type_name}" # nosec B608 - test-owned name + assert db.query("sql", sql).first().get("c") == 3 + + +def test_an_integer_past_64_bits_is_refused_by_both_routes_alike(db, monkeypatch): + rows = [{"i": 0, "a": 2**70}] + errors = [] + for type_name, columnar in (("A", True), ("B", False)): + with pytest.raises(Exception) as raised: + _load(db, monkeypatch, type_name, rows, columnar) + errors.append( + str(raised.value).replace(f"'{type_name}'", "'T'") + ) # the type name is the only difference + assert errors[0] == errors[1] + assert db.query("sql", "SELECT count(*) AS c FROM A").first().get("c") == 0 + + +def test_an_empty_batch_and_a_batch_of_one(db, monkeypatch): + rids, calls = _load(db, monkeypatch, "A", [], True) + assert rids == [] and COLP not in calls + rids, calls = _load(db, monkeypatch, "A", [{"i": 0, "s": "one"}], True) + assert COLP in calls and len(rids) == 1 and rids[0].startswith("#") + + +def test_a_failing_chunk_stores_nothing_of_itself_and_keeps_the_chunks_before_it( + db, monkeypatch +): + db.command("sql", "CREATE INDEX ON A (i) UNIQUE") + rows = [{"i": i, "s": "x"} for i in range(1000)] + [ + {"i": i, "s": "y"} for i in range(5) + ] # the 2nd chunk repeats keys + with pytest.raises(Exception): + _load(db, monkeypatch, "A", rows, True) + assert db.query("sql", "SELECT count(*) AS c FROM A").first().get("c") == 1000 + + +def test_the_paper_shaped_loads_take_the_columnar_route_end_to_end( + temp_db, monkeypatch +): + """l2 (Person rows in chunks) and e2 (Product rows with a float32 vector): every chunk is one arcadedb_n_gb_colp crossing, no create_vertices + crossing of the JSON entries, no codec hook; the vertices and the vectors are stored, the RIDs come back as a list of str. + """ + from arcadedb._native import codec + + temp_db.command("sql", "CREATE VERTEX TYPE Person") + temp_db.command("sql", "CREATE VERTEX TYPE Product") + temp_db.command("sql", "CREATE PROPERTY Product.embedding ARRAY_OF_FLOATS") + calls, hooks = [], {"prep": 0, "default": 0} + real_call, real_prep, real_default = lib.call, codec._prep, codec._default + + def spy(name, *args): + calls.append( + (name, args[1] if name in ("arcadedb_n_gb", "arcadedb_n_gb_bin") else None) + ) + return real_call(name, *args) + + def counting(key, real): + def wrapper(*args, **kwargs): + hooks[key] += 1 + return real(*args, **kwargs) + + return wrapper + + people = [ + {"id": i, "name": f"p{i}", "age": 20 + i % 50, "city": f"c{i % 7}"} + for i in range(2500) + ] + x = _vectors(2100, 16) + products = [ + {"id": i, "title": f"t{i}", "price": i * 0.25, "embedding": x[i]} + for i in range(2100) + ] + with ( + temp_db.graph_batch() as gb + ): # the default commit_every: 1000 rows per crossing + monkeypatch.setattr(lib, "call", spy) + monkeypatch.setattr(codec, "_prep", counting("prep", real_prep)) + monkeypatch.setattr(codec, "_default", counting("default", real_default)) + rids_people = gb.create_vertices("Person", people) + rids_products = gb.create_vertices("Product", products) + monkeypatch.undo() + assert [ + c for c in calls if c[1] == b"create_vertices" + ] == [] # no JSON entry carried a chunk + assert [c[0] for c in calls].count( + COLP + ) == 3 + 3 # 1000, 1000, 500 and 1000, 1000, 100 + assert hooks == {"prep": 0, "default": 0} + assert len(rids_people) == 2500 and len(rids_products) == 2100 + assert all(type(r) is str for r in rids_people + rids_products) + stored = { + r["id"]: r["embedding"] + for r in temp_db.query("sql", "SELECT id, embedding FROM Product").to_list() + } + assert len(stored) == 2100 + assert all( + np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + for i in range(2100) + ) + assert ( + temp_db.query("sql", "SELECT count(*) AS c FROM Person").first().get("c") + == 2500 + ) + + +@pytest.mark.parametrize("n", [1, 63, 64, 65, 100]) +def test_the_floor_picks_the_route_by_size_and_both_routes_store_the_same( + db, monkeypatch, n +): + """#388: the default floor of 64 rows: below it the JSON route of post3, from it the columns; the stored data and the RIDs are those of the JSON route either way.""" + rows = _rows(n, SHAPES["scalars"][0]) + rids_floor, calls_floor = _load(db, monkeypatch, "A", rows, True, floor=64) + rids_json, calls_json = _load(db, monkeypatch, "B", rows, False) + assert (COLP in calls_floor) is (n >= 64), (n, calls_floor) + assert COLP not in calls_json + assert _dump(db, "A") == _dump(db, "B") + for rids in (rids_floor, rids_json): + assert type(rids) is list and len(rids) == n + assert all(type(r) is str and re.fullmatch(r"#\d+:\d+", r) for r in rids) + rids_cols, calls_cols = _load( + db, monkeypatch, "C", rows, True, floor=0 + ) # the columns forced, even for one row + assert COLP in calls_cols and len(rids_cols) == n + assert _dump(db, "C") == _dump(db, "B") diff --git a/tests/test_native_graph_batch_frame_pack.py b/tests/test_native_graph_batch_frame_pack.py new file mode 100644 index 0000000..8a7e935 --- /dev/null +++ b/tests/test_native_graph_batch_frame_pack.py @@ -0,0 +1,415 @@ +"""Native GraphBatch.create_vertices sends a chunk of rows as columns and gets the RIDs back in an int64 buffer (ArcadeData/arcadedb-python #388, rank 7). + +The route is ``bulk.create_vertices_columns``: ``columnar.rows_to_columns`` and ``columnar.pack`` lay the rows out as the columns of +``arcadedb_n_insert_colp``, one crossing of ``arcadedb_n_gb_colp`` per chunk, and the library writes ``(bucket, position)`` as two int64 per +row into a buffer that Python hands over; no JSON text goes out and none comes back. A chunk the columns cannot carry (rows with other keys or +the same keys in another order, a nested list or map, a column of mixed types, a set, a NaT, a numpy scalar, an integer past 64 bits, a lone +surrogate, an empty chunk) keeps the JSON route it had, so nothing is lost or changed. + +No native library is needed here: ``lib.call`` is replaced by a stand-in that records the crossing, reads the addresses it is given and +writes RIDs into the buffer, as the library does. What the library stores is compared between the two routes in +``test_native_graph_batch_frame.py`` (native backend only). +""" + +import ctypes +import json +from datetime import date, datetime +from decimal import Decimal +from types import SimpleNamespace + +import numpy as np +import pytest + +from arcadedb._native import bulk, codec, columnar, extras, lib + +COLP = "arcadedb_n_gb_colp" +OLD = ("arcadedb_n_gb", "arcadedb_n_gb_bin") +BASE = 2**40 # a position that needs more than 32 bits + + +class _Db: + def is_transaction_active(self): + return False + + def query(self, language, text): # the schema read of columnar.Declared + self.schema_reads = getattr(self, "schema_reads", 0) + 1 + rows = [ + {"name": "P", "properties": [{"name": "v", "type": "ARRAY_OF_FLOATS"}]}, + ] + return SimpleNamespace(to_list=lambda: rows) + + +class _Library: + """Stands in for ``lib.call``: records every GraphBatch crossing and answers as the library does.""" + + def __init__(self, monkeypatch): + self.crossings, self.columns, self.next = [], [], 0 + monkeypatch.setattr(lib, "call", self.call) + monkeypatch.setattr( + lib, "_lib", SimpleNamespace(arcadedb_n_insert_colp=1, arcadedb_n_gb_colp=1) + ) + monkeypatch.setattr(lib, "GB_COLP", True) + monkeypatch.setattr( + lib, "GB_COLP_MIN_ROWS", 0 + ) # the tests below pin the columnar route on small batches; the floor has its own tests at the end + monkeypatch.setattr(lib, "GB_BIN", True) + monkeypatch.setattr(bulk, "_ENABLED", True) + + def call(self, name, *args): + if name == COLP: + _gb, type_name, header, spec, nspec, out, slots = args + head = json.loads(header) + n = head["n"] + assert slots == 2 * n and nspec == 6 * len(head["cols"]) + self.crossings.append((name, type_name.decode(), head)) + self.columns.append( + [ + ctypes.string_at(spec[6 * c], spec[6 * c + 1]) + for c in range(len(head["cols"])) + ] + ) + buf = (ctypes.c_int64 * slots).from_address(out) + for i in range( + n + ): # row i is #12:(BASE + next + i), as the engine hands them out + buf[2 * i], buf[2 * i + 1] = 12, BASE + self.next + i + self.next += n + return json.dumps({"ok": True, "n": n}) + assert name in OLD, name + body = json.loads(args[2]) + rows = ( + len(body["rows"]) if "rows" in body else body["count"] + ) # the count form has no rows + rids = [f"#1:{self.next + i}" for i in range(rows)] + self.crossings.append((name, args[1].decode(), None)) + self.next += rows + return json.dumps({"ok": True, "rids": rids}) + + def entries(self): + return [c[0] for c in self.crossings if c[0] in (COLP, *OLD)] + + +@pytest.fixture +def library(monkeypatch): + return _Library(monkeypatch) + + +def _batch(chunk=100): + return extras.NativeGraphBatch(_Db(), 7, commit_every=chunk) + + +def _expect_columns(library, rows, chunk=100): + gb = _batch(chunk) + rids = gb.create_vertices("P", rows) + assert set(library.entries()) == {COLP}, library.entries() + return rids + + +def test_plain_rows_are_one_columnar_crossing_and_the_rids_come_back_as_text(library): + rows = [ + {"id": i, "name": f"né{i}", "score": i * 0.5, "flag": i % 2 == 0} + for i in range(30) + ] + rids = _expect_columns(library, rows) + assert library.entries() == [COLP] + head = library.crossings[0][2] + assert head["n"] == 30 + assert head["cols"] == [ + ["id", "q", 0], + ["name", "s", 0], + ["score", "d", 0], + ["flag", "b", 0], + ] + assert library.columns[0][0] == np.arange(30, dtype=np.int64).tobytes() + assert type(rids) is list and all(type(r) is str for r in rids) + assert rids == [f"#12:{BASE + i}" for i in range(30)] # row order, bucket:position + + +def test_the_chunks_of_one_call_are_one_crossing_each_and_the_rids_stay_in_order( + library, +): + rows = [{"id": i} for i in range(250)] + rids = _expect_columns(library, rows, chunk=100) + assert [c[2]["n"] for c in library.crossings] == [100, 100, 50] + assert rids == [f"#12:{BASE + i}" for i in range(250)] + library.crossings.clear() + assert _batch(0)._vertex_chunk == extras.NativeGraphBatch._BULK_CHUNK + _batch(0).create_vertices("P", rows) # commit_every 0: all 250 rows in one chunk + assert [c[2]["n"] for c in library.crossings] == [250] + + +def test_a_batch_of_one_row_and_no_per_row_python_in_the_codec(library, monkeypatch): + seen = {"prep": 0, "default": 0} + for key, module, name in (("prep", codec, "_prep"), ("default", codec, "_default")): + real = getattr(module, name) + + def counting(*args, _real=real, _key=key, **kwargs): + seen[_key] += 1 + return _real(*args, **kwargs) + + monkeypatch.setattr(module, name, counting) + assert _expect_columns(library, [{"id": 1}]) == [f"#12:{BASE}"] + rows = [ + {"id": i, "name": f"p{i}", "born": datetime(2020, 1, 2, 3, 4, 5, i)} + for i in range(100) + ] + _expect_columns(library, rows) + assert seen == { + "prep": 0, + "default": 0, + } # the rows were never walked or written as JSON + + +@pytest.mark.parametrize( + "rows, kinds", + [ + ([{"a": 2**53 + 1}, {"a": -(2**63)}, {"a": 2**63 - 1}], "q"), + ([{"a": float("nan")}, {"a": float("inf")}, {"a": -0.0}], "d"), + ([{"a": None}, {"a": 1}, {"a": None}], "q"), + ([{"a": None}, {"a": None}], "s"), + ([{"a": date(2020, 1, 2)}, {"a": date(1969, 12, 31)}], "D"), + ([{"a": datetime(2020, 1, 2, 3, 4, 5, 6)}, {"a": None}], "T"), + ([{"a": Decimal("1.50")}, {"a": Decimal("-0.001")}], "n"), + ([{"a": b"\x00\xff"}, {"a": b""}], "x"), + ([{"a": True}, {"a": False}], "b"), + ([{"a": "x", "b": 1}, {"a": "y", "b": 2}], "s"), + ], +) +def test_every_value_a_column_holds_crosses_as_a_column(library, rows, kinds): + _expect_columns(library, rows) + assert library.crossings[0][2]["cols"][0][1] == kinds + + +def test_float32_vectors_cross_as_one_matrix_column(library): + x = np.random.default_rng(1).standard_normal((12, 8)).astype(np.float32) + rows = [{"id": i, "embedding": x[i]} for i in range(12)] + _expect_columns(library, rows) + cols = library.crossings[0][2]["cols"] + assert cols == [["id", "q", 0], ["embedding", "f", 8]] + assert library.crossings[0][2]["last"] == [ + "embedding" + ] # stored after the other properties, as the old route did + assert library.columns[0][1] == x.tobytes() + + +def test_a_declared_vector_property_is_read_once_for_all_chunks_of_a_call(library): + db = _Db() + gb = extras.NativeGraphBatch(db, 7, commit_every=10) + gb.create_vertices("P", [{"v": [i * 0.5, i + 0.25, 3.0]} for i in range(25)]) + assert library.entries() == [COLP] * 3 + assert db.schema_reads == 1 # the schema of the type, once for the three chunks + assert [c[2]["cols"][0] for c in library.crossings] == [["v", "f", 3]] * 3 + assert ( + "last" not in library.crossings[0][2] + ) # a list column stays in the HashMap order, as in the JSON rows + expected = np.array([[i * 0.5, i + 0.25, 3.0] for i in range(10)], np.float32) + assert library.columns[0][0] == expected.tobytes() + + +FALLBACKS = { + "other keys": [{"a": 1}, {"a": 2, "b": 3}], + "same keys, another order": [{"a": 1, "b": 2}, {"b": 3, "a": 4}], + "missing key": [{"a": 1, "b": 2}, {"a": 3}], + "a none row among dict rows": [{"a": 1}, None], + "only none rows": [None, None], + "nested list": [{"a": [1, 2]}, {"a": [3]}], + "nested map": [{"a": {"x": 1}}, {"a": {"x": 2}}], + "int and float in one column": [{"a": 1}, {"a": 2.5}], + "bool and int in one column": [{"a": True}, {"a": 2}], + "numpy scalars": [{"a": np.int64(1)}, {"a": np.int64(2)}], + "a set": [{"a": {1, 2}}, {"a": {3}}], + "integer past 64 bits": [{"a": 1}, {"a": 2**70}], + "non-finite decimal": [{"a": Decimal("NaN")}, {"a": Decimal(1)}], + "lone surrogate": [{"a": "ok"}, {"a": "\ud800"}], + "float64 vectors": [{"a": np.zeros(3)}, {"a": np.ones(3)}], + "ragged vectors": [{"a": np.zeros(3, np.float32)}, {"a": np.zeros(4, np.float32)}], + "two-dimensional value": [{"a": np.zeros((2, 2), np.float32)}] * 2, + "list of floats on an undeclared property": [{"a": [0.5, 1.5]}, {"a": [2.5, 3.5]}], +} + + +@pytest.mark.parametrize("shape", sorted(FALLBACKS)) +def test_a_shape_the_columns_cannot_carry_keeps_the_json_route(library, shape): + rows = FALLBACKS[shape] + rids = _batch().create_vertices("P", list(rows)) + assert set(library.entries()) <= set(OLD) and library.entries(), library.entries() + assert len(rids) == len(rows) and all(type(r) is str for r in rids) + + +def test_no_rows_and_a_count_keep_the_json_route(library): + assert _batch().create_vertices("P", []) == [] + assert len(_batch().create_vertices("P", 3)) == 3 + assert set(library.entries()) <= set(OLD) + + +def test_a_chunk_that_cannot_be_carried_does_not_stop_the_others(library): + rows = ( + [{"id": i} for i in range(10)] + + [ + {"id": 10 + i, **({"x": 1} if i == 3 else {})} for i in range(10) + ] # a key the others lack + + [{"id": i} for i in range(20, 30)] + ) + rids = _batch(chunk=10).create_vertices("P", rows) + entries = library.entries() + assert ( + len(entries) == 3 + and entries[0] == COLP + and entries[1] in OLD + and entries[2] == COLP + ) + assert len(rids) == 30 + + +@pytest.mark.parametrize("switch", ["env", "bulk", "library"]) +def test_the_switches_turn_the_route_off(library, monkeypatch, switch): + if switch == "env": + monkeypatch.setattr(lib, "GB_COLP", False) + elif switch == "bulk": + monkeypatch.setattr(bulk, "_ENABLED", False) + else: + monkeypatch.setattr(lib, "_lib", SimpleNamespace(arcadedb_n_insert_colp=1)) + monkeypatch.setattr( + lib, "GB_COLP", False + ) # what lib._load sets for a library without the entry + _batch().create_vertices("P", [{"id": 1}, {"id": 2}]) + assert set(library.entries()) <= set(OLD) and library.entries() + + +def test_a_count_that_disagrees_is_an_error_not_a_short_list(library, monkeypatch): + real = library.call + + def short(name, *args): + if name == COLP: + return json.dumps({"ok": True, "n": 1}) + return real(name, *args) + + monkeypatch.setattr(lib, "call", short) + with pytest.raises(Exception, match="created 1 vertices for 2 rows"): + _batch().create_vertices("P", [{"id": 1}, {"id": 2}]) + + +def test_the_rid_text_is_the_engines_text_for_any_bucket_and_position(): + out = np.array([0, 0, 7, 2**62, 2**31 - 1, 2**31], dtype=np.int64) + assert bulk.rids_of(out) == ["#0:0", f"#7:{2**62}", f"#{2**31 - 1}:{2**31}"] + assert bulk.rids_of(np.empty(0, dtype=np.int64)) == [] + + +def test_same_order_is_a_rule_of_the_row_check_only(): + rows = [{"a": 1, "b": 2}, {"b": 3, "a": 4}] + assert columnar.rows_to_columns(rows) == { + "a": [1, 4], + "b": [2, 3], + } # insert_many's rule: the same keys + assert columnar.rows_to_columns(rows, same_order=True) is None + assert columnar.rows_to_columns(rows[:1] * 3, same_order=True) == { + "a": [1] * 3, + "b": [2] * 3, + } + + +def test_a_pandas_timestamp_with_nanoseconds_keeps_the_json_route_of_a_vertex_load( + library, +): + """GraphBatch's JSON route writes a Timestamp with isoformat(), which keeps the nanoseconds, and microsecond ticks cannot; insert_many's + JSON route keeps microseconds (test_native_columnar_temporal.py), so only this route asks the packer to refuse them. + """ + import pandas as pd + + declared = columnar.Declared(known={}) + stamp = pd.Timestamp("2020-01-01 00:00:00.000000123") + assert columnar.pack({"t": [stamp]}, declared) is not None # insert_many + assert columnar.pack({"t": [stamp]}, declared, exact_nanos=True) is None + micros = pd.Timestamp("2020-01-01 00:00:00.000123") + assert columnar.pack({"t": [micros]}, declared, exact_nanos=True) is not None + _batch().create_vertices("P", [{"t": stamp}, {"t": stamp}]) + assert set(library.entries()) <= set(OLD) and library.entries() + library.crossings.clear() + _batch().create_vertices("P", [{"t": micros}, {"t": micros}]) + assert library.entries() == [COLP] + + +# the floor (ArcadeData/arcadedb-python #388, the laptop A/B of this route: one row 1.63 times slower than post3, 100 rows faster): a chunk below +# lib.GB_COLP_MIN_ROWS (64) takes the JSON route of 26.11.1.post3, exactly as before this change, and a chunk at or above it the columnar route + + +@pytest.mark.parametrize( + "n,route", [(1, OLD), (63, OLD), (64, (COLP,)), (65, (COLP,)), (100, (COLP,))] +) +def test_the_floor_keeps_small_chunks_on_the_json_route_and_the_rest_on_columns( + library, monkeypatch, n, route +): + monkeypatch.setattr( + lib, "GB_COLP_MIN_ROWS", 64 + ) # the default, whatever the environment says + rows = [{"id": i, "name": f"n{i}"} for i in range(n)] + rids = _batch(1000).create_vertices("P", rows) + assert library.entries()[0] in route and len(library.entries()) == 1, ( + n, + library.entries(), + ) + assert type(rids) is list and len(rids) == n and all(type(r) is str for r in rids) + + +def test_a_small_chunk_below_the_floor_reads_no_schema_and_builds_no_columns( + library, monkeypatch +): + """The post3 path costs nothing more than before for a small chunk: no schema read, no column layout, no packing.""" + monkeypatch.setattr(lib, "GB_COLP_MIN_ROWS", 64) + for name in ("rows_to_columns", "pack"): + monkeypatch.setattr( + columnar, + name, + lambda *a, _n=name, **k: pytest.fail( + f"{_n} ran for a chunk below the floor" + ), + ) + db = _Db() + extras.NativeGraphBatch(db, 7, commit_every=100).create_vertices( + "P", [{"id": i} for i in range(63)] + ) + assert getattr(db, "schema_reads", 0) == 0 + assert library.entries() == [OLD[0]] or library.entries() == [OLD[1]] + + +def test_the_chunks_of_a_load_are_judged_one_by_one(library, monkeypatch): + monkeypatch.setattr(lib, "GB_COLP_MIN_ROWS", 64) + _batch(100).create_vertices( + "P", [{"id": i} for i in range(230)] + ) # 100, 100 and a last chunk of 30 + assert [c[0] for c in library.crossings] == [COLP, COLP, OLD[0]] + + +def test_zero_turns_the_floor_off_and_the_switch_still_turns_the_route_off( + library, monkeypatch +): + monkeypatch.setattr(lib, "GB_COLP_MIN_ROWS", 0) + _batch().create_vertices("P", [{"id": 1}]) + assert library.entries() == [COLP] + library.crossings.clear() + monkeypatch.setattr(lib, "GB_COLP", False) + _batch().create_vertices("P", [{"id": i} for i in range(100)]) + assert COLP not in library.entries() + + +def test_the_floor_is_read_from_the_environment_and_is_64_by_default(): + import os + import subprocess # nosec B404 - runs this interpreter on a fixed one-line program + import sys + + code = "from arcadedb._native import lib; print(lib.GB_COLP_MIN_ROWS)" + env = { + k: v for k, v in os.environ.items() if k != "ARCADEDB_NATIVE_GB_COLP_MIN_ROWS" + } + + def run(e): + return subprocess.run( # nosec B603 - argv list, no shell: this interpreter, a fixed program + [sys.executable, "-c", code], + env=e, + capture_output=True, + text=True, + timeout=120, + ).stdout.strip() + + assert run(env) == "64" + assert run({**env, "ARCADEDB_NATIVE_GB_COLP_MIN_ROWS": "5"}) == "5" diff --git a/tests/test_native_ingest_defaults.py b/tests/test_native_ingest_defaults.py index 9234bb7..e7519f6 100644 --- a/tests/test_native_ingest_defaults.py +++ b/tests/test_native_ingest_defaults.py @@ -9,7 +9,7 @@ import pytest import arcadedb -from arcadedb._native import extras +from arcadedb._native import bulk, extras from arcadedb.exceptions import ArcadeDBError _COUNT_SQL = { # fixed statements, one for each type the tests create @@ -85,6 +85,16 @@ def op(this, prefix, name, **args): monkeypatch.setattr(extras, "_enc", enc) monkeypatch.setattr(extras.NativeGraphBatch, "_op", op) + real_columns = bulk.create_vertices_columns + + def columns(gb, type_name, rows, declared, prefix): + # plain rows now cross as columns (arcadedb_n_gb_colp), not through _op: the rows per crossing are read here, same numbers + rids = real_columns(gb, type_name, rows, declared, prefix) + if rids is not None: + self.vertex_calls.append(len(rows)) + return rids + + monkeypatch.setattr(bulk, "create_vertices_columns", columns) def commit_every_given(self): return [d["commit_every"] for d in self.created if "commit_every" in d] diff --git a/tests/test_native_vertex_vector_crossings.py b/tests/test_native_vertex_vector_crossings.py index e684845..4db0e5a 100644 --- a/tests/test_native_vertex_vector_crossings.py +++ b/tests/test_native_vertex_vector_crossings.py @@ -79,7 +79,10 @@ def spy(monkeypatch): @pytest.fixture -def db(temp_db): +def db(temp_db, monkeypatch): + # this file pins the vector-matrix route of arcadedb_n_gb_bin ("vecs"), which the chunks that the columnar entry cannot carry still take; a + # plain vector load takes arcadedb_n_gb_colp now (test_native_graph_batch_frame*.py), so the columnar route is off here + monkeypatch.setattr(lib, "GB_COLP", False) for name in ("Declared", "Free"): temp_db.command("sql", f"CREATE VERTEX TYPE {name}") temp_db.command("sql", f"CREATE PROPERTY {name}.id LONG") diff --git a/tests/test_reader_identity_crossings.py b/tests/test_reader_identity_crossings.py new file mode 100644 index 0000000..46af764 --- /dev/null +++ b/tests/test_reader_identity_crossings.py @@ -0,0 +1,184 @@ +"""Crossings of ``Result.get_rid()``, ``get_vertex()``, ``get_edge()``, ``get_element()`` and ``Document.get_rid()`` on the JVM build. + +``Result.get_rid()`` made four JPype calls (``getIdentity``, ``isPresent``, ``get``, ``toString``) and each of the three element getters made three +(``getVertex()``, ``isPresent()``, ``get()``): ``ReaderCalls`` now answers each in one call. ``Document.get_rid()`` makes two (``getIdentity()`` and +``str()`` of the RID) and stays on them: a bridge call is two operations as well (the call and the conversion of its String), and a timed A/B +read the one-call version 1.6 times slower per call (figure ``j_rid_docobj_100k``). Every test here pins two +things: the calls into the JVM (counted on a proxy of the Java row or record and of the bridge class, the technique of ``test_reader_crossings.py``; +a call on the Java row itself would show up in the log) and that the answer is the one the per-call path gives (the oracle is that path, written out +below), so a count cannot be bought with a change of behaviour. +""" + +import pytest + +from arcadedb import results +from arcadedb.graph import Document, Edge, Vertex + + +class Counting: + """Counts the method calls made on a Java object or a Java class; a ``Counting`` argument is unwrapped, return values are the real objects.""" + + def __init__(self, target, log, label): + self._target = target + self._log = log + self._label = label + + def __getattr__(self, name): + attr = getattr(self._target, name) + if not callable(attr): + return attr + + def call(*args): + self._log.append(f"{self._label}.{name}") + args = [a._target if isinstance(a, Counting) else a for a in args] + return attr(*args) + + return call + + +def _old_rid(java_row): + """``Result.get_rid()`` before the bridge call.""" + identity = java_row.getIdentity() + return str(identity.get().toString()) if identity.isPresent() else None + + +def _old_element(java_row, getter): + """``getVertex()``, ``getEdge()`` or ``getElement()`` of a Java row, read as the three calls did.""" + optional = getattr(java_row, getter)() + return optional.get() if optional.isPresent() else None + + +@pytest.fixture +def graph_db(temp_db): + temp_db.command("sql", "CREATE VERTEX TYPE V") + temp_db.command("sql", "CREATE EDGE TYPE E") + temp_db.command("sql", "CREATE DOCUMENT TYPE D") + with temp_db.transaction(): + a = temp_db.command("sql", "CREATE VERTEX V SET name = 'a'").first() + b = temp_db.command("sql", "CREATE VERTEX V SET name = 'b'").first() + temp_db.command( + "sql", + "CREATE EDGE E FROM :a TO :b SET w = 1", + {"a": a.get_rid(), "b": b.get_rid()}, + ).close() + temp_db.command("sql", "INSERT INTO D SET k = 1").close() + return temp_db + + +@pytest.fixture +def log(graph_db, monkeypatch): + """The list every counted call is appended to; the bridge class ``ReaderCalls`` is counted.""" + calls = [] + real = results._bridge_class("ReaderCalls") + assert real is not None, "the bridge jar must carry ReaderCalls" + monkeypatch.setitem( + results._BRIDGE_CLASSES, "ReaderCalls", Counting(real, calls, "ReaderCalls") + ) + return calls + + +# a vertex, an edge, a plain document, a projection that is no record, an aggregate +ROWS = { + "vertex": "SELECT FROM V WHERE name = 'a'", + "edge": "SELECT FROM E", + "document": "SELECT FROM D", + "projection": "SELECT 1 AS x", + "aggregate": "SELECT count(*) AS c FROM V", +} + + +def _row(db, kind): + return db.query("sql", ROWS[kind]).first() + + +@pytest.mark.parametrize("kind", ROWS) +def test_get_rid_is_one_call_and_the_rid_the_per_call_path_gives(graph_db, log, kind): + row = _row(graph_db, kind) + want = _old_rid(row._java_result) + row._java_result = Counting(row._java_result, log, "row") + del log[:] + got = row.get_rid() + assert log == ["ReaderCalls.ridOf"] + assert got == want + assert (got is None) == (kind in ("projection", "aggregate")) + + +@pytest.mark.parametrize("kind", ROWS) +@pytest.mark.parametrize( + "method, getter, bridge, wrapper", + [ + ("get_vertex", "getVertex", "vertexOrNull", Vertex), + ("get_edge", "getEdge", "edgeOrNull", Edge), + ("get_element", "getElement", "elementOrNull", Document), + ], +) +def test_the_element_getters_are_one_call_and_return_what_the_per_call_path_does( + graph_db, log, kind, method, getter, bridge, wrapper +): + row = _row(graph_db, kind) + want = _old_element(row._java_result, getter) + row._java_result = Counting(row._java_result, log, "row") + del log[:] + got = getattr(row, method)() + assert log == [f"ReaderCalls.{bridge}"] + if want is None: + assert got is None + return + assert isinstance(got, wrapper) + assert str(got._java_document.getClass().getName()) == str( + want.getClass().getName() + ) + assert str(got.get_rid()) == str(want.getIdentity()) + + +def test_each_kind_of_row_has_the_element_it_should(graph_db): + vertex = _row(graph_db, "vertex") + assert isinstance(vertex.get_vertex(), Vertex) and vertex.get_edge() is None + edge = _row(graph_db, "edge") + assert isinstance(edge.get_edge(), Edge) and edge.get_vertex() is None + assert isinstance(_row(graph_db, "document").get_element(), Document) + assert _row(graph_db, "projection").get_element() is None + + +def test_document_get_rid_stays_on_the_plain_path_and_gives_the_text_str_of_the_identity_gives( + graph_db, log +): + """No bridge call on purpose (see the module docstring): the one call on the Java record is ``getIdentity()``, and the text is ``str()`` of it.""" + doc = _row(graph_db, "document").get_element() + want = str(doc._java_document.getIdentity()) + doc._java_document = Counting(doc._java_document, log, "doc") + del log[:] + assert doc.get_rid() == want + assert log == ["doc.getIdentity"] + + +def test_a_record_with_no_identity_yet_reads_as_it_did(graph_db): + with graph_db.transaction(): + doc = graph_db.new_document("D") + # str() of whatever the engine gives an unsaved record: the same text as before, "None" for no identity at all + assert doc.get_rid() == str(doc._java_document.getIdentity()) + + +@pytest.mark.parametrize("kind", ROWS) +def test_without_the_bridge_class_every_getter_answers_as_with_it( + graph_db, monkeypatch, kind +): + """No ``ReaderCalls`` (a broken checkout, the case ``_bridge_class`` answers None for): the plain path, the same answers, no AttributeError.""" + + def answers(): + row = _row(graph_db, kind) + element = row.get_element() + return { + "rid": row.get_rid(), + "vertex": None if row.get_vertex() is None else row.get_vertex().get_rid(), + "edge": None if row.get_edge() is None else row.get_edge().get_rid(), + "element": None if element is None else element.get_rid(), + "types": [ + type(x).__name__ for x in (row.get_vertex(), row.get_edge(), element) + ], + } + + with_bridge = answers() + monkeypatch.setitem(results._BRIDGE_CLASSES, "ReaderCalls", None) + assert answers() == with_bridge + assert (with_bridge["rid"] is None) == (kind in ("projection", "aggregate"))