diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 987f1c1..389737d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,10 +51,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + # Run cargo from packages/deckd so rustup picks up its rust-toolchain.toml. - name: Show Rust toolchain + working-directory: packages/deckd run: rustc --version && cargo --version - name: Build deckd - run: cargo build --locked --manifest-path packages/deckd/Cargo.toml + working-directory: packages/deckd + run: cargo build --locked + - name: Test deckd + working-directory: packages/deckd + run: cargo test --locked - name: Preserve deckd for collaboration replay uses: actions/upload-artifact@v4 with: diff --git a/docs/deck-daemon.md b/docs/deck-daemon.md index e5365e9..67c6011 100644 --- a/docs/deck-daemon.md +++ b/docs/deck-daemon.md @@ -194,6 +194,18 @@ History (compat with today’s historyd client in | `POST /file-list-at` | `{ dir, revisionId }` | paths — **close the CLI gap** | | `POST /restore-plan` | `{ dir, revisionId }` | `{ writes, deletes }` | +Explicit commits, for a writer that stages content itself and must record each +write exactly once. These address a bare history repo by path rather than a +depot; the [deckd README](../packages/deckd/README.md#explicit-commits) has the +full contract: + +| Route | Body (sketch) | Result | +| ----------------- | -------------------------------------------------------------------- | --------------------------------------------------------- | +| `POST /commit` | `{ repo, parent, operationId, author, message, trailers?, changes }` | `{ revision, tree, created, replayed }`, or a typed error | +| `POST /outcome` | `{ repo, operationId }` | `committed` + revision, or `absent` (fences the ID) | +| `POST /head` | `{ repo }` | `{ revision, tree }` | +| `POST /tree-hash` | `{ repo, revision }` | `{ tree }`, the git tree ID | + Build (deferred — **not** on the Hub serve path): | Route | Body (sketch) | Result | diff --git a/packages/deckd/Cargo.lock b/packages/deckd/Cargo.lock index 654c319..43450b7 100644 --- a/packages/deckd/Cargo.lock +++ b/packages/deckd/Cargo.lock @@ -148,20 +148,20 @@ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "blake2" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" dependencies = [ "digest", ] [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -216,7 +216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures", "rand_core", ] @@ -242,19 +242,16 @@ dependencies = [ ] [[package]] -name = "core-foundation-sys" -version = "0.8.7" +name = "cmov" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] -name = "cpufeatures" -version = "0.2.17" +name = "core-foundation-sys" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" @@ -310,12 +307,20 @@ checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a" +dependencies = [ + "cmov", ] [[package]] @@ -340,16 +345,28 @@ dependencies = [ "base64", "chrono", "futures", + "gix", "jj-lib", "notify", - "pollster", + "pollster 0.4.0", "serde", "serde_json", + "sha2", "thiserror", "tokio", "tower-http", "tracing", "tracing-subscriber", + "ureq", +] + +[[package]] +name = "defmt" +version = "0.3.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4deef977660e373c458df605a913d4ab121db0ccd18d1ad6976dff7d37cddb53" +dependencies = [ + "defmt 1.1.1", ] [[package]] @@ -385,13 +402,24 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.7" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", "crypto-common", - "subtle", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] @@ -431,22 +459,14 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "etcetera" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" -dependencies = [ - "cfg-if", - "windows-sys 0.61.2", -] - [[package]] name = "faster-hex" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +checksum = "04839bdf9d8c10f66806fad16b852fc72aab80873aebc3cb69d85b4fa41543ed" dependencies = [ + "autocfg", + "defmt 0.3.101", "heapless", "serde", ] @@ -591,16 +611,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.4.3" @@ -615,9 +625,9 @@ dependencies = [ [[package]] name = "gix" -version = "0.85.0" +version = "0.88.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8b2e38ebfc4484dfef8580ddcaf8abb7285e6f3eb6413ff6775d104ae96ca6" +checksum = "32d6dd6028308ad2f2662207506e396b97f9ec899f74bd537c0775248c774f21" dependencies = [ "gix-actor", "gix-attributes", @@ -637,12 +647,14 @@ dependencies = [ "gix-ignore", "gix-index", "gix-lock", + "gix-note", "gix-object", "gix-odb", "gix-pack", "gix-path", "gix-pathspec", "gix-protocol", + "gix-quote", "gix-ref", "gix-refspec", "gix-revision", @@ -658,16 +670,16 @@ dependencies = [ "gix-validate", "gix-worktree", "gix-worktree-stream", + "gix-zlib", "nonempty", "smallvec", - "thiserror", ] [[package]] name = "gix-actor" -version = "0.41.2" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33f9308ad6fd35b2a865cbe4117ac61b2be59e4a9ef1621c7a9794f7c8e52c5b" +checksum = "8e046e9a929e8e1f40f9a34f736408c4c079f9cf004429f3e718d270e847be96" dependencies = [ "bstr", "gix-date", @@ -676,57 +688,57 @@ dependencies = [ [[package]] name = "gix-attributes" -version = "0.33.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39b40888d0ed415c0744a6cdc61eebf0304c9d26ab726725b718443c322e5ba4" +checksum = "a3d57bcf18768dc259868e335fae99a9063174178b78b00348603f0791cd780a" dependencies = [ "bstr", + "gix-error", + "gix-features", "gix-glob", "gix-path", "gix-quote", "gix-trace", - "kstring", "smallvec", - "thiserror", "unicode-bom", ] [[package]] name = "gix-bitmap" -version = "0.3.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd1d118d0f5d88b96e6f6e13b566475fef4797ead4a02c26fed36c1375066f7" +checksum = "4e9d28a8333a322df5c4fdc7b86a9928965e50794523c0edff57d1fbf2a520cf" dependencies = [ "gix-error", ] [[package]] name = "gix-chunk" -version = "0.7.3" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a871e5cab12ba568845714473505deefffb3c04eb47f4708ce344cd459c1cc" +checksum = "25303d07ce2b8f98aab48ce4251317d785a3208b7ed65698928e4a2f31889936" dependencies = [ "gix-error", ] [[package]] name = "gix-command" -version = "0.9.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4363accdf6ef7ba861871d2d521ab7418a04aaaed919fadb022af71d379b12" +checksum = "eb3c2e61f97adc08764060e9becb0c819d8f2ba27896434710e039ae57b1d2a7" dependencies = [ "bstr", + "gix-error", "gix-path", "gix-quote", "gix-trace", - "shell-words", ] [[package]] name = "gix-commitgraph" -version = "0.37.1" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +checksum = "0462cbffd5828447f376559d397d9b29f1979dac3d980b5212b17fa174b174eb" dependencies = [ "bstr", "gix-chunk", @@ -738,40 +750,41 @@ dependencies = [ [[package]] name = "gix-config" -version = "0.58.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29bf266c4cdaf759e535c24ad4ce655b987aeb6911075643403cc7cc5ade583" +checksum = "eb2cb454dce2f1895cf9def1df8b0a6bbc579d86d8313bdf60f1ab429d52df12" dependencies = [ "bstr", "gix-config-value", + "gix-error", "gix-features", "gix-glob", "gix-path", "gix-ref", "gix-sec", + "gix-utils", "smallvec", - "thiserror", "unicode-bom", ] [[package]] name = "gix-config-value" -version = "0.18.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +checksum = "1894e816c774650b5157d965853b50d7e8f7137ea74055a6bac2f2ec051cf997" dependencies = [ "bitflags 2.13.1", "bstr", + "gix-error", "gix-path", "libc", - "thiserror", ] [[package]] name = "gix-date" -version = "0.15.6" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e47b9e8cdc688296609b706428de570f88b1e0eed7156dde7b4a89d26fa4567" +checksum = "cb8beceb16a77fb222923592864b94beb4ad4c2b175a7055c2cac6815ee0370a" dependencies = [ "bstr", "gix-error", @@ -781,12 +794,13 @@ dependencies = [ [[package]] name = "gix-diff" -version = "0.65.0" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c6d56c94edf92d78203a1cd416f770e35e10b6955ede6b9d7d0c22ff88a5f3" +checksum = "939aee7bbc9f233de858839e865c82886e0baceb8f1fe792ea1703fa895f09e1" dependencies = [ "bstr", "gix-command", + "gix-error", "gix-filter", "gix-fs", "gix-hash", @@ -797,38 +811,37 @@ dependencies = [ "gix-trace", "gix-traverse", "gix-worktree", - "thiserror", ] [[package]] name = "gix-discover" -version = "0.53.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d624d5b23b10c1d85337645227abe353ac95ab8ff66a7bdd5ce689b2db33a722" +checksum = "4476ba055ec5fbb05f59dd98380b596428da060beac5f9dccd7374302ad1fbfb" dependencies = [ "bstr", "dunce", + "gix-error", "gix-fs", "gix-path", "gix-ref", "gix-sec", - "thiserror", ] [[package]] name = "gix-error" -version = "0.2.5" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9292309fd944e71b2a3c96d3c03a6feb8852db646febdde7cbb9f79cb5f329" +checksum = "2c4a5c4cd326ddfe7f046ad9eed05461c59af6ea98b681648bac73954317addd" dependencies = [ "bstr", ] [[package]] name = "gix-features" -version = "0.48.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +checksum = "bb3a430dbc02d6c5e30a9f92083459593495ea22a476e837d248b22d800eed80" dependencies = [ "bytes", "crc32fast", @@ -840,21 +853,20 @@ dependencies = [ "once_cell", "parking_lot", "prodash", - "thiserror", "walkdir", - "zlib-rs", ] [[package]] name = "gix-filter" -version = "0.32.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6644fb2ef97928c278675b239f366b457103d7e436f811d27331a8daf212759c" +checksum = "6f31a39e2c9b04843709d1f72b2c39c98dc0ee591e2f67d21302dbfbd06d2956" dependencies = [ "bstr", "encoding_rs", "gix-attributes", "gix-command", + "gix-error", "gix-hash", "gix-object", "gix-packetline", @@ -863,52 +875,52 @@ dependencies = [ "gix-trace", "gix-utils", "smallvec", - "thiserror", ] [[package]] name = "gix-fs" -version = "0.21.2" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +checksum = "0105ab1cef4542d092c27766d2daf1548fcbad7e460dc2208642c120bb19b3ad" dependencies = [ "bstr", - "fastrand", + "gix-error", "gix-features", "gix-path", "gix-utils", - "thiserror", ] [[package]] name = "gix-glob" -version = "0.26.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +checksum = "bba50803dea16646cafb9a2d67e95b8cdf5b21c7e8c45bcd18f38bafe02fb81f" dependencies = [ "bitflags 2.13.1", "bstr", "gix-features", "gix-path", + "gix-utils", ] [[package]] name = "gix-hash" -version = "0.25.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +checksum = "7a22b6febaaee8546eb0c9dd21ecdd93aca269a6cb3897e1250f0f60933c0174" dependencies = [ "faster-hex", + "gix-error", "gix-features", - "sha1-checked", - "thiserror", + "sha1dc", + "sha2", ] [[package]] name = "gix-hashtable" -version = "0.15.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e261d54091f0d1c729bc83f54548c071bdec60a697de1e58e88bdfd7a99d24e" +checksum = "c55bff2f32f6f5a95bdd43e5e5e01b0656250cffbc48eda19db0afa3d73adf2f" dependencies = [ "gix-hash", "hashbrown 0.17.1", @@ -917,9 +929,9 @@ dependencies = [ [[package]] name = "gix-ignore" -version = "0.21.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +checksum = "5bfd910e271ecd1a20e4df4009a0d9d844a0c51ee5444aaa3fadd1b1b077930b" dependencies = [ "bstr", "gix-glob", @@ -930,9 +942,9 @@ dependencies = [ [[package]] name = "gix-imara-diff" -version = "0.2.4" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a791e6620676a875f362f3156ed213e73ca099a09bf992c18812abe65cc37b1" +checksum = "7518b930f6aeb75e871d32ce873c2ae46e35cef19b5a0b039d4337f651d897cd" dependencies = [ "bstr", "hashbrown 0.17.1", @@ -940,15 +952,16 @@ dependencies = [ [[package]] name = "gix-index" -version = "0.53.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d45f82ec5a4d7542ea595e9ad16e03e26c8cb4f221e5bc9fcdcf469f63a681" +checksum = "dfd5b484b5b4e218a5ac114f0982949f9bd7582ff67986665a0cf85895332433" dependencies = [ "bitflags 2.13.1", "bstr", "filetime", "fnv", "gix-bitmap", + "gix-error", "gix-features", "gix-fs", "gix-hash", @@ -963,46 +976,71 @@ dependencies = [ "memmap2", "rustix", "smallvec", - "thiserror", ] [[package]] name = "gix-lock" -version = "23.0.1" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +checksum = "8fd5ddfd0cc476fef744ac544783b96a9ff5d49f8de541e6e9d6d6002aa90720" dependencies = [ + "gix-error", "gix-tempfile", "gix-utils", - "thiserror", +] + +[[package]] +name = "gix-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f11f5429ca73cc3758c9f53af56a9fba100a8701c5b4bc30e29b8b0b15288991" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "gix-note" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6879cc9e5a9a0429e40b05970af6c8dad8734098a3142fec23e32eeb567580" +dependencies = [ + "gix-error", + "gix-hash", + "gix-object", ] [[package]] name = "gix-object" -version = "0.62.0" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "019b38afc3eac1e41f9fe09a327664b313ba4a120fa5f40e3678795d0e42783e" +checksum = "a72909eeeed845721a143ef3dbeb4d6cd5df79c839fe7eefc138cb4f982eba3e" dependencies = [ "bstr", "gix-actor", + "gix-command", "gix-date", + "gix-error", "gix-features", "gix-hash", "gix-hashtable", + "gix-path", + "gix-tempfile", "gix-utils", "gix-validate", "itoa", "smallvec", - "thiserror", ] [[package]] name = "gix-odb" -version = "0.82.0" +version = "0.85.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fadc59f6fa0f9dd445eceee61060a2b59ca557f48da9fc677f567db535b782a" +checksum = "5794b85ff563a290ea18867fa1f0a5b0b84709f2f34dddbb3614556fbe88c629" dependencies = [ "arc-swap", + "gix-error", "gix-features", "gix-fs", "gix-hash", @@ -1011,19 +1049,20 @@ dependencies = [ "gix-pack", "gix-path", "gix-quote", + "gix-zlib", "memmap2", "parking_lot", "tempfile", - "thiserror", ] [[package]] name = "gix-pack" -version = "0.72.0" +version = "0.75.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3e7f1726cd2c0cd1cf1fc20be8a8e623f0b163f1f8d6fc836cfb9bc8cd758b" +checksum = "0451a48faef8e0e7a8f6d903c8329c7d18a793b3030907ca841f889dc695d1df" dependencies = [ "clru", + "crossbeam-deque", "gix-chunk", "gix-error", "gix-features", @@ -1031,75 +1070,75 @@ dependencies = [ "gix-hashtable", "gix-object", "gix-path", + "gix-zlib", "memmap2", "smallvec", - "thiserror", "uluru", ] [[package]] name = "gix-packetline" -version = "0.21.5" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b217dd0ee0c4021ecf169a4a519b1b4f80d15e3f3765f3dc466223dc0ac891d7" +checksum = "0ba540112a3fcc6388eb2aa4dc4837040478c285a35178cd9a44b07345cd6636" dependencies = [ "bstr", "faster-hex", + "gix-error", "gix-trace", - "thiserror", ] [[package]] name = "gix-path" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751d6bd162106f8c1e7e9aaccb5bbdd605267e91a930a17a4560c46e33a9100c" +checksum = "fd051ac60c5dbce2228daabf25b6a76408ac732d416c678cfa0a4914d13f60b8" dependencies = [ "bstr", + "gix-error", "gix-trace", "gix-validate", - "thiserror", ] [[package]] name = "gix-pathspec" -version = "0.18.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +checksum = "1ee883ab64bad641bf31ff338044edd7aa010605f19e866d98c9164e635c38b1" dependencies = [ "bitflags 2.13.1", "bstr", "gix-attributes", "gix-config-value", + "gix-error", "gix-glob", "gix-path", - "thiserror", ] [[package]] name = "gix-protocol" -version = "0.63.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978468bae4ea2df20c72db3b20d0bdb548a0c1090b85a83643b553e6e0e041f2" +checksum = "331ac489c461790b6b692afb322085c42041c6ff7621c30ac3dbb00b12e05e89" dependencies = [ "bstr", "gix-date", + "gix-error", "gix-features", "gix-hash", + "gix-macros", "gix-ref", "gix-shallow", "gix-transport", "gix-utils", - "maybe-async", "nonempty", - "thiserror", ] [[package]] name = "gix-quote" -version = "0.7.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +checksum = "e6b040cfa7af50f131bfa34119cceee804c3cc40c766daaba8e0c8149340aea7" dependencies = [ "bstr", "gix-error", @@ -1108,11 +1147,12 @@ dependencies = [ [[package]] name = "gix-ref" -version = "0.65.0" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bbfbce1dfd7d7f8469ddef6d3518376aff664348f153cbe0fc3e58ef993d24e" +checksum = "c90ceeec67de03b737c30b1b69116ca56fec803edde022f3d25181f0f597083d" dependencies = [ "gix-actor", + "gix-error", "gix-features", "gix-fs", "gix-hash", @@ -1123,30 +1163,26 @@ dependencies = [ "gix-utils", "gix-validate", "memmap2", - "thiserror", ] [[package]] name = "gix-refspec" -version = "0.43.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc36a4fb1a1540b59cf2da498783080743fa274b02a3f19ca444fc4015a9d4f" +checksum = "b4caa89a034a055c0578150092d3edb2f1c3f74cca86b883263a28caa26c5812" dependencies = [ "bstr", "gix-error", - "gix-glob", "gix-hash", - "gix-revision", "gix-validate", "smallvec", - "thiserror", ] [[package]] name = "gix-revision" -version = "0.47.0" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "885075c3c21eb9c06e0be3b3728ba5932c04e1c1011dcee7c81801980e3e986f" +checksum = "d8438d38e197fa3ea7bb297302db1701879a64a343f23a3d150a602a609a8751" dependencies = [ "bstr", "gix-commitgraph", @@ -1160,9 +1196,9 @@ dependencies = [ [[package]] name = "gix-revwalk" -version = "0.33.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f11fe7ca2585193d3d70bbe0be175a2008d883a704cc7a55e454e113e689455" +checksum = "2fa6d913e7cf8c6407d3954f9794b4b148c2113c672c34b63c59ea654a1e060e" dependencies = [ "gix-commitgraph", "gix-date", @@ -1171,14 +1207,13 @@ dependencies = [ "gix-hashtable", "gix-object", "smallvec", - "thiserror", ] [[package]] name = "gix-sec" -version = "0.14.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4fe6c152c1d50aea36f299825702cd37e303307832fec1d0fdd5844e47ce2f" +checksum = "7d8eb983a830ba586b61f7005deaf44609e1a4186cbf13a0e8668154ca910461" dependencies = [ "bitflags 2.13.1", "gix-path", @@ -1188,102 +1223,106 @@ dependencies = [ [[package]] name = "gix-shallow" -version = "0.12.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +checksum = "119bae1301715df40e8fd136deeff4fe8a3be4cfc001ed5e72bad7b47ebbc090" dependencies = [ "bstr", + "gix-error", "gix-hash", "gix-lock", "nonempty", - "thiserror", ] [[package]] name = "gix-submodule" -version = "0.32.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f9f594f7cbda0b38ba6b633b3e9a7b7901acdc5d27bc186a16633800cd1ac8" +checksum = "0b498192e5f89453fbd728263d52088aeec634a390e732f1da6d0a491c552091" dependencies = [ "bstr", "gix-config", + "gix-error", "gix-path", "gix-pathspec", "gix-refspec", "gix-url", - "thiserror", ] [[package]] name = "gix-tempfile" -version = "23.0.2" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef60812443484e67bf84e444cc71b4c78ae62deb822221774a4fa0c57fdb17f" +checksum = "3dd7dd8df3345933caddd6ce67821c9eec17192564f1011cf0bd0b8cb0d07ffb" dependencies = [ "dashmap", "gix-fs", "libc", "parking_lot", "tempfile", + "windows-sys 0.61.2", ] [[package]] name = "gix-trace" -version = "0.1.21" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3eb81d9dc914335923e50d52829c551feefd6a72d176c4130c546b67a60814" +checksum = "df45b2cf6052d9e52681130c17d67505b023e8933ad974f94a2be0b0d24f636a" [[package]] name = "gix-transport" -version = "0.57.2" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186874f7ad1fb2f9a2f2aa9c2dabc7f9dd087bef74c1a0eee2b4a9cf0248fcb3" +checksum = "5d2beb7b0387bbf4d7c13763c6e168f9957319aa811f09349dc59601be3b2951" dependencies = [ "bstr", "gix-command", + "gix-error", "gix-features", "gix-packetline", + "gix-path", "gix-quote", "gix-sec", "gix-url", - "thiserror", ] [[package]] name = "gix-traverse" -version = "0.59.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5062cca8f2977565bbaf666ec31dbdb9bc9d9293beb65f9bec52e6c1121b62a1" +checksum = "dd4864934e4efd92082ecc37107bce0c389b58d82888db634817f9c711efb7ed" dependencies = [ "bitflags 2.13.1", "gix-commitgraph", "gix-date", + "gix-error", "gix-hash", "gix-hashtable", "gix-object", "gix-revwalk", "smallvec", - "thiserror", ] [[package]] name = "gix-url" -version = "0.36.2" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d68e70e96da0e5f9c871f1566349e0fd0e1a20bb483c7f54af1dd0b85b4b29" +checksum = "1f30fe2a228d7de5f9d582463d9b724debcb870162d9fbe908cd5d76ccdecf0d" dependencies = [ "bstr", + "gix-error", "gix-path", + "gix-utils", "percent-encoding", - "thiserror", ] [[package]] name = "gix-utils" -version = "0.3.5" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1795bd2a970ca8b2185318c2abb97d955c71992f1cf28de73ad3b593a9f3ce8" +checksum = "70525766f53426ca4cd2ea4d58b0b31488dfedb416b3254e2efdd9dc184cc880" dependencies = [ + "bstr", "fastrand", "getrandom", "unicode-normalization", @@ -1291,21 +1330,24 @@ dependencies = [ [[package]] name = "gix-validate" -version = "0.11.3" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a034e84d1e04e1b1f20f51f12491da230b6ac8b925d0c8e1b89bcd87a7c5ccc" +checksum = "30574f81fc41b2ca3582ece2bdf8696bdfe9c170aa7e8bcafb81199c86b1ceea" dependencies = [ "bstr", + "gix-error", ] [[package]] name = "gix-worktree" -version = "0.54.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92399ed66f259592050c6ed9dc80105e095a2f8e87e6b83d98aa2e21d8e27036" +checksum = "04fa19b9b3961aafb3b9e60005b9bcef0100f8b211f46ab5269a6ac896073780" dependencies = [ "bstr", "gix-attributes", + "gix-error", + "gix-features", "gix-fs", "gix-glob", "gix-hash", @@ -1318,9 +1360,9 @@ dependencies = [ [[package]] name = "gix-worktree-stream" -version = "0.34.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f3a878c89a05470ad98c644b0015777c530da24854dd29e41fe4f41176840f" +checksum = "2021e1609beae0edbeefa76ac641be0571b725afacb77ac5b12f33b436e0d3d9" dependencies = [ "gix-attributes", "gix-error", @@ -1334,6 +1376,16 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "gix-zlib" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44a1a9c82ec5b712082566cbef8f5782472a0d3973132911e5a83380a5298fe2" +dependencies = [ + "gix-error", + "zlib-rs", +] + [[package]] name = "globset" version = "0.4.20" @@ -1439,6 +1491,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.0" @@ -1498,6 +1559,110 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1579,7 +1744,7 @@ version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ - "defmt", + "defmt 1.1.1", "jiff-core", "jiff-static", "jiff-tzdb-platform", @@ -1596,7 +1761,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" dependencies = [ - "defmt", + "defmt 1.1.1", ] [[package]] @@ -1626,11 +1791,52 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jj-core" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cc6c2549b3eeb6b87063e49786e3c73abea653d9b88b20c982967a8b112db40" +dependencies = [ + "async-trait", + "blake2", + "bstr", + "chrono", + "clru", + "digest", + "either", + "futures", + "globset", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "jj-core-proc-macros", + "ref-cast", + "regex", + "same-file", + "serde", + "smallvec", + "tempfile", + "thiserror", + "tracing", + "winreg", +] + +[[package]] +name = "jj-core-proc-macros" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4936b0a5244527dd7e2370c3553920b402a7b44cfecf6748131dedaa1890bb81" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "jj-lib" -version = "0.43.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22e2ced34a1a51f0f449bfff2cbacda1c338b2611268b5684a80251e4c8bf71" +checksum = "4b82d2d9b1f8400db1dda9a22bccfa6c3118f5592dfe6578452e56dcb378e9ad" dependencies = [ "async-trait", "blake2", @@ -1640,21 +1846,20 @@ dependencies = [ "digest", "dunce", "either", - "etcetera", "futures", "gix", "gix-ignore", "globset", - "hashbrown 0.17.1", "indexmap", "interim", "itertools 0.15.0", - "jj-lib-proc-macros", + "jj-core", "maplit", + "memchr", "once_cell", "pest", "pest_derive", - "pollster", + "pollster 1.0.1", "prost", "rand", "rand_chacha", @@ -1662,7 +1867,6 @@ dependencies = [ "ref-cast", "regex", "rustix", - "same-file", "serde", "smallvec", "strsim", @@ -1670,18 +1874,7 @@ dependencies = [ "thiserror", "toml_edit", "tracing", - "winreg", -] - -[[package]] -name = "jj-lib-proc-macros" -version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c21554e3755e5ecfec934d84ac64d0295d9c2f2b96f6a740a1ddb66addf4142e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "windows-sys 0.61.2", ] [[package]] @@ -1715,15 +1908,6 @@ dependencies = [ "libc", ] -[[package]] -name = "kstring" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b609e7ca5ea38f093c20a4a102335b247221c9643b7a6bc3510f196f99499a9e" -dependencies = [ - "static_assertions", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -1742,6 +1926,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + [[package]] name = "lock_api" version = "0.4.14" @@ -1812,17 +2002,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" -[[package]] -name = "maybe-async" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "memchr" version = "2.8.3" @@ -1997,6 +2176,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" +[[package]] +name = "pollster" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6355899e1c9462875b6757c79f3caa011a1fdae12bbb1a2e72dd1f234f8336" + [[package]] name = "portable-atomic" version = "1.15.0" @@ -2012,6 +2197,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2313,24 +2507,20 @@ dependencies = [ ] [[package]] -name = "sha1" -version = "0.10.7" +name = "sha1dc" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] +checksum = "c5b0ebf6537e3e8db8bb7d21ecb6d7399738c953c757945c6b95e46c4e95df7b" [[package]] -name = "sha1-checked" -version = "0.10.0" +name = "sha2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ + "cfg-if", + "cpufeatures", "digest", - "sha1", ] [[package]] @@ -2342,12 +2532,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shell-words" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" - [[package]] name = "shlex" version = "2.0.1" @@ -2395,24 +2579,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "syn" version = "2.0.119" @@ -2441,6 +2613,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -2483,6 +2666,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.12.0" @@ -2713,16 +2906,42 @@ dependencies = [ ] [[package]] -name = "valuable" -version = "0.1.1" +name = "ureq" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "log", + "once_cell", + "serde", + "serde_json", + "url", +] [[package]] -name = "version_check" -version = "0.9.5" +name = "url" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "walkdir" @@ -2954,6 +3173,35 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.56" @@ -2974,6 +3222,60 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "zlib-rs" version = "0.6.7" diff --git a/packages/deckd/Cargo.toml b/packages/deckd/Cargo.toml index 3529114..ec66f73 100644 --- a/packages/deckd/Cargo.toml +++ b/packages/deckd/Cargo.toml @@ -16,12 +16,15 @@ axum = { version = "0.8", features = ["json"] } base64 = "0.22" chrono = { version = "0.4", default-features = false, features = ["clock"] } futures = "0.3" -# Same crates.io pin as boxel-labs jj-historyd (not a fork of the jj git tree). -jj-lib = "0.43.0" +# Only for `gix::hash::Kind` when initializing a repo; must match jj-lib's gix. +gix = { version = "0.88.0", default-features = false } +# From crates.io, not a fork of the jj git tree. 0.46 needs rustc 1.97.1. +jj-lib = "0.46.0" notify = "7" pollster = "0.4" serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = { version = "0.11", default-features = false } thiserror = "2" tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "process", "sync", "fs", "io-util", "time"] } tower-http = { version = "0.6", features = ["trace"] } @@ -33,3 +36,6 @@ default = [] # Derived esbuild AMD plane (`/ensure-build`, `/artifact-at`). Off by default — # History is the product; enable with `cargo run --features build`. build = [] + +[dev-dependencies] +ureq = { version = "2", default-features = false, features = ["json"] } diff --git a/packages/deckd/README.md b/packages/deckd/README.md index 306ddca..adf8e61 100644 --- a/packages/deckd/README.md +++ b/packages/deckd/README.md @@ -11,7 +11,8 @@ jj-lib path copied closely from boxel-labs [`jj-historyd`](file:///Users/chris/Projects/boxel-labs/examples/jj-lib-node/rust) (`HistoryService` + axum + `spawn_blocking`/`pollster`). -Dependency: crates.io `jj-lib = "0.43.0"` (same pin as historyd). +Dependency: crates.io `jj-lib = "0.46.0"`, which needs rustc 1.97.1; +`rust-toolchain.toml` pins the compiler, so run cargo from this directory. ## Features @@ -70,6 +71,181 @@ When Hub uses `DECKD_URL`, Node watchers publish + HMR only — they do not | ----------------------------- | ---------------------------------------- | | ensure / seal / restore | per-workspace **write** lock | | list / file-at / file-list-at | per-workspace **read** lock (concurrent) | +| commit / outcome | per-repo **file** lock (cross-process) | + +## Explicit commits + +`/commit`, `/outcome`, `/head` and `/tree-hash` record a revision from content +the caller staged, never from a working copy. A writer such as realm-server +can then record each write exactly once, even when deckd's response is lost. + +Each call names `repo`, the absolute path of one history directory (for +example `/.history`). The directory's parent must exist; deckd creates +the directory and a bare jj repo inside it on first use. When `DECKD_REPO_ROOT` +is set, `repo` must resolve, symlinks followed, to a directory inside it; +deployments should set it to the realms directory. + +```text +/ + jj/ # jj repo, internal git backend, no working copy + deckd/HEAD # recorded HEAD; locked around every call but /tree-hash + deckd/operations/ # pending | committed | fenced +``` + +A revision is the git commit ID (40 hex). HEAD is the repo's single visible +head; an empty repo's HEAD is the root commit, `0000…0000`. + +### Guarantees + +- **Exactly the listed paths.** A revision is HEAD's tree plus the request's + `changes`; nothing on disk is read. A change that would turn a file into a + directory or back is refused, since it would remove paths it doesn't list. +- **Compare-and-swap on HEAD.** A commit whose `parent` isn't HEAD is refused + with `parent-mismatch` and the current `head`, never re-parented. +- **Idempotent by operation ID.** Committing an ID that already committed + returns the original revision with `replayed: true` and writes nothing, + even when the retry's `parent` is now stale. The retry must ask for the + same thing: an ID that committed a different author, message, trailers or + changes is refused with `operation-mismatch`. +- **Fencing outcome query.** `/outcome` reports `committed` or `absent`. + Reporting `absent` fences the ID: a later commit with it is refused with + `operation-fenced`. +- **Durable, not in-memory.** Outcomes and fences are files under ``, + so they survive a restart and are shared by every deckd on the same + filesystem. Each record and its directory entry are synced before the call + returns. Record existence is tested with `O_EXCL` creates, which an NFS + server answers, so a stale negative lookup can't hide one. +- **No-op writes record nothing.** When the changes leave the tree as it was, + no revision is created; the response returns HEAD with `created: false`. + +Correctness doesn't depend on deckd's in-process state: + +- **One lock file, never deleted.** An exclusive lock on `deckd/HEAD` + serializes deckd processes. Deleting a lock file on release lets two NFS + clients each hold the lock, one on the unlinked inode and one on its + replacement, so this file stays. Within a process, calls for one repo also + share a mutex keyed by the directory's device and inode, so a symlinked + spelling can't bypass it. Both locks share the 30 s `lock-timeout`. +- **A recorded HEAD.** `deckd/HEAD` holds the revision deckd last recorded, + written just before jj publishes it. Taking an NFS lock revalidates that + file's cached data, so it is current even when this host's listing of jj's + operations is stale. jj's HEAD must match it before anything is written. + - The file holds two checksummed slots, and each write goes to the slot + without the latest value, so a write torn by a crash leaves the previous + revision readable. + - A revision is recorded only after its git objects are synced, so a power + loss can't leave `deckd/HEAD` naming a commit the store lost. + - When jj is one commit behind (a crash between recording and publishing, or + a stale listing), the recorded commit is published, which is idempotent. + - Any other mismatch is reloaded for a few seconds, then refused with + `history-unsettled` and nothing written. +- **Fences hold.** A `fenced` record for an operation found committed means + the lock failed to exclude a writer. It is reported as an error that needs an + operator, never overwritten. + +A writer running on several hosts should still serialize its own commits +(realm-server uses a Postgres advisory lock). + +Each commit's message ends with the caller's trailers, then +`Deckd-Operation-Id: `, which callers may not set themselves. + +### `POST /commit` + +```json +{ + "repo": "/realms/acme/.history", + "parent": "0000000000000000000000000000000000000000", + "operationId": "01JB8ZQ4T6W3", + "author": { "name": "Jane", "email": "jane@example.com" }, + "message": "Update Author/jane.json", + "trailers": [{ "key": "Boxel-Channel", "value": "host" }], + "changes": [ + { "path": "Author/jane.json", "contentBase64": "eyJuYW1lIjoiSmFuZSJ9" }, + { "path": "Author/old.json", "delete": true } + ] +} +``` + +```json +{ + "revision": "6f1c2d8e0a9b4c3d2e1f0a9b8c7d6e5f4a3b2c1d", + "tree": "9a3e1b7c5d2f4e6a8b0c1d3e5f7a9b1c3d5e7f9a", + "created": true, + "replayed": false +} +``` + +- `operationId`: 1–128 characters from `[A-Za-z0-9._-]`, not starting with `.`. +- `changes`: each entry has `path` plus exactly one of `contentBase64` or + `delete: true`. A path appears at most once. Paths are relative and may not + contain empty, `.`, `..` or `.git` segments. No listed path may be inside + another listed path, inside a file in HEAD, or name a directory in HEAD. + Deleting an absent path changes nothing. Writing an existing file keeps its + executable bit. +- `trailers`: optional. Keys match `[A-Za-z0-9-]+`; values are one line. +- `message` is required. `author.name` and `author.email` are required and may + not contain newlines, `<` or `>`. + +### `POST /outcome` + +```json +{ "repo": "/realms/acme/.history", "operationId": "01JB8ZQ4T6W3" } +``` + +```json +{ "state": "committed", "revision": "6f1c2d8e0a9b4c3d2e1f0a9b8c7d6e5f4a3b2c1d" } +``` + +or `{ "state": "absent" }`, after which the ID is fenced. It is a POST because +it can change state. + +### `POST /head` + +```json +{ "repo": "/realms/acme/.history" } +``` + +```json +{ + "revision": "6f1c2d8e0a9b4c3d2e1f0a9b8c7d6e5f4a3b2c1d", + "tree": "9a3e1b7c5d2f4e6a8b0c1d3e5f7a9b1c3d5e7f9a" +} +``` + +### `POST /tree-hash` + +```json +{ + "repo": "/realms/acme/.history", + "revision": "6f1c2d8e0a9b4c3d2e1f0a9b8c7d6e5f4a3b2c1d" +} +``` + +```json +{ "tree": "9a3e1b7c5d2f4e6a8b0c1d3e5f7a9b1c3d5e7f9a" } +``` + +The tree hash is the git tree ID, so identical content has the same hash in +any repo, whoever committed it and whatever the message. The revision must be +in the repo's history; a commit object that was written but never published +is `revision-not-found`. + +### Errors + +Every error body is `{ "code", "error" }`, plus fields named below, on every +route, including a request deckd can't parse. + +| Status | `code` | When | +| ----------- | -------------------- | ----------------------------------------------------------------------- | +| 400/415/422 | `invalid-request` | Unparseable JSON, a missing field, the wrong content type, or bad input | +| 404 | `revision-not-found` | `/tree-hash` for a revision the repo doesn't have | +| 409 | `parent-mismatch` | `parent` isn't HEAD; body has `head` | +| 409 | `operation-fenced` | The operation ID was reported `absent` by `/outcome` | +| 409 | `operation-mismatch` | The operation ID committed a different request; body has `revision` | +| 413 | `payload-too-large` | A `/commit` body over 64 MiB; split the change into several commits | +| 503 | `history-unsettled` | jj's HEAD didn't match `deckd/HEAD` within a few seconds; retry | +| 503 | `lock-timeout` | Other calls held the repo's locks for 30 s | +| 500 | `internal` | jj or filesystem failure, or a fenced operation found committed | ## Run diff --git a/packages/deckd/rust-toolchain.toml b/packages/deckd/rust-toolchain.toml new file mode 100644 index 0000000..a1eeb29 --- /dev/null +++ b/packages/deckd/rust-toolchain.toml @@ -0,0 +1,4 @@ +# jj-lib 0.46 needs rustc 1.97.1 or newer. rustup reads this file from the +# current directory, so run cargo from packages/deckd. +[toolchain] +channel = "1.99.1" diff --git a/packages/deckd/src/api.rs b/packages/deckd/src/api.rs index 20e09e3..d38d581 100644 --- a/packages/deckd/src/api.rs +++ b/packages/deckd/src/api.rs @@ -2,7 +2,8 @@ use std::sync::Arc; -use axum::extract::State; +use axum::extract::rejection::JsonRejection; +use axum::extract::{DefaultBodyLimit, FromRequest, Request, State}; use axum::http::StatusCode; use axum::routing::{get, post}; use axum::{Json, Router}; @@ -10,6 +11,10 @@ use base64::Engine; use serde::{Deserialize, Serialize}; use tower_http::trace::TraceLayer; +use crate::history::commit::{ + Author, Change, CommitError, CommitOutcome, CommitRequest, CommitService, CommitTrailer, + OperationOutcome, RevisionTree, +}; use crate::history::{Actor, HistoryBackend, HistoryBackendError, HistoryEntry, RestorePlan}; use crate::watch::WatchPlane; @@ -21,6 +26,8 @@ use crate::build::{ #[derive(Clone)] pub struct AppState { pub history: Arc, + /// Explicit-content commits for repos addressed by path. + pub commits: Arc, /// Present only with `--features build`. #[cfg(feature = "build")] pub build: Option>, @@ -40,6 +47,13 @@ pub fn router(state: AppState) -> Router { .route("/file-at", post(file_at)) .route("/file-list-at", post(file_list_at)) .route("/restore-plan", post(restore_plan)) + .route( + "/commit", + post(commit).layer(DefaultBodyLimit::max(COMMIT_BODY_LIMIT)), + ) + .route("/outcome", post(outcome)) + .route("/head", post(head)) + .route("/tree-hash", post(tree_hash)) .route("/ensure-build", post(ensure_build)) .route("/artifact-at", post(artifact_at)) .layer(TraceLayer::new_for_http()) @@ -56,12 +70,53 @@ pub fn router(state: AppState) -> Router { .route("/file-at", post(file_at)) .route("/file-list-at", post(file_list_at)) .route("/restore-plan", post(restore_plan)) + .route( + "/commit", + post(commit).layer(DefaultBodyLimit::max(COMMIT_BODY_LIMIT)), + ) + .route("/outcome", post(outcome)) + .route("/head", post(head)) + .route("/tree-hash", post(tree_hash)) .layer(TraceLayer::new_for_http()) .with_state(state); app } +/// A commit carries every staged file base64-encoded in one JSON body, so it +/// needs more than axum's 2 MB default. The whole body and its decoded copy +/// are held in memory, so this stays well under what the sidecar can spare; +/// a larger push is split into several commits by the caller. +const COMMIT_BODY_LIMIT: usize = 64 * 1024 * 1024; + +/// A JSON request body whose rejections (malformed JSON, a missing field, the +/// wrong content type, a body over the limit) answer as typed `ApiError`s, +/// like every other error. +struct Body(T); + +impl FromRequest for Body +where + Json: FromRequest, + S: Send + Sync, +{ + type Rejection = ApiError; + + async fn from_request(req: Request, state: &S) -> Result { + match Json::::from_request(req, state).await { + Ok(Json(value)) => Ok(Body(value)), + Err(rejection) => { + let status = rejection.status(); + let code = if status == StatusCode::PAYLOAD_TOO_LARGE { + "payload-too-large" + } else { + "invalid-request" + }; + Err(ApiError::new(status, code, rejection.body_text())) + } + } + } +} + #[derive(Debug, Deserialize)] struct DirBody { dir: String, @@ -123,7 +178,7 @@ struct RestorePlanBody { async fn ensure( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { st.history.ensure(&body.dir).await?; if let Some(watch) = &st.watch { @@ -136,7 +191,7 @@ async fn ensure( async fn note( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { st.history.note(&body.dir, &body.path).await?; Ok(Json(serde_json::json!({}))) @@ -144,7 +199,7 @@ async fn note( async fn flush( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let change_id = st.history.flush_notes(&body.dir).await?; Ok(Json(SealResponse { change_id })) @@ -152,7 +207,7 @@ async fn flush( async fn seal( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let change_id = st .history @@ -163,14 +218,14 @@ async fn seal( async fn list( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result>, ApiError> { Ok(Json(st.history.list(&body.dir).await?)) } async fn file_at( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let bytes = st .history @@ -190,7 +245,7 @@ async fn file_at( async fn file_list_at( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { Ok(Json(FileListAtResponse { paths: st @@ -202,7 +257,7 @@ async fn file_list_at( async fn restore_plan( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { Ok(Json( st.history @@ -211,10 +266,141 @@ async fn restore_plan( )) } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CommitBody { + repo: String, + parent: String, + operation_id: String, + author: Author, + message: String, + #[serde(default)] + trailers: Vec, + changes: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ChangeBody { + path: String, + #[serde(default)] + content_base64: Option, + #[serde(default)] + delete: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct OutcomeBody { + repo: String, + operation_id: String, +} + +#[derive(Debug, Deserialize)] +struct RepoBody { + repo: String, +} + +#[derive(Debug, Deserialize)] +struct TreeHashBody { + repo: String, + revision: String, +} + +#[derive(Debug, Serialize)] +struct TreeHashResponse { + tree: String, +} + +async fn commit( + State(st): State, + Body(body): Body, +) -> Result, ApiError> { + let mut changes = Vec::with_capacity(body.changes.len()); + for change in body.changes { + let content = match (change.content_base64, change.delete) { + (Some(encoded), false) => Some( + base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|e| { + CommitError::Invalid(format!("contentBase64 for {:?}: {e}", change.path)) + })?, + ), + (None, true) => None, + _ => { + return Err(CommitError::Invalid(format!( + "change {:?} needs exactly one of contentBase64 or delete: true", + change.path + )) + .into()) + } + }; + changes.push(Change { + path: change.path, + content, + }); + } + let request = CommitRequest { + parent: body.parent, + operation_id: body.operation_id, + author: body.author, + message: body.message, + trailers: body.trailers, + changes, + }; + let repo = std::path::PathBuf::from(body.repo); + Ok(Json( + run_commits(&st, move |c| c.commit(&repo, request)).await?, + )) +} + +async fn outcome( + State(st): State, + Body(body): Body, +) -> Result, ApiError> { + let repo = std::path::PathBuf::from(body.repo); + let operation_id = body.operation_id; + Ok(Json( + run_commits(&st, move |c| c.outcome(&repo, &operation_id)).await?, + )) +} + +async fn head( + State(st): State, + Body(body): Body, +) -> Result, ApiError> { + let repo = std::path::PathBuf::from(body.repo); + Ok(Json(run_commits(&st, move |c| c.head(&repo)).await?)) +} + +async fn tree_hash( + State(st): State, + Body(body): Body, +) -> Result, ApiError> { + let repo = std::path::PathBuf::from(body.repo); + let revision = body.revision; + let found = run_commits(&st, move |c| c.tree_hash(&repo, &revision)).await?; + Ok(Json(TreeHashResponse { tree: found.tree })) +} + +/// Commit work waits on repo locks and drives jj-lib's `!Send` futures, so it +/// runs on a blocking thread. +async fn run_commits(st: &AppState, f: F) -> Result +where + T: Send + 'static, + F: FnOnce(Arc) -> Result + Send + 'static, +{ + let commits = st.commits.clone(); + tokio::task::spawn_blocking(move || f(commits)) + .await + .map_err(|e| ApiError::from(format!("join error: {e}")))? + .map_err(ApiError::from) +} + #[cfg(feature = "build")] async fn ensure_build( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let Some(build) = &st.build else { return Err("build plane unavailable".to_string().into()); @@ -225,7 +411,7 @@ async fn ensure_build( #[cfg(feature = "build")] async fn artifact_at( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let Some(build) = &st.build else { return Err("build plane unavailable".to_string().into()); @@ -235,37 +421,86 @@ async fn artifact_at( struct ApiError { status: StatusCode, + /// Stable, machine-readable error kind. + code: &'static str, message: String, + /// Extra fields merged into the error body, e.g. `head` on a conflict. + details: serde_json::Map, +} + +impl ApiError { + fn new(status: StatusCode, code: &'static str, message: String) -> Self { + Self { + status, + code, + message, + details: serde_json::Map::new(), + } + } +} + +impl From for ApiError { + fn from(err: CommitError) -> Self { + let message = err.to_string(); + match err { + CommitError::Invalid(_) => { + Self::new(StatusCode::BAD_REQUEST, "invalid-request", message) + } + CommitError::ParentMismatch { head, .. } => { + let mut error = Self::new(StatusCode::CONFLICT, "parent-mismatch", message); + error.details.insert("head".into(), head.into()); + error + } + CommitError::OperationFenced(_) => { + Self::new(StatusCode::CONFLICT, "operation-fenced", message) + } + CommitError::OperationMismatch { revision, .. } => { + let mut error = Self::new(StatusCode::CONFLICT, "operation-mismatch", message); + error.details.insert("revision".into(), revision.into()); + error + } + CommitError::RevisionNotFound(_) => { + Self::new(StatusCode::NOT_FOUND, "revision-not-found", message) + } + CommitError::HistoryUnsettled { .. } => Self::new( + StatusCode::SERVICE_UNAVAILABLE, + "history-unsettled", + message, + ), + CommitError::LockTimeout(_) => { + Self::new(StatusCode::SERVICE_UNAVAILABLE, "lock-timeout", message) + } + CommitError::Internal(_) => { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal", message) + } + } + } } impl From for ApiError { fn from(err: HistoryBackendError) -> Self { - let status = match &err { - HistoryBackendError::NotFound(_) => StatusCode::NOT_FOUND, - HistoryBackendError::Invalid(_) => StatusCode::BAD_REQUEST, + let (status, code) = match &err { + HistoryBackendError::NotFound(_) => (StatusCode::NOT_FOUND, "not-found"), + HistoryBackendError::Invalid(_) => (StatusCode::BAD_REQUEST, "invalid-request"), HistoryBackendError::Io(_) | HistoryBackendError::Jj(_) => { - StatusCode::INTERNAL_SERVER_ERROR + (StatusCode::INTERNAL_SERVER_ERROR, "internal") } }; - Self { - status, - message: err.to_string(), - } + Self::new(status, code, err.to_string()) } } impl From for ApiError { fn from(message: String) -> Self { - Self { - status: StatusCode::INTERNAL_SERVER_ERROR, - message, - } + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal", message) } } impl axum::response::IntoResponse for ApiError { fn into_response(self) -> axum::response::Response { - let body = Json(serde_json::json!({ "error": self.message })); - (self.status, body).into_response() + let mut body = self.details; + body.insert("code".into(), self.code.into()); + body.insert("error".into(), self.message.into()); + (self.status, Json(serde_json::Value::Object(body))).into_response() } } diff --git a/packages/deckd/src/history/commit.rs b/packages/deckd/src/history/commit.rs new file mode 100644 index 0000000..26c2375 --- /dev/null +++ b/packages/deckd/src/history/commit.rs @@ -0,0 +1,2382 @@ +//! Explicit-content commits for a history repo addressed by path. +//! +//! The caller (realm-server) stages every file's final bytes and asks deckd to +//! record exactly those changes as one revision on top of `parent`. Nothing is +//! read from a working copy: the repo has no workspace, so a revision can never +//! pick up bytes some other request left on disk. +//! +//! Layout under `repo` (the caller chooses where that directory lives): +//! +//! ```text +//! / +//! jj/ # bare jj repo, internal git backend +//! deckd/HEAD # recorded HEAD; locked around every call +//! deckd/operations/ # `pending ` | `committed ` | `fenced` +//! ``` +//! +//! Correctness rests on files in the repo, never on this process's memory: +//! +//! - `commit`, `outcome` and `head` hold an exclusive lock on `deckd/HEAD`, +//! which is never deleted, so deckd processes on different hosts sharing the +//! directory serialize. A per-repo in-process mutex sits in front of it +//! because Linux NFS emulates flock with POSIX locks, which don't exclude +//! threads of one process. The mutex is keyed by the directory's device and +//! inode, so every spelling of one repo shares it. +//! - `deckd/HEAD` holds the revision deckd last recorded, written before jj +//! publishes it. It is read through the locked descriptor, and taking an NFS +//! lock revalidates that file's cached data, so it is current even when this +//! host's listing of jj's operation heads is stale. jj's HEAD must equal it +//! before anything is written. When jj is exactly one commit behind (a crash +//! between recording and publishing, or a stale listing), the recorded +//! commit is published again, which is idempotent. Any other disagreement is +//! reloaded until it settles, then refused as `HistoryUnsettled`. +//! - A revision is recorded only after its git objects are on stable storage, +//! so `deckd/HEAD` never names a commit a power loss could take away. +//! - Whether an operation record exists is only ever learned from +//! `create_new` (O_EXCL), which the NFS server answers. A peer's cached +//! negative lookup can't make an existing record look absent. Records and +//! the directory entries naming them are synced before a call returns. +//! - Every commit carries a `Deckd-Operation-Id` trailer. Before acting, each +//! locked call sets HEAD's operation record to `committed HEAD`. A crash +//! after publishing but before writing the record can only leave HEAD +//! unrecorded, so after that repair every committed operation has a record. +//! - A record keeps a digest of the request that claimed it. Replaying a +//! committed operation with a different request is refused, so a reused +//! operation ID can't silently drop a write. + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fs::{File, OpenOptions, TryLockError}; +use std::io::{ErrorKind, Read as _, Seek as _, SeekFrom, Write as _}; +use std::os::unix::fs::MetadataExt as _; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, MutexGuard, TryLockError as MutexTryLockError, Weak}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use jj_lib::backend::{BackendError, CommitId, CopyId, Signature, Timestamp, TreeValue}; +use jj_lib::commit::Commit; +use jj_lib::default_backend_factories::default_backend_factories; +use jj_lib::merge::Merge; +use jj_lib::merged_tree::MergedTree; +use jj_lib::merged_tree_builder::MergedTreeBuilder; +use jj_lib::object_id::ObjectId as _; +use jj_lib::repo::{ReadonlyRepo, Repo as _, RepoLoader}; +use jj_lib::repo_path::{RepoPath, RepoPathBuf}; +use jj_lib::signing_factory::signer_from_settings; +use jj_lib::trailer::parse_description_trailers; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; + +use super::jj_lib::{hermetic_git_env, is_valid_history_path, user_settings}; + +/// Trailer deckd adds to every explicit commit. Callers may not set it. +const OPERATION_ID_TRAILER: &str = "Deckd-Operation-Id"; + +/// How long a call waits for the repo's locks before giving up. +const LOCK_TIMEOUT: Duration = if cfg!(test) { + Duration::from_secs(5) +} else { + Duration::from_secs(30) +}; + +/// How long a call waits for jj's HEAD to agree with `deckd/HEAD`. +const SETTLE_TIMEOUT: Duration = if cfg!(test) { + Duration::from_millis(300) +} else { + Duration::from_secs(5) +}; + +#[derive(Debug, thiserror::Error)] +pub enum CommitError { + #[error("{0}")] + Invalid(String), + #[error("parent {parent} is not HEAD {head}")] + ParentMismatch { parent: String, head: String }, + #[error("operation {0} was reported absent and can never commit")] + OperationFenced(String), + #[error("operation {operation_id} already committed a different request as {revision}")] + OperationMismatch { + operation_id: String, + revision: String, + }, + #[error("revision {0} not found")] + RevisionNotFound(String), + #[error("timed out waiting for the commit lock on {0}")] + LockTimeout(String), + #[error("history HEAD {observed} does not match recorded HEAD {recorded}; retry")] + HistoryUnsettled { recorded: String, observed: String }, + #[error("{0}")] + Internal(String), +} + +type CommitResult = Result; + +fn internal(error: impl std::fmt::Display) -> CommitError { + CommitError::Internal(error.to_string()) +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Author { + pub name: String, + pub email: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct CommitTrailer { + pub key: String, + pub value: String, +} + +/// One staged path: new bytes, or a delete. +#[derive(Debug, Clone)] +pub struct Change { + pub path: String, + pub content: Option>, +} + +#[derive(Debug, Clone)] +pub struct CommitRequest { + pub parent: String, + pub operation_id: String, + pub author: Author, + pub message: String, + pub trailers: Vec, + pub changes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct CommitOutcome { + pub revision: String, + pub tree: String, + /// False when the changes left the tree as it was and no revision was + /// written; `revision` is then the unchanged HEAD. + pub created: bool, + /// True when this operation ID had already committed and nothing new + /// was written. + pub replayed: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "state", rename_all = "camelCase")] +pub enum OperationOutcome { + Committed { revision: String }, + Absent, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RevisionTree { + pub revision: String, + pub tree: String, +} + +/// Explicit-content commits, operation outcomes and tree hashes. Holds no +/// state that correctness depends on; a fresh instance behaves exactly like +/// one that has been running. +pub struct CommitService { + /// When set, every repo must resolve to a directory inside it. + root: Option, + gates: Mutex, +} + +/// In-process gates by the repo directory's (device, inode). Entries no call +/// holds are pruned, so the map only grows with concurrent repos. +type Gates = HashMap<(u64, u64), Weak>>; + +impl Default for CommitService { + fn default() -> Self { + Self::new() + } +} + +impl CommitService { + /// A service that accepts any absolute repo path. + pub fn new() -> Self { + hermetic_git_env(); + Self { + root: None, + gates: Mutex::new(HashMap::new()), + } + } + + /// A service that refuses repos outside `root`, which must exist. + pub fn with_root(root: &Path) -> std::io::Result { + Ok(Self { + root: Some(std::fs::canonicalize(root)?), + ..Self::new() + }) + } + + /// Record `request.changes` as one revision on top of `request.parent`. + /// Runs on a blocking thread: it waits on locks and drives jj-lib's + /// `!Send` futures to completion while holding them. + pub fn commit(&self, repo_dir: &Path, request: CommitRequest) -> CommitResult { + validate_operation_id(&request.operation_id)?; + validate_commit_request(&request)?; + self.locked(repo_dir, |repo_dir, head_file| { + pollster::block_on(commit_locked(repo_dir, head_file, request)) + }) + } + + /// Report whether `operation_id` committed. Reporting it absent fences it: + /// from then on a commit with that ID is refused. + pub fn outcome(&self, repo_dir: &Path, operation_id: &str) -> CommitResult { + validate_operation_id(operation_id)?; + self.locked(repo_dir, |repo_dir, head_file| { + pollster::block_on(outcome_locked(repo_dir, head_file, operation_id)) + }) + } + + /// The current HEAD, creating an empty repo (HEAD is the root commit) if + /// there is none yet. + pub fn head(&self, repo_dir: &Path) -> CommitResult { + let head = self.locked(repo_dir, |repo_dir, head_file| { + pollster::block_on(settled_head(repo_dir, head_file)).map(|(_repo, head)| head) + })?; + Ok(RevisionTree { + revision: head.id().hex(), + tree: tree_hash(&head), + }) + } + + /// The tree of `revision`, which must be in the repo's history: a commit + /// object that was written but never published is not found. + pub fn tree_hash(&self, repo_dir: &Path, revision: &str) -> CommitResult { + let repo_dir = self.resolve_repo_dir(repo_dir)?; + let not_found = || CommitError::RevisionNotFound(revision.to_owned()); + let id = CommitId::try_from_hex(revision).ok_or_else(not_found)?; + // Load first rather than testing for `jj/`: a cached negative lookup + // must not turn an existing revision into a 404. + let repo = match pollster::block_on(load_repo(&repo_dir)) { + Ok(repo) => repo, + Err(_) if !repo_dir.join("jj").exists() => return Err(not_found()), + Err(error) => return Err(error), + }; + let heads: Vec = repo.view().heads().iter().cloned().collect(); + let commit = match pollster::block_on(published_commit(&repo, &heads, &id))? { + Some(commit) => commit, + // Unlocked, this host's listing may not show a peer's latest + // commit yet. Only a miss pays for the lock, which settles HEAD. + None => self + .locked(&repo_dir, |repo_dir, head_file| { + pollster::block_on(async { + let (repo, head) = settled_head(repo_dir, head_file).await?; + published_commit(&repo, &[head.id().clone()], &id).await + }) + })? + .ok_or_else(not_found)?, + }; + Ok(RevisionTree { + revision: commit.id().hex(), + tree: tree_hash(&commit), + }) + } + + /// Run `body` holding both repo locks, with the resolved repo directory + /// and the locked `deckd/HEAD`. Both locks share one `LOCK_TIMEOUT`, so a + /// busy or stuck repo can't park blocking threads indefinitely. + fn locked( + &self, + repo_dir: &Path, + body: impl FnOnce(&Path, &mut File) -> CommitResult, + ) -> CommitResult { + let repo_dir = self.resolve_repo_dir(repo_dir)?; + create_repo_dir(&repo_dir)?; + let deadline = Instant::now() + LOCK_TIMEOUT; + let gate = self.gate(&repo_dir)?; + let _in_process = lock_gate(&gate, &repo_dir, deadline)?; + let mut head_file = lock_repo(&repo_dir, deadline)?; + body(&repo_dir, &mut head_file) + } + + /// `repo_dir` with symlinks resolved. Its parent (normally the realm + /// directory) must exist, so a mistyped path fails instead of quietly + /// growing a new tree; the repo directory itself need not. + fn resolve_repo_dir(&self, repo_dir: &Path) -> CommitResult { + validate_repo_dir(repo_dir)?; + let (Some(parent), Some(name)) = (repo_dir.parent(), repo_dir.file_name()) else { + return Err(CommitError::Invalid("repo must name a directory".into())); + }; + let missing_parent = || { + CommitError::Invalid(format!( + "parent directory of {} does not exist", + repo_dir.display() + )) + }; + let parent = std::fs::canonicalize(parent).map_err(|_| missing_parent())?; + let mut resolved = parent.join(name); + if resolved.is_symlink() { + resolved = std::fs::canonicalize(&resolved).map_err(|_| missing_parent())?; + } + if let Some(root) = &self.root { + if !resolved.starts_with(root) || resolved == *root { + return Err(CommitError::Invalid(format!( + "repo must be inside {}", + root.display() + ))); + } + } + Ok(resolved) + } + + fn gate(&self, repo_dir: &Path) -> CommitResult>> { + let metadata = std::fs::metadata(repo_dir).map_err(internal)?; + let key = (metadata.dev(), metadata.ino()); + let mut gates = self.gates.lock().unwrap_or_else(|e| e.into_inner()); + gates.retain(|_, gate| gate.strong_count() > 0); + if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) { + return Ok(gate); + } + let gate = Arc::new(Mutex::new(())); + gates.insert(key, Arc::downgrade(&gate)); + Ok(gate) + } +} + +/// Take a repo's in-process gate, giving up at `deadline`. +fn lock_gate<'a>( + gate: &'a Mutex<()>, + repo_dir: &Path, + deadline: Instant, +) -> CommitResult> { + loop { + match gate.try_lock() { + Ok(guard) => return Ok(guard), + Err(MutexTryLockError::Poisoned(poisoned)) => return Ok(poisoned.into_inner()), + Err(MutexTryLockError::WouldBlock) if Instant::now() >= deadline => { + return Err(CommitError::LockTimeout(repo_dir.display().to_string())) + } + Err(MutexTryLockError::WouldBlock) => std::thread::sleep(Duration::from_millis(10)), + } + } +} + +/// The body of [`CommitService::commit`]. Callers hold both repo locks. +async fn commit_locked( + repo_dir: &Path, + head_file: &mut File, + request: CommitRequest, +) -> CommitResult { + let (repo, head) = settled_head(repo_dir, head_file).await?; + repair_head_record(repo_dir, &head)?; + + let digest = request_digest(&request); + let record_path = operation_record_path(repo_dir, &request.operation_id); + if !claim_record(&record_path, &Record::Pending(Some(digest.clone())))? { + match read_record(&record_path)? { + Record::Committed { + revision, + digest: recorded, + } => { + if recorded.is_some_and(|recorded| recorded != digest) { + return Err(CommitError::OperationMismatch { + operation_id: request.operation_id, + revision, + }); + } + let commit = get_commit(&repo, &revision)?; + return Ok(CommitOutcome { + revision, + tree: tree_hash(&commit), + created: false, + replayed: true, + }); + } + Record::Fenced => { + return Err(CommitError::OperationFenced(request.operation_id)); + } + // A crashed attempt that never published: the repair above + // would have marked it committed if it had. Take it over, so a + // replay is checked against this request. + Record::Pending(_) => { + write_record(&record_path, &Record::Pending(Some(digest.clone())))?; + } + } + } + + let head_hex = head.id().hex(); + if CommitId::try_from_hex(&request.parent).as_ref() != Some(head.id()) { + // Leave no claim behind, so the caller can retry this operation + // on the new HEAD. + remove_record(&record_path)?; + return Err(CommitError::ParentMismatch { + parent: request.parent, + head: head_hex, + }); + } + + let tree = match build_tree(&repo, &head, &request.changes).await { + Ok(tree) => tree, + Err(error) => { + remove_record(&record_path)?; + return Err(error); + } + }; + if tree.tree_ids() == head.tree_ids() { + write_record( + &record_path, + &Record::Committed { + revision: head_hex.clone(), + digest: Some(digest), + }, + )?; + return Ok(CommitOutcome { + revision: head_hex, + tree: tree_hash(&head), + created: false, + replayed: false, + }); + } + + let signature = Signature { + name: request.author.name.clone(), + email: request.author.email.clone(), + timestamp: Timestamp::now(), + }; + let mut tx = repo.start_transaction(); + let commit = tx + .repo_mut() + .new_commit(vec![head.id().clone()], tree) + .set_description(description(&request)) + .set_author(signature.clone()) + .set_committer(signature) + .write() + .await + .map_err(internal)?; + let revision = commit.id().hex(); + sync_new_objects(repo_dir, &commit, &request.changes).await?; + // The commit point: from here a crash leaves this revision as HEAD, and + // the next locked call publishes it if jj hasn't. + write_recorded_head(head_file, &revision)?; + tx.commit(format!( + "commit {revision} for operation {}", + request.operation_id + )) + .await + .map_err(internal)?; + + write_record( + &record_path, + &Record::Committed { + revision: revision.clone(), + digest: Some(digest), + }, + )?; + Ok(CommitOutcome { + revision, + tree: tree_hash(&commit), + created: true, + replayed: false, + }) +} + +/// The body of [`CommitService::outcome`]. Callers hold both repo locks. +async fn outcome_locked( + repo_dir: &Path, + head_file: &mut File, + operation_id: &str, +) -> CommitResult { + let (_repo, head) = settled_head(repo_dir, head_file).await?; + repair_head_record(repo_dir, &head)?; + + let record_path = operation_record_path(repo_dir, operation_id); + if claim_record(&record_path, &Record::Fenced)? { + return Ok(OperationOutcome::Absent); + } + match read_record(&record_path)? { + Record::Committed { revision, .. } => Ok(OperationOutcome::Committed { revision }), + Record::Fenced => Ok(OperationOutcome::Absent), + Record::Pending(_) => { + write_record(&record_path, &Record::Fenced)?; + Ok(OperationOutcome::Absent) + } + } +} + +/// Take the cross-process lock on an existing repo directory, giving up at +/// `deadline`, and return the locked `deckd/HEAD`. Callers hold the repo's +/// in-process gate first. Dropping the file releases the lock. +fn lock_repo(repo_dir: &Path, deadline: Instant) -> CommitResult { + let deckd_dir = repo_dir.join("deckd"); + std::fs::create_dir_all(deckd_dir.join("operations")).map_err(internal)?; + // The lock file is never deleted. Deleting one on release (as jj-lib's + // `FileLock` does) lets two NFS clients each hold "the" lock: one on the + // unlinked inode, which NFS keeps alive under a hidden name, and one on + // its replacement. + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(deckd_dir.join("HEAD")) + .map_err(internal)?; + loop { + match file.try_lock() { + Ok(()) => return Ok(file), + Err(TryLockError::WouldBlock) if Instant::now() >= deadline => { + return Err(CommitError::LockTimeout(repo_dir.display().to_string())) + } + Err(TryLockError::WouldBlock) => std::thread::sleep(Duration::from_millis(10)), + Err(TryLockError::Error(error)) => return Err(internal(error)), + } + } +} + +/// `deckd/HEAD` can't be replaced by a rename, because it is the lock, so it +/// holds two fixed-size slots, each ` \n`. A write goes +/// to the slot that doesn't hold the latest value, so a write torn by a crash +/// leaves the previous value intact in the other slot. +const HEAD_SLOT_LEN: usize = 16 + 1 + 40 + 1 + 16 + 1; + +/// The latest intact slot in the locked `deckd/HEAD`, as (sequence, revision). +/// With none (no write ever finished), HEAD is the root commit. +fn read_head_slots(head_file: &mut File) -> CommitResult> { + let mut bytes = Vec::new(); + head_file.seek(SeekFrom::Start(0)).map_err(internal)?; + head_file.read_to_end(&mut bytes).map_err(internal)?; + Ok(bytes + .chunks(HEAD_SLOT_LEN) + .take(2) + .filter_map(parse_head_slot) + .max_by_key(|(seq, _)| *seq)) +} + +fn parse_head_slot(slot: &[u8]) -> Option<(u64, CommitId)> { + let text = std::str::from_utf8(slot).ok()?.strip_suffix('\n')?; + let (body, check) = text.rsplit_once(' ')?; + if slot.len() != HEAD_SLOT_LEN || check != format!("{:016x}", fnv1a(body.as_bytes())) { + return None; + } + let (seq, revision) = body.split_once(' ')?; + Some(( + u64::from_str_radix(seq, 16).ok()?, + CommitId::try_from_hex(revision)?, + )) +} + +/// The revision in the locked `deckd/HEAD`. +fn read_recorded_head(head_file: &mut File, root: &CommitId) -> CommitResult { + Ok(read_head_slots(head_file)?.map_or_else(|| root.clone(), |(_, revision)| revision)) +} + +/// Record `revision` in the locked `deckd/HEAD` and sync it. +fn write_recorded_head(head_file: &mut File, revision: &str) -> CommitResult<()> { + if revision.len() != 40 { + return Err(CommitError::Internal(format!( + "cannot record revision {revision:?}" + ))); + } + let seq = read_head_slots(head_file)?.map_or(0, |(seq, _)| seq) + 1; + let body = format!("{seq:016x} {revision}"); + let slot = format!("{body} {:016x}\n", fnv1a(body.as_bytes())); + let offset = (seq % 2) * HEAD_SLOT_LEN as u64; + head_file.seek(SeekFrom::Start(offset)).map_err(internal)?; + head_file.write_all(slot.as_bytes()).map_err(internal)?; + head_file.sync_all().map_err(internal) +} + +/// FNV-1a, enough to tell a torn slot from an intact one. +fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0100_0000_01b3) + }) +} + +/// Load the repo and return its HEAD once it agrees with `deckd/HEAD`. Callers +/// hold both repo locks. +async fn settled_head( + repo_dir: &Path, + head_file: &mut File, +) -> CommitResult<(Arc, Commit)> { + let started = Instant::now(); + loop { + let repo = load_or_init(repo_dir).await?; + let recorded = read_recorded_head(head_file, repo.store().root_commit_id())?; + let head = head_commit(&repo)?; + if head.id() == &recorded { + return Ok((repo, head)); + } + // jj one commit behind: the recorded revision was written and then + // either never published (a crash) or published where this host's + // listing can't see it yet. Publishing it is right in both cases, and + // publishing it twice merges to the same single head. + if let Ok(commit) = repo.store().get_commit(&recorded) { + if commit.parent_ids() == std::slice::from_ref(head.id()) { + let mut tx = repo.start_transaction(); + tx.repo_mut().add_head(&commit).await.map_err(internal)?; + let repo = tx + .commit(format!("publish recorded HEAD {}", recorded.hex())) + .await + .map_err(internal)?; + return Ok((repo, commit)); + } + } + if started.elapsed() >= SETTLE_TIMEOUT { + return Err(CommitError::HistoryUnsettled { + recorded: recorded.hex(), + observed: head.id().hex(), + }); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn validate_repo_dir(repo_dir: &Path) -> CommitResult<()> { + if !repo_dir.is_absolute() { + return Err(CommitError::Invalid("repo must be an absolute path".into())); + } + if repo_dir + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return Err(CommitError::Invalid("repo must not contain `..`".into())); + } + Ok(()) +} + +/// Operation IDs name files, so they're limited to a filename-safe alphabet +/// and may not start with `.` (temporary records do). +fn validate_operation_id(operation_id: &str) -> CommitResult<()> { + let valid = (1..=128).contains(&operation_id.len()) + && !operation_id.starts_with('.') + && operation_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')); + if valid { + Ok(()) + } else { + Err(CommitError::Invalid(format!( + "invalid operation id {operation_id:?}: use 1-128 of [A-Za-z0-9._-], not starting with `.`" + ))) + } +} + +/// A path a commit may write or delete. Beyond what reads accept, it has no +/// `.` segment (git rejects such trees) and no `.git` segment (git refuses to +/// check one out). History only grows, so a bad tree could never be removed. +fn is_valid_commit_path(path: &str) -> bool { + is_valid_history_path(path) + && !path + .split('/') + .any(|segment| segment == "." || segment.eq_ignore_ascii_case(".git")) +} + +fn validate_commit_request(request: &CommitRequest) -> CommitResult<()> { + if CommitId::try_from_hex(&request.parent).is_none() { + return Err(CommitError::Invalid(format!( + "parent {:?} is not a revision id", + request.parent + ))); + } + if request.author.name.trim().is_empty() || request.author.email.trim().is_empty() { + return Err(CommitError::Invalid( + "author name and email are required".into(), + )); + } + // These would corrupt the git signature line. + let unsafe_author_char = |c: char| matches!(c, '\n' | '\r' | '<' | '>'); + if request.author.name.contains(unsafe_author_char) + || request.author.email.contains(unsafe_author_char) + { + return Err(CommitError::Invalid( + "author must not contain newlines, `<` or `>`".into(), + )); + } + // With no message, the trailer block would be the first paragraph, and + // trailer parsing (as in git) ignores trailers there. + if request.message.trim().is_empty() { + return Err(CommitError::Invalid("message is required".into())); + } + for trailer in &request.trailers { + let key_ok = !trailer.key.is_empty() + && trailer + .key + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-'); + if !key_ok || trailer.value.contains('\n') || trailer.value.contains('\r') { + return Err(CommitError::Invalid(format!( + "invalid trailer {:?}", + trailer.key + ))); + } + if trailer.key.eq_ignore_ascii_case(OPERATION_ID_TRAILER) { + return Err(CommitError::Invalid(format!( + "{OPERATION_ID_TRAILER} is set by deckd from operationId" + ))); + } + } + let mut seen = HashSet::new(); + for change in &request.changes { + if !is_valid_commit_path(&change.path) { + return Err(CommitError::Invalid(format!( + "invalid path {:?}", + change.path + ))); + } + if !seen.insert(change.path.as_str()) { + return Err(CommitError::Invalid(format!( + "path {:?} appears more than once", + change.path + ))); + } + } + // A listed path inside another listed path would turn a file into a + // directory or back, changing what the other entry says. + for change in &request.changes { + let mut ancestor = change.path.as_str(); + while let Some((parent, _)) = ancestor.rsplit_once('/') { + if seen.contains(parent) { + return Err(CommitError::Invalid(format!( + "path {:?} is inside listed path {parent:?}", + change.path + ))); + } + ancestor = parent; + } + } + Ok(()) +} + +fn description(request: &CommitRequest) -> String { + let mut description = request.message.trim_end().to_owned(); + description.push_str("\n\n"); + for trailer in &request.trailers { + description.push_str(&format!("{}: {}\n", trailer.key, trailer.value)); + } + description.push_str(&format!( + "{OPERATION_ID_TRAILER}: {}\n", + request.operation_id + )); + description +} + +fn create_repo_dir(repo_dir: &Path) -> CommitResult<()> { + // The parent (normally the realm directory) must already exist, so a + // mistyped path fails instead of quietly growing a new tree. + match std::fs::create_dir(repo_dir) { + Ok(()) => Ok(()), + Err(e) if e.kind() == ErrorKind::AlreadyExists => Ok(()), + Err(e) if e.kind() == ErrorKind::NotFound => Err(CommitError::Invalid(format!( + "parent directory of {} does not exist", + repo_dir.display() + ))), + Err(e) => Err(internal(e)), + } +} + +/// Load the repo, creating it first if needed. Callers hold the repo lock. +/// The repo is built in a temporary directory and renamed into place, so a +/// crash mid-init never leaves a half-made `jj/`. +async fn load_or_init(repo_dir: &Path) -> CommitResult> { + let jj_dir = repo_dir.join("jj"); + // A cached negative lookup can make an existing `jj/` look absent; the + // rename in `install_repo` is what decides. + if !jj_dir.is_dir() { + let staging = repo_dir.join("deckd").join(format!("init-{}", nonce())); + std::fs::create_dir_all(&staging).map_err(internal)?; + let settings = user_settings(None).map_err(internal)?; + let backend_initializer: &jj_lib::repo::BackendInitializer = &|settings, store_path| { + Ok(Box::new(jj_lib::git_backend::GitBackend::init_internal( + settings, + store_path, + gix::hash::Kind::Sha1, + )?)) + }; + ReadonlyRepo::init( + &settings, + &staging, + backend_initializer, + signer_from_settings(&settings).map_err(internal)?, + ReadonlyRepo::default_workspace_store_initializer(), + ReadonlyRepo::default_op_store_initializer(), + ReadonlyRepo::default_op_heads_store_initializer(), + ReadonlyRepo::default_index_store_initializer(), + ReadonlyRepo::default_submodule_store_initializer(), + ) + .await + .map_err(internal)?; + install_repo(&staging, &jj_dir)?; + } + load_repo(repo_dir).await +} + +/// Move a freshly initialized repo into place. When `jj/` already exists the +/// staged copy is discarded and the existing repo wins. +fn install_repo(staging: &Path, jj_dir: &Path) -> CommitResult<()> { + match std::fs::rename(staging, jj_dir) { + Ok(()) => Ok(()), + Err(e) + if matches!( + e.kind(), + ErrorKind::AlreadyExists | ErrorKind::DirectoryNotEmpty + ) => + { + std::fs::remove_dir_all(staging).map_err(internal) + } + Err(e) => Err(internal(e)), + } +} + +async fn load_repo(repo_dir: &Path) -> CommitResult> { + let settings = user_settings(None).map_err(internal)?; + RepoLoader::init_from_file_system( + &settings, + &repo_dir.join("jj"), + &default_backend_factories(), + ) + .map_err(internal)? + .load_at_head() + .await + .map_err(internal) +} + +/// HEAD is the single visible head. History is linear, so more than one head +/// means two writers raced past the lock; that is reported, never resolved. +fn head_commit(repo: &Arc) -> CommitResult { + let heads = repo.view().heads(); + let mut iter = heads.iter(); + match (iter.next(), iter.next()) { + (Some(head), None) => repo.store().get_commit(head).map_err(internal), + _ => Err(CommitError::Internal(format!( + "history has {} heads; expected exactly one", + heads.len() + ))), + } +} + +fn get_commit(repo: &Arc, revision: &str) -> CommitResult { + let not_found = || CommitError::RevisionNotFound(revision.to_owned()); + let id = CommitId::try_from_hex(revision).ok_or_else(not_found)?; + match repo.store().get_commit(&id) { + Ok(commit) => Ok(commit), + Err(BackendError::ObjectNotFound { .. }) => Err(not_found()), + Err(e) => Err(internal(e)), + } +} + +fn tree_hash(commit: &Commit) -> String { + match commit.tree_ids().as_resolved() { + Some(tree_id) => tree_id.hex(), + // Explicit commits never write conflicted trees. + None => String::new(), + } +} + +/// `id` as a commit in history: an ancestor of, or one of, `heads`. A commit +/// object that was written but never published isn't in the index. +async fn published_commit( + repo: &Arc, + heads: &[CommitId], + id: &CommitId, +) -> CommitResult> { + let index = repo.index(); + if !index.has_id(id).await.map_err(internal)? { + return Ok(None); + } + for head in heads { + if index.is_ancestor(id, head).await.map_err(internal)? { + return Ok(Some(repo.store().get_commit(id).map_err(internal)?)); + } + } + Ok(None) +} + +/// Sync the git objects `commit` added: the commit, every tree on the way to +/// a changed path, and each written file. git doesn't sync loose objects, and +/// recording a commit whose objects a power loss could drop would leave +/// `deckd/HEAD` naming a revision the store doesn't have. An object with no +/// loose file was already packed, and so already durable. +async fn sync_new_objects( + repo_dir: &Path, + commit: &Commit, + changes: &[Change], +) -> CommitResult<()> { + let tree = commit.tree(); + let mut ids = BTreeSet::from([commit.id().hex(), tree_hash(commit)]); + for change in changes { + let path = RepoPathBuf::from_internal_string(change.path.clone()).map_err(internal)?; + for ancestor in path.ancestors().skip(1) { + if ancestor.is_root() { + break; + } + if let Some(Some(TreeValue::Tree(id))) = tree + .path_value(ancestor) + .await + .map_err(internal)? + .as_resolved() + { + ids.insert(id.hex()); + } + } + if let Some(Some(TreeValue::File { id, .. })) = tree + .path_value(&path) + .await + .map_err(internal)? + .as_resolved() + { + ids.insert(id.hex()); + } + } + let mut dirs = BTreeSet::new(); + for id in &ids { + let path = loose_object_path(repo_dir, id); + match File::open(&path) { + Ok(file) => file.sync_all().map_err(internal)?, + Err(e) if e.kind() == ErrorKind::NotFound => continue, + Err(e) => return Err(internal(e)), + } + // The fan-out directory, and `objects/` in case that was new too. + let dir = path + .parent() + .ok_or_else(|| internal("object has no parent"))?; + dirs.insert(dir.to_path_buf()); + dirs.extend(dir.parent().map(Path::to_path_buf)); + } + dirs.iter().try_for_each(|dir| sync_dir(dir)) +} + +fn loose_object_path(repo_dir: &Path, hex: &str) -> PathBuf { + repo_dir + .join("jj/store/git/objects") + .join(&hex[..2]) + .join(&hex[2..]) +} + +fn sync_dir(dir: &Path) -> CommitResult<()> { + File::open(dir).and_then(|d| d.sync_all()).map_err(internal) +} + +/// What the parent tree holds at `path`. +enum Existing { + Absent, + File { + executable: bool, + }, + Directory, + /// A symlink, submodule or conflict. + Other, +} + +async fn existing(tree: &MergedTree, path: &RepoPath) -> CommitResult { + let value = tree.path_value(path).await.map_err(internal)?; + Ok(match value.as_resolved() { + Some(None) => Existing::Absent, + Some(Some(TreeValue::Tree(_))) => Existing::Directory, + Some(Some(TreeValue::File { executable, .. })) => Existing::File { + executable: *executable, + }, + _ => Existing::Other, + }) +} + +/// HEAD's tree with `changes` applied. A change may not turn a file into a +/// directory or back: that would remove paths the request didn't list. A +/// write keeps an existing file's executable bit; it replaces a symlink or +/// submodule with a regular file holding the given bytes. +async fn build_tree( + repo: &Arc, + parent: &Commit, + changes: &[Change], +) -> CommitResult { + let store = repo.store(); + let base = parent.tree(); + let mut builder = MergedTreeBuilder::new(base.clone()); + for change in changes { + let path = RepoPathBuf::from_internal_string(change.path.clone()) + .map_err(|e| CommitError::Invalid(format!("invalid path {:?}: {e}", change.path)))?; + for ancestor in path.ancestors().skip(1) { + if ancestor.is_root() { + break; + } + if let Existing::File { .. } | Existing::Other = existing(&base, ancestor).await? { + return Err(CommitError::Invalid(format!( + "path {:?} is inside file {:?}", + change.path, + ancestor.as_internal_file_string() + ))); + } + } + let executable = match existing(&base, &path).await? { + Existing::Directory => { + return Err(CommitError::Invalid(format!( + "path {:?} is a directory; list the files inside it instead", + change.path + ))); + } + Existing::File { executable } => executable, + Existing::Absent | Existing::Other => false, + }; + let value = match &change.content { + Some(bytes) => { + let id = store + .write_file(&path, &mut bytes.as_slice()) + .await + .map_err(internal)?; + Merge::normal(TreeValue::File { + id, + executable, + copy_id: CopyId::placeholder(), + }) + } + None => Merge::absent(), + }; + builder.set_or_remove(path, value); + } + builder.write_tree().await.map_err(internal) +} + +/// An operation record. `Pending` and `Committed` carry the digest of the +/// request that claimed the operation; it is absent only when the record was +/// rebuilt from HEAD without a readable claim. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Record { + Pending(Option), + Committed { + revision: String, + digest: Option, + }, + Fenced, +} + +impl Record { + fn encode(&self) -> String { + let with_digest = |text: String, digest: &Option| match digest { + Some(digest) => format!("{text} {digest}\n"), + None => format!("{text}\n"), + }; + match self { + Record::Pending(digest) => with_digest("pending".into(), digest), + Record::Committed { revision, digest } => { + with_digest(format!("committed {revision}"), digest) + } + Record::Fenced => "fenced\n".into(), + } + } + + fn decode(text: &str) -> CommitResult { + let words: Vec<&str> = text.split_whitespace().collect(); + let digest = |word: Option<&&str>| word.map(|w| w.to_string()); + match words.as_slice() { + [] => Ok(Record::Pending(None)), + ["pending", rest @ ..] if rest.len() <= 1 => Ok(Record::Pending(digest(rest.first()))), + ["fenced"] => Ok(Record::Fenced), + ["committed", revision, rest @ ..] if rest.len() <= 1 => Ok(Record::Committed { + revision: revision.to_string(), + digest: digest(rest.first()), + }), + _ => Err(CommitError::Internal(format!( + "unreadable operation record {:?}", + text.trim() + ))), + } + } +} + +/// A digest of everything a commit request asks for except its parent. A +/// retry may name a newer parent after a conflict; anything else differing +/// means the operation ID was reused for another write. +fn request_digest(request: &CommitRequest) -> String { + fn field(hasher: &mut Sha256, bytes: &[u8]) { + hasher.update((bytes.len() as u64).to_be_bytes()); + hasher.update(bytes); + } + let mut hasher = Sha256::new(); + field(&mut hasher, request.author.name.as_bytes()); + field(&mut hasher, request.author.email.as_bytes()); + field(&mut hasher, request.message.as_bytes()); + hasher.update((request.trailers.len() as u64).to_be_bytes()); + for trailer in &request.trailers { + field(&mut hasher, trailer.key.as_bytes()); + field(&mut hasher, trailer.value.as_bytes()); + } + // Paths are unique, so their order carries no meaning. + let mut changes: Vec<&Change> = request.changes.iter().collect(); + changes.sort_by(|a, b| a.path.cmp(&b.path)); + hasher.update((changes.len() as u64).to_be_bytes()); + for change in changes { + field(&mut hasher, change.path.as_bytes()); + match &change.content { + Some(bytes) => { + field(&mut hasher, b"write"); + field(&mut hasher, bytes); + } + None => field(&mut hasher, b"delete"), + } + } + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn operation_record_path(repo_dir: &Path, operation_id: &str) -> PathBuf { + repo_dir.join("deckd").join("operations").join(operation_id) +} + +fn record_dir(path: &Path) -> CommitResult<&Path> { + path.parent() + .ok_or_else(|| CommitError::Internal("operation record has no parent".into())) +} + +/// Create the record with `create_new`. Returns false when it already exists. +/// An empty record (a crash between create and write) reads as `pending`. +/// The record and its directory entry are synced, so a crash can't undo a +/// claim this call reported. +fn claim_record(path: &Path, record: &Record) -> CommitResult { + let mut file = match OpenOptions::new().write(true).create_new(true).open(path) { + Ok(file) => file, + Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(false), + Err(e) => return Err(internal(e)), + }; + file.write_all(record.encode().as_bytes()) + .map_err(internal)?; + file.sync_all().map_err(internal)?; + sync_dir(record_dir(path)?)?; + Ok(true) +} + +fn read_record(path: &Path) -> CommitResult { + Record::decode(&std::fs::read_to_string(path).map_err(internal)?) +} + +/// Replace a record atomically: write a temporary file, then rename it over. +fn write_record(path: &Path, record: &Record) -> CommitResult<()> { + let dir = record_dir(path)?; + let name = path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| CommitError::Internal("operation record has no name".into()))?; + let tmp = dir.join(format!(".{name}.{}", nonce())); + { + let mut file = File::create(&tmp).map_err(internal)?; + file.write_all(record.encode().as_bytes()) + .map_err(internal)?; + file.sync_all().map_err(internal)?; + } + std::fs::rename(&tmp, path).map_err(internal)?; + sync_dir(dir) +} + +/// Remove a claim that never published. +fn remove_record(path: &Path) -> CommitResult<()> { + std::fs::remove_file(path).map_err(internal)?; + sync_dir(record_dir(path)?) +} + +/// Make HEAD's operation record say `committed HEAD`, keeping the digest a +/// pending claim recorded. The record is written whenever it doesn't already +/// say so, including when reading it fails: a missing file may only be a +/// stale NFS lookup, and the write is the truth. A `fenced` record for HEAD's +/// operation means the lock failed to exclude a writer; that is refused +/// loudly, never overwritten. +fn repair_head_record(repo_dir: &Path, head: &Commit) -> CommitResult<()> { + let Some(operation_id) = parse_description_trailers(head.description()) + .into_iter() + .find(|t| t.key == OPERATION_ID_TRAILER) + .map(|t| t.value) + else { + return Ok(()); + }; + if validate_operation_id(&operation_id).is_err() { + return Ok(()); + } + let path = operation_record_path(repo_dir, &operation_id); + let revision = head.id().hex(); + let digest = match read_record(&path).ok() { + Some(Record::Committed { + revision: recorded, .. + }) if recorded == revision => return Ok(()), + Some(Record::Fenced) => { + let message = format!( + "operation {operation_id} was reported absent but committed as {revision}; \ + history needs an operator" + ); + tracing::error!(repo = %repo_dir.display(), "{message}"); + return Err(CommitError::Internal(message)); + } + Some(Record::Pending(digest) | Record::Committed { digest, .. }) => digest, + None => None, + }; + write_record(&path, &Record::Committed { revision, digest }) +} + +fn nonce() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or_default(); + format!("{}-{nanos}", std::process::id()) +} + +#[cfg(test)] +mod tests { + use futures::AsyncReadExt as _; + + use super::*; + + /// A realm directory whose history repo is `/.history`. The repo + /// directory itself doesn't exist yet; deckd creates it. + struct Realm { + root: PathBuf, + } + + impl Realm { + fn new(name: &str) -> Self { + let root = std::env::temp_dir().join(format!("deckd-commit-{name}-{}", nonce())); + std::fs::create_dir_all(&root).unwrap(); + Self { root } + } + + fn repo(&self) -> PathBuf { + self.root.join(".history") + } + } + + impl Drop for Realm { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.root); + } + } + + fn write(path: &str, content: &str) -> Change { + Change { + path: path.into(), + content: Some(content.as_bytes().to_vec()), + } + } + + fn delete(path: &str) -> Change { + Change { + path: path.into(), + content: None, + } + } + + fn request(parent: &str, operation_id: &str, changes: Vec) -> CommitRequest { + CommitRequest { + parent: parent.into(), + operation_id: operation_id.into(), + author: Author { + name: "Jane".into(), + email: "jane@example.com".into(), + }, + message: format!("Write for {operation_id}"), + trailers: vec![CommitTrailer { + key: "Boxel-Channel".into(), + value: "host".into(), + }], + changes, + } + } + + fn head(service: &CommitService, realm: &Realm) -> String { + service.head(&realm.repo()).unwrap().revision + } + + fn commit( + service: &CommitService, + realm: &Realm, + req: CommitRequest, + ) -> CommitResult { + service.commit(&realm.repo(), req) + } + + fn outcome(service: &CommitService, realm: &Realm, operation_id: &str) -> OperationOutcome { + service.outcome(&realm.repo(), operation_id).unwrap() + } + + /// Every file at `revision`, with its content. + fn files_at(realm: &Realm, revision: &str) -> Vec<(String, String)> { + pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let commit = get_commit(&repo, revision).unwrap(); + let mut files = Vec::new(); + for (path, value) in commit.tree().entries() { + let Some(Some(TreeValue::File { id, .. })) = value.unwrap().into_resolved().ok() + else { + panic!("unexpected tree value at {path:?}"); + }; + let mut content = String::new(); + repo.store() + .read_file(&path, &id) + .await + .unwrap() + .read_to_string(&mut content) + .await + .unwrap(); + files.push((path.as_internal_file_string().to_owned(), content)); + } + files + }) + } + + fn description_at(realm: &Realm, revision: &str) -> String { + pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + get_commit(&repo, revision) + .unwrap() + .description() + .to_owned() + }) + } + + fn files(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(p, c)| (p.to_string(), c.to_string())) + .collect() + } + + #[test] + fn commit_contains_exactly_the_listed_paths() { + let realm = Realm::new("explicit"); + let service = CommitService::new(); + let root = head(&service, &realm); + // Bytes on disk that no request listed: a working-copy snapshot would + // pick these up. + std::fs::write(realm.root.join("unlisted.json"), "{}").unwrap(); + std::fs::write(realm.repo().join("stray.txt"), "stray").unwrap(); + + let done = commit( + &service, + &realm, + request( + &root, + "op-1", + vec![write("Author/jane.json", "{\"name\":\"Jane\"}")], + ), + ) + .unwrap(); + + assert!(done.created); + assert!(!done.replayed); + assert_eq!(head(&service, &realm), done.revision); + assert_eq!( + files_at(&realm, &done.revision), + files(&[("Author/jane.json", "{\"name\":\"Jane\"}")]) + ); + } + + #[test] + fn commit_records_message_author_and_trailers() { + let realm = Realm::new("description"); + let service = CommitService::new(); + let root = head(&service, &realm); + + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "a")]), + ) + .unwrap(); + + assert_eq!( + description_at(&realm, &done.revision), + "Write for op-1\n\nBoxel-Channel: host\nDeckd-Operation-Id: op-1\n" + ); + let author = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + get_commit(&repo, &done.revision).unwrap().author().clone() + }); + assert_eq!( + (author.name.as_str(), author.email.as_str()), + ("Jane", "jane@example.com") + ); + } + + #[test] + fn stale_parent_is_refused_with_current_head() { + let realm = Realm::new("cas"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + + let error = commit( + &service, + &realm, + request(&root, "op-2", vec![write("b.txt", "2")]), + ) + .unwrap_err(); + + match error { + CommitError::ParentMismatch { + parent, + head: current, + } => { + assert_eq!(parent, root); + assert_eq!(current, first.revision); + } + other => panic!("expected parent mismatch, got {other:?}"), + } + assert_eq!(head(&service, &realm), first.revision); + } + + #[test] + fn conflicted_operation_can_retry_on_the_new_head() { + let realm = Realm::new("cas-retry"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + commit( + &service, + &realm, + request(&root, "op-2", vec![write("b.txt", "2")]), + ) + .unwrap_err(); + + let retried = commit( + &service, + &realm, + request(&first.revision, "op-2", vec![write("b.txt", "2")]), + ) + .unwrap(); + + assert!(retried.created); + assert_eq!( + files_at(&realm, &retried.revision), + files(&[("a.txt", "1"), ("b.txt", "2")]) + ); + } + + #[test] + fn retried_operation_returns_the_original_revision() { + let realm = Realm::new("idempotent"); + let service = CommitService::new(); + let root = head(&service, &realm); + let req = request(&root, "op-1", vec![write("a.txt", "1")]); + let first = commit(&service, &realm, req.clone()).unwrap(); + + // The retry's parent is now stale; the replay still wins. + let again = commit(&service, &realm, req).unwrap(); + + assert_eq!(again.revision, first.revision); + assert_eq!(again.tree, first.tree); + assert!(again.replayed); + assert!(!again.created); + assert_eq!(head(&service, &realm), first.revision); + let parent = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + get_commit(&repo, &first.revision) + .unwrap() + .parent_ids() + .to_vec() + }); + assert_eq!( + parent.iter().map(|id| id.hex()).collect::>(), + vec![root] + ); + } + + #[test] + fn absent_outcome_fences_the_operation_across_restarts() { + let realm = Realm::new("fence"); + let service = CommitService::new(); + let root = head(&service, &realm); + + assert_eq!(outcome(&service, &realm, "op-1"), OperationOutcome::Absent); + let refused = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ); + assert!(matches!(refused, Err(CommitError::OperationFenced(_)))); + + // A new instance has no memory of the fence; only the repo does. + let restarted = CommitService::new(); + let refused = commit( + &restarted, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ); + assert!(matches!(refused, Err(CommitError::OperationFenced(_)))); + assert_eq!( + outcome(&restarted, &realm, "op-1"), + OperationOutcome::Absent + ); + assert_eq!(head(&restarted, &realm), root); + } + + #[test] + fn outcome_reports_a_committed_operation() { + let realm = Realm::new("outcome-committed"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + + let restarted = CommitService::new(); + + assert_eq!( + outcome(&restarted, &realm, "op-1"), + OperationOutcome::Committed { + revision: done.revision.clone() + } + ); + // Asking doesn't fence a committed operation; a retry still replays. + let again = commit( + &restarted, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + assert_eq!(again.revision, done.revision); + } + + #[test] + fn commit_published_without_its_record_is_reported_committed() { + let realm = Realm::new("repair"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + let record = operation_record_path(&realm.repo(), "op-1"); + + // Crash after publish: the record still holds the claim, or is gone. + std::fs::write(&record, "pending\n").unwrap(); + assert_eq!( + outcome(&service, &realm, "op-1"), + OperationOutcome::Committed { + revision: done.revision.clone() + } + ); + std::fs::remove_file(&record).unwrap(); + let again = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + assert_eq!(again.revision, done.revision); + assert!(again.replayed); + } + + #[test] + fn claim_left_by_a_crash_before_publish_resolves_absent() { + let realm = Realm::new("pending"); + let service = CommitService::new(); + let root = head(&service, &realm); + // Create the repo layout, then leave a claim as a crashed commit would. + outcome(&service, &realm, "unrelated"); + std::fs::write(operation_record_path(&realm.repo(), "op-1"), "pending\n").unwrap(); + std::fs::write(operation_record_path(&realm.repo(), "op-2"), "").unwrap(); + + assert_eq!(outcome(&service, &realm, "op-1"), OperationOutcome::Absent); + let refused = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ); + assert!(matches!(refused, Err(CommitError::OperationFenced(_)))); + + // Retrying the crashed operation instead takes its claim over. + let done = commit( + &service, + &realm, + request(&root, "op-2", vec![write("b.txt", "2")]), + ) + .unwrap(); + assert!(done.created); + } + + #[test] + fn racing_writers_stay_linear() { + let realm = Realm::new("race"); + let root = head(&CommitService::new(), &realm); + let writers = 8; + + // Each writer is its own service, as separate deckd processes would be. + let first_round: Vec<_> = std::thread::scope(|s| { + let handles: Vec<_> = (0..writers) + .map(|i| { + let realm = &realm; + let root = root.clone(); + s.spawn(move || { + let service = CommitService::new(); + commit( + &service, + realm, + request( + &root, + &format!("op-{i}"), + vec![write(&format!("{i}.txt"), "x")], + ), + ) + }) + }) + .collect(); + handles.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert_eq!(first_round.iter().filter(|r| r.is_ok()).count(), 1); + assert!(first_round + .iter() + .filter_map(|r| r.as_ref().err()) + .all(|e| matches!(e, CommitError::ParentMismatch { .. }))); + + // Losers retry the same operation on whatever HEAD the conflict reports. + std::thread::scope(|s| { + for i in 0..writers { + let realm = &realm; + s.spawn(move || { + let service = CommitService::new(); + let mut parent = head(&service, realm); + loop { + let req = request( + &parent, + &format!("op-{i}"), + vec![write(&format!("{i}.txt"), "x")], + ); + match commit(&service, realm, req) { + Ok(_) => return, + Err(CommitError::ParentMismatch { head, .. }) => parent = head, + Err(other) => panic!("writer {i}: {other:?}"), + } + } + }); + } + }); + + let service = CommitService::new(); + let tip = head(&service, &realm); + assert_eq!(files_at(&realm, &tip).len(), writers); + let depth = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let mut depth = 0; + let mut commit = get_commit(&repo, &tip).unwrap(); + while commit.id() != repo.store().root_commit_id() { + assert_eq!(commit.parent_ids().len(), 1); + commit = repo.store().get_commit(&commit.parent_ids()[0]).unwrap(); + depth += 1; + } + depth + }); + assert_eq!(depth, writers); + } + + #[test] + fn parent_matches_regardless_of_hex_case() { + let realm = Realm::new("hex-case"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + + let second = commit( + &service, + &realm, + request( + &first.revision.to_uppercase(), + "op-2", + vec![write("b.txt", "2")], + ), + ) + .unwrap(); + + assert!(second.created); + } + + /// Overwrite the locked `deckd/HEAD`, as a crashed or partitioned peer + /// would leave it. + fn set_recorded_head(realm: &Realm, revision: &str) { + let mut file = lock_repo(&realm.repo(), Instant::now() + LOCK_TIMEOUT).unwrap(); + write_recorded_head(&mut file, revision).unwrap(); + } + + /// Write a commit for `operation_id` without publishing it, as a writer + /// that crashed between recording and publishing would. + fn unpublished_commit(realm: &Realm, parent: &str, operation_id: &str) -> String { + pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let head = get_commit(&repo, parent).unwrap(); + let req = request(parent, operation_id, vec![write("crashed.txt", "x")]); + let tree = build_tree(&repo, &head, &req.changes).await.unwrap(); + let mut tx = repo.start_transaction(); + let commit = tx + .repo_mut() + .new_commit(vec![head.id().clone()], tree) + .set_description(description(&req)) + .write() + .await + .unwrap(); + commit.id().hex() + }) + } + + fn assert_invalid(result: CommitResult, needle: &str) { + match result { + Err(CommitError::Invalid(message)) => { + assert!(message.contains(needle), "{message:?} lacks {needle:?}") + } + other => panic!("expected invalid request, got {other:?}"), + } + } + + #[test] + fn changes_may_not_cross_a_file_directory_boundary() { + let realm = Realm::new("boundary"); + let service = CommitService::new(); + let root = head(&service, &realm); + let base = commit( + &service, + &realm, + request( + &root, + "op-base", + vec![write("a", "1"), write("d/x", "1"), write("d/y", "1")], + ), + ) + .unwrap(); + let attempt = + |op: &str, changes| commit(&service, &realm, request(&base.revision, op, changes)); + + assert_invalid(attempt("op-1", vec![write("a/b", "2")]), "inside file"); + assert_invalid(attempt("op-2", vec![write("d", "2")]), "is a directory"); + assert_invalid(attempt("op-3", vec![delete("d")]), "is a directory"); + assert_invalid( + attempt("op-4", vec![write("e", "2"), write("e/f", "2")]), + "inside listed path", + ); + assert_invalid( + attempt("op-5", vec![delete("a"), write("a/b", "2")]), + "inside listed path", + ); + + // Refused requests leave no claim, so the IDs stay usable. + let fixed = attempt("op-1", vec![write("a2/b", "2")]).unwrap(); + assert!(fixed.created); + assert_eq!( + files_at(&realm, &fixed.revision), + files(&[("a", "1"), ("a2/b", "2"), ("d/x", "1"), ("d/y", "1")]) + ); + } + + #[test] + fn dot_and_git_segments_are_refused() { + let realm = Realm::new("dot-paths"); + let service = CommitService::new(); + let root = head(&service, &realm); + + for path in [".", "x/./y", ".git/config", "x/.GIT/hooks", "x/.git"] { + assert_invalid( + commit( + &service, + &realm, + request(&root, "op", vec![write(path, "")]), + ), + "invalid path", + ); + } + // Names that only start with those segments are ordinary files. + let done = commit( + &service, + &realm, + request( + &root, + "op", + vec![write(".gitignore", ""), write("x/.github/ci", "")], + ), + ) + .unwrap(); + assert!(done.created); + } + + #[test] + fn lock_file_records_head_and_persists() { + let realm = Realm::new("head-file"); + let service = CommitService::new(); + let root = head(&service, &realm); + + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + + assert_eq!(recorded_head(&realm), Some(done.revision.clone())); + let next = commit( + &service, + &realm, + request(&done.revision, "op-2", vec![write("b", "2")]), + ) + .unwrap(); + assert_eq!(recorded_head(&realm), Some(next.revision)); + } + + fn recorded_head(realm: &Realm) -> Option { + let mut file = File::open(realm.repo().join("deckd/HEAD")).unwrap(); + read_head_slots(&mut file) + .unwrap() + .map(|(_, revision)| revision.hex()) + } + + #[test] + fn torn_head_write_keeps_the_previous_revision() { + let realm = Realm::new("torn-head"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + let second = commit( + &service, + &realm, + request(&first.revision, "op-2", vec![write("b", "2")]), + ) + .unwrap(); + // A crash partway through recording a third revision: half the + // bytes that write changes are new, half are still the old ones. + let head_path = realm.repo().join("deckd/HEAD"); + let before = std::fs::read(&head_path).unwrap(); + set_recorded_head(&realm, &"cd".repeat(20)); + let after = std::fs::read(&head_path).unwrap(); + let changed: Vec = (0..after.len()) + .filter(|&i| before.get(i) != Some(&after[i])) + .collect(); + let mut torn = after.clone(); + for &i in &changed[changed.len() / 2..] { + torn[i] = before.get(i).copied().unwrap_or(0); + } + std::fs::write(&head_path, torn).unwrap(); + + assert_eq!(recorded_head(&realm), Some(second.revision.clone())); + assert_eq!(head(&service, &realm), second.revision); + let next = commit( + &service, + &realm, + request(&second.revision, "op-3", vec![write("c", "3")]), + ) + .unwrap(); + assert_eq!(recorded_head(&realm), Some(next.revision)); + } + + #[test] + fn new_objects_are_loose_where_they_are_synced() { + let realm = Realm::new("loose-objects"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("dir/a.json", "{}")]), + ) + .unwrap(); + + let (dir_tree, blob) = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let tree = get_commit(&repo, &done.revision).unwrap().tree(); + let value = |path: &str| { + let path = RepoPathBuf::from_internal_string(path.to_owned()).unwrap(); + let tree = tree.clone(); + async move { + match tree.path_value(&path).await.unwrap().into_resolved() { + Ok(Some(TreeValue::Tree(id))) => id.hex(), + Ok(Some(TreeValue::File { id, .. })) => id.hex(), + other => panic!("unexpected {other:?}"), + } + } + }; + (value("dir").await, value("dir/a.json").await) + }); + for id in [&done.revision, &done.tree, &dir_tree, &blob] { + assert!(loose_object_path(&realm.repo(), id).is_file(), "{id}"); + } + } + + #[test] + fn commit_recorded_but_not_published_is_published_by_the_next_call() { + let realm = Realm::new("roll-forward"); + let service = CommitService::new(); + let root = head(&service, &realm); + let revision = unpublished_commit(&realm, &root, "op-crashed"); + set_recorded_head(&realm, &revision); + let crashed = request(&root, "op-crashed", vec![write("crashed.txt", "x")]); + std::fs::write( + operation_record_path(&realm.repo(), "op-crashed"), + Record::Pending(Some(request_digest(&crashed))).encode(), + ) + .unwrap(); + + let restarted = CommitService::new(); + + assert_eq!( + outcome(&restarted, &realm, "op-crashed"), + OperationOutcome::Committed { + revision: revision.clone() + } + ); + assert_eq!(head(&restarted, &realm), revision); + assert_eq!(files_at(&realm, &revision), files(&[("crashed.txt", "x")])); + // The repaired record kept the claim's digest. + let replay = commit(&restarted, &realm, crashed.clone()).unwrap(); + assert!(replay.replayed); + let mut different = crashed; + different.changes = vec![write("crashed.txt", "y")]; + assert!(matches!( + commit(&restarted, &realm, different), + Err(CommitError::OperationMismatch { .. }) + )); + let next = commit( + &restarted, + &realm, + request(&revision, "op-next", vec![write("b", "2")]), + ) + .unwrap(); + assert!(next.created); + } + + #[test] + fn head_that_disagrees_with_the_record_is_refused() { + let realm = Realm::new("unsettled"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + // A recorded HEAD jj can't reach, as a stale listing would look. + set_recorded_head(&realm, &"ab".repeat(20)); + + let error = commit( + &service, + &realm, + request(&first.revision, "op-2", vec![write("b", "2")]), + ) + .unwrap_err(); + + assert!( + matches!(error, CommitError::HistoryUnsettled { .. }), + "{error:?}" + ); + // Nothing was claimed or written. + set_recorded_head(&realm, &first.revision); + assert_eq!(outcome(&service, &realm, "op-2"), OperationOutcome::Absent); + assert_eq!(head(&service, &realm), first.revision); + } + + #[test] + fn fenced_record_for_head_is_reported_not_overwritten() { + let realm = Realm::new("fence-violated"); + let service = CommitService::new(); + let root = head(&service, &realm); + commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + let record = operation_record_path(&realm.repo(), "op-1"); + std::fs::write(&record, "fenced\n").unwrap(); + + let error = service.outcome(&realm.repo(), "op-2").unwrap_err(); + + assert!( + error.to_string().contains("reported absent but committed"), + "{error}" + ); + assert_eq!(std::fs::read_to_string(&record).unwrap(), "fenced\n"); + } + + #[test] + fn initializing_over_an_existing_repo_keeps_it() { + let realm = Realm::new("init-race"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + let staging = realm.repo().join("deckd/init-late"); + std::fs::create_dir_all(staging.join("store")).unwrap(); + + install_repo(&staging, &realm.repo().join("jj")).unwrap(); + + assert!(!staging.exists()); + assert_eq!(head(&service, &realm), done.revision); + } + + #[test] + fn tree_hash_in_a_missing_repo_is_not_found() { + let realm = Realm::new("tree-no-repo"); + let error = CommitService::new() + .tree_hash(&realm.repo(), &"ab".repeat(20)) + .unwrap_err(); + assert!(matches!(error, CommitError::RevisionNotFound(_))); + } + + #[test] + fn delete_removes_the_path() { + let realm = Realm::new("delete"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request( + &root, + "op-1", + vec![write("dir/a.txt", "a"), write("b.txt", "b")], + ), + ) + .unwrap(); + + let second = commit( + &service, + &realm, + request(&first.revision, "op-2", vec![delete("dir/a.txt")]), + ) + .unwrap(); + + assert!(second.created); + assert_eq!(files_at(&realm, &second.revision), files(&[("b.txt", "b")])); + assert_eq!( + files_at(&realm, &first.revision), + files(&[("b.txt", "b"), ("dir/a.txt", "a")]) + ); + } + + #[test] + fn unchanged_content_records_no_revision() { + let realm = Realm::new("noop"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + + let same = commit( + &service, + &realm, + request( + &first.revision, + "op-2", + vec![write("a.txt", "1"), delete("missing.txt")], + ), + ) + .unwrap(); + + assert!(!same.created); + assert_eq!(same.revision, first.revision); + assert_eq!(head(&service, &realm), first.revision); + assert_eq!( + outcome(&service, &realm, "op-2"), + OperationOutcome::Committed { + revision: first.revision + } + ); + } + + #[test] + fn identical_trees_have_identical_hashes() { + let one = Realm::new("tree-one"); + let two = Realm::new("tree-two"); + let service = CommitService::new(); + let changes = || vec![write("Author/jane.json", "{}"), write("index.json", "[]")]; + let root_one = head(&service, &one); + let root_two = head(&service, &two); + + let a = commit(&service, &one, request(&root_one, "op-a", changes())).unwrap(); + let mut other = request(&root_two, "op-b", changes()); + other.author.name = "Someone Else".into(); + other.message = "A different message".into(); + let b = commit(&service, &two, other).unwrap(); + + assert_ne!(a.revision, b.revision); + assert_eq!(a.tree, b.tree); + let looked_up = service.tree_hash(&one.repo(), &a.revision).unwrap(); + assert_eq!(looked_up.tree, a.tree); + let again = CommitService::new() + .tree_hash(&one.repo(), &a.revision) + .unwrap(); + assert_eq!(again.tree, a.tree); + } + + #[test] + fn tree_hash_of_an_unknown_revision_is_not_found() { + let realm = Realm::new("tree-missing"); + let service = CommitService::new(); + head(&service, &realm); + + let error = service + .tree_hash(&realm.repo(), &"ab".repeat(20)) + .unwrap_err(); + + assert!(matches!(error, CommitError::RevisionNotFound(_))); + } + + #[test] + fn invalid_requests_are_refused() { + let realm = Realm::new("invalid"); + let service = CommitService::new(); + let root = head(&service, &realm); + let invalid = |req: CommitRequest| { + matches!(commit(&service, &realm, req), Err(CommitError::Invalid(_))) + }; + + assert!(invalid(request(&root, "../escape", vec![write("a", "")]))); + assert!(invalid(request(&root, ".hidden", vec![write("a", "")]))); + assert!(invalid(request("not-hex", "op", vec![write("a", "")]))); + assert!(invalid(request(&root, "op", vec![write("/abs", "")]))); + assert!(invalid(request(&root, "op", vec![write("a/../b", "")]))); + assert!(invalid(request( + &root, + "op", + vec![write("a", "1"), delete("a")] + ))); + let mut no_message = request(&root, "op", vec![write("a", "")]); + no_message.message = " \n".into(); + assert!(invalid(no_message)); + let mut bad_author = request(&root, "op", vec![write("a", "")]); + bad_author.author.email = "jane@example.com> evil".into(); + assert!(invalid(bad_author)); + let mut reserved = request(&root, "op", vec![write("a", "")]); + reserved.trailers.push(CommitTrailer { + key: "deckd-operation-id".into(), + value: "other".into(), + }); + assert!(invalid(reserved)); + let relative = service.head(Path::new("relative/.history")); + assert!(matches!(relative, Err(CommitError::Invalid(_)))); + let orphan = service.head(&realm.root.join("missing/.history")); + assert!(matches!(orphan, Err(CommitError::Invalid(_)))); + // None of those wrote anything. + assert_eq!(head(&service, &realm), root); + } + + #[test] + fn reused_operation_id_with_a_different_request_is_refused() { + let realm = Realm::new("op-reuse"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1"), delete("gone")]), + ) + .unwrap(); + + // The same request replays, from a newer parent and in any order. + let replay = commit( + &service, + &realm, + request( + &done.revision, + "op-1", + vec![delete("gone"), write("a", "1")], + ), + ) + .unwrap(); + assert!(replay.replayed); + assert_eq!(replay.revision, done.revision); + + let mut other_message = request(&root, "op-1", vec![write("a", "1"), delete("gone")]); + other_message.message = "Something else".into(); + for reused in [ + request(&root, "op-1", vec![write("a", "2"), delete("gone")]), + request(&root, "op-1", vec![write("a", "1")]), + other_message, + ] { + match commit(&service, &realm, reused) { + Err(CommitError::OperationMismatch { revision, .. }) => { + assert_eq!(revision, done.revision) + } + other => panic!("expected operation-mismatch, got {other:?}"), + } + } + assert_eq!(head(&service, &realm), done.revision); + } + + #[test] + fn unchanged_content_replay_is_checked_too() { + let realm = Realm::new("noop-reuse"); + let service = CommitService::new(); + let root = head(&service, &realm); + let base = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + commit( + &service, + &realm, + request(&base.revision, "op-noop", vec![write("a", "1")]), + ) + .unwrap(); + + let reused = commit( + &service, + &realm, + request(&base.revision, "op-noop", vec![write("a", "2")]), + ); + assert!(matches!(reused, Err(CommitError::OperationMismatch { .. }))); + } + + #[test] + fn spellings_of_one_repo_share_a_gate_that_is_pruned() { + let realm = Realm::new("gate"); + let service = CommitService::new(); + head(&service, &realm); + let alias = std::env::temp_dir().join(format!("deckd-commit-alias-{}", nonce())); + std::os::unix::fs::symlink(&realm.root, &alias).unwrap(); + + let direct = service.gate(&realm.repo()).unwrap(); + let via_alias = service + .gate(&service.resolve_repo_dir(&alias.join(".history")).unwrap()) + .unwrap(); + assert!(Arc::ptr_eq(&direct, &via_alias)); + + drop((direct, via_alias)); + let other = Realm::new("gate-other"); + head(&service, &other); + // Only the gate of the latest call is left, and nothing holds it. + let gates = service.gates.lock().unwrap(); + assert_eq!(gates.len(), 1); + assert!(gates.values().all(|gate| gate.strong_count() == 0)); + drop(gates); + std::fs::remove_file(alias).unwrap(); + } + + #[test] + fn busy_gate_times_out() { + let realm = Realm::new("gate-timeout"); + let service = CommitService::new(); + head(&service, &realm); + let gate = service.gate(&realm.repo()).unwrap(); + let _held = gate.lock().unwrap(); + + let started = Instant::now(); + let error = std::thread::scope(|s| { + s.spawn(|| service.head(&realm.repo()).unwrap_err()) + .join() + .unwrap() + }); + + assert!(matches!(error, CommitError::LockTimeout(_)), "{error:?}"); + assert!(started.elapsed() >= LOCK_TIMEOUT); + } + + #[test] + fn repos_outside_the_root_are_refused() { + let inside = Realm::new("root-inside"); + let outside = Realm::new("root-outside"); + let service = CommitService::with_root(&inside.root).unwrap(); + let escape = inside.root.join("escape"); + std::os::unix::fs::symlink(&outside.root, &escape).unwrap(); + + assert!(service.head(&inside.repo()).is_ok()); + for repo in [outside.repo(), escape, inside.root.clone()] { + assert!( + matches!(service.head(&repo), Err(CommitError::Invalid(_))), + "{repo:?}" + ); + } + assert!(matches!( + service.tree_hash(&outside.repo(), &"ab".repeat(20)), + Err(CommitError::Invalid(_)) + )); + assert!(!outside.repo().exists()); + } + + #[test] + fn unpublished_commit_has_no_tree_hash() { + let realm = Realm::new("orphan"); + let service = CommitService::new(); + let root = head(&service, &realm); + let orphan = unpublished_commit(&realm, &root, "op-crashed"); + + let error = service.tree_hash(&realm.repo(), &orphan).unwrap_err(); + + assert!( + matches!(error, CommitError::RevisionNotFound(_)), + "{error:?}" + ); + assert!(service.tree_hash(&realm.repo(), &root).is_ok()); + } + + #[test] + fn rewriting_an_executable_file_keeps_it_executable() { + let realm = Realm::new("executable"); + let service = CommitService::new(); + let root = head(&service, &realm); + let path = RepoPathBuf::from_internal_string("run.sh".to_owned()).unwrap(); + // Explicit commits write only regular files; imported history may not. + let seeded = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let parent = get_commit(&repo, &root).unwrap(); + let id = repo + .store() + .write_file(&path, &mut &b"#!/bin/sh\n"[..]) + .await + .unwrap(); + let mut builder = MergedTreeBuilder::new(parent.tree()); + builder.set_or_remove( + path.clone(), + Merge::normal(TreeValue::File { + id, + executable: true, + copy_id: CopyId::placeholder(), + }), + ); + let tree = builder.write_tree().await.unwrap(); + let mut tx = repo.start_transaction(); + let commit = tx + .repo_mut() + .new_commit(vec![parent.id().clone()], tree) + .set_description("seed\n") + .write() + .await + .unwrap(); + tx.commit("seed").await.unwrap(); + commit.id().hex() + }); + set_recorded_head(&realm, &seeded); + + let done = commit( + &service, + &realm, + request(&seeded, "op-1", vec![write("run.sh", "#!/bin/sh\necho\n")]), + ) + .unwrap(); + + let executable = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let tree = get_commit(&repo, &done.revision).unwrap().tree(); + match tree.path_value(&path).await.unwrap().into_resolved() { + Ok(Some(TreeValue::File { executable, .. })) => executable, + other => panic!("unexpected {other:?}"), + } + }); + assert!(executable); + } +} diff --git a/packages/deckd/src/history/jj_lib.rs b/packages/deckd/src/history/jj_lib.rs index eb749ba..9da75a3 100644 --- a/packages/deckd/src/history/jj_lib.rs +++ b/packages/deckd/src/history/jj_lib.rs @@ -22,6 +22,9 @@ use futures::StreamExt as _; use jj_lib::backend::{FileId, TreeValue}; use jj_lib::commit::Commit; use jj_lib::config::{ConfigLayer, ConfigSource, StackedConfig}; +use jj_lib::default_backend_factories::{ + default_backend_factories, default_working_copy_factories, +}; use jj_lib::fileset::FilesetExpression; use jj_lib::gitignore::GitIgnoreFile; use jj_lib::matchers::{EverythingMatcher, NothingMatcher}; @@ -29,14 +32,12 @@ use jj_lib::merge::Diff; use jj_lib::merged_tree::MergedTree; use jj_lib::object_id::{HexPrefix, ObjectId, PrefixResolution}; use jj_lib::ref_name::WorkspaceName; -use jj_lib::repo::{ReadonlyRepo, Repo as _, StoreFactories}; +use jj_lib::repo::{ReadonlyRepo, Repo as _}; use jj_lib::repo_path::{RepoPath, RepoPathBuf}; use jj_lib::revset::{RevsetExpression, RevsetFilterPredicate}; use jj_lib::settings::UserSettings; use jj_lib::working_copy::SnapshotOptions; -use jj_lib::workspace::{ - default_working_copy_factories, Workspace, WorkspaceInitError, WorkspaceLoadError, -}; +use jj_lib::workspace::{Workspace, WorkspaceInitError, WorkspaceLoadError}; use tokio::sync::Mutex; /// Per-depot async note debounce. Realm servers fire `/note` and return; @@ -432,7 +433,7 @@ pub fn is_valid_history_path(path: &str) -> bool { .any(|segment| segment.is_empty() || segment == "..") } -fn hermetic_git_env() { +pub(super) fn hermetic_git_env() { // Prevent GitBackend from reading host git config (same as historyd). unsafe { std::env::set_var("GIT_CONFIG_SYSTEM", "/dev/null"); @@ -440,7 +441,7 @@ fn hermetic_git_env() { } } -fn user_settings(actor: Option<&Actor>) -> HistoryResult { +pub(super) fn user_settings(actor: Option<&Actor>) -> HistoryResult { let name = actor .map(|a| sanitize(&a.name)) .unwrap_or_else(|| "deck-history".into()); @@ -461,7 +462,8 @@ fn user_settings(actor: Option<&Actor>) -> HistoryResult { ConfigLayer::parse(ConfigSource::User, &config_text) .map_err(|e| HistoryError::msg(e.to_string()))?, ); - UserSettings::from_config(config).map_err(|e| HistoryError::msg(e.to_string())) + UserSettings::from_config_and_home_dir(config, None) + .map_err(|e| HistoryError::msg(e.to_string())) } fn sanitize(value: &str) -> String { @@ -543,7 +545,8 @@ async fn ensure_repo_inner(dir: &Path) -> HistoryResult<()> { if !jj.is_dir() { // historyd used init_colocated_git; Deck forbids depot .git → internal. let settings = user_settings(None)?; - let (_workspace, _repo) = Workspace::init_internal_git(&settings, dir).await?; + let (_workspace, _repo) = + Workspace::init_internal_git(&settings, dir, gix::hash::Kind::Sha1).await?; } if !jj_repo.is_dir() { @@ -595,7 +598,7 @@ async fn load_workspace(dir: &Path) -> HistoryResult<(Workspace, Arc Ok(paths @@ -851,7 +863,7 @@ async fn files_summary_from_index( async fn file_at_inner(dir: &Path, revision_id: &str, path: &str) -> HistoryResult>> { let (_workspace, repo) = load_workspace(dir).await?; - let commit = match resolve_revision(repo.as_ref(), revision_id)? { + let commit = match resolve_revision(repo.as_ref(), revision_id).await? { Some(c) => c, None => return Ok(None), }; @@ -872,7 +884,8 @@ async fn file_at_inner(dir: &Path, revision_id: &str, path: &str) -> HistoryResu async fn file_list_at_inner(dir: &Path, revision_id: &str) -> HistoryResult> { let (_workspace, repo) = load_workspace(dir).await?; - let commit = resolve_revision(repo.as_ref(), revision_id)? + let commit = resolve_revision(repo.as_ref(), revision_id) + .await? .ok_or_else(|| HistoryError::msg("revision not found"))?; let root = repo.store().root_commit(); let mut paths = Vec::new(); @@ -896,7 +909,8 @@ async fn file_list_at_inner(dir: &Path, revision_id: &str) -> HistoryResult HistoryResult { let (_workspace, repo) = load_workspace(dir).await?; - let target = resolve_revision(repo.as_ref(), revision_id)? + let target = resolve_revision(repo.as_ref(), revision_id) + .await? .ok_or_else(|| HistoryError::msg("revision not found"))?; let wc_id = repo .view() @@ -931,10 +945,14 @@ async fn restore_plan_inner(dir: &Path, revision_id: &str) -> HistoryResult HistoryResult> { +async fn resolve_revision( + repo: &ReadonlyRepo, + revision_id: &str, +) -> HistoryResult> { if let Some(prefix) = HexPrefix::try_from_reverse_hex(revision_id) { match repo .resolve_change_id_prefix(&prefix) + .await .map_err(|e| HistoryError::msg(e.to_string()))? { PrefixResolution::SingleMatch(targets) => { @@ -957,6 +975,7 @@ fn resolve_revision(repo: &ReadonlyRepo, revision_id: &str) -> HistoryResult { diff --git a/packages/deckd/src/history/mod.rs b/packages/deckd/src/history/mod.rs index 125b9e4..b0dfae4 100644 --- a/packages/deckd/src/history/mod.rs +++ b/packages/deckd/src/history/mod.rs @@ -7,6 +7,7 @@ use std::pin::Pin; use serde::{Deserialize, Serialize}; use thiserror::Error; +pub mod commit; pub mod jj_lib; pub mod jj_lib_notes; diff --git a/packages/deckd/src/main.rs b/packages/deckd/src/main.rs index 262d762..13c8dc2 100644 --- a/packages/deckd/src/main.rs +++ b/packages/deckd/src/main.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use tracing_subscriber::EnvFilter; use crate::api::AppState; +use crate::history::commit::CommitService; use crate::history::jj_lib::JjLibHistory; use crate::history::HistoryBackend; use crate::watch::WatchPlane; @@ -42,17 +43,27 @@ async fn main() -> Result<(), Box> { }; let history: Arc = Arc::new(jj_history); + // When set, explicit-commit repos must live under this directory. + let commits = Arc::new(match std::env::var_os("DECKD_REPO_ROOT") { + Some(root) => CommitService::with_root(std::path::Path::new(&root))?, + None => CommitService::new(), + }); #[cfg(feature = "build")] let state = { let jj_bin = std::env::var("JJ_BIN").unwrap_or_else(|_| "jj".into()); AppState { history, + commits, build: Some(Arc::new(crate::build::BuildPlane::new(jj_bin))), watch, } }; #[cfg(not(feature = "build"))] - let state = AppState { history, watch }; + let state = AppState { + history, + commits, + watch, + }; let app = api::router(state); let addr: SocketAddr = bind.parse()?; diff --git a/packages/deckd/tests/commit_restart.rs b/packages/deckd/tests/commit_restart.rs new file mode 100644 index 0000000..a63a744 --- /dev/null +++ b/packages/deckd/tests/commit_restart.rs @@ -0,0 +1,83 @@ +//! The commit API against a real deckd process that is killed and restarted +//! between calls: operation outcomes and fences live in the repo, so a new +//! process gives the same answers. + +use std::path::Path; + +use serde_json::{json, Value}; + +mod common; + +use common::{realm, Deckd}; + +fn commit_body(repo: &Path, parent: &str, operation_id: &str) -> Value { + json!({ + "repo": repo, + "parent": parent, + "operationId": operation_id, + "author": { "name": "Jane", "email": "jane@example.com" }, + "message": "Update Author/jane.json", + "trailers": [{ "key": "Boxel-Channel", "value": "host" }], + "changes": [ + { "path": "Author/jane.json", "contentBase64": "eyJuYW1lIjoiSmFuZSJ9" }, + { "path": "Author/old.json", "delete": true } + ] + }) +} + +#[test] +fn committed_outcome_survives_a_restart() { + let realm = realm("committed"); + let repo = realm.join(".history"); + let deckd = Deckd::start(); + let (_, head) = deckd.post("/head", json!({ "repo": repo })); + let root = head["revision"].as_str().unwrap().to_owned(); + + let (status, committed) = deckd.post("/commit", commit_body(&repo, &root, "op-1")); + assert_eq!(status, 200, "{committed}"); + deckd.kill(); + + let deckd = Deckd::start(); + let (status, outcome) = deckd.post("/outcome", json!({ "repo": repo, "operationId": "op-1" })); + assert_eq!(status, 200); + assert_eq!( + outcome, + json!({ "state": "committed", "revision": committed["revision"] }) + ); + let (_, retried) = deckd.post("/commit", commit_body(&repo, &root, "op-1")); + assert_eq!(retried["revision"], committed["revision"]); + assert_eq!(retried["replayed"], json!(true)); + let (_, tree) = deckd.post( + "/tree-hash", + json!({ "repo": repo, "revision": committed["revision"] }), + ); + assert_eq!(tree["tree"], committed["tree"]); + std::fs::remove_dir_all(realm).unwrap(); +} + +#[test] +fn fence_survives_a_restart() { + let realm = realm("fence"); + let repo = realm.join(".history"); + let deckd = Deckd::start(); + let (status, outcome) = deckd.post("/outcome", json!({ "repo": repo, "operationId": "op-1" })); + assert_eq!(status, 200); + assert_eq!(outcome, json!({ "state": "absent" })); + let (_, head) = deckd.post("/head", json!({ "repo": repo })); + let root = head["revision"].as_str().unwrap().to_owned(); + deckd.kill(); + + let deckd = Deckd::start(); + let (status, refused) = deckd.post("/commit", commit_body(&repo, &root, "op-1")); + assert_eq!(status, 409); + assert_eq!(refused["code"], json!("operation-fenced")); + + // The fence is per operation: other operations still commit, under CAS. + let (status, first) = deckd.post("/commit", commit_body(&repo, &root, "op-2")); + assert_eq!(status, 200, "{first}"); + let (status, conflict) = deckd.post("/commit", commit_body(&repo, &root, "op-3")); + assert_eq!(status, 409); + assert_eq!(conflict["code"], json!("parent-mismatch")); + assert_eq!(conflict["head"], first["revision"]); + std::fs::remove_dir_all(realm).unwrap(); +} diff --git a/packages/deckd/tests/common/mod.rs b/packages/deckd/tests/common/mod.rs new file mode 100644 index 0000000..0bceb9c --- /dev/null +++ b/packages/deckd/tests/common/mod.rs @@ -0,0 +1,89 @@ +//! Helpers for tests that drive a real deckd process over HTTP. + +#![allow(dead_code)] + +use std::net::TcpListener; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use serde_json::Value; + +pub struct Deckd { + child: Child, + pub base: String, +} + +impl Deckd { + /// Start deckd on a fresh port. A restart needs no particular port: all + /// state is in the repo. + pub fn start() -> Self { + let port = free_port(); + let child = Command::new(env!("CARGO_BIN_EXE_deckd")) + .env("DECKD_BIND", format!("127.0.0.1:{port}")) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start deckd"); + let base = format!("http://127.0.0.1:{port}"); + let started = Instant::now(); + while ureq::get(&format!("{base}/health")).call().is_err() { + assert!( + started.elapsed() < Duration::from_secs(20), + "deckd did not start" + ); + std::thread::sleep(Duration::from_millis(50)); + } + Self { child, base } + } + + /// POST `body` and return the status and JSON response, error or not. + pub fn post(&self, route: &str, body: Value) -> (u16, Value) { + match ureq::post(&format!("{}{route}", self.base)).send_json(body) { + Ok(response) => (response.status(), response.into_json().unwrap()), + Err(ureq::Error::Status(status, response)) => (status, response.into_json().unwrap()), + Err(error) => panic!("{route}: {error}"), + } + } + + pub fn kill(mut self) { + self.child.kill().unwrap(); + self.child.wait().unwrap(); + } +} + +impl Drop for Deckd { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +pub fn realm(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = + std::env::temp_dir().join(format!("deckd-http-{name}-{}-{nanos}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// POST raw bytes and return the status and JSON response, error or not. +pub fn post_raw(deckd: &Deckd, route: &str, content_type: &str, body: &str) -> (u16, Value) { + let request = ureq::post(&format!("{}{route}", deckd.base)).set("content-type", content_type); + match request.send_string(body) { + Ok(response) => (response.status(), response.into_json().unwrap()), + Err(ureq::Error::Status(status, response)) => (status, response.into_json().unwrap()), + Err(error) => panic!("{route}: {error}"), + } +} diff --git a/packages/deckd/tests/http_errors.rs b/packages/deckd/tests/http_errors.rs new file mode 100644 index 0000000..0272f0d --- /dev/null +++ b/packages/deckd/tests/http_errors.rs @@ -0,0 +1,85 @@ +//! Every error deckd returns, including a request it can't parse, is a JSON +//! body with a stable `code`. + +use std::io::{Read as _, Write as _}; +use std::net::TcpStream; +use std::time::Duration; + +use serde_json::{json, Value}; + +mod common; + +use common::{post_raw, realm, Deckd}; + +#[test] +fn unparseable_requests_get_typed_errors() { + let deckd = Deckd::start(); + let repo = realm("parse").join(".history"); + + let (status, body) = post_raw(&deckd, "/commit", "application/json", "{\"repo\":"); + assert_eq!( + (status, &body["code"]), + (400, &json!("invalid-request")), + "{body}" + ); + + let missing = json!({ "repo": repo }).to_string(); + let (status, body) = post_raw(&deckd, "/commit", "application/json", &missing); + assert_eq!( + (status, &body["code"]), + (422, &json!("invalid-request")), + "{body}" + ); + assert!(body["error"].as_str().unwrap().contains("parent"), "{body}"); + + let (status, body) = post_raw(&deckd, "/outcome", "text/plain", "{}"); + assert_eq!( + (status, &body["code"]), + (415, &json!("invalid-request")), + "{body}" + ); + + // Routes that predate the commit API answer the same way. + let (status, body) = post_raw(&deckd, "/list", "application/json", "[]"); + assert_eq!( + (status, &body["code"]), + (422, &json!("invalid-request")), + "{body}" + ); +} + +#[test] +fn oversized_commit_is_refused_as_too_large() { + let deckd = Deckd::start(); + let length = 65 * 1024 * 1024; + let mut stream = TcpStream::connect(deckd.base.trim_start_matches("http://")).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(30))) + .unwrap(); + write!( + stream, + "POST /commit HTTP/1.1\r\nHost: deckd\r\nContent-Type: application/json\r\n\ + Content-Length: {length}\r\nConnection: close\r\n\r\n" + ) + .unwrap(); + // deckd answers once the body passes the limit and stops reading, so the + // upload runs on its own thread and its write errors are expected. + let mut writer = stream.try_clone().unwrap(); + let upload = std::thread::spawn(move || { + let chunk = vec![b' '; 1024 * 1024]; + for _ in 0..length / chunk.len() { + if writer.write_all(&chunk).is_err() { + return; + } + } + }); + let mut response = Vec::new(); + let _ = stream.read_to_end(&mut response); + let _ = upload.join(); + + let response = String::from_utf8_lossy(&response); + let (head, body) = response.split_once("\r\n\r\n").expect("a response"); + assert!(head.starts_with("HTTP/1.1 413"), "{head}"); + let body: Value = serde_json::from_str(body).unwrap(); + assert_eq!(body["code"], json!("payload-too-large"), "{body}"); +}