From ec26cb62d854ded2a557c888f07de4b63b1975ef Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 26 Oct 2026 14:58:30 -0400 Subject: [PATCH 1/3] Commit staged content idempotently by operation ID with HEAD CAS deckd can now record a write exactly once, from content the caller staged, against a bare history repo addressed by path: - POST /commit builds a revision from an explicit {path, bytes | delete} list on top of `parent`, refusing with `parent-mismatch` (and the current HEAD) unless `parent` is HEAD. Retrying an operation ID that already committed returns the original revision. - POST /outcome reports `committed` or `absent`; `absent` fences the ID so it can never commit later. - POST /head and POST /tree-hash report HEAD and a revision's git tree ID. Operation records are files under the repo, tested with O_EXCL creates, and every commit carries a Deckd-Operation-Id trailer that repairs a record lost to a crash. A flock serializes deckd processes. Also bumps jj-lib to 0.46, pins the Rust toolchain it needs, and runs cargo test in CI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tg2ZC6YEPfRcNZSoWKq2HS --- .github/workflows/ci.yml | 8 +- packages/deckd/Cargo.lock | 803 ++++++++++----- packages/deckd/Cargo.toml | 9 +- packages/deckd/README.md | 139 ++- packages/deckd/rust-toolchain.toml | 4 + packages/deckd/src/api.rs | 224 +++- packages/deckd/src/history/commit.rs | 1302 ++++++++++++++++++++++++ packages/deckd/src/history/jj_lib.rs | 37 +- packages/deckd/src/history/mod.rs | 1 + packages/deckd/src/main.rs | 9 +- packages/deckd/tests/commit_restart.rs | 153 +++ 11 files changed, 2405 insertions(+), 284 deletions(-) create mode 100644 packages/deckd/rust-toolchain.toml create mode 100644 packages/deckd/src/history/commit.rs create mode 100644 packages/deckd/tests/commit_restart.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 987f1c1..389737d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,10 +51,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + # Run cargo from packages/deckd so rustup picks up its rust-toolchain.toml. - name: Show Rust toolchain + working-directory: packages/deckd run: rustc --version && cargo --version - name: Build deckd - run: cargo build --locked --manifest-path packages/deckd/Cargo.toml + working-directory: packages/deckd + run: cargo build --locked + - name: Test deckd + working-directory: packages/deckd + run: cargo test --locked - name: Preserve deckd for collaboration replay uses: actions/upload-artifact@v4 with: diff --git a/packages/deckd/Cargo.lock b/packages/deckd/Cargo.lock index 654c319..d8196c5 100644 --- a/packages/deckd/Cargo.lock +++ b/packages/deckd/Cargo.lock @@ -148,20 +148,20 @@ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "blake2" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" dependencies = [ "digest", ] [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -216,7 +216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures", "rand_core", ] @@ -242,19 +242,16 @@ dependencies = [ ] [[package]] -name = "core-foundation-sys" -version = "0.8.7" +name = "cmov" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] -name = "cpufeatures" -version = "0.2.17" +name = "core-foundation-sys" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" @@ -310,12 +307,20 @@ checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a" +dependencies = [ + "cmov", ] [[package]] @@ -340,9 +345,10 @@ dependencies = [ "base64", "chrono", "futures", + "gix", "jj-lib", "notify", - "pollster", + "pollster 0.4.0", "serde", "serde_json", "thiserror", @@ -350,6 +356,16 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "ureq", +] + +[[package]] +name = "defmt" +version = "0.3.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4deef977660e373c458df605a913d4ab121db0ccd18d1ad6976dff7d37cddb53" +dependencies = [ + "defmt 1.1.1", ] [[package]] @@ -385,13 +401,24 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.7" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", "crypto-common", - "subtle", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] @@ -431,22 +458,14 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "etcetera" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" -dependencies = [ - "cfg-if", - "windows-sys 0.61.2", -] - [[package]] name = "faster-hex" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +checksum = "04839bdf9d8c10f66806fad16b852fc72aab80873aebc3cb69d85b4fa41543ed" dependencies = [ + "autocfg", + "defmt 0.3.101", "heapless", "serde", ] @@ -591,16 +610,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.4.3" @@ -615,9 +624,9 @@ dependencies = [ [[package]] name = "gix" -version = "0.85.0" +version = "0.88.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8b2e38ebfc4484dfef8580ddcaf8abb7285e6f3eb6413ff6775d104ae96ca6" +checksum = "32d6dd6028308ad2f2662207506e396b97f9ec899f74bd537c0775248c774f21" dependencies = [ "gix-actor", "gix-attributes", @@ -637,12 +646,14 @@ dependencies = [ "gix-ignore", "gix-index", "gix-lock", + "gix-note", "gix-object", "gix-odb", "gix-pack", "gix-path", "gix-pathspec", "gix-protocol", + "gix-quote", "gix-ref", "gix-refspec", "gix-revision", @@ -658,16 +669,16 @@ dependencies = [ "gix-validate", "gix-worktree", "gix-worktree-stream", + "gix-zlib", "nonempty", "smallvec", - "thiserror", ] [[package]] name = "gix-actor" -version = "0.41.2" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33f9308ad6fd35b2a865cbe4117ac61b2be59e4a9ef1621c7a9794f7c8e52c5b" +checksum = "8e046e9a929e8e1f40f9a34f736408c4c079f9cf004429f3e718d270e847be96" dependencies = [ "bstr", "gix-date", @@ -676,57 +687,57 @@ dependencies = [ [[package]] name = "gix-attributes" -version = "0.33.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39b40888d0ed415c0744a6cdc61eebf0304c9d26ab726725b718443c322e5ba4" +checksum = "a3d57bcf18768dc259868e335fae99a9063174178b78b00348603f0791cd780a" dependencies = [ "bstr", + "gix-error", + "gix-features", "gix-glob", "gix-path", "gix-quote", "gix-trace", - "kstring", "smallvec", - "thiserror", "unicode-bom", ] [[package]] name = "gix-bitmap" -version = "0.3.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd1d118d0f5d88b96e6f6e13b566475fef4797ead4a02c26fed36c1375066f7" +checksum = "4e9d28a8333a322df5c4fdc7b86a9928965e50794523c0edff57d1fbf2a520cf" dependencies = [ "gix-error", ] [[package]] name = "gix-chunk" -version = "0.7.3" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a871e5cab12ba568845714473505deefffb3c04eb47f4708ce344cd459c1cc" +checksum = "25303d07ce2b8f98aab48ce4251317d785a3208b7ed65698928e4a2f31889936" dependencies = [ "gix-error", ] [[package]] name = "gix-command" -version = "0.9.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4363accdf6ef7ba861871d2d521ab7418a04aaaed919fadb022af71d379b12" +checksum = "eb3c2e61f97adc08764060e9becb0c819d8f2ba27896434710e039ae57b1d2a7" dependencies = [ "bstr", + "gix-error", "gix-path", "gix-quote", "gix-trace", - "shell-words", ] [[package]] name = "gix-commitgraph" -version = "0.37.1" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +checksum = "0462cbffd5828447f376559d397d9b29f1979dac3d980b5212b17fa174b174eb" dependencies = [ "bstr", "gix-chunk", @@ -738,40 +749,41 @@ dependencies = [ [[package]] name = "gix-config" -version = "0.58.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29bf266c4cdaf759e535c24ad4ce655b987aeb6911075643403cc7cc5ade583" +checksum = "eb2cb454dce2f1895cf9def1df8b0a6bbc579d86d8313bdf60f1ab429d52df12" dependencies = [ "bstr", "gix-config-value", + "gix-error", "gix-features", "gix-glob", "gix-path", "gix-ref", "gix-sec", + "gix-utils", "smallvec", - "thiserror", "unicode-bom", ] [[package]] name = "gix-config-value" -version = "0.18.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +checksum = "1894e816c774650b5157d965853b50d7e8f7137ea74055a6bac2f2ec051cf997" dependencies = [ "bitflags 2.13.1", "bstr", + "gix-error", "gix-path", "libc", - "thiserror", ] [[package]] name = "gix-date" -version = "0.15.6" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e47b9e8cdc688296609b706428de570f88b1e0eed7156dde7b4a89d26fa4567" +checksum = "cb8beceb16a77fb222923592864b94beb4ad4c2b175a7055c2cac6815ee0370a" dependencies = [ "bstr", "gix-error", @@ -781,12 +793,13 @@ dependencies = [ [[package]] name = "gix-diff" -version = "0.65.0" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c6d56c94edf92d78203a1cd416f770e35e10b6955ede6b9d7d0c22ff88a5f3" +checksum = "939aee7bbc9f233de858839e865c82886e0baceb8f1fe792ea1703fa895f09e1" dependencies = [ "bstr", "gix-command", + "gix-error", "gix-filter", "gix-fs", "gix-hash", @@ -797,38 +810,37 @@ dependencies = [ "gix-trace", "gix-traverse", "gix-worktree", - "thiserror", ] [[package]] name = "gix-discover" -version = "0.53.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d624d5b23b10c1d85337645227abe353ac95ab8ff66a7bdd5ce689b2db33a722" +checksum = "4476ba055ec5fbb05f59dd98380b596428da060beac5f9dccd7374302ad1fbfb" dependencies = [ "bstr", "dunce", + "gix-error", "gix-fs", "gix-path", "gix-ref", "gix-sec", - "thiserror", ] [[package]] name = "gix-error" -version = "0.2.5" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9292309fd944e71b2a3c96d3c03a6feb8852db646febdde7cbb9f79cb5f329" +checksum = "2c4a5c4cd326ddfe7f046ad9eed05461c59af6ea98b681648bac73954317addd" dependencies = [ "bstr", ] [[package]] name = "gix-features" -version = "0.48.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +checksum = "bb3a430dbc02d6c5e30a9f92083459593495ea22a476e837d248b22d800eed80" dependencies = [ "bytes", "crc32fast", @@ -840,21 +852,20 @@ dependencies = [ "once_cell", "parking_lot", "prodash", - "thiserror", "walkdir", - "zlib-rs", ] [[package]] name = "gix-filter" -version = "0.32.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6644fb2ef97928c278675b239f366b457103d7e436f811d27331a8daf212759c" +checksum = "6f31a39e2c9b04843709d1f72b2c39c98dc0ee591e2f67d21302dbfbd06d2956" dependencies = [ "bstr", "encoding_rs", "gix-attributes", "gix-command", + "gix-error", "gix-hash", "gix-object", "gix-packetline", @@ -863,52 +874,52 @@ dependencies = [ "gix-trace", "gix-utils", "smallvec", - "thiserror", ] [[package]] name = "gix-fs" -version = "0.21.2" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +checksum = "0105ab1cef4542d092c27766d2daf1548fcbad7e460dc2208642c120bb19b3ad" dependencies = [ "bstr", - "fastrand", + "gix-error", "gix-features", "gix-path", "gix-utils", - "thiserror", ] [[package]] name = "gix-glob" -version = "0.26.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +checksum = "bba50803dea16646cafb9a2d67e95b8cdf5b21c7e8c45bcd18f38bafe02fb81f" dependencies = [ "bitflags 2.13.1", "bstr", "gix-features", "gix-path", + "gix-utils", ] [[package]] name = "gix-hash" -version = "0.25.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +checksum = "7a22b6febaaee8546eb0c9dd21ecdd93aca269a6cb3897e1250f0f60933c0174" dependencies = [ "faster-hex", + "gix-error", "gix-features", - "sha1-checked", - "thiserror", + "sha1dc", + "sha2", ] [[package]] name = "gix-hashtable" -version = "0.15.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e261d54091f0d1c729bc83f54548c071bdec60a697de1e58e88bdfd7a99d24e" +checksum = "c55bff2f32f6f5a95bdd43e5e5e01b0656250cffbc48eda19db0afa3d73adf2f" dependencies = [ "gix-hash", "hashbrown 0.17.1", @@ -917,9 +928,9 @@ dependencies = [ [[package]] name = "gix-ignore" -version = "0.21.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +checksum = "5bfd910e271ecd1a20e4df4009a0d9d844a0c51ee5444aaa3fadd1b1b077930b" dependencies = [ "bstr", "gix-glob", @@ -930,9 +941,9 @@ dependencies = [ [[package]] name = "gix-imara-diff" -version = "0.2.4" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a791e6620676a875f362f3156ed213e73ca099a09bf992c18812abe65cc37b1" +checksum = "7518b930f6aeb75e871d32ce873c2ae46e35cef19b5a0b039d4337f651d897cd" dependencies = [ "bstr", "hashbrown 0.17.1", @@ -940,15 +951,16 @@ dependencies = [ [[package]] name = "gix-index" -version = "0.53.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d45f82ec5a4d7542ea595e9ad16e03e26c8cb4f221e5bc9fcdcf469f63a681" +checksum = "dfd5b484b5b4e218a5ac114f0982949f9bd7582ff67986665a0cf85895332433" dependencies = [ "bitflags 2.13.1", "bstr", "filetime", "fnv", "gix-bitmap", + "gix-error", "gix-features", "gix-fs", "gix-hash", @@ -963,46 +975,71 @@ dependencies = [ "memmap2", "rustix", "smallvec", - "thiserror", ] [[package]] name = "gix-lock" -version = "23.0.1" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +checksum = "8fd5ddfd0cc476fef744ac544783b96a9ff5d49f8de541e6e9d6d6002aa90720" dependencies = [ + "gix-error", "gix-tempfile", "gix-utils", - "thiserror", +] + +[[package]] +name = "gix-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f11f5429ca73cc3758c9f53af56a9fba100a8701c5b4bc30e29b8b0b15288991" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "gix-note" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6879cc9e5a9a0429e40b05970af6c8dad8734098a3142fec23e32eeb567580" +dependencies = [ + "gix-error", + "gix-hash", + "gix-object", ] [[package]] name = "gix-object" -version = "0.62.0" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "019b38afc3eac1e41f9fe09a327664b313ba4a120fa5f40e3678795d0e42783e" +checksum = "a72909eeeed845721a143ef3dbeb4d6cd5df79c839fe7eefc138cb4f982eba3e" dependencies = [ "bstr", "gix-actor", + "gix-command", "gix-date", + "gix-error", "gix-features", "gix-hash", "gix-hashtable", + "gix-path", + "gix-tempfile", "gix-utils", "gix-validate", "itoa", "smallvec", - "thiserror", ] [[package]] name = "gix-odb" -version = "0.82.0" +version = "0.85.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fadc59f6fa0f9dd445eceee61060a2b59ca557f48da9fc677f567db535b782a" +checksum = "5794b85ff563a290ea18867fa1f0a5b0b84709f2f34dddbb3614556fbe88c629" dependencies = [ "arc-swap", + "gix-error", "gix-features", "gix-fs", "gix-hash", @@ -1011,19 +1048,20 @@ dependencies = [ "gix-pack", "gix-path", "gix-quote", + "gix-zlib", "memmap2", "parking_lot", "tempfile", - "thiserror", ] [[package]] name = "gix-pack" -version = "0.72.0" +version = "0.75.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3e7f1726cd2c0cd1cf1fc20be8a8e623f0b163f1f8d6fc836cfb9bc8cd758b" +checksum = "0451a48faef8e0e7a8f6d903c8329c7d18a793b3030907ca841f889dc695d1df" dependencies = [ "clru", + "crossbeam-deque", "gix-chunk", "gix-error", "gix-features", @@ -1031,75 +1069,75 @@ dependencies = [ "gix-hashtable", "gix-object", "gix-path", + "gix-zlib", "memmap2", "smallvec", - "thiserror", "uluru", ] [[package]] name = "gix-packetline" -version = "0.21.5" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b217dd0ee0c4021ecf169a4a519b1b4f80d15e3f3765f3dc466223dc0ac891d7" +checksum = "0ba540112a3fcc6388eb2aa4dc4837040478c285a35178cd9a44b07345cd6636" dependencies = [ "bstr", "faster-hex", + "gix-error", "gix-trace", - "thiserror", ] [[package]] name = "gix-path" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751d6bd162106f8c1e7e9aaccb5bbdd605267e91a930a17a4560c46e33a9100c" +checksum = "fd051ac60c5dbce2228daabf25b6a76408ac732d416c678cfa0a4914d13f60b8" dependencies = [ "bstr", + "gix-error", "gix-trace", "gix-validate", - "thiserror", ] [[package]] name = "gix-pathspec" -version = "0.18.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +checksum = "1ee883ab64bad641bf31ff338044edd7aa010605f19e866d98c9164e635c38b1" dependencies = [ "bitflags 2.13.1", "bstr", "gix-attributes", "gix-config-value", + "gix-error", "gix-glob", "gix-path", - "thiserror", ] [[package]] name = "gix-protocol" -version = "0.63.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978468bae4ea2df20c72db3b20d0bdb548a0c1090b85a83643b553e6e0e041f2" +checksum = "331ac489c461790b6b692afb322085c42041c6ff7621c30ac3dbb00b12e05e89" dependencies = [ "bstr", "gix-date", + "gix-error", "gix-features", "gix-hash", + "gix-macros", "gix-ref", "gix-shallow", "gix-transport", "gix-utils", - "maybe-async", "nonempty", - "thiserror", ] [[package]] name = "gix-quote" -version = "0.7.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +checksum = "e6b040cfa7af50f131bfa34119cceee804c3cc40c766daaba8e0c8149340aea7" dependencies = [ "bstr", "gix-error", @@ -1108,11 +1146,12 @@ dependencies = [ [[package]] name = "gix-ref" -version = "0.65.0" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bbfbce1dfd7d7f8469ddef6d3518376aff664348f153cbe0fc3e58ef993d24e" +checksum = "c90ceeec67de03b737c30b1b69116ca56fec803edde022f3d25181f0f597083d" dependencies = [ "gix-actor", + "gix-error", "gix-features", "gix-fs", "gix-hash", @@ -1123,30 +1162,26 @@ dependencies = [ "gix-utils", "gix-validate", "memmap2", - "thiserror", ] [[package]] name = "gix-refspec" -version = "0.43.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc36a4fb1a1540b59cf2da498783080743fa274b02a3f19ca444fc4015a9d4f" +checksum = "b4caa89a034a055c0578150092d3edb2f1c3f74cca86b883263a28caa26c5812" dependencies = [ "bstr", "gix-error", - "gix-glob", "gix-hash", - "gix-revision", "gix-validate", "smallvec", - "thiserror", ] [[package]] name = "gix-revision" -version = "0.47.0" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "885075c3c21eb9c06e0be3b3728ba5932c04e1c1011dcee7c81801980e3e986f" +checksum = "d8438d38e197fa3ea7bb297302db1701879a64a343f23a3d150a602a609a8751" dependencies = [ "bstr", "gix-commitgraph", @@ -1160,9 +1195,9 @@ dependencies = [ [[package]] name = "gix-revwalk" -version = "0.33.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f11fe7ca2585193d3d70bbe0be175a2008d883a704cc7a55e454e113e689455" +checksum = "2fa6d913e7cf8c6407d3954f9794b4b148c2113c672c34b63c59ea654a1e060e" dependencies = [ "gix-commitgraph", "gix-date", @@ -1171,14 +1206,13 @@ dependencies = [ "gix-hashtable", "gix-object", "smallvec", - "thiserror", ] [[package]] name = "gix-sec" -version = "0.14.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4fe6c152c1d50aea36f299825702cd37e303307832fec1d0fdd5844e47ce2f" +checksum = "7d8eb983a830ba586b61f7005deaf44609e1a4186cbf13a0e8668154ca910461" dependencies = [ "bitflags 2.13.1", "gix-path", @@ -1188,102 +1222,106 @@ dependencies = [ [[package]] name = "gix-shallow" -version = "0.12.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +checksum = "119bae1301715df40e8fd136deeff4fe8a3be4cfc001ed5e72bad7b47ebbc090" dependencies = [ "bstr", + "gix-error", "gix-hash", "gix-lock", "nonempty", - "thiserror", ] [[package]] name = "gix-submodule" -version = "0.32.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f9f594f7cbda0b38ba6b633b3e9a7b7901acdc5d27bc186a16633800cd1ac8" +checksum = "0b498192e5f89453fbd728263d52088aeec634a390e732f1da6d0a491c552091" dependencies = [ "bstr", "gix-config", + "gix-error", "gix-path", "gix-pathspec", "gix-refspec", "gix-url", - "thiserror", ] [[package]] name = "gix-tempfile" -version = "23.0.2" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef60812443484e67bf84e444cc71b4c78ae62deb822221774a4fa0c57fdb17f" +checksum = "3dd7dd8df3345933caddd6ce67821c9eec17192564f1011cf0bd0b8cb0d07ffb" dependencies = [ "dashmap", "gix-fs", "libc", "parking_lot", "tempfile", + "windows-sys 0.61.2", ] [[package]] name = "gix-trace" -version = "0.1.21" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3eb81d9dc914335923e50d52829c551feefd6a72d176c4130c546b67a60814" +checksum = "df45b2cf6052d9e52681130c17d67505b023e8933ad974f94a2be0b0d24f636a" [[package]] name = "gix-transport" -version = "0.57.2" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186874f7ad1fb2f9a2f2aa9c2dabc7f9dd087bef74c1a0eee2b4a9cf0248fcb3" +checksum = "5d2beb7b0387bbf4d7c13763c6e168f9957319aa811f09349dc59601be3b2951" dependencies = [ "bstr", "gix-command", + "gix-error", "gix-features", "gix-packetline", + "gix-path", "gix-quote", "gix-sec", "gix-url", - "thiserror", ] [[package]] name = "gix-traverse" -version = "0.59.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5062cca8f2977565bbaf666ec31dbdb9bc9d9293beb65f9bec52e6c1121b62a1" +checksum = "dd4864934e4efd92082ecc37107bce0c389b58d82888db634817f9c711efb7ed" dependencies = [ "bitflags 2.13.1", "gix-commitgraph", "gix-date", + "gix-error", "gix-hash", "gix-hashtable", "gix-object", "gix-revwalk", "smallvec", - "thiserror", ] [[package]] name = "gix-url" -version = "0.36.2" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d68e70e96da0e5f9c871f1566349e0fd0e1a20bb483c7f54af1dd0b85b4b29" +checksum = "1f30fe2a228d7de5f9d582463d9b724debcb870162d9fbe908cd5d76ccdecf0d" dependencies = [ "bstr", + "gix-error", "gix-path", + "gix-utils", "percent-encoding", - "thiserror", ] [[package]] name = "gix-utils" -version = "0.3.5" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1795bd2a970ca8b2185318c2abb97d955c71992f1cf28de73ad3b593a9f3ce8" +checksum = "70525766f53426ca4cd2ea4d58b0b31488dfedb416b3254e2efdd9dc184cc880" dependencies = [ + "bstr", "fastrand", "getrandom", "unicode-normalization", @@ -1291,21 +1329,24 @@ dependencies = [ [[package]] name = "gix-validate" -version = "0.11.3" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a034e84d1e04e1b1f20f51f12491da230b6ac8b925d0c8e1b89bcd87a7c5ccc" +checksum = "30574f81fc41b2ca3582ece2bdf8696bdfe9c170aa7e8bcafb81199c86b1ceea" dependencies = [ "bstr", + "gix-error", ] [[package]] name = "gix-worktree" -version = "0.54.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92399ed66f259592050c6ed9dc80105e095a2f8e87e6b83d98aa2e21d8e27036" +checksum = "04fa19b9b3961aafb3b9e60005b9bcef0100f8b211f46ab5269a6ac896073780" dependencies = [ "bstr", "gix-attributes", + "gix-error", + "gix-features", "gix-fs", "gix-glob", "gix-hash", @@ -1318,9 +1359,9 @@ dependencies = [ [[package]] name = "gix-worktree-stream" -version = "0.34.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f3a878c89a05470ad98c644b0015777c530da24854dd29e41fe4f41176840f" +checksum = "2021e1609beae0edbeefa76ac641be0571b725afacb77ac5b12f33b436e0d3d9" dependencies = [ "gix-attributes", "gix-error", @@ -1334,6 +1375,16 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "gix-zlib" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44a1a9c82ec5b712082566cbef8f5782472a0d3973132911e5a83380a5298fe2" +dependencies = [ + "gix-error", + "zlib-rs", +] + [[package]] name = "globset" version = "0.4.20" @@ -1439,6 +1490,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.0" @@ -1498,6 +1558,110 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1579,7 +1743,7 @@ version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ - "defmt", + "defmt 1.1.1", "jiff-core", "jiff-static", "jiff-tzdb-platform", @@ -1596,7 +1760,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" dependencies = [ - "defmt", + "defmt 1.1.1", ] [[package]] @@ -1626,11 +1790,52 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jj-core" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cc6c2549b3eeb6b87063e49786e3c73abea653d9b88b20c982967a8b112db40" +dependencies = [ + "async-trait", + "blake2", + "bstr", + "chrono", + "clru", + "digest", + "either", + "futures", + "globset", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "jj-core-proc-macros", + "ref-cast", + "regex", + "same-file", + "serde", + "smallvec", + "tempfile", + "thiserror", + "tracing", + "winreg", +] + +[[package]] +name = "jj-core-proc-macros" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4936b0a5244527dd7e2370c3553920b402a7b44cfecf6748131dedaa1890bb81" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "jj-lib" -version = "0.43.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22e2ced34a1a51f0f449bfff2cbacda1c338b2611268b5684a80251e4c8bf71" +checksum = "4b82d2d9b1f8400db1dda9a22bccfa6c3118f5592dfe6578452e56dcb378e9ad" dependencies = [ "async-trait", "blake2", @@ -1640,21 +1845,20 @@ dependencies = [ "digest", "dunce", "either", - "etcetera", "futures", "gix", "gix-ignore", "globset", - "hashbrown 0.17.1", "indexmap", "interim", "itertools 0.15.0", - "jj-lib-proc-macros", + "jj-core", "maplit", + "memchr", "once_cell", "pest", "pest_derive", - "pollster", + "pollster 1.0.1", "prost", "rand", "rand_chacha", @@ -1662,7 +1866,6 @@ dependencies = [ "ref-cast", "regex", "rustix", - "same-file", "serde", "smallvec", "strsim", @@ -1670,18 +1873,7 @@ dependencies = [ "thiserror", "toml_edit", "tracing", - "winreg", -] - -[[package]] -name = "jj-lib-proc-macros" -version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c21554e3755e5ecfec934d84ac64d0295d9c2f2b96f6a740a1ddb66addf4142e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "windows-sys 0.61.2", ] [[package]] @@ -1715,15 +1907,6 @@ dependencies = [ "libc", ] -[[package]] -name = "kstring" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b609e7ca5ea38f093c20a4a102335b247221c9643b7a6bc3510f196f99499a9e" -dependencies = [ - "static_assertions", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -1742,6 +1925,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + [[package]] name = "lock_api" version = "0.4.14" @@ -1812,17 +2001,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" -[[package]] -name = "maybe-async" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "memchr" version = "2.8.3" @@ -1997,6 +2175,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" +[[package]] +name = "pollster" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6355899e1c9462875b6757c79f3caa011a1fdae12bbb1a2e72dd1f234f8336" + [[package]] name = "portable-atomic" version = "1.15.0" @@ -2012,6 +2196,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2313,24 +2506,20 @@ dependencies = [ ] [[package]] -name = "sha1" -version = "0.10.7" +name = "sha1dc" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] +checksum = "c5b0ebf6537e3e8db8bb7d21ecb6d7399738c953c757945c6b95e46c4e95df7b" [[package]] -name = "sha1-checked" -version = "0.10.0" +name = "sha2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ + "cfg-if", + "cpufeatures", "digest", - "sha1", ] [[package]] @@ -2342,12 +2531,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shell-words" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" - [[package]] name = "shlex" version = "2.0.1" @@ -2395,24 +2578,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "syn" version = "2.0.119" @@ -2441,6 +2612,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -2483,6 +2665,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.12.0" @@ -2713,16 +2905,42 @@ dependencies = [ ] [[package]] -name = "valuable" -version = "0.1.1" +name = "ureq" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "log", + "once_cell", + "serde", + "serde_json", + "url", +] [[package]] -name = "version_check" -version = "0.9.5" +name = "url" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "walkdir" @@ -2954,6 +3172,35 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.56" @@ -2974,6 +3221,60 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "zlib-rs" version = "0.6.7" diff --git a/packages/deckd/Cargo.toml b/packages/deckd/Cargo.toml index 3529114..f13fbca 100644 --- a/packages/deckd/Cargo.toml +++ b/packages/deckd/Cargo.toml @@ -16,8 +16,10 @@ axum = { version = "0.8", features = ["json"] } base64 = "0.22" chrono = { version = "0.4", default-features = false, features = ["clock"] } futures = "0.3" -# Same crates.io pin as boxel-labs jj-historyd (not a fork of the jj git tree). -jj-lib = "0.43.0" +# Only for `gix::hash::Kind` when initializing a repo; must match jj-lib's gix. +gix = { version = "0.88.0", default-features = false } +# From crates.io, not a fork of the jj git tree. 0.46 needs rustc 1.97.1. +jj-lib = "0.46.0" notify = "7" pollster = "0.4" serde = { version = "1", features = ["derive"] } @@ -33,3 +35,6 @@ default = [] # Derived esbuild AMD plane (`/ensure-build`, `/artifact-at`). Off by default — # History is the product; enable with `cargo run --features build`. build = [] + +[dev-dependencies] +ureq = { version = "2", default-features = false, features = ["json"] } diff --git a/packages/deckd/README.md b/packages/deckd/README.md index 306ddca..12067b8 100644 --- a/packages/deckd/README.md +++ b/packages/deckd/README.md @@ -11,7 +11,8 @@ jj-lib path copied closely from boxel-labs [`jj-historyd`](file:///Users/chris/Projects/boxel-labs/examples/jj-lib-node/rust) (`HistoryService` + axum + `spawn_blocking`/`pollster`). -Dependency: crates.io `jj-lib = "0.43.0"` (same pin as historyd). +Dependency: crates.io `jj-lib = "0.46.0"`, which needs rustc 1.97.1; +`rust-toolchain.toml` pins the compiler, so run cargo from this directory. ## Features @@ -70,6 +71,142 @@ When Hub uses `DECKD_URL`, Node watchers publish + HMR only — they do not | ----------------------------- | ---------------------------------------- | | ensure / seal / restore | per-workspace **write** lock | | list / file-at / file-list-at | per-workspace **read** lock (concurrent) | +| commit / outcome | per-repo **file** lock (cross-process) | + +## Explicit commits + +`/commit`, `/outcome`, `/head` and `/tree-hash` record a revision from content +the caller staged, never from a working copy. A writer such as realm-server +can then record each write exactly once, even when deckd's response is lost. + +Each call names `repo`, the absolute path of one history directory (for +example `/.history`). The directory's parent must exist; deckd creates +the directory and a bare jj repo inside it on first use: + +```text +/ + jj/ # jj repo, internal git backend, no working copy + deckd/commit.lock # flock held around /commit and /outcome + deckd/operations/ # pending | committed | fenced +``` + +A revision is the git commit ID (40 hex). HEAD is the repo's single visible +head; an empty repo's HEAD is the root commit, `0000…0000`. + +### Guarantees + +- **Exactly the listed paths.** A revision is HEAD's tree plus the request's + `changes`; nothing on disk is read. +- **Compare-and-swap on HEAD.** A commit whose `parent` isn't HEAD is refused + with `parent-mismatch` and the current `head`, never re-parented. +- **Idempotent by operation ID.** Committing an ID that already committed + returns the original revision with `replayed: true` and writes nothing, + even when the retry's `parent` is now stale. +- **Fencing outcome query.** `/outcome` reports `committed` or `absent`. + Reporting `absent` fences the ID: a later commit with it is refused with + `operation-fenced`. +- **Durable, not in-memory.** Outcomes and fences are files under ``, + so they survive a restart and are shared by every deckd on the same + filesystem. Record existence is tested with `O_EXCL` creates, which an NFS + server answers, so a stale negative lookup can't hide one. +- **No-op writes record nothing.** When the changes leave the tree as it was, + no revision is created; the response returns HEAD with `created: false`. + +Correctness doesn't depend on deckd's in-process state. The file lock +serializes deckd processes; a writer running on several hosts should still +serialize its own commits (realm-server uses a Postgres advisory lock). + +Each commit's message ends with the caller's trailers, then +`Deckd-Operation-Id: `, which callers may not set themselves. + +### `POST /commit` + +```json +{ + "repo": "/realms/acme/.history", + "parent": "0000000000000000000000000000000000000000", + "operationId": "01JB8ZQ4T6W3", + "author": { "name": "Jane", "email": "jane@example.com" }, + "message": "Update Author/jane.json", + "trailers": [{ "key": "Boxel-Channel", "value": "host" }], + "changes": [ + { "path": "Author/jane.json", "contentBase64": "eyJuYW1lIjoiSmFuZSJ9" }, + { "path": "Author/old.json", "delete": true } + ] +} +``` + +```json +{ + "revision": "6f1c2d8e0a9b4c3d2e1f0a9b8c7d6e5f4a3b2c1d", + "tree": "9a3e1b7c5d2f4e6a8b0c1d3e5f7a9b1c3d5e7f9a", + "created": true, + "replayed": false +} +``` + +- `operationId`: 1–128 characters from `[A-Za-z0-9._-]`, not starting with `.`. +- `changes`: each entry has `path` plus exactly one of `contentBase64` or + `delete: true`. A path appears at most once. Paths are relative and may not + contain `..` or empty segments. Deleting an absent path changes nothing. +- `trailers`: optional. Keys match `[A-Za-z0-9-]+`; values are one line. +- `message` is required. `author.name` and `author.email` are required and may + not contain newlines, `<` or `>`. + +### `POST /outcome` + +```json +{ "repo": "/realms/acme/.history", "operationId": "01JB8ZQ4T6W3" } +``` + +```json +{ "state": "committed", "revision": "6f1c2d8e0a9b4c3d2e1f0a9b8c7d6e5f4a3b2c1d" } +``` + +or `{ "state": "absent" }`, after which the ID is fenced. It is a POST because +it can change state. + +### `POST /head` + +```json +{ "repo": "/realms/acme/.history" } +``` + +```json +{ + "revision": "6f1c2d8e0a9b4c3d2e1f0a9b8c7d6e5f4a3b2c1d", + "tree": "9a3e1b7c5d2f4e6a8b0c1d3e5f7a9b1c3d5e7f9a" +} +``` + +### `POST /tree-hash` + +```json +{ + "repo": "/realms/acme/.history", + "revision": "6f1c2d8e0a9b4c3d2e1f0a9b8c7d6e5f4a3b2c1d" +} +``` + +```json +{ "tree": "9a3e1b7c5d2f4e6a8b0c1d3e5f7a9b1c3d5e7f9a" } +``` + +The tree hash is the git tree ID, so identical content has the same hash in +any repo, whoever committed it and whatever the message. + +### Errors + +Every error body is `{ "code", "error" }`, plus fields named below. + +| Status | `code` | When | +| ------ | -------------------- | ---------------------------------------------------- | +| 400 | `invalid-request` | Bad repo path, operation ID, path, trailer or base64 | +| 404 | `revision-not-found` | `/tree-hash` for a revision the repo doesn't have | +| 409 | `parent-mismatch` | `parent` isn't HEAD; body has `head` | +| 409 | `operation-fenced` | The operation ID was reported `absent` by `/outcome` | +| 503 | `lock-timeout` | Another process held the repo's lock for 30 s | +| 500 | `internal` | jj or filesystem failure | ## Run diff --git a/packages/deckd/rust-toolchain.toml b/packages/deckd/rust-toolchain.toml new file mode 100644 index 0000000..a1eeb29 --- /dev/null +++ b/packages/deckd/rust-toolchain.toml @@ -0,0 +1,4 @@ +# jj-lib 0.46 needs rustc 1.97.1 or newer. rustup reads this file from the +# current directory, so run cargo from packages/deckd. +[toolchain] +channel = "1.99.1" diff --git a/packages/deckd/src/api.rs b/packages/deckd/src/api.rs index 20e09e3..7ee2c78 100644 --- a/packages/deckd/src/api.rs +++ b/packages/deckd/src/api.rs @@ -2,7 +2,7 @@ use std::sync::Arc; -use axum::extract::State; +use axum::extract::{DefaultBodyLimit, State}; use axum::http::StatusCode; use axum::routing::{get, post}; use axum::{Json, Router}; @@ -10,6 +10,10 @@ use base64::Engine; use serde::{Deserialize, Serialize}; use tower_http::trace::TraceLayer; +use crate::history::commit::{ + Author, Change, CommitError, CommitOutcome, CommitRequest, CommitService, CommitTrailer, + OperationOutcome, RevisionTree, +}; use crate::history::{Actor, HistoryBackend, HistoryBackendError, HistoryEntry, RestorePlan}; use crate::watch::WatchPlane; @@ -21,6 +25,8 @@ use crate::build::{ #[derive(Clone)] pub struct AppState { pub history: Arc, + /// Explicit-content commits for repos addressed by path. + pub commits: Arc, /// Present only with `--features build`. #[cfg(feature = "build")] pub build: Option>, @@ -40,6 +46,13 @@ pub fn router(state: AppState) -> Router { .route("/file-at", post(file_at)) .route("/file-list-at", post(file_list_at)) .route("/restore-plan", post(restore_plan)) + .route( + "/commit", + post(commit).layer(DefaultBodyLimit::max(COMMIT_BODY_LIMIT)), + ) + .route("/outcome", post(outcome)) + .route("/head", post(head)) + .route("/tree-hash", post(tree_hash)) .route("/ensure-build", post(ensure_build)) .route("/artifact-at", post(artifact_at)) .layer(TraceLayer::new_for_http()) @@ -56,12 +69,23 @@ pub fn router(state: AppState) -> Router { .route("/file-at", post(file_at)) .route("/file-list-at", post(file_list_at)) .route("/restore-plan", post(restore_plan)) + .route( + "/commit", + post(commit).layer(DefaultBodyLimit::max(COMMIT_BODY_LIMIT)), + ) + .route("/outcome", post(outcome)) + .route("/head", post(head)) + .route("/tree-hash", post(tree_hash)) .layer(TraceLayer::new_for_http()) .with_state(state); app } +/// A commit carries every staged file base64-encoded in one JSON body, so it +/// needs far more than axum's 2 MB default. Batch limits are the caller's. +const COMMIT_BODY_LIMIT: usize = 512 * 1024 * 1024; + #[derive(Debug, Deserialize)] struct DirBody { dir: String, @@ -211,6 +235,137 @@ async fn restore_plan( )) } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CommitBody { + repo: String, + parent: String, + operation_id: String, + author: Author, + message: String, + #[serde(default)] + trailers: Vec, + changes: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ChangeBody { + path: String, + #[serde(default)] + content_base64: Option, + #[serde(default)] + delete: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct OutcomeBody { + repo: String, + operation_id: String, +} + +#[derive(Debug, Deserialize)] +struct RepoBody { + repo: String, +} + +#[derive(Debug, Deserialize)] +struct TreeHashBody { + repo: String, + revision: String, +} + +#[derive(Debug, Serialize)] +struct TreeHashResponse { + tree: String, +} + +async fn commit( + State(st): State, + Json(body): Json, +) -> Result, ApiError> { + let mut changes = Vec::with_capacity(body.changes.len()); + for change in body.changes { + let content = match (change.content_base64, change.delete) { + (Some(encoded), false) => Some( + base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|e| { + CommitError::Invalid(format!("contentBase64 for {:?}: {e}", change.path)) + })?, + ), + (None, true) => None, + _ => { + return Err(CommitError::Invalid(format!( + "change {:?} needs exactly one of contentBase64 or delete: true", + change.path + )) + .into()) + } + }; + changes.push(Change { + path: change.path, + content, + }); + } + let request = CommitRequest { + parent: body.parent, + operation_id: body.operation_id, + author: body.author, + message: body.message, + trailers: body.trailers, + changes, + }; + let repo = std::path::PathBuf::from(body.repo); + Ok(Json( + run_commits(&st, move |c| c.commit(&repo, request)).await?, + )) +} + +async fn outcome( + State(st): State, + Json(body): Json, +) -> Result, ApiError> { + let repo = std::path::PathBuf::from(body.repo); + let operation_id = body.operation_id; + Ok(Json( + run_commits(&st, move |c| c.outcome(&repo, &operation_id)).await?, + )) +} + +async fn head( + State(st): State, + Json(body): Json, +) -> Result, ApiError> { + let repo = std::path::PathBuf::from(body.repo); + Ok(Json(run_commits(&st, move |c| c.head(&repo)).await?)) +} + +async fn tree_hash( + State(st): State, + Json(body): Json, +) -> Result, ApiError> { + let repo = std::path::PathBuf::from(body.repo); + let revision = body.revision; + let found = run_commits(&st, move |c| c.tree_hash(&repo, &revision)).await?; + Ok(Json(TreeHashResponse { tree: found.tree })) +} + +/// Commit work waits on repo locks and drives jj-lib's `!Send` futures, so it +/// runs on a blocking thread. +async fn run_commits(st: &AppState, f: F) -> Result +where + T: Send + 'static, + F: FnOnce(Arc) -> Result + Send + 'static, +{ + let commits = st.commits.clone(); + tokio::task::spawn_blocking(move || f(commits)) + .await + .map_err(|e| ApiError::from(format!("join error: {e}")))? + .map_err(ApiError::from) +} + #[cfg(feature = "build")] async fn ensure_build( State(st): State, @@ -235,37 +390,76 @@ async fn artifact_at( struct ApiError { status: StatusCode, + /// Stable, machine-readable error kind. + code: &'static str, message: String, + /// Extra fields merged into the error body, e.g. `head` on a conflict. + details: serde_json::Map, +} + +impl ApiError { + fn new(status: StatusCode, code: &'static str, message: String) -> Self { + Self { + status, + code, + message, + details: serde_json::Map::new(), + } + } +} + +impl From for ApiError { + fn from(err: CommitError) -> Self { + let message = err.to_string(); + match err { + CommitError::Invalid(_) => { + Self::new(StatusCode::BAD_REQUEST, "invalid-request", message) + } + CommitError::ParentMismatch { head, .. } => { + let mut error = Self::new(StatusCode::CONFLICT, "parent-mismatch", message); + error.details.insert("head".into(), head.into()); + error + } + CommitError::OperationFenced(_) => { + Self::new(StatusCode::CONFLICT, "operation-fenced", message) + } + CommitError::RevisionNotFound(_) => { + Self::new(StatusCode::NOT_FOUND, "revision-not-found", message) + } + CommitError::LockTimeout(_) => { + Self::new(StatusCode::SERVICE_UNAVAILABLE, "lock-timeout", message) + } + CommitError::Internal(_) => { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal", message) + } + } + } } impl From for ApiError { fn from(err: HistoryBackendError) -> Self { - let status = match &err { - HistoryBackendError::NotFound(_) => StatusCode::NOT_FOUND, - HistoryBackendError::Invalid(_) => StatusCode::BAD_REQUEST, + let (status, code) = match &err { + HistoryBackendError::NotFound(_) => (StatusCode::NOT_FOUND, "not-found"), + HistoryBackendError::Invalid(_) => (StatusCode::BAD_REQUEST, "invalid-request"), HistoryBackendError::Io(_) | HistoryBackendError::Jj(_) => { - StatusCode::INTERNAL_SERVER_ERROR + (StatusCode::INTERNAL_SERVER_ERROR, "internal") } }; - Self { - status, - message: err.to_string(), - } + Self::new(status, code, err.to_string()) } } impl From for ApiError { fn from(message: String) -> Self { - Self { - status: StatusCode::INTERNAL_SERVER_ERROR, - message, - } + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal", message) } } impl axum::response::IntoResponse for ApiError { fn into_response(self) -> axum::response::Response { - let body = Json(serde_json::json!({ "error": self.message })); - (self.status, body).into_response() + let mut body = self.details; + body.insert("code".into(), self.code.into()); + body.insert("error".into(), self.message.into()); + (self.status, Json(serde_json::Value::Object(body))).into_response() } } diff --git a/packages/deckd/src/history/commit.rs b/packages/deckd/src/history/commit.rs new file mode 100644 index 0000000..3758b6e --- /dev/null +++ b/packages/deckd/src/history/commit.rs @@ -0,0 +1,1302 @@ +//! Explicit-content commits for a history repo addressed by path. +//! +//! The caller (realm-server) stages every file's final bytes and asks deckd to +//! record exactly those changes as one revision on top of `parent`. Nothing is +//! read from a working copy: the repo has no workspace, so a revision can never +//! pick up bytes some other request left on disk. +//! +//! Layout under `repo` (the caller chooses where that directory lives): +//! +//! ```text +//! / +//! jj/ # bare jj repo, internal git backend +//! deckd/commit.lock # cross-process lock around commit / outcome +//! deckd/operations/ # `pending` | `committed ` | `fenced` +//! ``` +//! +//! Correctness rests on files in the repo, never on this process's memory: +//! +//! - `commit` and `outcome` run under a flock on `deckd/commit.lock`, so +//! deckd processes on different hosts sharing the directory serialize. A +//! per-repo in-process mutex sits in front of it because Linux NFS emulates +//! flock with POSIX locks, which don't exclude threads of one process. +//! - Whether an operation record exists is only ever learned from +//! `create_new` (O_EXCL), which the NFS server answers. A peer's cached +//! negative lookup can't make an existing record look absent. +//! - Every commit carries a `Deckd-Operation-Id` trailer. Before acting, each +//! locked call sets HEAD's operation record to `committed HEAD`. A crash +//! after publishing but before writing the record can only leave HEAD +//! unrecorded, so after that repair every committed operation has a record. + +use std::collections::{HashMap, HashSet}; +use std::fs::{File, OpenOptions}; +use std::io::{ErrorKind, Write as _}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use jj_lib::backend::{BackendError, CommitId, CopyId, Signature, Timestamp, TreeValue}; +use jj_lib::commit::Commit; +use jj_lib::default_backend_factories::default_backend_factories; +use jj_lib::lock::FileLock; +use jj_lib::merge::Merge; +use jj_lib::merged_tree_builder::MergedTreeBuilder; +use jj_lib::object_id::ObjectId as _; +use jj_lib::repo::{ReadonlyRepo, Repo as _, RepoLoader}; +use jj_lib::repo_path::RepoPathBuf; +use jj_lib::signing_factory::signer_from_settings; +use jj_lib::trailer::parse_description_trailers; +use serde::{Deserialize, Serialize}; + +use super::jj_lib::{hermetic_git_env, is_valid_history_path, user_settings}; + +/// Trailer deckd adds to every explicit commit. Callers may not set it. +const OPERATION_ID_TRAILER: &str = "Deckd-Operation-Id"; + +/// How long a call waits for another process's commit lock before giving up. +const LOCK_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, thiserror::Error)] +pub enum CommitError { + #[error("{0}")] + Invalid(String), + #[error("parent {parent} is not HEAD {head}")] + ParentMismatch { parent: String, head: String }, + #[error("operation {0} was reported absent and can never commit")] + OperationFenced(String), + #[error("revision {0} not found")] + RevisionNotFound(String), + #[error("timed out waiting for the commit lock on {0}")] + LockTimeout(String), + #[error("{0}")] + Internal(String), +} + +type CommitResult = Result; + +fn internal(error: impl std::fmt::Display) -> CommitError { + CommitError::Internal(error.to_string()) +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Author { + pub name: String, + pub email: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct CommitTrailer { + pub key: String, + pub value: String, +} + +/// One staged path: new bytes, or a delete. +#[derive(Debug, Clone)] +pub struct Change { + pub path: String, + pub content: Option>, +} + +#[derive(Debug, Clone)] +pub struct CommitRequest { + pub parent: String, + pub operation_id: String, + pub author: Author, + pub message: String, + pub trailers: Vec, + pub changes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct CommitOutcome { + pub revision: String, + pub tree: String, + /// False when the changes left the tree as it was and no revision was + /// written; `revision` is then the unchanged HEAD. + pub created: bool, + /// True when this operation ID had already committed and nothing new + /// was written. + pub replayed: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "state", rename_all = "camelCase")] +pub enum OperationOutcome { + Committed { revision: String }, + Absent, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RevisionTree { + pub revision: String, + pub tree: String, +} + +/// Explicit-content commits, operation outcomes and tree hashes. Holds no +/// state that correctness depends on; a fresh instance behaves exactly like +/// one that has been running. +pub struct CommitService { + gates: Mutex>>>, +} + +impl Default for CommitService { + fn default() -> Self { + Self::new() + } +} + +impl CommitService { + pub fn new() -> Self { + hermetic_git_env(); + Self { + gates: Mutex::new(HashMap::new()), + } + } + + /// Record `request.changes` as one revision on top of `request.parent`. + /// Runs on a blocking thread: it waits on locks and drives jj-lib's + /// `!Send` futures to completion while holding them. + pub fn commit(&self, repo_dir: &Path, request: CommitRequest) -> CommitResult { + validate_repo_dir(repo_dir)?; + validate_operation_id(&request.operation_id)?; + validate_commit_request(&request)?; + let gate = self.gate(repo_dir); + let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); + let _file_lock = lock_repo(repo_dir)?; + pollster::block_on(commit_locked(repo_dir, request)) + } + + /// Report whether `operation_id` committed. Reporting it absent fences it: + /// from then on a commit with that ID is refused. + pub fn outcome(&self, repo_dir: &Path, operation_id: &str) -> CommitResult { + validate_repo_dir(repo_dir)?; + validate_operation_id(operation_id)?; + let gate = self.gate(repo_dir); + let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); + let _file_lock = lock_repo(repo_dir)?; + pollster::block_on(outcome_locked(repo_dir, operation_id)) + } + + /// The current HEAD, creating an empty repo (HEAD is the root commit) if + /// there is none yet. + pub fn head(&self, repo_dir: &Path) -> CommitResult { + validate_repo_dir(repo_dir)?; + let repo = if repo_dir.join("jj").is_dir() { + pollster::block_on(load_repo(repo_dir))? + } else { + let gate = self.gate(repo_dir); + let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); + let _file_lock = lock_repo(repo_dir)?; + pollster::block_on(load_or_init(repo_dir))? + }; + let head = head_commit(&repo)?; + Ok(RevisionTree { + revision: head.id().hex(), + tree: tree_hash(&head), + }) + } + + pub fn tree_hash(&self, repo_dir: &Path, revision: &str) -> CommitResult { + validate_repo_dir(repo_dir)?; + if !repo_dir.join("jj").is_dir() { + return Err(CommitError::RevisionNotFound(revision.to_owned())); + } + let repo = pollster::block_on(load_repo(repo_dir))?; + let commit = get_commit(&repo, revision)?; + Ok(RevisionTree { + revision: commit.id().hex(), + tree: tree_hash(&commit), + }) + } + + fn gate(&self, repo_dir: &Path) -> Arc> { + let mut gates = self.gates.lock().unwrap_or_else(|e| e.into_inner()); + gates + .entry(repo_dir.to_path_buf()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + } +} + +/// The body of [`CommitService::commit`]. Callers hold both repo locks. +async fn commit_locked(repo_dir: &Path, request: CommitRequest) -> CommitResult { + let repo = load_or_init(repo_dir).await?; + let head = head_commit(&repo)?; + repair_head_record(repo_dir, &head)?; + + let record_path = operation_record_path(repo_dir, &request.operation_id); + if !claim_record(&record_path, Record::Pending)? { + match read_record(&record_path)? { + Record::Committed(revision) => { + let commit = get_commit(&repo, &revision)?; + return Ok(CommitOutcome { + revision, + tree: tree_hash(&commit), + created: false, + replayed: true, + }); + } + Record::Fenced => { + return Err(CommitError::OperationFenced(request.operation_id)); + } + // A crashed attempt that never published: the repair above + // would have marked it committed if it had. Take it over. + Record::Pending => {} + } + } + + let head_hex = head.id().hex(); + if CommitId::try_from_hex(&request.parent).as_ref() != Some(head.id()) { + // Leave no claim behind, so the caller can retry this operation + // on the new HEAD. + std::fs::remove_file(&record_path).map_err(internal)?; + return Err(CommitError::ParentMismatch { + parent: request.parent, + head: head_hex, + }); + } + + let tree = build_tree(&repo, &head, &request.changes).await?; + if tree.tree_ids() == head.tree_ids() { + write_record(&record_path, &Record::Committed(head_hex.clone()))?; + return Ok(CommitOutcome { + revision: head_hex, + tree: tree_hash(&head), + created: false, + replayed: false, + }); + } + + let signature = Signature { + name: request.author.name.clone(), + email: request.author.email.clone(), + timestamp: Timestamp::now(), + }; + let mut tx = repo.start_transaction(); + let commit = tx + .repo_mut() + .new_commit(vec![head.id().clone()], tree) + .set_description(description(&request)) + .set_author(signature.clone()) + .set_committer(signature) + .write() + .await + .map_err(internal)?; + tx.commit(format!( + "commit {} for operation {}", + commit.id().hex(), + request.operation_id + )) + .await + .map_err(internal)?; + + let revision = commit.id().hex(); + write_record(&record_path, &Record::Committed(revision.clone()))?; + Ok(CommitOutcome { + revision, + tree: tree_hash(&commit), + created: true, + replayed: false, + }) +} + +/// The body of [`CommitService::outcome`]. Callers hold both repo locks. +async fn outcome_locked(repo_dir: &Path, operation_id: &str) -> CommitResult { + let repo = load_or_init(repo_dir).await?; + let head = head_commit(&repo)?; + repair_head_record(repo_dir, &head)?; + + let record_path = operation_record_path(repo_dir, operation_id); + if claim_record(&record_path, Record::Fenced)? { + return Ok(OperationOutcome::Absent); + } + match read_record(&record_path)? { + Record::Committed(revision) => Ok(OperationOutcome::Committed { revision }), + Record::Fenced => Ok(OperationOutcome::Absent), + Record::Pending => { + write_record(&record_path, &Record::Fenced)?; + Ok(OperationOutcome::Absent) + } + } +} + +/// Take the cross-process commit lock on `repo_dir`, creating the repo +/// directory if needed. Callers hold the repo's in-process gate first. +fn lock_repo(repo_dir: &Path) -> CommitResult { + create_repo_dir(repo_dir)?; + let deckd_dir = repo_dir.join("deckd"); + std::fs::create_dir_all(deckd_dir.join("operations")).map_err(internal)?; + let lock_path = deckd_dir.join("commit.lock"); + let started = Instant::now(); + loop { + match FileLock::try_lock(lock_path.clone()).map_err(internal)? { + Some(file_lock) => return Ok(file_lock), + None if started.elapsed() >= LOCK_TIMEOUT => { + return Err(CommitError::LockTimeout(repo_dir.display().to_string())) + } + None => std::thread::sleep(Duration::from_millis(10)), + } + } +} + +fn validate_repo_dir(repo_dir: &Path) -> CommitResult<()> { + if !repo_dir.is_absolute() { + return Err(CommitError::Invalid("repo must be an absolute path".into())); + } + if repo_dir + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return Err(CommitError::Invalid("repo must not contain `..`".into())); + } + Ok(()) +} + +/// Operation IDs name files, so they're limited to a filename-safe alphabet +/// and may not start with `.` (temporary records do). +fn validate_operation_id(operation_id: &str) -> CommitResult<()> { + let valid = (1..=128).contains(&operation_id.len()) + && !operation_id.starts_with('.') + && operation_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')); + if valid { + Ok(()) + } else { + Err(CommitError::Invalid(format!( + "invalid operation id {operation_id:?}: use 1-128 of [A-Za-z0-9._-], not starting with `.`" + ))) + } +} + +fn validate_commit_request(request: &CommitRequest) -> CommitResult<()> { + if CommitId::try_from_hex(&request.parent).is_none() { + return Err(CommitError::Invalid(format!( + "parent {:?} is not a revision id", + request.parent + ))); + } + if request.author.name.trim().is_empty() || request.author.email.trim().is_empty() { + return Err(CommitError::Invalid( + "author name and email are required".into(), + )); + } + // These would corrupt the git signature line. + let unsafe_author_char = |c: char| matches!(c, '\n' | '\r' | '<' | '>'); + if request.author.name.contains(unsafe_author_char) + || request.author.email.contains(unsafe_author_char) + { + return Err(CommitError::Invalid( + "author must not contain newlines, `<` or `>`".into(), + )); + } + // With no message, the trailer block would be the first paragraph, and + // trailer parsing (as in git) ignores trailers there. + if request.message.trim().is_empty() { + return Err(CommitError::Invalid("message is required".into())); + } + for trailer in &request.trailers { + let key_ok = !trailer.key.is_empty() + && trailer + .key + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-'); + if !key_ok || trailer.value.contains('\n') || trailer.value.contains('\r') { + return Err(CommitError::Invalid(format!( + "invalid trailer {:?}", + trailer.key + ))); + } + if trailer.key.eq_ignore_ascii_case(OPERATION_ID_TRAILER) { + return Err(CommitError::Invalid(format!( + "{OPERATION_ID_TRAILER} is set by deckd from operationId" + ))); + } + } + let mut seen = HashSet::new(); + for change in &request.changes { + if !is_valid_history_path(&change.path) { + return Err(CommitError::Invalid(format!( + "invalid path {:?}", + change.path + ))); + } + if !seen.insert(change.path.as_str()) { + return Err(CommitError::Invalid(format!( + "path {:?} appears more than once", + change.path + ))); + } + } + Ok(()) +} + +fn description(request: &CommitRequest) -> String { + let mut description = request.message.trim_end().to_owned(); + description.push_str("\n\n"); + for trailer in &request.trailers { + description.push_str(&format!("{}: {}\n", trailer.key, trailer.value)); + } + description.push_str(&format!( + "{OPERATION_ID_TRAILER}: {}\n", + request.operation_id + )); + description +} + +fn create_repo_dir(repo_dir: &Path) -> CommitResult<()> { + // The parent (normally the realm directory) must already exist, so a + // mistyped path fails instead of quietly growing a new tree. + match std::fs::create_dir(repo_dir) { + Ok(()) => Ok(()), + Err(e) if e.kind() == ErrorKind::AlreadyExists => Ok(()), + Err(e) if e.kind() == ErrorKind::NotFound => Err(CommitError::Invalid(format!( + "parent directory of {} does not exist", + repo_dir.display() + ))), + Err(e) => Err(internal(e)), + } +} + +/// Load the repo, creating it first if needed. Callers hold the repo lock. +/// The repo is built in a temporary directory and renamed into place, so a +/// crash mid-init never leaves a half-made `jj/`. +async fn load_or_init(repo_dir: &Path) -> CommitResult> { + let jj_dir = repo_dir.join("jj"); + if !jj_dir.is_dir() { + let staging = repo_dir.join("deckd").join(format!("init-{}", nonce())); + std::fs::create_dir_all(&staging).map_err(internal)?; + let settings = user_settings(None).map_err(internal)?; + let backend_initializer: &jj_lib::repo::BackendInitializer = &|settings, store_path| { + Ok(Box::new(jj_lib::git_backend::GitBackend::init_internal( + settings, + store_path, + gix::hash::Kind::Sha1, + )?)) + }; + ReadonlyRepo::init( + &settings, + &staging, + backend_initializer, + signer_from_settings(&settings).map_err(internal)?, + ReadonlyRepo::default_workspace_store_initializer(), + ReadonlyRepo::default_op_store_initializer(), + ReadonlyRepo::default_op_heads_store_initializer(), + ReadonlyRepo::default_index_store_initializer(), + ReadonlyRepo::default_submodule_store_initializer(), + ) + .await + .map_err(internal)?; + std::fs::rename(&staging, &jj_dir).map_err(internal)?; + } + load_repo(repo_dir).await +} + +async fn load_repo(repo_dir: &Path) -> CommitResult> { + let settings = user_settings(None).map_err(internal)?; + RepoLoader::init_from_file_system( + &settings, + &repo_dir.join("jj"), + &default_backend_factories(), + ) + .map_err(internal)? + .load_at_head() + .await + .map_err(internal) +} + +/// HEAD is the single visible head. History is linear, so more than one head +/// means two writers raced past the lock; that is reported, never resolved. +fn head_commit(repo: &Arc) -> CommitResult { + let heads = repo.view().heads(); + let mut iter = heads.iter(); + match (iter.next(), iter.next()) { + (Some(head), None) => repo.store().get_commit(head).map_err(internal), + _ => Err(CommitError::Internal(format!( + "history has {} heads; expected exactly one", + heads.len() + ))), + } +} + +fn get_commit(repo: &Arc, revision: &str) -> CommitResult { + let not_found = || CommitError::RevisionNotFound(revision.to_owned()); + let id = CommitId::try_from_hex(revision).ok_or_else(not_found)?; + match repo.store().get_commit(&id) { + Ok(commit) => Ok(commit), + Err(BackendError::ObjectNotFound { .. }) => Err(not_found()), + Err(e) => Err(internal(e)), + } +} + +fn tree_hash(commit: &Commit) -> String { + match commit.tree_ids().as_resolved() { + Some(tree_id) => tree_id.hex(), + // Explicit commits never write conflicted trees. + None => String::new(), + } +} + +async fn build_tree( + repo: &Arc, + parent: &Commit, + changes: &[Change], +) -> CommitResult { + let store = repo.store(); + let mut builder = MergedTreeBuilder::new(parent.tree()); + for change in changes { + let path = RepoPathBuf::from_internal_string(change.path.clone()) + .map_err(|e| CommitError::Invalid(format!("invalid path {:?}: {e}", change.path)))?; + let value = match &change.content { + Some(bytes) => { + let id = store + .write_file(&path, &mut bytes.as_slice()) + .await + .map_err(internal)?; + Merge::normal(TreeValue::File { + id, + executable: false, + copy_id: CopyId::placeholder(), + }) + } + None => Merge::absent(), + }; + builder.set_or_remove(path, value); + } + builder.write_tree().await.map_err(internal) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum Record { + Pending, + Committed(String), + Fenced, +} + +impl Record { + fn encode(&self) -> String { + match self { + Record::Pending => "pending\n".into(), + Record::Committed(revision) => format!("committed {revision}\n"), + Record::Fenced => "fenced\n".into(), + } + } + + fn decode(text: &str) -> CommitResult { + let text = text.trim(); + match text.split_once(' ') { + None if text == "pending" || text.is_empty() => Ok(Record::Pending), + None if text == "fenced" => Ok(Record::Fenced), + Some(("committed", revision)) => Ok(Record::Committed(revision.to_owned())), + _ => Err(CommitError::Internal(format!( + "unreadable operation record {text:?}" + ))), + } + } +} + +fn operation_record_path(repo_dir: &Path, operation_id: &str) -> PathBuf { + repo_dir.join("deckd").join("operations").join(operation_id) +} + +/// Create the record with `create_new`. Returns false when it already exists. +/// An empty record (a crash between create and write) reads as `pending`. +fn claim_record(path: &Path, record: Record) -> CommitResult { + let mut file = match OpenOptions::new().write(true).create_new(true).open(path) { + Ok(file) => file, + Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(false), + Err(e) => return Err(internal(e)), + }; + file.write_all(record.encode().as_bytes()) + .map_err(internal)?; + file.sync_all().map_err(internal)?; + Ok(true) +} + +fn read_record(path: &Path) -> CommitResult { + Record::decode(&std::fs::read_to_string(path).map_err(internal)?) +} + +/// Replace a record atomically: write a temporary file, then rename it over. +fn write_record(path: &Path, record: &Record) -> CommitResult<()> { + let dir = path + .parent() + .ok_or_else(|| CommitError::Internal("operation record has no parent".into()))?; + let name = path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| CommitError::Internal("operation record has no name".into()))?; + let tmp = dir.join(format!(".{name}.{}", nonce())); + { + let mut file = File::create(&tmp).map_err(internal)?; + file.write_all(record.encode().as_bytes()) + .map_err(internal)?; + file.sync_all().map_err(internal)?; + } + std::fs::rename(&tmp, path).map_err(internal)?; + File::open(dir) + .and_then(|d| d.sync_all()) + .map_err(internal)?; + Ok(()) +} + +/// Make HEAD's operation record say `committed HEAD`. The record is written +/// whenever it doesn't already say so, including when reading it fails: a +/// missing file may only be a stale NFS lookup, and the write is the truth. +fn repair_head_record(repo_dir: &Path, head: &Commit) -> CommitResult<()> { + let Some(operation_id) = parse_description_trailers(head.description()) + .into_iter() + .find(|t| t.key == OPERATION_ID_TRAILER) + .map(|t| t.value) + else { + return Ok(()); + }; + if validate_operation_id(&operation_id).is_err() { + return Ok(()); + } + let path = operation_record_path(repo_dir, &operation_id); + let expected = Record::Committed(head.id().hex()); + if read_record(&path).ok().as_ref() != Some(&expected) { + write_record(&path, &expected)?; + } + Ok(()) +} + +fn nonce() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or_default(); + format!("{}-{nanos}", std::process::id()) +} + +#[cfg(test)] +mod tests { + use futures::AsyncReadExt as _; + + use super::*; + + /// A realm directory whose history repo is `/.history`. The repo + /// directory itself doesn't exist yet; deckd creates it. + struct Realm { + root: PathBuf, + } + + impl Realm { + fn new(name: &str) -> Self { + let root = std::env::temp_dir().join(format!("deckd-commit-{name}-{}", nonce())); + std::fs::create_dir_all(&root).unwrap(); + Self { root } + } + + fn repo(&self) -> PathBuf { + self.root.join(".history") + } + } + + impl Drop for Realm { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.root); + } + } + + fn write(path: &str, content: &str) -> Change { + Change { + path: path.into(), + content: Some(content.as_bytes().to_vec()), + } + } + + fn delete(path: &str) -> Change { + Change { + path: path.into(), + content: None, + } + } + + fn request(parent: &str, operation_id: &str, changes: Vec) -> CommitRequest { + CommitRequest { + parent: parent.into(), + operation_id: operation_id.into(), + author: Author { + name: "Jane".into(), + email: "jane@example.com".into(), + }, + message: format!("Write for {operation_id}"), + trailers: vec![CommitTrailer { + key: "Boxel-Channel".into(), + value: "host".into(), + }], + changes, + } + } + + fn head(service: &CommitService, realm: &Realm) -> String { + service.head(&realm.repo()).unwrap().revision + } + + fn commit( + service: &CommitService, + realm: &Realm, + req: CommitRequest, + ) -> CommitResult { + service.commit(&realm.repo(), req) + } + + fn outcome(service: &CommitService, realm: &Realm, operation_id: &str) -> OperationOutcome { + service.outcome(&realm.repo(), operation_id).unwrap() + } + + /// Every file at `revision`, with its content. + fn files_at(realm: &Realm, revision: &str) -> Vec<(String, String)> { + pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let commit = get_commit(&repo, revision).unwrap(); + let mut files = Vec::new(); + for (path, value) in commit.tree().entries() { + let Some(Some(TreeValue::File { id, .. })) = value.unwrap().into_resolved().ok() + else { + panic!("unexpected tree value at {path:?}"); + }; + let mut content = String::new(); + repo.store() + .read_file(&path, &id) + .await + .unwrap() + .read_to_string(&mut content) + .await + .unwrap(); + files.push((path.as_internal_file_string().to_owned(), content)); + } + files + }) + } + + fn description_at(realm: &Realm, revision: &str) -> String { + pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + get_commit(&repo, revision) + .unwrap() + .description() + .to_owned() + }) + } + + fn files(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(p, c)| (p.to_string(), c.to_string())) + .collect() + } + + #[test] + fn commit_contains_exactly_the_listed_paths() { + let realm = Realm::new("explicit"); + let service = CommitService::new(); + let root = head(&service, &realm); + // Bytes on disk that no request listed: a working-copy snapshot would + // pick these up. + std::fs::write(realm.root.join("unlisted.json"), "{}").unwrap(); + std::fs::write(realm.repo().join("stray.txt"), "stray").unwrap(); + + let done = commit( + &service, + &realm, + request( + &root, + "op-1", + vec![write("Author/jane.json", "{\"name\":\"Jane\"}")], + ), + ) + .unwrap(); + + assert!(done.created); + assert!(!done.replayed); + assert_eq!(head(&service, &realm), done.revision); + assert_eq!( + files_at(&realm, &done.revision), + files(&[("Author/jane.json", "{\"name\":\"Jane\"}")]) + ); + } + + #[test] + fn commit_records_message_author_and_trailers() { + let realm = Realm::new("description"); + let service = CommitService::new(); + let root = head(&service, &realm); + + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "a")]), + ) + .unwrap(); + + assert_eq!( + description_at(&realm, &done.revision), + "Write for op-1\n\nBoxel-Channel: host\nDeckd-Operation-Id: op-1\n" + ); + let author = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + get_commit(&repo, &done.revision).unwrap().author().clone() + }); + assert_eq!( + (author.name.as_str(), author.email.as_str()), + ("Jane", "jane@example.com") + ); + } + + #[test] + fn stale_parent_is_refused_with_current_head() { + let realm = Realm::new("cas"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + + let error = commit( + &service, + &realm, + request(&root, "op-2", vec![write("b.txt", "2")]), + ) + .unwrap_err(); + + match error { + CommitError::ParentMismatch { + parent, + head: current, + } => { + assert_eq!(parent, root); + assert_eq!(current, first.revision); + } + other => panic!("expected parent mismatch, got {other:?}"), + } + assert_eq!(head(&service, &realm), first.revision); + } + + #[test] + fn conflicted_operation_can_retry_on_the_new_head() { + let realm = Realm::new("cas-retry"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + commit( + &service, + &realm, + request(&root, "op-2", vec![write("b.txt", "2")]), + ) + .unwrap_err(); + + let retried = commit( + &service, + &realm, + request(&first.revision, "op-2", vec![write("b.txt", "2")]), + ) + .unwrap(); + + assert!(retried.created); + assert_eq!( + files_at(&realm, &retried.revision), + files(&[("a.txt", "1"), ("b.txt", "2")]) + ); + } + + #[test] + fn retried_operation_returns_the_original_revision() { + let realm = Realm::new("idempotent"); + let service = CommitService::new(); + let root = head(&service, &realm); + let req = request(&root, "op-1", vec![write("a.txt", "1")]); + let first = commit(&service, &realm, req.clone()).unwrap(); + + // The retry's parent is now stale; the replay still wins. + let again = commit(&service, &realm, req).unwrap(); + + assert_eq!(again.revision, first.revision); + assert_eq!(again.tree, first.tree); + assert!(again.replayed); + assert!(!again.created); + assert_eq!(head(&service, &realm), first.revision); + let parent = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + get_commit(&repo, &first.revision) + .unwrap() + .parent_ids() + .to_vec() + }); + assert_eq!( + parent.iter().map(|id| id.hex()).collect::>(), + vec![root] + ); + } + + #[test] + fn absent_outcome_fences_the_operation_across_restarts() { + let realm = Realm::new("fence"); + let service = CommitService::new(); + let root = head(&service, &realm); + + assert_eq!(outcome(&service, &realm, "op-1"), OperationOutcome::Absent); + let refused = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ); + assert!(matches!(refused, Err(CommitError::OperationFenced(_)))); + + // A new instance has no memory of the fence; only the repo does. + let restarted = CommitService::new(); + let refused = commit( + &restarted, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ); + assert!(matches!(refused, Err(CommitError::OperationFenced(_)))); + assert_eq!( + outcome(&restarted, &realm, "op-1"), + OperationOutcome::Absent + ); + assert_eq!(head(&restarted, &realm), root); + } + + #[test] + fn outcome_reports_a_committed_operation() { + let realm = Realm::new("outcome-committed"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + + let restarted = CommitService::new(); + + assert_eq!( + outcome(&restarted, &realm, "op-1"), + OperationOutcome::Committed { + revision: done.revision.clone() + } + ); + // Asking doesn't fence a committed operation; a retry still replays. + let again = commit( + &restarted, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + assert_eq!(again.revision, done.revision); + } + + #[test] + fn commit_published_without_its_record_is_reported_committed() { + let realm = Realm::new("repair"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + let record = operation_record_path(&realm.repo(), "op-1"); + + // Crash after publish: the record still holds the claim, or is gone. + std::fs::write(&record, "pending\n").unwrap(); + assert_eq!( + outcome(&service, &realm, "op-1"), + OperationOutcome::Committed { + revision: done.revision.clone() + } + ); + std::fs::remove_file(&record).unwrap(); + let again = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + assert_eq!(again.revision, done.revision); + assert!(again.replayed); + } + + #[test] + fn claim_left_by_a_crash_before_publish_resolves_absent() { + let realm = Realm::new("pending"); + let service = CommitService::new(); + let root = head(&service, &realm); + // Create the repo layout, then leave a claim as a crashed commit would. + outcome(&service, &realm, "unrelated"); + std::fs::write(operation_record_path(&realm.repo(), "op-1"), "pending\n").unwrap(); + std::fs::write(operation_record_path(&realm.repo(), "op-2"), "").unwrap(); + + assert_eq!(outcome(&service, &realm, "op-1"), OperationOutcome::Absent); + let refused = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ); + assert!(matches!(refused, Err(CommitError::OperationFenced(_)))); + + // Retrying the crashed operation instead takes its claim over. + let done = commit( + &service, + &realm, + request(&root, "op-2", vec![write("b.txt", "2")]), + ) + .unwrap(); + assert!(done.created); + } + + #[test] + fn racing_writers_stay_linear() { + let realm = Realm::new("race"); + let root = head(&CommitService::new(), &realm); + let writers = 8; + + // Each writer is its own service, as separate deckd processes would be. + let first_round: Vec<_> = std::thread::scope(|s| { + let handles: Vec<_> = (0..writers) + .map(|i| { + let realm = &realm; + let root = root.clone(); + s.spawn(move || { + let service = CommitService::new(); + commit( + &service, + realm, + request( + &root, + &format!("op-{i}"), + vec![write(&format!("{i}.txt"), "x")], + ), + ) + }) + }) + .collect(); + handles.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert_eq!(first_round.iter().filter(|r| r.is_ok()).count(), 1); + assert!(first_round + .iter() + .filter_map(|r| r.as_ref().err()) + .all(|e| matches!(e, CommitError::ParentMismatch { .. }))); + + // Losers retry the same operation on whatever HEAD the conflict reports. + std::thread::scope(|s| { + for i in 0..writers { + let realm = &realm; + s.spawn(move || { + let service = CommitService::new(); + let mut parent = head(&service, realm); + loop { + let req = request( + &parent, + &format!("op-{i}"), + vec![write(&format!("{i}.txt"), "x")], + ); + match commit(&service, realm, req) { + Ok(_) => return, + Err(CommitError::ParentMismatch { head, .. }) => parent = head, + Err(other) => panic!("writer {i}: {other:?}"), + } + } + }); + } + }); + + let service = CommitService::new(); + let tip = head(&service, &realm); + assert_eq!(files_at(&realm, &tip).len(), writers); + let depth = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let mut depth = 0; + let mut commit = get_commit(&repo, &tip).unwrap(); + while commit.id() != repo.store().root_commit_id() { + assert_eq!(commit.parent_ids().len(), 1); + commit = repo.store().get_commit(&commit.parent_ids()[0]).unwrap(); + depth += 1; + } + depth + }); + assert_eq!(depth, writers); + } + + #[test] + fn parent_matches_regardless_of_hex_case() { + let realm = Realm::new("hex-case"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + + let second = commit( + &service, + &realm, + request( + &first.revision.to_uppercase(), + "op-2", + vec![write("b.txt", "2")], + ), + ) + .unwrap(); + + assert!(second.created); + } + + #[test] + fn delete_removes_the_path() { + let realm = Realm::new("delete"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request( + &root, + "op-1", + vec![write("dir/a.txt", "a"), write("b.txt", "b")], + ), + ) + .unwrap(); + + let second = commit( + &service, + &realm, + request(&first.revision, "op-2", vec![delete("dir/a.txt")]), + ) + .unwrap(); + + assert!(second.created); + assert_eq!(files_at(&realm, &second.revision), files(&[("b.txt", "b")])); + assert_eq!( + files_at(&realm, &first.revision), + files(&[("b.txt", "b"), ("dir/a.txt", "a")]) + ); + } + + #[test] + fn unchanged_content_records_no_revision() { + let realm = Realm::new("noop"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a.txt", "1")]), + ) + .unwrap(); + + let same = commit( + &service, + &realm, + request( + &first.revision, + "op-2", + vec![write("a.txt", "1"), delete("missing.txt")], + ), + ) + .unwrap(); + + assert!(!same.created); + assert_eq!(same.revision, first.revision); + assert_eq!(head(&service, &realm), first.revision); + assert_eq!( + outcome(&service, &realm, "op-2"), + OperationOutcome::Committed { + revision: first.revision + } + ); + } + + #[test] + fn identical_trees_have_identical_hashes() { + let one = Realm::new("tree-one"); + let two = Realm::new("tree-two"); + let service = CommitService::new(); + let changes = || vec![write("Author/jane.json", "{}"), write("index.json", "[]")]; + let root_one = head(&service, &one); + let root_two = head(&service, &two); + + let a = commit(&service, &one, request(&root_one, "op-a", changes())).unwrap(); + let mut other = request(&root_two, "op-b", changes()); + other.author.name = "Someone Else".into(); + other.message = "A different message".into(); + let b = commit(&service, &two, other).unwrap(); + + assert_ne!(a.revision, b.revision); + assert_eq!(a.tree, b.tree); + let looked_up = service.tree_hash(&one.repo(), &a.revision).unwrap(); + assert_eq!(looked_up.tree, a.tree); + let again = CommitService::new() + .tree_hash(&one.repo(), &a.revision) + .unwrap(); + assert_eq!(again.tree, a.tree); + } + + #[test] + fn tree_hash_of_an_unknown_revision_is_not_found() { + let realm = Realm::new("tree-missing"); + let service = CommitService::new(); + head(&service, &realm); + + let error = service + .tree_hash(&realm.repo(), &"ab".repeat(20)) + .unwrap_err(); + + assert!(matches!(error, CommitError::RevisionNotFound(_))); + } + + #[test] + fn invalid_requests_are_refused() { + let realm = Realm::new("invalid"); + let service = CommitService::new(); + let root = head(&service, &realm); + let invalid = |req: CommitRequest| { + matches!(commit(&service, &realm, req), Err(CommitError::Invalid(_))) + }; + + assert!(invalid(request(&root, "../escape", vec![write("a", "")]))); + assert!(invalid(request(&root, ".hidden", vec![write("a", "")]))); + assert!(invalid(request("not-hex", "op", vec![write("a", "")]))); + assert!(invalid(request(&root, "op", vec![write("/abs", "")]))); + assert!(invalid(request(&root, "op", vec![write("a/../b", "")]))); + assert!(invalid(request( + &root, + "op", + vec![write("a", "1"), delete("a")] + ))); + let mut no_message = request(&root, "op", vec![write("a", "")]); + no_message.message = " \n".into(); + assert!(invalid(no_message)); + let mut bad_author = request(&root, "op", vec![write("a", "")]); + bad_author.author.email = "jane@example.com> evil".into(); + assert!(invalid(bad_author)); + let mut reserved = request(&root, "op", vec![write("a", "")]); + reserved.trailers.push(CommitTrailer { + key: "deckd-operation-id".into(), + value: "other".into(), + }); + assert!(invalid(reserved)); + let relative = service.head(Path::new("relative/.history")); + assert!(matches!(relative, Err(CommitError::Invalid(_)))); + let orphan = service.head(&realm.root.join("missing/.history")); + assert!(matches!(orphan, Err(CommitError::Invalid(_)))); + // None of those wrote anything. + assert_eq!(head(&service, &realm), root); + } +} diff --git a/packages/deckd/src/history/jj_lib.rs b/packages/deckd/src/history/jj_lib.rs index eb749ba..d5470aa 100644 --- a/packages/deckd/src/history/jj_lib.rs +++ b/packages/deckd/src/history/jj_lib.rs @@ -22,6 +22,9 @@ use futures::StreamExt as _; use jj_lib::backend::{FileId, TreeValue}; use jj_lib::commit::Commit; use jj_lib::config::{ConfigLayer, ConfigSource, StackedConfig}; +use jj_lib::default_backend_factories::{ + default_backend_factories, default_working_copy_factories, +}; use jj_lib::fileset::FilesetExpression; use jj_lib::gitignore::GitIgnoreFile; use jj_lib::matchers::{EverythingMatcher, NothingMatcher}; @@ -29,14 +32,12 @@ use jj_lib::merge::Diff; use jj_lib::merged_tree::MergedTree; use jj_lib::object_id::{HexPrefix, ObjectId, PrefixResolution}; use jj_lib::ref_name::WorkspaceName; -use jj_lib::repo::{ReadonlyRepo, Repo as _, StoreFactories}; +use jj_lib::repo::{ReadonlyRepo, Repo as _}; use jj_lib::repo_path::{RepoPath, RepoPathBuf}; use jj_lib::revset::{RevsetExpression, RevsetFilterPredicate}; use jj_lib::settings::UserSettings; use jj_lib::working_copy::SnapshotOptions; -use jj_lib::workspace::{ - default_working_copy_factories, Workspace, WorkspaceInitError, WorkspaceLoadError, -}; +use jj_lib::workspace::{Workspace, WorkspaceInitError, WorkspaceLoadError}; use tokio::sync::Mutex; /// Per-depot async note debounce. Realm servers fire `/note` and return; @@ -432,7 +433,7 @@ pub fn is_valid_history_path(path: &str) -> bool { .any(|segment| segment.is_empty() || segment == "..") } -fn hermetic_git_env() { +pub(super) fn hermetic_git_env() { // Prevent GitBackend from reading host git config (same as historyd). unsafe { std::env::set_var("GIT_CONFIG_SYSTEM", "/dev/null"); @@ -440,7 +441,7 @@ fn hermetic_git_env() { } } -fn user_settings(actor: Option<&Actor>) -> HistoryResult { +pub(super) fn user_settings(actor: Option<&Actor>) -> HistoryResult { let name = actor .map(|a| sanitize(&a.name)) .unwrap_or_else(|| "deck-history".into()); @@ -461,7 +462,8 @@ fn user_settings(actor: Option<&Actor>) -> HistoryResult { ConfigLayer::parse(ConfigSource::User, &config_text) .map_err(|e| HistoryError::msg(e.to_string()))?, ); - UserSettings::from_config(config).map_err(|e| HistoryError::msg(e.to_string())) + UserSettings::from_config_and_home_dir(config, None) + .map_err(|e| HistoryError::msg(e.to_string())) } fn sanitize(value: &str) -> String { @@ -543,7 +545,8 @@ async fn ensure_repo_inner(dir: &Path) -> HistoryResult<()> { if !jj.is_dir() { // historyd used init_colocated_git; Deck forbids depot .git → internal. let settings = user_settings(None)?; - let (_workspace, _repo) = Workspace::init_internal_git(&settings, dir).await?; + let (_workspace, _repo) = + Workspace::init_internal_git(&settings, dir, gix::hash::Kind::Sha1).await?; } if !jj_repo.is_dir() { @@ -595,7 +598,7 @@ async fn load_workspace(dir: &Path) -> HistoryResult<(Workspace, Arc Ok(paths @@ -851,7 +855,7 @@ async fn files_summary_from_index( async fn file_at_inner(dir: &Path, revision_id: &str, path: &str) -> HistoryResult>> { let (_workspace, repo) = load_workspace(dir).await?; - let commit = match resolve_revision(repo.as_ref(), revision_id)? { + let commit = match resolve_revision(repo.as_ref(), revision_id).await? { Some(c) => c, None => return Ok(None), }; @@ -872,7 +876,8 @@ async fn file_at_inner(dir: &Path, revision_id: &str, path: &str) -> HistoryResu async fn file_list_at_inner(dir: &Path, revision_id: &str) -> HistoryResult> { let (_workspace, repo) = load_workspace(dir).await?; - let commit = resolve_revision(repo.as_ref(), revision_id)? + let commit = resolve_revision(repo.as_ref(), revision_id) + .await? .ok_or_else(|| HistoryError::msg("revision not found"))?; let root = repo.store().root_commit(); let mut paths = Vec::new(); @@ -896,7 +901,8 @@ async fn file_list_at_inner(dir: &Path, revision_id: &str) -> HistoryResult HistoryResult { let (_workspace, repo) = load_workspace(dir).await?; - let target = resolve_revision(repo.as_ref(), revision_id)? + let target = resolve_revision(repo.as_ref(), revision_id) + .await? .ok_or_else(|| HistoryError::msg("revision not found"))?; let wc_id = repo .view() @@ -931,10 +937,14 @@ async fn restore_plan_inner(dir: &Path, revision_id: &str) -> HistoryResult HistoryResult> { +async fn resolve_revision( + repo: &ReadonlyRepo, + revision_id: &str, +) -> HistoryResult> { if let Some(prefix) = HexPrefix::try_from_reverse_hex(revision_id) { match repo .resolve_change_id_prefix(&prefix) + .await .map_err(|e| HistoryError::msg(e.to_string()))? { PrefixResolution::SingleMatch(targets) => { @@ -957,6 +967,7 @@ fn resolve_revision(repo: &ReadonlyRepo, revision_id: &str) -> HistoryResult { diff --git a/packages/deckd/src/history/mod.rs b/packages/deckd/src/history/mod.rs index 125b9e4..b0dfae4 100644 --- a/packages/deckd/src/history/mod.rs +++ b/packages/deckd/src/history/mod.rs @@ -7,6 +7,7 @@ use std::pin::Pin; use serde::{Deserialize, Serialize}; use thiserror::Error; +pub mod commit; pub mod jj_lib; pub mod jj_lib_notes; diff --git a/packages/deckd/src/main.rs b/packages/deckd/src/main.rs index 262d762..1da551f 100644 --- a/packages/deckd/src/main.rs +++ b/packages/deckd/src/main.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use tracing_subscriber::EnvFilter; use crate::api::AppState; +use crate::history::commit::CommitService; use crate::history::jj_lib::JjLibHistory; use crate::history::HistoryBackend; use crate::watch::WatchPlane; @@ -42,17 +43,23 @@ async fn main() -> Result<(), Box> { }; let history: Arc = Arc::new(jj_history); + let commits = Arc::new(CommitService::new()); #[cfg(feature = "build")] let state = { let jj_bin = std::env::var("JJ_BIN").unwrap_or_else(|_| "jj".into()); AppState { history, + commits, build: Some(Arc::new(crate::build::BuildPlane::new(jj_bin))), watch, } }; #[cfg(not(feature = "build"))] - let state = AppState { history, watch }; + let state = AppState { + history, + commits, + watch, + }; let app = api::router(state); let addr: SocketAddr = bind.parse()?; diff --git a/packages/deckd/tests/commit_restart.rs b/packages/deckd/tests/commit_restart.rs new file mode 100644 index 0000000..416774e --- /dev/null +++ b/packages/deckd/tests/commit_restart.rs @@ -0,0 +1,153 @@ +//! The commit API against a real deckd process that is killed and restarted +//! between calls: operation outcomes and fences live in the repo, so a new +//! process gives the same answers. + +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use serde_json::{json, Value}; + +struct Deckd { + child: Child, + base: String, +} + +impl Deckd { + /// Start deckd on a fresh port. A restart needs no particular port: all + /// state is in the repo. + fn start() -> Self { + let port = free_port(); + let child = Command::new(env!("CARGO_BIN_EXE_deckd")) + .env("DECKD_BIND", format!("127.0.0.1:{port}")) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start deckd"); + let base = format!("http://127.0.0.1:{port}"); + let started = Instant::now(); + while ureq::get(&format!("{base}/health")).call().is_err() { + assert!( + started.elapsed() < Duration::from_secs(20), + "deckd did not start" + ); + std::thread::sleep(Duration::from_millis(50)); + } + Self { child, base } + } + + /// POST `body` and return the status and JSON response, error or not. + fn post(&self, route: &str, body: Value) -> (u16, Value) { + match ureq::post(&format!("{}{route}", self.base)).send_json(body) { + Ok(response) => (response.status(), response.into_json().unwrap()), + Err(ureq::Error::Status(status, response)) => (status, response.into_json().unwrap()), + Err(error) => panic!("{route}: {error}"), + } + } + + fn kill(mut self) { + self.child.kill().unwrap(); + self.child.wait().unwrap(); + } +} + +impl Drop for Deckd { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +fn realm(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!( + "deckd-restart-{name}-{}-{nanos}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn commit_body(repo: &Path, parent: &str, operation_id: &str) -> Value { + json!({ + "repo": repo, + "parent": parent, + "operationId": operation_id, + "author": { "name": "Jane", "email": "jane@example.com" }, + "message": "Update Author/jane.json", + "trailers": [{ "key": "Boxel-Channel", "value": "host" }], + "changes": [ + { "path": "Author/jane.json", "contentBase64": "eyJuYW1lIjoiSmFuZSJ9" }, + { "path": "Author/old.json", "delete": true } + ] + }) +} + +#[test] +fn committed_outcome_survives_a_restart() { + let realm = realm("committed"); + let repo = realm.join(".history"); + let deckd = Deckd::start(); + let (_, head) = deckd.post("/head", json!({ "repo": repo })); + let root = head["revision"].as_str().unwrap().to_owned(); + + let (status, committed) = deckd.post("/commit", commit_body(&repo, &root, "op-1")); + assert_eq!(status, 200, "{committed}"); + deckd.kill(); + + let deckd = Deckd::start(); + let (status, outcome) = deckd.post("/outcome", json!({ "repo": repo, "operationId": "op-1" })); + assert_eq!(status, 200); + assert_eq!( + outcome, + json!({ "state": "committed", "revision": committed["revision"] }) + ); + let (_, retried) = deckd.post("/commit", commit_body(&repo, &root, "op-1")); + assert_eq!(retried["revision"], committed["revision"]); + assert_eq!(retried["replayed"], json!(true)); + let (_, tree) = deckd.post( + "/tree-hash", + json!({ "repo": repo, "revision": committed["revision"] }), + ); + assert_eq!(tree["tree"], committed["tree"]); + std::fs::remove_dir_all(realm).unwrap(); +} + +#[test] +fn fence_survives_a_restart() { + let realm = realm("fence"); + let repo = realm.join(".history"); + let deckd = Deckd::start(); + let (status, outcome) = deckd.post("/outcome", json!({ "repo": repo, "operationId": "op-1" })); + assert_eq!(status, 200); + assert_eq!(outcome, json!({ "state": "absent" })); + let (_, head) = deckd.post("/head", json!({ "repo": repo })); + let root = head["revision"].as_str().unwrap().to_owned(); + deckd.kill(); + + let deckd = Deckd::start(); + let (status, refused) = deckd.post("/commit", commit_body(&repo, &root, "op-1")); + assert_eq!(status, 409); + assert_eq!(refused["code"], json!("operation-fenced")); + + // The fence is per operation: other operations still commit, under CAS. + let (status, first) = deckd.post("/commit", commit_body(&repo, &root, "op-2")); + assert_eq!(status, 200, "{first}"); + let (status, conflict) = deckd.post("/commit", commit_body(&repo, &root, "op-3")); + assert_eq!(status, 409); + assert_eq!(conflict["code"], json!("parent-mismatch")); + assert_eq!(conflict["head"], first["revision"]); + std::fs::remove_dir_all(realm).unwrap(); +} From c8ccedb641845ff47c5489f0cb31a0dfb24eefff Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 26 Oct 2026 16:04:40 -0400 Subject: [PATCH 2/3] Harden explicit commits: paths, NFS locking, init races, typed errors - Refuse changes that cross a file/directory boundary (a listed path inside another listed path, inside a file in HEAD, or naming a directory in HEAD) and paths with `.` or `.git` segments, so a revision holds exactly the listed paths and only valid git trees. - Lock a persistent deckd/HEAD instead of jj-lib's FileLock, which deletes its file on release and so can be held twice over NFS. The file records the revision about to be published; jj's HEAD must match it before a write, a recorded-but-unpublished revision is published by the next call, and any other mismatch is refused as history-unsettled. - Never overwrite a fenced record for HEAD's operation; report it. - Treat a lost race to initialize jj/ as already initialized, and let /tree-hash load before deciding a repo is missing. - Answer unparseable requests with typed {code, error} bodies on every route, and lower the /commit body limit to 64 MiB. - Warn when a snapshot skips a file whose name isn't valid UTF-8. - List the explicit-commit routes in docs/deck-daemon.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tg2ZC6YEPfRcNZSoWKq2HS --- docs/deck-daemon.md | 12 + packages/deckd/README.md | 55 ++- packages/deckd/src/api.rs | 70 +++- packages/deckd/src/history/commit.rs | 539 ++++++++++++++++++++++--- packages/deckd/src/history/jj_lib.rs | 10 +- packages/deckd/tests/commit_restart.rs | 76 +--- packages/deckd/tests/common/mod.rs | 89 ++++ packages/deckd/tests/http_errors.rs | 85 ++++ 8 files changed, 777 insertions(+), 159 deletions(-) create mode 100644 packages/deckd/tests/common/mod.rs create mode 100644 packages/deckd/tests/http_errors.rs diff --git a/docs/deck-daemon.md b/docs/deck-daemon.md index e5365e9..67c6011 100644 --- a/docs/deck-daemon.md +++ b/docs/deck-daemon.md @@ -194,6 +194,18 @@ History (compat with today’s historyd client in | `POST /file-list-at` | `{ dir, revisionId }` | paths — **close the CLI gap** | | `POST /restore-plan` | `{ dir, revisionId }` | `{ writes, deletes }` | +Explicit commits, for a writer that stages content itself and must record each +write exactly once. These address a bare history repo by path rather than a +depot; the [deckd README](../packages/deckd/README.md#explicit-commits) has the +full contract: + +| Route | Body (sketch) | Result | +| ----------------- | -------------------------------------------------------------------- | --------------------------------------------------------- | +| `POST /commit` | `{ repo, parent, operationId, author, message, trailers?, changes }` | `{ revision, tree, created, replayed }`, or a typed error | +| `POST /outcome` | `{ repo, operationId }` | `committed` + revision, or `absent` (fences the ID) | +| `POST /head` | `{ repo }` | `{ revision, tree }` | +| `POST /tree-hash` | `{ repo, revision }` | `{ tree }`, the git tree ID | + Build (deferred — **not** on the Hub serve path): | Route | Body (sketch) | Result | diff --git a/packages/deckd/README.md b/packages/deckd/README.md index 12067b8..04d48e3 100644 --- a/packages/deckd/README.md +++ b/packages/deckd/README.md @@ -86,7 +86,7 @@ the directory and a bare jj repo inside it on first use: ```text / jj/ # jj repo, internal git backend, no working copy - deckd/commit.lock # flock held around /commit and /outcome + deckd/HEAD # recorded HEAD; locked around every call but /tree-hash deckd/operations/ # pending | committed | fenced ``` @@ -96,7 +96,8 @@ head; an empty repo's HEAD is the root commit, `0000…0000`. ### Guarantees - **Exactly the listed paths.** A revision is HEAD's tree plus the request's - `changes`; nothing on disk is read. + `changes`; nothing on disk is read. A change that would turn a file into a + directory or back is refused, since it would remove paths it doesn't list. - **Compare-and-swap on HEAD.** A commit whose `parent` isn't HEAD is refused with `parent-mismatch` and the current `head`, never re-parented. - **Idempotent by operation ID.** Committing an ID that already committed @@ -112,9 +113,26 @@ head; an empty repo's HEAD is the root commit, `0000…0000`. - **No-op writes record nothing.** When the changes leave the tree as it was, no revision is created; the response returns HEAD with `created: false`. -Correctness doesn't depend on deckd's in-process state. The file lock -serializes deckd processes; a writer running on several hosts should still -serialize its own commits (realm-server uses a Postgres advisory lock). +Correctness doesn't depend on deckd's in-process state: + +- **One lock file, never deleted.** An exclusive lock on `deckd/HEAD` + serializes deckd processes. Deleting a lock file on release lets two NFS + clients each hold the lock, one on the unlinked inode and one on its + replacement, so this file stays. +- **A recorded HEAD.** `deckd/HEAD` holds the revision deckd last recorded, + written just before jj publishes it. Taking an NFS lock revalidates that + file's cached data, so it is current even when this host's listing of jj's + operations is stale. jj's HEAD must match it before anything is written. + - When jj is one commit behind (a crash between recording and publishing, or + a stale listing), the recorded commit is published, which is idempotent. + - Any other mismatch is reloaded for a few seconds, then refused with + `history-unsettled` and nothing written. +- **Fences hold.** A `fenced` record for an operation found committed means + the lock failed to exclude a writer. It is reported as an error that needs an + operator, never overwritten. + +A writer running on several hosts should still serialize its own commits +(realm-server uses a Postgres advisory lock). Each commit's message ends with the caller's trailers, then `Deckd-Operation-Id: `, which callers may not set themselves. @@ -148,7 +166,9 @@ Each commit's message ends with the caller's trailers, then - `operationId`: 1–128 characters from `[A-Za-z0-9._-]`, not starting with `.`. - `changes`: each entry has `path` plus exactly one of `contentBase64` or `delete: true`. A path appears at most once. Paths are relative and may not - contain `..` or empty segments. Deleting an absent path changes nothing. + contain empty, `.`, `..` or `.git` segments. No listed path may be inside + another listed path, inside a file in HEAD, or name a directory in HEAD. + Deleting an absent path changes nothing. - `trailers`: optional. Keys match `[A-Za-z0-9-]+`; values are one line. - `message` is required. `author.name` and `author.email` are required and may not contain newlines, `<` or `>`. @@ -197,16 +217,19 @@ any repo, whoever committed it and whatever the message. ### Errors -Every error body is `{ "code", "error" }`, plus fields named below. - -| Status | `code` | When | -| ------ | -------------------- | ---------------------------------------------------- | -| 400 | `invalid-request` | Bad repo path, operation ID, path, trailer or base64 | -| 404 | `revision-not-found` | `/tree-hash` for a revision the repo doesn't have | -| 409 | `parent-mismatch` | `parent` isn't HEAD; body has `head` | -| 409 | `operation-fenced` | The operation ID was reported `absent` by `/outcome` | -| 503 | `lock-timeout` | Another process held the repo's lock for 30 s | -| 500 | `internal` | jj or filesystem failure | +Every error body is `{ "code", "error" }`, plus fields named below, on every +route, including a request deckd can't parse. + +| Status | `code` | When | +| ----------- | -------------------- | ----------------------------------------------------------------------- | +| 400/415/422 | `invalid-request` | Unparseable JSON, a missing field, the wrong content type, or bad input | +| 404 | `revision-not-found` | `/tree-hash` for a revision the repo doesn't have | +| 409 | `parent-mismatch` | `parent` isn't HEAD; body has `head` | +| 409 | `operation-fenced` | The operation ID was reported `absent` by `/outcome` | +| 413 | `payload-too-large` | A `/commit` body over 64 MiB; split the change into several commits | +| 503 | `history-unsettled` | jj's HEAD didn't match `deckd/HEAD` within a few seconds; retry | +| 503 | `lock-timeout` | Another process held the repo's lock for 30 s | +| 500 | `internal` | jj or filesystem failure, or a fenced operation found committed | ## Run diff --git a/packages/deckd/src/api.rs b/packages/deckd/src/api.rs index 7ee2c78..3cb8d4b 100644 --- a/packages/deckd/src/api.rs +++ b/packages/deckd/src/api.rs @@ -2,7 +2,8 @@ use std::sync::Arc; -use axum::extract::{DefaultBodyLimit, State}; +use axum::extract::rejection::JsonRejection; +use axum::extract::{DefaultBodyLimit, FromRequest, Request, State}; use axum::http::StatusCode; use axum::routing::{get, post}; use axum::{Json, Router}; @@ -83,8 +84,38 @@ pub fn router(state: AppState) -> Router { } /// A commit carries every staged file base64-encoded in one JSON body, so it -/// needs far more than axum's 2 MB default. Batch limits are the caller's. -const COMMIT_BODY_LIMIT: usize = 512 * 1024 * 1024; +/// needs more than axum's 2 MB default. The whole body and its decoded copy +/// are held in memory, so this stays well under what the sidecar can spare; +/// a larger push is split into several commits by the caller. +const COMMIT_BODY_LIMIT: usize = 64 * 1024 * 1024; + +/// A JSON request body whose rejections (malformed JSON, a missing field, the +/// wrong content type, a body over the limit) answer as typed `ApiError`s, +/// like every other error. +struct Body(T); + +impl FromRequest for Body +where + Json: FromRequest, + S: Send + Sync, +{ + type Rejection = ApiError; + + async fn from_request(req: Request, state: &S) -> Result { + match Json::::from_request(req, state).await { + Ok(Json(value)) => Ok(Body(value)), + Err(rejection) => { + let status = rejection.status(); + let code = if status == StatusCode::PAYLOAD_TOO_LARGE { + "payload-too-large" + } else { + "invalid-request" + }; + Err(ApiError::new(status, code, rejection.body_text())) + } + } + } +} #[derive(Debug, Deserialize)] struct DirBody { @@ -147,7 +178,7 @@ struct RestorePlanBody { async fn ensure( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { st.history.ensure(&body.dir).await?; if let Some(watch) = &st.watch { @@ -160,7 +191,7 @@ async fn ensure( async fn note( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { st.history.note(&body.dir, &body.path).await?; Ok(Json(serde_json::json!({}))) @@ -168,7 +199,7 @@ async fn note( async fn flush( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let change_id = st.history.flush_notes(&body.dir).await?; Ok(Json(SealResponse { change_id })) @@ -176,7 +207,7 @@ async fn flush( async fn seal( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let change_id = st .history @@ -187,14 +218,14 @@ async fn seal( async fn list( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result>, ApiError> { Ok(Json(st.history.list(&body.dir).await?)) } async fn file_at( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let bytes = st .history @@ -214,7 +245,7 @@ async fn file_at( async fn file_list_at( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { Ok(Json(FileListAtResponse { paths: st @@ -226,7 +257,7 @@ async fn file_list_at( async fn restore_plan( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { Ok(Json( st.history @@ -283,7 +314,7 @@ struct TreeHashResponse { async fn commit( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let mut changes = Vec::with_capacity(body.changes.len()); for change in body.changes { @@ -325,7 +356,7 @@ async fn commit( async fn outcome( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let repo = std::path::PathBuf::from(body.repo); let operation_id = body.operation_id; @@ -336,7 +367,7 @@ async fn outcome( async fn head( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let repo = std::path::PathBuf::from(body.repo); Ok(Json(run_commits(&st, move |c| c.head(&repo)).await?)) @@ -344,7 +375,7 @@ async fn head( async fn tree_hash( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let repo = std::path::PathBuf::from(body.repo); let revision = body.revision; @@ -369,7 +400,7 @@ where #[cfg(feature = "build")] async fn ensure_build( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let Some(build) = &st.build else { return Err("build plane unavailable".to_string().into()); @@ -380,7 +411,7 @@ async fn ensure_build( #[cfg(feature = "build")] async fn artifact_at( State(st): State, - Json(body): Json, + Body(body): Body, ) -> Result, ApiError> { let Some(build) = &st.build else { return Err("build plane unavailable".to_string().into()); @@ -426,6 +457,11 @@ impl From for ApiError { CommitError::RevisionNotFound(_) => { Self::new(StatusCode::NOT_FOUND, "revision-not-found", message) } + CommitError::HistoryUnsettled { .. } => Self::new( + StatusCode::SERVICE_UNAVAILABLE, + "history-unsettled", + message, + ), CommitError::LockTimeout(_) => { Self::new(StatusCode::SERVICE_UNAVAILABLE, "lock-timeout", message) } diff --git a/packages/deckd/src/history/commit.rs b/packages/deckd/src/history/commit.rs index 3758b6e..4bc3312 100644 --- a/packages/deckd/src/history/commit.rs +++ b/packages/deckd/src/history/commit.rs @@ -10,16 +10,25 @@ //! ```text //! / //! jj/ # bare jj repo, internal git backend -//! deckd/commit.lock # cross-process lock around commit / outcome +//! deckd/HEAD # recorded HEAD; locked around every call //! deckd/operations/ # `pending` | `committed ` | `fenced` //! ``` //! //! Correctness rests on files in the repo, never on this process's memory: //! -//! - `commit` and `outcome` run under a flock on `deckd/commit.lock`, so -//! deckd processes on different hosts sharing the directory serialize. A -//! per-repo in-process mutex sits in front of it because Linux NFS emulates -//! flock with POSIX locks, which don't exclude threads of one process. +//! - `commit`, `outcome` and `head` hold an exclusive lock on `deckd/HEAD`, +//! which is never deleted, so deckd processes on different hosts sharing the +//! directory serialize. A per-repo in-process mutex sits in front of it +//! because Linux NFS emulates flock with POSIX locks, which don't exclude +//! threads of one process. +//! - `deckd/HEAD` holds the revision deckd last recorded, written before jj +//! publishes it. It is read through the locked descriptor, and taking an NFS +//! lock revalidates that file's cached data, so it is current even when this +//! host's listing of jj's operation heads is stale. jj's HEAD must equal it +//! before anything is written. When jj is exactly one commit behind (a crash +//! between recording and publishing, or a stale listing), the recorded +//! commit is published again, which is idempotent. Any other disagreement is +//! reloaded until it settles, then refused as `HistoryUnsettled`. //! - Whether an operation record exists is only ever learned from //! `create_new` (O_EXCL), which the NFS server answers. A peer's cached //! negative lookup can't make an existing record look absent. @@ -29,8 +38,8 @@ //! unrecorded, so after that repair every committed operation has a record. use std::collections::{HashMap, HashSet}; -use std::fs::{File, OpenOptions}; -use std::io::{ErrorKind, Write as _}; +use std::fs::{File, OpenOptions, TryLockError}; +use std::io::{ErrorKind, Read as _, Seek as _, SeekFrom, Write as _}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -38,12 +47,12 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use jj_lib::backend::{BackendError, CommitId, CopyId, Signature, Timestamp, TreeValue}; use jj_lib::commit::Commit; use jj_lib::default_backend_factories::default_backend_factories; -use jj_lib::lock::FileLock; use jj_lib::merge::Merge; +use jj_lib::merged_tree::MergedTree; use jj_lib::merged_tree_builder::MergedTreeBuilder; use jj_lib::object_id::ObjectId as _; use jj_lib::repo::{ReadonlyRepo, Repo as _, RepoLoader}; -use jj_lib::repo_path::RepoPathBuf; +use jj_lib::repo_path::{RepoPath, RepoPathBuf}; use jj_lib::signing_factory::signer_from_settings; use jj_lib::trailer::parse_description_trailers; use serde::{Deserialize, Serialize}; @@ -53,9 +62,16 @@ use super::jj_lib::{hermetic_git_env, is_valid_history_path, user_settings}; /// Trailer deckd adds to every explicit commit. Callers may not set it. const OPERATION_ID_TRAILER: &str = "Deckd-Operation-Id"; -/// How long a call waits for another process's commit lock before giving up. +/// How long a call waits for another process's lock before giving up. const LOCK_TIMEOUT: Duration = Duration::from_secs(30); +/// How long a call waits for jj's HEAD to agree with `deckd/HEAD`. +const SETTLE_TIMEOUT: Duration = if cfg!(test) { + Duration::from_millis(300) +} else { + Duration::from_secs(5) +}; + #[derive(Debug, thiserror::Error)] pub enum CommitError { #[error("{0}")] @@ -68,6 +84,8 @@ pub enum CommitError { RevisionNotFound(String), #[error("timed out waiting for the commit lock on {0}")] LockTimeout(String), + #[error("history HEAD {observed} does not match recorded HEAD {recorded}; retry")] + HistoryUnsettled { recorded: String, observed: String }, #[error("{0}")] Internal(String), } @@ -162,8 +180,8 @@ impl CommitService { validate_commit_request(&request)?; let gate = self.gate(repo_dir); let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); - let _file_lock = lock_repo(repo_dir)?; - pollster::block_on(commit_locked(repo_dir, request)) + let mut head_file = lock_repo(repo_dir)?; + pollster::block_on(commit_locked(repo_dir, &mut head_file, request)) } /// Report whether `operation_id` committed. Reporting it absent fences it: @@ -173,23 +191,18 @@ impl CommitService { validate_operation_id(operation_id)?; let gate = self.gate(repo_dir); let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); - let _file_lock = lock_repo(repo_dir)?; - pollster::block_on(outcome_locked(repo_dir, operation_id)) + let mut head_file = lock_repo(repo_dir)?; + pollster::block_on(outcome_locked(repo_dir, &mut head_file, operation_id)) } /// The current HEAD, creating an empty repo (HEAD is the root commit) if /// there is none yet. pub fn head(&self, repo_dir: &Path) -> CommitResult { validate_repo_dir(repo_dir)?; - let repo = if repo_dir.join("jj").is_dir() { - pollster::block_on(load_repo(repo_dir))? - } else { - let gate = self.gate(repo_dir); - let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); - let _file_lock = lock_repo(repo_dir)?; - pollster::block_on(load_or_init(repo_dir))? - }; - let head = head_commit(&repo)?; + let gate = self.gate(repo_dir); + let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); + let mut head_file = lock_repo(repo_dir)?; + let (_repo, head) = pollster::block_on(settled_head(repo_dir, &mut head_file))?; Ok(RevisionTree { revision: head.id().hex(), tree: tree_hash(&head), @@ -198,10 +211,15 @@ impl CommitService { pub fn tree_hash(&self, repo_dir: &Path, revision: &str) -> CommitResult { validate_repo_dir(repo_dir)?; - if !repo_dir.join("jj").is_dir() { - return Err(CommitError::RevisionNotFound(revision.to_owned())); - } - let repo = pollster::block_on(load_repo(repo_dir))?; + // Load first rather than testing for `jj/`: a cached negative lookup + // must not turn an existing revision into a 404. + let repo = match pollster::block_on(load_repo(repo_dir)) { + Ok(repo) => repo, + Err(_) if !repo_dir.join("jj").exists() => { + return Err(CommitError::RevisionNotFound(revision.to_owned())) + } + Err(error) => return Err(error), + }; let commit = get_commit(&repo, revision)?; Ok(RevisionTree { revision: commit.id().hex(), @@ -219,9 +237,12 @@ impl CommitService { } /// The body of [`CommitService::commit`]. Callers hold both repo locks. -async fn commit_locked(repo_dir: &Path, request: CommitRequest) -> CommitResult { - let repo = load_or_init(repo_dir).await?; - let head = head_commit(&repo)?; +async fn commit_locked( + repo_dir: &Path, + head_file: &mut File, + request: CommitRequest, +) -> CommitResult { + let (repo, head) = settled_head(repo_dir, head_file).await?; repair_head_record(repo_dir, &head)?; let record_path = operation_record_path(repo_dir, &request.operation_id); @@ -256,7 +277,13 @@ async fn commit_locked(repo_dir: &Path, request: CommitRequest) -> CommitResult< }); } - let tree = build_tree(&repo, &head, &request.changes).await?; + let tree = match build_tree(&repo, &head, &request.changes).await { + Ok(tree) => tree, + Err(error) => { + std::fs::remove_file(&record_path).map_err(internal)?; + return Err(error); + } + }; if tree.tree_ids() == head.tree_ids() { write_record(&record_path, &Record::Committed(head_hex.clone()))?; return Ok(CommitOutcome { @@ -282,15 +309,17 @@ async fn commit_locked(repo_dir: &Path, request: CommitRequest) -> CommitResult< .write() .await .map_err(internal)?; + let revision = commit.id().hex(); + // The commit point: from here a crash leaves this revision as HEAD, and + // the next locked call publishes it if jj hasn't. + write_recorded_head(head_file, &revision)?; tx.commit(format!( - "commit {} for operation {}", - commit.id().hex(), + "commit {revision} for operation {}", request.operation_id )) .await .map_err(internal)?; - let revision = commit.id().hex(); write_record(&record_path, &Record::Committed(revision.clone()))?; Ok(CommitOutcome { revision, @@ -301,9 +330,12 @@ async fn commit_locked(repo_dir: &Path, request: CommitRequest) -> CommitResult< } /// The body of [`CommitService::outcome`]. Callers hold both repo locks. -async fn outcome_locked(repo_dir: &Path, operation_id: &str) -> CommitResult { - let repo = load_or_init(repo_dir).await?; - let head = head_commit(&repo)?; +async fn outcome_locked( + repo_dir: &Path, + head_file: &mut File, + operation_id: &str, +) -> CommitResult { + let (_repo, head) = settled_head(repo_dir, head_file).await?; repair_head_record(repo_dir, &head)?; let record_path = operation_record_path(repo_dir, operation_id); @@ -320,22 +352,96 @@ async fn outcome_locked(repo_dir: &Path, operation_id: &str) -> CommitResult CommitResult { +/// Take the cross-process lock on `repo_dir`, creating the repo directory if +/// needed, and return the locked `deckd/HEAD`. Callers hold the repo's +/// in-process gate first. Dropping the file releases the lock. +fn lock_repo(repo_dir: &Path) -> CommitResult { create_repo_dir(repo_dir)?; let deckd_dir = repo_dir.join("deckd"); std::fs::create_dir_all(deckd_dir.join("operations")).map_err(internal)?; - let lock_path = deckd_dir.join("commit.lock"); + // The lock file is never deleted. Deleting one on release (as jj-lib's + // `FileLock` does) lets two NFS clients each hold "the" lock: one on the + // unlinked inode, which NFS keeps alive under a hidden name, and one on + // its replacement. + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(deckd_dir.join("HEAD")) + .map_err(internal)?; let started = Instant::now(); loop { - match FileLock::try_lock(lock_path.clone()).map_err(internal)? { - Some(file_lock) => return Ok(file_lock), - None if started.elapsed() >= LOCK_TIMEOUT => { + match file.try_lock() { + Ok(()) => return Ok(file), + Err(TryLockError::WouldBlock) if started.elapsed() >= LOCK_TIMEOUT => { return Err(CommitError::LockTimeout(repo_dir.display().to_string())) } - None => std::thread::sleep(Duration::from_millis(10)), + Err(TryLockError::WouldBlock) => std::thread::sleep(Duration::from_millis(10)), + Err(TryLockError::Error(error)) => return Err(internal(error)), + } + } +} + +/// The revision in the locked `deckd/HEAD`; empty means the root commit. +fn read_recorded_head(head_file: &mut File, root: &CommitId) -> CommitResult { + let mut text = String::new(); + head_file.seek(SeekFrom::Start(0)).map_err(internal)?; + head_file.read_to_string(&mut text).map_err(internal)?; + let text = text.trim(); + if text.is_empty() { + return Ok(root.clone()); + } + CommitId::try_from_hex(text) + .ok_or_else(|| CommitError::Internal(format!("unreadable deckd/HEAD {text:?}"))) +} + +/// Overwrite `deckd/HEAD` in place. Every revision has the same length, so +/// the file is never truncated and a reader never sees it empty. +fn write_recorded_head(head_file: &mut File, revision: &str) -> CommitResult<()> { + head_file.seek(SeekFrom::Start(0)).map_err(internal)?; + head_file + .write_all(format!("{revision}\n").as_bytes()) + .map_err(internal)?; + head_file.sync_all().map_err(internal) +} + +/// Load the repo and return its HEAD once it agrees with `deckd/HEAD`. Callers +/// hold both repo locks. +async fn settled_head( + repo_dir: &Path, + head_file: &mut File, +) -> CommitResult<(Arc, Commit)> { + let started = Instant::now(); + loop { + let repo = load_or_init(repo_dir).await?; + let recorded = read_recorded_head(head_file, repo.store().root_commit_id())?; + let head = head_commit(&repo)?; + if head.id() == &recorded { + return Ok((repo, head)); + } + // jj one commit behind: the recorded revision was written and then + // either never published (a crash) or published where this host's + // listing can't see it yet. Publishing it is right in both cases, and + // publishing it twice merges to the same single head. + if let Ok(commit) = repo.store().get_commit(&recorded) { + if commit.parent_ids() == std::slice::from_ref(head.id()) { + let mut tx = repo.start_transaction(); + tx.repo_mut().add_head(&commit).await.map_err(internal)?; + let repo = tx + .commit(format!("publish recorded HEAD {}", recorded.hex())) + .await + .map_err(internal)?; + return Ok((repo, commit)); + } + } + if started.elapsed() >= SETTLE_TIMEOUT { + return Err(CommitError::HistoryUnsettled { + recorded: recorded.hex(), + observed: head.id().hex(), + }); } + std::thread::sleep(Duration::from_millis(50)); } } @@ -369,6 +475,16 @@ fn validate_operation_id(operation_id: &str) -> CommitResult<()> { } } +/// A path a commit may write or delete. Beyond what reads accept, it has no +/// `.` segment (git rejects such trees) and no `.git` segment (git refuses to +/// check one out). History only grows, so a bad tree could never be removed. +fn is_valid_commit_path(path: &str) -> bool { + is_valid_history_path(path) + && !path + .split('/') + .any(|segment| segment == "." || segment.eq_ignore_ascii_case(".git")) +} + fn validate_commit_request(request: &CommitRequest) -> CommitResult<()> { if CommitId::try_from_hex(&request.parent).is_none() { return Err(CommitError::Invalid(format!( @@ -415,7 +531,7 @@ fn validate_commit_request(request: &CommitRequest) -> CommitResult<()> { } let mut seen = HashSet::new(); for change in &request.changes { - if !is_valid_history_path(&change.path) { + if !is_valid_commit_path(&change.path) { return Err(CommitError::Invalid(format!( "invalid path {:?}", change.path @@ -428,6 +544,20 @@ fn validate_commit_request(request: &CommitRequest) -> CommitResult<()> { ))); } } + // A listed path inside another listed path would turn a file into a + // directory or back, changing what the other entry says. + for change in &request.changes { + let mut ancestor = change.path.as_str(); + while let Some((parent, _)) = ancestor.rsplit_once('/') { + if seen.contains(parent) { + return Err(CommitError::Invalid(format!( + "path {:?} is inside listed path {parent:?}", + change.path + ))); + } + ancestor = parent; + } + } Ok(()) } @@ -463,6 +593,8 @@ fn create_repo_dir(repo_dir: &Path) -> CommitResult<()> { /// crash mid-init never leaves a half-made `jj/`. async fn load_or_init(repo_dir: &Path) -> CommitResult> { let jj_dir = repo_dir.join("jj"); + // A cached negative lookup can make an existing `jj/` look absent; the + // rename in `install_repo` is what decides. if !jj_dir.is_dir() { let staging = repo_dir.join("deckd").join(format!("init-{}", nonce())); std::fs::create_dir_all(&staging).map_err(internal)?; @@ -487,11 +619,28 @@ async fn load_or_init(repo_dir: &Path) -> CommitResult> { ) .await .map_err(internal)?; - std::fs::rename(&staging, &jj_dir).map_err(internal)?; + install_repo(&staging, &jj_dir)?; } load_repo(repo_dir).await } +/// Move a freshly initialized repo into place. When `jj/` already exists the +/// staged copy is discarded and the existing repo wins. +fn install_repo(staging: &Path, jj_dir: &Path) -> CommitResult<()> { + match std::fs::rename(staging, jj_dir) { + Ok(()) => Ok(()), + Err(e) + if matches!( + e.kind(), + ErrorKind::AlreadyExists | ErrorKind::DirectoryNotEmpty + ) => + { + std::fs::remove_dir_all(staging).map_err(internal) + } + Err(e) => Err(internal(e)), + } +} + async fn load_repo(repo_dir: &Path) -> CommitResult> { let settings = user_settings(None).map_err(internal)?; RepoLoader::init_from_file_system( @@ -537,16 +686,54 @@ fn tree_hash(commit: &Commit) -> String { } } +/// What the parent tree holds at `path`. +enum Existing { + Absent, + File, + Directory, +} + +async fn existing(tree: &MergedTree, path: &RepoPath) -> CommitResult { + let value = tree.path_value(path).await.map_err(internal)?; + Ok(match value.as_resolved() { + Some(None) => Existing::Absent, + Some(Some(TreeValue::Tree(_))) => Existing::Directory, + // Files, symlinks, submodules and conflicts all block a directory. + _ => Existing::File, + }) +} + +/// HEAD's tree with `changes` applied. A change may not turn a file into a +/// directory or back: that would remove paths the request didn't list. async fn build_tree( repo: &Arc, parent: &Commit, changes: &[Change], -) -> CommitResult { +) -> CommitResult { let store = repo.store(); - let mut builder = MergedTreeBuilder::new(parent.tree()); + let base = parent.tree(); + let mut builder = MergedTreeBuilder::new(base.clone()); for change in changes { let path = RepoPathBuf::from_internal_string(change.path.clone()) .map_err(|e| CommitError::Invalid(format!("invalid path {:?}: {e}", change.path)))?; + for ancestor in path.ancestors().skip(1) { + if ancestor.is_root() { + break; + } + if let Existing::File = existing(&base, ancestor).await? { + return Err(CommitError::Invalid(format!( + "path {:?} is inside file {:?}", + change.path, + ancestor.as_internal_file_string() + ))); + } + } + if let Existing::Directory = existing(&base, &path).await? { + return Err(CommitError::Invalid(format!( + "path {:?} is a directory; list the files inside it instead", + change.path + ))); + } let value = match &change.content { Some(bytes) => { let id = store @@ -643,6 +830,8 @@ fn write_record(path: &Path, record: &Record) -> CommitResult<()> { /// Make HEAD's operation record say `committed HEAD`. The record is written /// whenever it doesn't already say so, including when reading it fails: a /// missing file may only be a stale NFS lookup, and the write is the truth. +/// A `fenced` record for HEAD's operation means the lock failed to exclude a +/// writer; that is refused loudly, never overwritten. fn repair_head_record(repo_dir: &Path, head: &Commit) -> CommitResult<()> { let Some(operation_id) = parse_description_trailers(head.description()) .into_iter() @@ -656,10 +845,19 @@ fn repair_head_record(repo_dir: &Path, head: &Commit) -> CommitResult<()> { } let path = operation_record_path(repo_dir, &operation_id); let expected = Record::Committed(head.id().hex()); - if read_record(&path).ok().as_ref() != Some(&expected) { - write_record(&path, &expected)?; + match read_record(&path).ok() { + Some(record) if record == expected => Ok(()), + Some(Record::Fenced) => { + let message = format!( + "operation {operation_id} was reported absent but committed as {}; \ + history needs an operator", + head.id().hex() + ); + tracing::error!(repo = %repo_dir.display(), "{message}"); + Err(CommitError::Internal(message)) + } + _ => write_record(&path, &expected), } - Ok(()) } fn nonce() -> String { @@ -1158,6 +1356,243 @@ mod tests { assert!(second.created); } + /// Overwrite the locked `deckd/HEAD`, as a crashed or partitioned peer + /// would leave it. + fn set_recorded_head(realm: &Realm, revision: &str) { + let mut file = lock_repo(&realm.repo()).unwrap(); + write_recorded_head(&mut file, revision).unwrap(); + } + + /// Write a commit for `operation_id` without publishing it, as a writer + /// that crashed between recording and publishing would. + fn unpublished_commit(realm: &Realm, parent: &str, operation_id: &str) -> String { + pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let head = get_commit(&repo, parent).unwrap(); + let req = request(parent, operation_id, vec![write("crashed.txt", "x")]); + let tree = build_tree(&repo, &head, &req.changes).await.unwrap(); + let mut tx = repo.start_transaction(); + let commit = tx + .repo_mut() + .new_commit(vec![head.id().clone()], tree) + .set_description(description(&req)) + .write() + .await + .unwrap(); + commit.id().hex() + }) + } + + fn assert_invalid(result: CommitResult, needle: &str) { + match result { + Err(CommitError::Invalid(message)) => { + assert!(message.contains(needle), "{message:?} lacks {needle:?}") + } + other => panic!("expected invalid request, got {other:?}"), + } + } + + #[test] + fn changes_may_not_cross_a_file_directory_boundary() { + let realm = Realm::new("boundary"); + let service = CommitService::new(); + let root = head(&service, &realm); + let base = commit( + &service, + &realm, + request( + &root, + "op-base", + vec![write("a", "1"), write("d/x", "1"), write("d/y", "1")], + ), + ) + .unwrap(); + let attempt = + |op: &str, changes| commit(&service, &realm, request(&base.revision, op, changes)); + + assert_invalid(attempt("op-1", vec![write("a/b", "2")]), "inside file"); + assert_invalid(attempt("op-2", vec![write("d", "2")]), "is a directory"); + assert_invalid(attempt("op-3", vec![delete("d")]), "is a directory"); + assert_invalid( + attempt("op-4", vec![write("e", "2"), write("e/f", "2")]), + "inside listed path", + ); + assert_invalid( + attempt("op-5", vec![delete("a"), write("a/b", "2")]), + "inside listed path", + ); + + // Refused requests leave no claim, so the IDs stay usable. + let fixed = attempt("op-1", vec![write("a2/b", "2")]).unwrap(); + assert!(fixed.created); + assert_eq!( + files_at(&realm, &fixed.revision), + files(&[("a", "1"), ("a2/b", "2"), ("d/x", "1"), ("d/y", "1")]) + ); + } + + #[test] + fn dot_and_git_segments_are_refused() { + let realm = Realm::new("dot-paths"); + let service = CommitService::new(); + let root = head(&service, &realm); + + for path in [".", "x/./y", ".git/config", "x/.GIT/hooks", "x/.git"] { + assert_invalid( + commit( + &service, + &realm, + request(&root, "op", vec![write(path, "")]), + ), + "invalid path", + ); + } + // Names that only start with those segments are ordinary files. + let done = commit( + &service, + &realm, + request( + &root, + "op", + vec![write(".gitignore", ""), write("x/.github/ci", "")], + ), + ) + .unwrap(); + assert!(done.created); + } + + #[test] + fn lock_file_records_head_and_persists() { + let realm = Realm::new("head-file"); + let service = CommitService::new(); + let root = head(&service, &realm); + + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + + let recorded = std::fs::read_to_string(realm.repo().join("deckd/HEAD")).unwrap(); + assert_eq!(recorded.trim(), done.revision); + } + + #[test] + fn commit_recorded_but_not_published_is_published_by_the_next_call() { + let realm = Realm::new("roll-forward"); + let service = CommitService::new(); + let root = head(&service, &realm); + let revision = unpublished_commit(&realm, &root, "op-crashed"); + set_recorded_head(&realm, &revision); + std::fs::write( + operation_record_path(&realm.repo(), "op-crashed"), + "pending\n", + ) + .unwrap(); + + let restarted = CommitService::new(); + + assert_eq!( + outcome(&restarted, &realm, "op-crashed"), + OperationOutcome::Committed { + revision: revision.clone() + } + ); + assert_eq!(head(&restarted, &realm), revision); + assert_eq!(files_at(&realm, &revision), files(&[("crashed.txt", "x")])); + let next = commit( + &restarted, + &realm, + request(&revision, "op-next", vec![write("b", "2")]), + ) + .unwrap(); + assert!(next.created); + } + + #[test] + fn head_that_disagrees_with_the_record_is_refused() { + let realm = Realm::new("unsettled"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + // A recorded HEAD jj can't reach, as a stale listing would look. + set_recorded_head(&realm, &"ab".repeat(20)); + + let error = commit( + &service, + &realm, + request(&first.revision, "op-2", vec![write("b", "2")]), + ) + .unwrap_err(); + + assert!( + matches!(error, CommitError::HistoryUnsettled { .. }), + "{error:?}" + ); + // Nothing was claimed or written. + set_recorded_head(&realm, &first.revision); + assert_eq!(outcome(&service, &realm, "op-2"), OperationOutcome::Absent); + assert_eq!(head(&service, &realm), first.revision); + } + + #[test] + fn fenced_record_for_head_is_reported_not_overwritten() { + let realm = Realm::new("fence-violated"); + let service = CommitService::new(); + let root = head(&service, &realm); + commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + let record = operation_record_path(&realm.repo(), "op-1"); + std::fs::write(&record, "fenced\n").unwrap(); + + let error = service.outcome(&realm.repo(), "op-2").unwrap_err(); + + assert!( + error.to_string().contains("reported absent but committed"), + "{error}" + ); + assert_eq!(std::fs::read_to_string(&record).unwrap(), "fenced\n"); + } + + #[test] + fn initializing_over_an_existing_repo_keeps_it() { + let realm = Realm::new("init-race"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + let staging = realm.repo().join("deckd/init-late"); + std::fs::create_dir_all(staging.join("store")).unwrap(); + + install_repo(&staging, &realm.repo().join("jj")).unwrap(); + + assert!(!staging.exists()); + assert_eq!(head(&service, &realm), done.revision); + } + + #[test] + fn tree_hash_in_a_missing_repo_is_not_found() { + let realm = Realm::new("tree-no-repo"); + let error = CommitService::new() + .tree_hash(&realm.repo(), &"ab".repeat(20)) + .unwrap_err(); + assert!(matches!(error, CommitError::RevisionNotFound(_))); + } + #[test] fn delete_removes_the_path() { let realm = Realm::new("delete"); diff --git a/packages/deckd/src/history/jj_lib.rs b/packages/deckd/src/history/jj_lib.rs index d5470aa..9da75a3 100644 --- a/packages/deckd/src/history/jj_lib.rs +++ b/packages/deckd/src/history/jj_lib.rs @@ -632,11 +632,19 @@ async fn snapshot_wc( .get_commit(&wc_commit_id) .map_err(|e| HistoryError::msg(e.to_string()))?; - let (new_tree, _stats) = locked_ws + let (new_tree, stats) = locked_ws .locked_wc() .snapshot(&options) .await .map_err(|e| HistoryError::msg(e.to_string()))?; + // jj skips names that aren't valid UTF-8; those files never enter History. + for (dir, name) in &stats.invalid_utf8_paths { + tracing::warn!( + dir = %dir.as_internal_file_string(), + name = ?name, + "snapshot skipped a file whose name isn't valid UTF-8" + ); + } let repo = if new_tree.tree_ids_and_labels() != wc_commit.tree().tree_ids_and_labels() { let mut tx = repo.start_transaction(); diff --git a/packages/deckd/tests/commit_restart.rs b/packages/deckd/tests/commit_restart.rs index 416774e..a63a744 100644 --- a/packages/deckd/tests/commit_restart.rs +++ b/packages/deckd/tests/commit_restart.rs @@ -2,83 +2,13 @@ //! between calls: operation outcomes and fences live in the repo, so a new //! process gives the same answers. -use std::net::TcpListener; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Stdio}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::path::Path; use serde_json::{json, Value}; -struct Deckd { - child: Child, - base: String, -} - -impl Deckd { - /// Start deckd on a fresh port. A restart needs no particular port: all - /// state is in the repo. - fn start() -> Self { - let port = free_port(); - let child = Command::new(env!("CARGO_BIN_EXE_deckd")) - .env("DECKD_BIND", format!("127.0.0.1:{port}")) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("start deckd"); - let base = format!("http://127.0.0.1:{port}"); - let started = Instant::now(); - while ureq::get(&format!("{base}/health")).call().is_err() { - assert!( - started.elapsed() < Duration::from_secs(20), - "deckd did not start" - ); - std::thread::sleep(Duration::from_millis(50)); - } - Self { child, base } - } - - /// POST `body` and return the status and JSON response, error or not. - fn post(&self, route: &str, body: Value) -> (u16, Value) { - match ureq::post(&format!("{}{route}", self.base)).send_json(body) { - Ok(response) => (response.status(), response.into_json().unwrap()), - Err(ureq::Error::Status(status, response)) => (status, response.into_json().unwrap()), - Err(error) => panic!("{route}: {error}"), - } - } - - fn kill(mut self) { - self.child.kill().unwrap(); - self.child.wait().unwrap(); - } -} +mod common; -impl Drop for Deckd { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - -fn free_port() -> u16 { - TcpListener::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap() - .port() -} - -fn realm(name: &str) -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - let dir = std::env::temp_dir().join(format!( - "deckd-restart-{name}-{}-{nanos}", - std::process::id() - )); - std::fs::create_dir_all(&dir).unwrap(); - dir -} +use common::{realm, Deckd}; fn commit_body(repo: &Path, parent: &str, operation_id: &str) -> Value { json!({ diff --git a/packages/deckd/tests/common/mod.rs b/packages/deckd/tests/common/mod.rs new file mode 100644 index 0000000..0bceb9c --- /dev/null +++ b/packages/deckd/tests/common/mod.rs @@ -0,0 +1,89 @@ +//! Helpers for tests that drive a real deckd process over HTTP. + +#![allow(dead_code)] + +use std::net::TcpListener; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use serde_json::Value; + +pub struct Deckd { + child: Child, + pub base: String, +} + +impl Deckd { + /// Start deckd on a fresh port. A restart needs no particular port: all + /// state is in the repo. + pub fn start() -> Self { + let port = free_port(); + let child = Command::new(env!("CARGO_BIN_EXE_deckd")) + .env("DECKD_BIND", format!("127.0.0.1:{port}")) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start deckd"); + let base = format!("http://127.0.0.1:{port}"); + let started = Instant::now(); + while ureq::get(&format!("{base}/health")).call().is_err() { + assert!( + started.elapsed() < Duration::from_secs(20), + "deckd did not start" + ); + std::thread::sleep(Duration::from_millis(50)); + } + Self { child, base } + } + + /// POST `body` and return the status and JSON response, error or not. + pub fn post(&self, route: &str, body: Value) -> (u16, Value) { + match ureq::post(&format!("{}{route}", self.base)).send_json(body) { + Ok(response) => (response.status(), response.into_json().unwrap()), + Err(ureq::Error::Status(status, response)) => (status, response.into_json().unwrap()), + Err(error) => panic!("{route}: {error}"), + } + } + + pub fn kill(mut self) { + self.child.kill().unwrap(); + self.child.wait().unwrap(); + } +} + +impl Drop for Deckd { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +pub fn realm(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = + std::env::temp_dir().join(format!("deckd-http-{name}-{}-{nanos}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// POST raw bytes and return the status and JSON response, error or not. +pub fn post_raw(deckd: &Deckd, route: &str, content_type: &str, body: &str) -> (u16, Value) { + let request = ureq::post(&format!("{}{route}", deckd.base)).set("content-type", content_type); + match request.send_string(body) { + Ok(response) => (response.status(), response.into_json().unwrap()), + Err(ureq::Error::Status(status, response)) => (status, response.into_json().unwrap()), + Err(error) => panic!("{route}: {error}"), + } +} diff --git a/packages/deckd/tests/http_errors.rs b/packages/deckd/tests/http_errors.rs new file mode 100644 index 0000000..0272f0d --- /dev/null +++ b/packages/deckd/tests/http_errors.rs @@ -0,0 +1,85 @@ +//! Every error deckd returns, including a request it can't parse, is a JSON +//! body with a stable `code`. + +use std::io::{Read as _, Write as _}; +use std::net::TcpStream; +use std::time::Duration; + +use serde_json::{json, Value}; + +mod common; + +use common::{post_raw, realm, Deckd}; + +#[test] +fn unparseable_requests_get_typed_errors() { + let deckd = Deckd::start(); + let repo = realm("parse").join(".history"); + + let (status, body) = post_raw(&deckd, "/commit", "application/json", "{\"repo\":"); + assert_eq!( + (status, &body["code"]), + (400, &json!("invalid-request")), + "{body}" + ); + + let missing = json!({ "repo": repo }).to_string(); + let (status, body) = post_raw(&deckd, "/commit", "application/json", &missing); + assert_eq!( + (status, &body["code"]), + (422, &json!("invalid-request")), + "{body}" + ); + assert!(body["error"].as_str().unwrap().contains("parent"), "{body}"); + + let (status, body) = post_raw(&deckd, "/outcome", "text/plain", "{}"); + assert_eq!( + (status, &body["code"]), + (415, &json!("invalid-request")), + "{body}" + ); + + // Routes that predate the commit API answer the same way. + let (status, body) = post_raw(&deckd, "/list", "application/json", "[]"); + assert_eq!( + (status, &body["code"]), + (422, &json!("invalid-request")), + "{body}" + ); +} + +#[test] +fn oversized_commit_is_refused_as_too_large() { + let deckd = Deckd::start(); + let length = 65 * 1024 * 1024; + let mut stream = TcpStream::connect(deckd.base.trim_start_matches("http://")).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(30))) + .unwrap(); + write!( + stream, + "POST /commit HTTP/1.1\r\nHost: deckd\r\nContent-Type: application/json\r\n\ + Content-Length: {length}\r\nConnection: close\r\n\r\n" + ) + .unwrap(); + // deckd answers once the body passes the limit and stops reading, so the + // upload runs on its own thread and its write errors are expected. + let mut writer = stream.try_clone().unwrap(); + let upload = std::thread::spawn(move || { + let chunk = vec![b' '; 1024 * 1024]; + for _ in 0..length / chunk.len() { + if writer.write_all(&chunk).is_err() { + return; + } + } + }); + let mut response = Vec::new(); + let _ = stream.read_to_end(&mut response); + let _ = upload.join(); + + let response = String::from_utf8_lossy(&response); + let (head, body) = response.split_once("\r\n\r\n").expect("a response"); + assert!(head.starts_with("HTTP/1.1 413"), "{head}"); + let body: Value = serde_json::from_str(body).unwrap(); + assert_eq!(body["code"], json!("payload-too-large"), "{body}"); +} From 812d3e3203821278042e961b0b9f3757721eb0a9 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 26 Oct 2026 17:27:12 -0400 Subject: [PATCH 3/3] Make explicit commits crash-safe and refuse reused operation IDs - deckd/HEAD holds two checksummed slots; a write goes to the slot without the latest value, so a torn write leaves the previous revision readable. - The new commit's git objects are synced before deckd/HEAD records it. - Claiming or removing an operation record syncs its directory entry. - Operation records keep a digest of the claiming request (everything but the parent); replaying a committed ID with a different request is a 409 operation-mismatch. The HEAD repair keeps a pending claim's digest. - The in-process gate is keyed by the repo directory's device and inode, is taken with the same deadline as the file lock, and is pruned once no call holds it. - DECKD_REPO_ROOT, when set, confines explicit-commit repos to a directory. - /tree-hash only answers for revisions in history; an unpublished commit object is revision-not-found. - Rewriting an existing file keeps its executable bit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tg2ZC6YEPfRcNZSoWKq2HS --- packages/deckd/Cargo.lock | 1 + packages/deckd/Cargo.toml | 1 + packages/deckd/README.md | 32 +- packages/deckd/src/api.rs | 5 + packages/deckd/src/history/commit.rs | 873 +++++++++++++++++++++++---- packages/deckd/src/main.rs | 6 +- 6 files changed, 795 insertions(+), 123 deletions(-) diff --git a/packages/deckd/Cargo.lock b/packages/deckd/Cargo.lock index d8196c5..43450b7 100644 --- a/packages/deckd/Cargo.lock +++ b/packages/deckd/Cargo.lock @@ -351,6 +351,7 @@ dependencies = [ "pollster 0.4.0", "serde", "serde_json", + "sha2", "thiserror", "tokio", "tower-http", diff --git a/packages/deckd/Cargo.toml b/packages/deckd/Cargo.toml index f13fbca..ec66f73 100644 --- a/packages/deckd/Cargo.toml +++ b/packages/deckd/Cargo.toml @@ -24,6 +24,7 @@ notify = "7" pollster = "0.4" serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = { version = "0.11", default-features = false } thiserror = "2" tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "process", "sync", "fs", "io-util", "time"] } tower-http = { version = "0.6", features = ["trace"] } diff --git a/packages/deckd/README.md b/packages/deckd/README.md index 04d48e3..adf8e61 100644 --- a/packages/deckd/README.md +++ b/packages/deckd/README.md @@ -81,13 +81,15 @@ can then record each write exactly once, even when deckd's response is lost. Each call names `repo`, the absolute path of one history directory (for example `/.history`). The directory's parent must exist; deckd creates -the directory and a bare jj repo inside it on first use: +the directory and a bare jj repo inside it on first use. When `DECKD_REPO_ROOT` +is set, `repo` must resolve, symlinks followed, to a directory inside it; +deployments should set it to the realms directory. ```text / jj/ # jj repo, internal git backend, no working copy deckd/HEAD # recorded HEAD; locked around every call but /tree-hash - deckd/operations/ # pending | committed | fenced + deckd/operations/ # pending | committed | fenced ``` A revision is the git commit ID (40 hex). HEAD is the repo's single visible @@ -102,13 +104,16 @@ head; an empty repo's HEAD is the root commit, `0000…0000`. with `parent-mismatch` and the current `head`, never re-parented. - **Idempotent by operation ID.** Committing an ID that already committed returns the original revision with `replayed: true` and writes nothing, - even when the retry's `parent` is now stale. + even when the retry's `parent` is now stale. The retry must ask for the + same thing: an ID that committed a different author, message, trailers or + changes is refused with `operation-mismatch`. - **Fencing outcome query.** `/outcome` reports `committed` or `absent`. Reporting `absent` fences the ID: a later commit with it is refused with `operation-fenced`. - **Durable, not in-memory.** Outcomes and fences are files under ``, so they survive a restart and are shared by every deckd on the same - filesystem. Record existence is tested with `O_EXCL` creates, which an NFS + filesystem. Each record and its directory entry are synced before the call + returns. Record existence is tested with `O_EXCL` creates, which an NFS server answers, so a stale negative lookup can't hide one. - **No-op writes record nothing.** When the changes leave the tree as it was, no revision is created; the response returns HEAD with `created: false`. @@ -118,11 +123,18 @@ Correctness doesn't depend on deckd's in-process state: - **One lock file, never deleted.** An exclusive lock on `deckd/HEAD` serializes deckd processes. Deleting a lock file on release lets two NFS clients each hold the lock, one on the unlinked inode and one on its - replacement, so this file stays. + replacement, so this file stays. Within a process, calls for one repo also + share a mutex keyed by the directory's device and inode, so a symlinked + spelling can't bypass it. Both locks share the 30 s `lock-timeout`. - **A recorded HEAD.** `deckd/HEAD` holds the revision deckd last recorded, written just before jj publishes it. Taking an NFS lock revalidates that file's cached data, so it is current even when this host's listing of jj's operations is stale. jj's HEAD must match it before anything is written. + - The file holds two checksummed slots, and each write goes to the slot + without the latest value, so a write torn by a crash leaves the previous + revision readable. + - A revision is recorded only after its git objects are synced, so a power + loss can't leave `deckd/HEAD` naming a commit the store lost. - When jj is one commit behind (a crash between recording and publishing, or a stale listing), the recorded commit is published, which is idempotent. - Any other mismatch is reloaded for a few seconds, then refused with @@ -168,7 +180,8 @@ Each commit's message ends with the caller's trailers, then `delete: true`. A path appears at most once. Paths are relative and may not contain empty, `.`, `..` or `.git` segments. No listed path may be inside another listed path, inside a file in HEAD, or name a directory in HEAD. - Deleting an absent path changes nothing. + Deleting an absent path changes nothing. Writing an existing file keeps its + executable bit. - `trailers`: optional. Keys match `[A-Za-z0-9-]+`; values are one line. - `message` is required. `author.name` and `author.email` are required and may not contain newlines, `<` or `>`. @@ -213,7 +226,9 @@ it can change state. ``` The tree hash is the git tree ID, so identical content has the same hash in -any repo, whoever committed it and whatever the message. +any repo, whoever committed it and whatever the message. The revision must be +in the repo's history; a commit object that was written but never published +is `revision-not-found`. ### Errors @@ -226,9 +241,10 @@ route, including a request deckd can't parse. | 404 | `revision-not-found` | `/tree-hash` for a revision the repo doesn't have | | 409 | `parent-mismatch` | `parent` isn't HEAD; body has `head` | | 409 | `operation-fenced` | The operation ID was reported `absent` by `/outcome` | +| 409 | `operation-mismatch` | The operation ID committed a different request; body has `revision` | | 413 | `payload-too-large` | A `/commit` body over 64 MiB; split the change into several commits | | 503 | `history-unsettled` | jj's HEAD didn't match `deckd/HEAD` within a few seconds; retry | -| 503 | `lock-timeout` | Another process held the repo's lock for 30 s | +| 503 | `lock-timeout` | Other calls held the repo's locks for 30 s | | 500 | `internal` | jj or filesystem failure, or a fenced operation found committed | ## Run diff --git a/packages/deckd/src/api.rs b/packages/deckd/src/api.rs index 3cb8d4b..d38d581 100644 --- a/packages/deckd/src/api.rs +++ b/packages/deckd/src/api.rs @@ -454,6 +454,11 @@ impl From for ApiError { CommitError::OperationFenced(_) => { Self::new(StatusCode::CONFLICT, "operation-fenced", message) } + CommitError::OperationMismatch { revision, .. } => { + let mut error = Self::new(StatusCode::CONFLICT, "operation-mismatch", message); + error.details.insert("revision".into(), revision.into()); + error + } CommitError::RevisionNotFound(_) => { Self::new(StatusCode::NOT_FOUND, "revision-not-found", message) } diff --git a/packages/deckd/src/history/commit.rs b/packages/deckd/src/history/commit.rs index 4bc3312..26c2375 100644 --- a/packages/deckd/src/history/commit.rs +++ b/packages/deckd/src/history/commit.rs @@ -11,7 +11,7 @@ //! / //! jj/ # bare jj repo, internal git backend //! deckd/HEAD # recorded HEAD; locked around every call -//! deckd/operations/ # `pending` | `committed ` | `fenced` +//! deckd/operations/ # `pending ` | `committed ` | `fenced` //! ``` //! //! Correctness rests on files in the repo, never on this process's memory: @@ -20,7 +20,8 @@ //! which is never deleted, so deckd processes on different hosts sharing the //! directory serialize. A per-repo in-process mutex sits in front of it //! because Linux NFS emulates flock with POSIX locks, which don't exclude -//! threads of one process. +//! threads of one process. The mutex is keyed by the directory's device and +//! inode, so every spelling of one repo shares it. //! - `deckd/HEAD` holds the revision deckd last recorded, written before jj //! publishes it. It is read through the locked descriptor, and taking an NFS //! lock revalidates that file's cached data, so it is current even when this @@ -29,19 +30,26 @@ //! between recording and publishing, or a stale listing), the recorded //! commit is published again, which is idempotent. Any other disagreement is //! reloaded until it settles, then refused as `HistoryUnsettled`. +//! - A revision is recorded only after its git objects are on stable storage, +//! so `deckd/HEAD` never names a commit a power loss could take away. //! - Whether an operation record exists is only ever learned from //! `create_new` (O_EXCL), which the NFS server answers. A peer's cached -//! negative lookup can't make an existing record look absent. +//! negative lookup can't make an existing record look absent. Records and +//! the directory entries naming them are synced before a call returns. //! - Every commit carries a `Deckd-Operation-Id` trailer. Before acting, each //! locked call sets HEAD's operation record to `committed HEAD`. A crash //! after publishing but before writing the record can only leave HEAD //! unrecorded, so after that repair every committed operation has a record. +//! - A record keeps a digest of the request that claimed it. Replaying a +//! committed operation with a different request is refused, so a reused +//! operation ID can't silently drop a write. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fs::{File, OpenOptions, TryLockError}; use std::io::{ErrorKind, Read as _, Seek as _, SeekFrom, Write as _}; +use std::os::unix::fs::MetadataExt as _; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard, TryLockError as MutexTryLockError, Weak}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use jj_lib::backend::{BackendError, CommitId, CopyId, Signature, Timestamp, TreeValue}; @@ -56,14 +64,19 @@ use jj_lib::repo_path::{RepoPath, RepoPathBuf}; use jj_lib::signing_factory::signer_from_settings; use jj_lib::trailer::parse_description_trailers; use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; use super::jj_lib::{hermetic_git_env, is_valid_history_path, user_settings}; /// Trailer deckd adds to every explicit commit. Callers may not set it. const OPERATION_ID_TRAILER: &str = "Deckd-Operation-Id"; -/// How long a call waits for another process's lock before giving up. -const LOCK_TIMEOUT: Duration = Duration::from_secs(30); +/// How long a call waits for the repo's locks before giving up. +const LOCK_TIMEOUT: Duration = if cfg!(test) { + Duration::from_secs(5) +} else { + Duration::from_secs(30) +}; /// How long a call waits for jj's HEAD to agree with `deckd/HEAD`. const SETTLE_TIMEOUT: Duration = if cfg!(test) { @@ -80,6 +93,11 @@ pub enum CommitError { ParentMismatch { parent: String, head: String }, #[error("operation {0} was reported absent and can never commit")] OperationFenced(String), + #[error("operation {operation_id} already committed a different request as {revision}")] + OperationMismatch { + operation_id: String, + revision: String, + }, #[error("revision {0} not found")] RevisionNotFound(String), #[error("timed out waiting for the commit lock on {0}")] @@ -154,9 +172,15 @@ pub struct RevisionTree { /// state that correctness depends on; a fresh instance behaves exactly like /// one that has been running. pub struct CommitService { - gates: Mutex>>>, + /// When set, every repo must resolve to a directory inside it. + root: Option, + gates: Mutex, } +/// In-process gates by the repo directory's (device, inode). Entries no call +/// holds are pruned, so the map only grows with concurrent repos. +type Gates = HashMap<(u64, u64), Weak>>; + impl Default for CommitService { fn default() -> Self { Self::new() @@ -164,75 +188,164 @@ impl Default for CommitService { } impl CommitService { + /// A service that accepts any absolute repo path. pub fn new() -> Self { hermetic_git_env(); Self { + root: None, gates: Mutex::new(HashMap::new()), } } + /// A service that refuses repos outside `root`, which must exist. + pub fn with_root(root: &Path) -> std::io::Result { + Ok(Self { + root: Some(std::fs::canonicalize(root)?), + ..Self::new() + }) + } + /// Record `request.changes` as one revision on top of `request.parent`. /// Runs on a blocking thread: it waits on locks and drives jj-lib's /// `!Send` futures to completion while holding them. pub fn commit(&self, repo_dir: &Path, request: CommitRequest) -> CommitResult { - validate_repo_dir(repo_dir)?; validate_operation_id(&request.operation_id)?; validate_commit_request(&request)?; - let gate = self.gate(repo_dir); - let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); - let mut head_file = lock_repo(repo_dir)?; - pollster::block_on(commit_locked(repo_dir, &mut head_file, request)) + self.locked(repo_dir, |repo_dir, head_file| { + pollster::block_on(commit_locked(repo_dir, head_file, request)) + }) } /// Report whether `operation_id` committed. Reporting it absent fences it: /// from then on a commit with that ID is refused. pub fn outcome(&self, repo_dir: &Path, operation_id: &str) -> CommitResult { - validate_repo_dir(repo_dir)?; validate_operation_id(operation_id)?; - let gate = self.gate(repo_dir); - let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); - let mut head_file = lock_repo(repo_dir)?; - pollster::block_on(outcome_locked(repo_dir, &mut head_file, operation_id)) + self.locked(repo_dir, |repo_dir, head_file| { + pollster::block_on(outcome_locked(repo_dir, head_file, operation_id)) + }) } /// The current HEAD, creating an empty repo (HEAD is the root commit) if /// there is none yet. pub fn head(&self, repo_dir: &Path) -> CommitResult { - validate_repo_dir(repo_dir)?; - let gate = self.gate(repo_dir); - let _in_process = gate.lock().unwrap_or_else(|e| e.into_inner()); - let mut head_file = lock_repo(repo_dir)?; - let (_repo, head) = pollster::block_on(settled_head(repo_dir, &mut head_file))?; + let head = self.locked(repo_dir, |repo_dir, head_file| { + pollster::block_on(settled_head(repo_dir, head_file)).map(|(_repo, head)| head) + })?; Ok(RevisionTree { revision: head.id().hex(), tree: tree_hash(&head), }) } + /// The tree of `revision`, which must be in the repo's history: a commit + /// object that was written but never published is not found. pub fn tree_hash(&self, repo_dir: &Path, revision: &str) -> CommitResult { - validate_repo_dir(repo_dir)?; + let repo_dir = self.resolve_repo_dir(repo_dir)?; + let not_found = || CommitError::RevisionNotFound(revision.to_owned()); + let id = CommitId::try_from_hex(revision).ok_or_else(not_found)?; // Load first rather than testing for `jj/`: a cached negative lookup // must not turn an existing revision into a 404. - let repo = match pollster::block_on(load_repo(repo_dir)) { + let repo = match pollster::block_on(load_repo(&repo_dir)) { Ok(repo) => repo, - Err(_) if !repo_dir.join("jj").exists() => { - return Err(CommitError::RevisionNotFound(revision.to_owned())) - } + Err(_) if !repo_dir.join("jj").exists() => return Err(not_found()), Err(error) => return Err(error), }; - let commit = get_commit(&repo, revision)?; + let heads: Vec = repo.view().heads().iter().cloned().collect(); + let commit = match pollster::block_on(published_commit(&repo, &heads, &id))? { + Some(commit) => commit, + // Unlocked, this host's listing may not show a peer's latest + // commit yet. Only a miss pays for the lock, which settles HEAD. + None => self + .locked(&repo_dir, |repo_dir, head_file| { + pollster::block_on(async { + let (repo, head) = settled_head(repo_dir, head_file).await?; + published_commit(&repo, &[head.id().clone()], &id).await + }) + })? + .ok_or_else(not_found)?, + }; Ok(RevisionTree { revision: commit.id().hex(), tree: tree_hash(&commit), }) } - fn gate(&self, repo_dir: &Path) -> Arc> { + /// Run `body` holding both repo locks, with the resolved repo directory + /// and the locked `deckd/HEAD`. Both locks share one `LOCK_TIMEOUT`, so a + /// busy or stuck repo can't park blocking threads indefinitely. + fn locked( + &self, + repo_dir: &Path, + body: impl FnOnce(&Path, &mut File) -> CommitResult, + ) -> CommitResult { + let repo_dir = self.resolve_repo_dir(repo_dir)?; + create_repo_dir(&repo_dir)?; + let deadline = Instant::now() + LOCK_TIMEOUT; + let gate = self.gate(&repo_dir)?; + let _in_process = lock_gate(&gate, &repo_dir, deadline)?; + let mut head_file = lock_repo(&repo_dir, deadline)?; + body(&repo_dir, &mut head_file) + } + + /// `repo_dir` with symlinks resolved. Its parent (normally the realm + /// directory) must exist, so a mistyped path fails instead of quietly + /// growing a new tree; the repo directory itself need not. + fn resolve_repo_dir(&self, repo_dir: &Path) -> CommitResult { + validate_repo_dir(repo_dir)?; + let (Some(parent), Some(name)) = (repo_dir.parent(), repo_dir.file_name()) else { + return Err(CommitError::Invalid("repo must name a directory".into())); + }; + let missing_parent = || { + CommitError::Invalid(format!( + "parent directory of {} does not exist", + repo_dir.display() + )) + }; + let parent = std::fs::canonicalize(parent).map_err(|_| missing_parent())?; + let mut resolved = parent.join(name); + if resolved.is_symlink() { + resolved = std::fs::canonicalize(&resolved).map_err(|_| missing_parent())?; + } + if let Some(root) = &self.root { + if !resolved.starts_with(root) || resolved == *root { + return Err(CommitError::Invalid(format!( + "repo must be inside {}", + root.display() + ))); + } + } + Ok(resolved) + } + + fn gate(&self, repo_dir: &Path) -> CommitResult>> { + let metadata = std::fs::metadata(repo_dir).map_err(internal)?; + let key = (metadata.dev(), metadata.ino()); let mut gates = self.gates.lock().unwrap_or_else(|e| e.into_inner()); - gates - .entry(repo_dir.to_path_buf()) - .or_insert_with(|| Arc::new(Mutex::new(()))) - .clone() + gates.retain(|_, gate| gate.strong_count() > 0); + if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) { + return Ok(gate); + } + let gate = Arc::new(Mutex::new(())); + gates.insert(key, Arc::downgrade(&gate)); + Ok(gate) + } +} + +/// Take a repo's in-process gate, giving up at `deadline`. +fn lock_gate<'a>( + gate: &'a Mutex<()>, + repo_dir: &Path, + deadline: Instant, +) -> CommitResult> { + loop { + match gate.try_lock() { + Ok(guard) => return Ok(guard), + Err(MutexTryLockError::Poisoned(poisoned)) => return Ok(poisoned.into_inner()), + Err(MutexTryLockError::WouldBlock) if Instant::now() >= deadline => { + return Err(CommitError::LockTimeout(repo_dir.display().to_string())) + } + Err(MutexTryLockError::WouldBlock) => std::thread::sleep(Duration::from_millis(10)), + } } } @@ -245,10 +358,20 @@ async fn commit_locked( let (repo, head) = settled_head(repo_dir, head_file).await?; repair_head_record(repo_dir, &head)?; + let digest = request_digest(&request); let record_path = operation_record_path(repo_dir, &request.operation_id); - if !claim_record(&record_path, Record::Pending)? { + if !claim_record(&record_path, &Record::Pending(Some(digest.clone())))? { match read_record(&record_path)? { - Record::Committed(revision) => { + Record::Committed { + revision, + digest: recorded, + } => { + if recorded.is_some_and(|recorded| recorded != digest) { + return Err(CommitError::OperationMismatch { + operation_id: request.operation_id, + revision, + }); + } let commit = get_commit(&repo, &revision)?; return Ok(CommitOutcome { revision, @@ -261,8 +384,11 @@ async fn commit_locked( return Err(CommitError::OperationFenced(request.operation_id)); } // A crashed attempt that never published: the repair above - // would have marked it committed if it had. Take it over. - Record::Pending => {} + // would have marked it committed if it had. Take it over, so a + // replay is checked against this request. + Record::Pending(_) => { + write_record(&record_path, &Record::Pending(Some(digest.clone())))?; + } } } @@ -270,7 +396,7 @@ async fn commit_locked( if CommitId::try_from_hex(&request.parent).as_ref() != Some(head.id()) { // Leave no claim behind, so the caller can retry this operation // on the new HEAD. - std::fs::remove_file(&record_path).map_err(internal)?; + remove_record(&record_path)?; return Err(CommitError::ParentMismatch { parent: request.parent, head: head_hex, @@ -280,12 +406,18 @@ async fn commit_locked( let tree = match build_tree(&repo, &head, &request.changes).await { Ok(tree) => tree, Err(error) => { - std::fs::remove_file(&record_path).map_err(internal)?; + remove_record(&record_path)?; return Err(error); } }; if tree.tree_ids() == head.tree_ids() { - write_record(&record_path, &Record::Committed(head_hex.clone()))?; + write_record( + &record_path, + &Record::Committed { + revision: head_hex.clone(), + digest: Some(digest), + }, + )?; return Ok(CommitOutcome { revision: head_hex, tree: tree_hash(&head), @@ -310,6 +442,7 @@ async fn commit_locked( .await .map_err(internal)?; let revision = commit.id().hex(); + sync_new_objects(repo_dir, &commit, &request.changes).await?; // The commit point: from here a crash leaves this revision as HEAD, and // the next locked call publishes it if jj hasn't. write_recorded_head(head_file, &revision)?; @@ -320,7 +453,13 @@ async fn commit_locked( .await .map_err(internal)?; - write_record(&record_path, &Record::Committed(revision.clone()))?; + write_record( + &record_path, + &Record::Committed { + revision: revision.clone(), + digest: Some(digest), + }, + )?; Ok(CommitOutcome { revision, tree: tree_hash(&commit), @@ -339,24 +478,23 @@ async fn outcome_locked( repair_head_record(repo_dir, &head)?; let record_path = operation_record_path(repo_dir, operation_id); - if claim_record(&record_path, Record::Fenced)? { + if claim_record(&record_path, &Record::Fenced)? { return Ok(OperationOutcome::Absent); } match read_record(&record_path)? { - Record::Committed(revision) => Ok(OperationOutcome::Committed { revision }), + Record::Committed { revision, .. } => Ok(OperationOutcome::Committed { revision }), Record::Fenced => Ok(OperationOutcome::Absent), - Record::Pending => { + Record::Pending(_) => { write_record(&record_path, &Record::Fenced)?; Ok(OperationOutcome::Absent) } } } -/// Take the cross-process lock on `repo_dir`, creating the repo directory if -/// needed, and return the locked `deckd/HEAD`. Callers hold the repo's +/// Take the cross-process lock on an existing repo directory, giving up at +/// `deadline`, and return the locked `deckd/HEAD`. Callers hold the repo's /// in-process gate first. Dropping the file releases the lock. -fn lock_repo(repo_dir: &Path) -> CommitResult { - create_repo_dir(repo_dir)?; +fn lock_repo(repo_dir: &Path, deadline: Instant) -> CommitResult { let deckd_dir = repo_dir.join("deckd"); std::fs::create_dir_all(deckd_dir.join("operations")).map_err(internal)?; // The lock file is never deleted. Deleting one on release (as jj-lib's @@ -370,11 +508,10 @@ fn lock_repo(repo_dir: &Path) -> CommitResult { .truncate(false) .open(deckd_dir.join("HEAD")) .map_err(internal)?; - let started = Instant::now(); loop { match file.try_lock() { Ok(()) => return Ok(file), - Err(TryLockError::WouldBlock) if started.elapsed() >= LOCK_TIMEOUT => { + Err(TryLockError::WouldBlock) if Instant::now() >= deadline => { return Err(CommitError::LockTimeout(repo_dir.display().to_string())) } Err(TryLockError::WouldBlock) => std::thread::sleep(Duration::from_millis(10)), @@ -383,29 +520,66 @@ fn lock_repo(repo_dir: &Path) -> CommitResult { } } -/// The revision in the locked `deckd/HEAD`; empty means the root commit. -fn read_recorded_head(head_file: &mut File, root: &CommitId) -> CommitResult { - let mut text = String::new(); +/// `deckd/HEAD` can't be replaced by a rename, because it is the lock, so it +/// holds two fixed-size slots, each ` \n`. A write goes +/// to the slot that doesn't hold the latest value, so a write torn by a crash +/// leaves the previous value intact in the other slot. +const HEAD_SLOT_LEN: usize = 16 + 1 + 40 + 1 + 16 + 1; + +/// The latest intact slot in the locked `deckd/HEAD`, as (sequence, revision). +/// With none (no write ever finished), HEAD is the root commit. +fn read_head_slots(head_file: &mut File) -> CommitResult> { + let mut bytes = Vec::new(); head_file.seek(SeekFrom::Start(0)).map_err(internal)?; - head_file.read_to_string(&mut text).map_err(internal)?; - let text = text.trim(); - if text.is_empty() { - return Ok(root.clone()); + head_file.read_to_end(&mut bytes).map_err(internal)?; + Ok(bytes + .chunks(HEAD_SLOT_LEN) + .take(2) + .filter_map(parse_head_slot) + .max_by_key(|(seq, _)| *seq)) +} + +fn parse_head_slot(slot: &[u8]) -> Option<(u64, CommitId)> { + let text = std::str::from_utf8(slot).ok()?.strip_suffix('\n')?; + let (body, check) = text.rsplit_once(' ')?; + if slot.len() != HEAD_SLOT_LEN || check != format!("{:016x}", fnv1a(body.as_bytes())) { + return None; } - CommitId::try_from_hex(text) - .ok_or_else(|| CommitError::Internal(format!("unreadable deckd/HEAD {text:?}"))) + let (seq, revision) = body.split_once(' ')?; + Some(( + u64::from_str_radix(seq, 16).ok()?, + CommitId::try_from_hex(revision)?, + )) +} + +/// The revision in the locked `deckd/HEAD`. +fn read_recorded_head(head_file: &mut File, root: &CommitId) -> CommitResult { + Ok(read_head_slots(head_file)?.map_or_else(|| root.clone(), |(_, revision)| revision)) } -/// Overwrite `deckd/HEAD` in place. Every revision has the same length, so -/// the file is never truncated and a reader never sees it empty. +/// Record `revision` in the locked `deckd/HEAD` and sync it. fn write_recorded_head(head_file: &mut File, revision: &str) -> CommitResult<()> { - head_file.seek(SeekFrom::Start(0)).map_err(internal)?; - head_file - .write_all(format!("{revision}\n").as_bytes()) - .map_err(internal)?; + if revision.len() != 40 { + return Err(CommitError::Internal(format!( + "cannot record revision {revision:?}" + ))); + } + let seq = read_head_slots(head_file)?.map_or(0, |(seq, _)| seq) + 1; + let body = format!("{seq:016x} {revision}"); + let slot = format!("{body} {:016x}\n", fnv1a(body.as_bytes())); + let offset = (seq % 2) * HEAD_SLOT_LEN as u64; + head_file.seek(SeekFrom::Start(offset)).map_err(internal)?; + head_file.write_all(slot.as_bytes()).map_err(internal)?; head_file.sync_all().map_err(internal) } +/// FNV-1a, enough to tell a torn slot from an intact one. +fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0100_0000_01b3) + }) +} + /// Load the repo and return its HEAD once it agrees with `deckd/HEAD`. Callers /// hold both repo locks. async fn settled_head( @@ -686,11 +860,99 @@ fn tree_hash(commit: &Commit) -> String { } } +/// `id` as a commit in history: an ancestor of, or one of, `heads`. A commit +/// object that was written but never published isn't in the index. +async fn published_commit( + repo: &Arc, + heads: &[CommitId], + id: &CommitId, +) -> CommitResult> { + let index = repo.index(); + if !index.has_id(id).await.map_err(internal)? { + return Ok(None); + } + for head in heads { + if index.is_ancestor(id, head).await.map_err(internal)? { + return Ok(Some(repo.store().get_commit(id).map_err(internal)?)); + } + } + Ok(None) +} + +/// Sync the git objects `commit` added: the commit, every tree on the way to +/// a changed path, and each written file. git doesn't sync loose objects, and +/// recording a commit whose objects a power loss could drop would leave +/// `deckd/HEAD` naming a revision the store doesn't have. An object with no +/// loose file was already packed, and so already durable. +async fn sync_new_objects( + repo_dir: &Path, + commit: &Commit, + changes: &[Change], +) -> CommitResult<()> { + let tree = commit.tree(); + let mut ids = BTreeSet::from([commit.id().hex(), tree_hash(commit)]); + for change in changes { + let path = RepoPathBuf::from_internal_string(change.path.clone()).map_err(internal)?; + for ancestor in path.ancestors().skip(1) { + if ancestor.is_root() { + break; + } + if let Some(Some(TreeValue::Tree(id))) = tree + .path_value(ancestor) + .await + .map_err(internal)? + .as_resolved() + { + ids.insert(id.hex()); + } + } + if let Some(Some(TreeValue::File { id, .. })) = tree + .path_value(&path) + .await + .map_err(internal)? + .as_resolved() + { + ids.insert(id.hex()); + } + } + let mut dirs = BTreeSet::new(); + for id in &ids { + let path = loose_object_path(repo_dir, id); + match File::open(&path) { + Ok(file) => file.sync_all().map_err(internal)?, + Err(e) if e.kind() == ErrorKind::NotFound => continue, + Err(e) => return Err(internal(e)), + } + // The fan-out directory, and `objects/` in case that was new too. + let dir = path + .parent() + .ok_or_else(|| internal("object has no parent"))?; + dirs.insert(dir.to_path_buf()); + dirs.extend(dir.parent().map(Path::to_path_buf)); + } + dirs.iter().try_for_each(|dir| sync_dir(dir)) +} + +fn loose_object_path(repo_dir: &Path, hex: &str) -> PathBuf { + repo_dir + .join("jj/store/git/objects") + .join(&hex[..2]) + .join(&hex[2..]) +} + +fn sync_dir(dir: &Path) -> CommitResult<()> { + File::open(dir).and_then(|d| d.sync_all()).map_err(internal) +} + /// What the parent tree holds at `path`. enum Existing { Absent, - File, + File { + executable: bool, + }, Directory, + /// A symlink, submodule or conflict. + Other, } async fn existing(tree: &MergedTree, path: &RepoPath) -> CommitResult { @@ -698,13 +960,17 @@ async fn existing(tree: &MergedTree, path: &RepoPath) -> CommitResult Ok(match value.as_resolved() { Some(None) => Existing::Absent, Some(Some(TreeValue::Tree(_))) => Existing::Directory, - // Files, symlinks, submodules and conflicts all block a directory. - _ => Existing::File, + Some(Some(TreeValue::File { executable, .. })) => Existing::File { + executable: *executable, + }, + _ => Existing::Other, }) } /// HEAD's tree with `changes` applied. A change may not turn a file into a -/// directory or back: that would remove paths the request didn't list. +/// directory or back: that would remove paths the request didn't list. A +/// write keeps an existing file's executable bit; it replaces a symlink or +/// submodule with a regular file holding the given bytes. async fn build_tree( repo: &Arc, parent: &Commit, @@ -720,7 +986,7 @@ async fn build_tree( if ancestor.is_root() { break; } - if let Existing::File = existing(&base, ancestor).await? { + if let Existing::File { .. } | Existing::Other = existing(&base, ancestor).await? { return Err(CommitError::Invalid(format!( "path {:?} is inside file {:?}", change.path, @@ -728,12 +994,16 @@ async fn build_tree( ))); } } - if let Existing::Directory = existing(&base, &path).await? { - return Err(CommitError::Invalid(format!( - "path {:?} is a directory; list the files inside it instead", - change.path - ))); - } + let executable = match existing(&base, &path).await? { + Existing::Directory => { + return Err(CommitError::Invalid(format!( + "path {:?} is a directory; list the files inside it instead", + change.path + ))); + } + Existing::File { executable } => executable, + Existing::Absent | Existing::Other => false, + }; let value = match &change.content { Some(bytes) => { let id = store @@ -742,7 +1012,7 @@ async fn build_tree( .map_err(internal)?; Merge::normal(TreeValue::File { id, - executable: false, + executable, copy_id: CopyId::placeholder(), }) } @@ -753,42 +1023,105 @@ async fn build_tree( builder.write_tree().await.map_err(internal) } +/// An operation record. `Pending` and `Committed` carry the digest of the +/// request that claimed the operation; it is absent only when the record was +/// rebuilt from HEAD without a readable claim. #[derive(Debug, Clone, PartialEq, Eq)] enum Record { - Pending, - Committed(String), + Pending(Option), + Committed { + revision: String, + digest: Option, + }, Fenced, } impl Record { fn encode(&self) -> String { + let with_digest = |text: String, digest: &Option| match digest { + Some(digest) => format!("{text} {digest}\n"), + None => format!("{text}\n"), + }; match self { - Record::Pending => "pending\n".into(), - Record::Committed(revision) => format!("committed {revision}\n"), + Record::Pending(digest) => with_digest("pending".into(), digest), + Record::Committed { revision, digest } => { + with_digest(format!("committed {revision}"), digest) + } Record::Fenced => "fenced\n".into(), } } fn decode(text: &str) -> CommitResult { - let text = text.trim(); - match text.split_once(' ') { - None if text == "pending" || text.is_empty() => Ok(Record::Pending), - None if text == "fenced" => Ok(Record::Fenced), - Some(("committed", revision)) => Ok(Record::Committed(revision.to_owned())), + let words: Vec<&str> = text.split_whitespace().collect(); + let digest = |word: Option<&&str>| word.map(|w| w.to_string()); + match words.as_slice() { + [] => Ok(Record::Pending(None)), + ["pending", rest @ ..] if rest.len() <= 1 => Ok(Record::Pending(digest(rest.first()))), + ["fenced"] => Ok(Record::Fenced), + ["committed", revision, rest @ ..] if rest.len() <= 1 => Ok(Record::Committed { + revision: revision.to_string(), + digest: digest(rest.first()), + }), _ => Err(CommitError::Internal(format!( - "unreadable operation record {text:?}" + "unreadable operation record {:?}", + text.trim() ))), } } } +/// A digest of everything a commit request asks for except its parent. A +/// retry may name a newer parent after a conflict; anything else differing +/// means the operation ID was reused for another write. +fn request_digest(request: &CommitRequest) -> String { + fn field(hasher: &mut Sha256, bytes: &[u8]) { + hasher.update((bytes.len() as u64).to_be_bytes()); + hasher.update(bytes); + } + let mut hasher = Sha256::new(); + field(&mut hasher, request.author.name.as_bytes()); + field(&mut hasher, request.author.email.as_bytes()); + field(&mut hasher, request.message.as_bytes()); + hasher.update((request.trailers.len() as u64).to_be_bytes()); + for trailer in &request.trailers { + field(&mut hasher, trailer.key.as_bytes()); + field(&mut hasher, trailer.value.as_bytes()); + } + // Paths are unique, so their order carries no meaning. + let mut changes: Vec<&Change> = request.changes.iter().collect(); + changes.sort_by(|a, b| a.path.cmp(&b.path)); + hasher.update((changes.len() as u64).to_be_bytes()); + for change in changes { + field(&mut hasher, change.path.as_bytes()); + match &change.content { + Some(bytes) => { + field(&mut hasher, b"write"); + field(&mut hasher, bytes); + } + None => field(&mut hasher, b"delete"), + } + } + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + fn operation_record_path(repo_dir: &Path, operation_id: &str) -> PathBuf { repo_dir.join("deckd").join("operations").join(operation_id) } +fn record_dir(path: &Path) -> CommitResult<&Path> { + path.parent() + .ok_or_else(|| CommitError::Internal("operation record has no parent".into())) +} + /// Create the record with `create_new`. Returns false when it already exists. /// An empty record (a crash between create and write) reads as `pending`. -fn claim_record(path: &Path, record: Record) -> CommitResult { +/// The record and its directory entry are synced, so a crash can't undo a +/// claim this call reported. +fn claim_record(path: &Path, record: &Record) -> CommitResult { let mut file = match OpenOptions::new().write(true).create_new(true).open(path) { Ok(file) => file, Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(false), @@ -797,6 +1130,7 @@ fn claim_record(path: &Path, record: Record) -> CommitResult { file.write_all(record.encode().as_bytes()) .map_err(internal)?; file.sync_all().map_err(internal)?; + sync_dir(record_dir(path)?)?; Ok(true) } @@ -806,9 +1140,7 @@ fn read_record(path: &Path) -> CommitResult { /// Replace a record atomically: write a temporary file, then rename it over. fn write_record(path: &Path, record: &Record) -> CommitResult<()> { - let dir = path - .parent() - .ok_or_else(|| CommitError::Internal("operation record has no parent".into()))?; + let dir = record_dir(path)?; let name = path .file_name() .and_then(|n| n.to_str()) @@ -821,17 +1153,21 @@ fn write_record(path: &Path, record: &Record) -> CommitResult<()> { file.sync_all().map_err(internal)?; } std::fs::rename(&tmp, path).map_err(internal)?; - File::open(dir) - .and_then(|d| d.sync_all()) - .map_err(internal)?; - Ok(()) + sync_dir(dir) +} + +/// Remove a claim that never published. +fn remove_record(path: &Path) -> CommitResult<()> { + std::fs::remove_file(path).map_err(internal)?; + sync_dir(record_dir(path)?) } -/// Make HEAD's operation record say `committed HEAD`. The record is written -/// whenever it doesn't already say so, including when reading it fails: a -/// missing file may only be a stale NFS lookup, and the write is the truth. -/// A `fenced` record for HEAD's operation means the lock failed to exclude a -/// writer; that is refused loudly, never overwritten. +/// Make HEAD's operation record say `committed HEAD`, keeping the digest a +/// pending claim recorded. The record is written whenever it doesn't already +/// say so, including when reading it fails: a missing file may only be a +/// stale NFS lookup, and the write is the truth. A `fenced` record for HEAD's +/// operation means the lock failed to exclude a writer; that is refused +/// loudly, never overwritten. fn repair_head_record(repo_dir: &Path, head: &Commit) -> CommitResult<()> { let Some(operation_id) = parse_description_trailers(head.description()) .into_iter() @@ -844,20 +1180,23 @@ fn repair_head_record(repo_dir: &Path, head: &Commit) -> CommitResult<()> { return Ok(()); } let path = operation_record_path(repo_dir, &operation_id); - let expected = Record::Committed(head.id().hex()); - match read_record(&path).ok() { - Some(record) if record == expected => Ok(()), + let revision = head.id().hex(); + let digest = match read_record(&path).ok() { + Some(Record::Committed { + revision: recorded, .. + }) if recorded == revision => return Ok(()), Some(Record::Fenced) => { let message = format!( - "operation {operation_id} was reported absent but committed as {}; \ - history needs an operator", - head.id().hex() + "operation {operation_id} was reported absent but committed as {revision}; \ + history needs an operator" ); tracing::error!(repo = %repo_dir.display(), "{message}"); - Err(CommitError::Internal(message)) + return Err(CommitError::Internal(message)); } - _ => write_record(&path, &expected), - } + Some(Record::Pending(digest) | Record::Committed { digest, .. }) => digest, + None => None, + }; + write_record(&path, &Record::Committed { revision, digest }) } fn nonce() -> String { @@ -1359,7 +1698,7 @@ mod tests { /// Overwrite the locked `deckd/HEAD`, as a crashed or partitioned peer /// would leave it. fn set_recorded_head(realm: &Realm, revision: &str) { - let mut file = lock_repo(&realm.repo()).unwrap(); + let mut file = lock_repo(&realm.repo(), Instant::now() + LOCK_TIMEOUT).unwrap(); write_recorded_head(&mut file, revision).unwrap(); } @@ -1474,8 +1813,97 @@ mod tests { ) .unwrap(); - let recorded = std::fs::read_to_string(realm.repo().join("deckd/HEAD")).unwrap(); - assert_eq!(recorded.trim(), done.revision); + assert_eq!(recorded_head(&realm), Some(done.revision.clone())); + let next = commit( + &service, + &realm, + request(&done.revision, "op-2", vec![write("b", "2")]), + ) + .unwrap(); + assert_eq!(recorded_head(&realm), Some(next.revision)); + } + + fn recorded_head(realm: &Realm) -> Option { + let mut file = File::open(realm.repo().join("deckd/HEAD")).unwrap(); + read_head_slots(&mut file) + .unwrap() + .map(|(_, revision)| revision.hex()) + } + + #[test] + fn torn_head_write_keeps_the_previous_revision() { + let realm = Realm::new("torn-head"); + let service = CommitService::new(); + let root = head(&service, &realm); + let first = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + let second = commit( + &service, + &realm, + request(&first.revision, "op-2", vec![write("b", "2")]), + ) + .unwrap(); + // A crash partway through recording a third revision: half the + // bytes that write changes are new, half are still the old ones. + let head_path = realm.repo().join("deckd/HEAD"); + let before = std::fs::read(&head_path).unwrap(); + set_recorded_head(&realm, &"cd".repeat(20)); + let after = std::fs::read(&head_path).unwrap(); + let changed: Vec = (0..after.len()) + .filter(|&i| before.get(i) != Some(&after[i])) + .collect(); + let mut torn = after.clone(); + for &i in &changed[changed.len() / 2..] { + torn[i] = before.get(i).copied().unwrap_or(0); + } + std::fs::write(&head_path, torn).unwrap(); + + assert_eq!(recorded_head(&realm), Some(second.revision.clone())); + assert_eq!(head(&service, &realm), second.revision); + let next = commit( + &service, + &realm, + request(&second.revision, "op-3", vec![write("c", "3")]), + ) + .unwrap(); + assert_eq!(recorded_head(&realm), Some(next.revision)); + } + + #[test] + fn new_objects_are_loose_where_they_are_synced() { + let realm = Realm::new("loose-objects"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("dir/a.json", "{}")]), + ) + .unwrap(); + + let (dir_tree, blob) = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let tree = get_commit(&repo, &done.revision).unwrap().tree(); + let value = |path: &str| { + let path = RepoPathBuf::from_internal_string(path.to_owned()).unwrap(); + let tree = tree.clone(); + async move { + match tree.path_value(&path).await.unwrap().into_resolved() { + Ok(Some(TreeValue::Tree(id))) => id.hex(), + Ok(Some(TreeValue::File { id, .. })) => id.hex(), + other => panic!("unexpected {other:?}"), + } + } + }; + (value("dir").await, value("dir/a.json").await) + }); + for id in [&done.revision, &done.tree, &dir_tree, &blob] { + assert!(loose_object_path(&realm.repo(), id).is_file(), "{id}"); + } } #[test] @@ -1485,9 +1913,10 @@ mod tests { let root = head(&service, &realm); let revision = unpublished_commit(&realm, &root, "op-crashed"); set_recorded_head(&realm, &revision); + let crashed = request(&root, "op-crashed", vec![write("crashed.txt", "x")]); std::fs::write( operation_record_path(&realm.repo(), "op-crashed"), - "pending\n", + Record::Pending(Some(request_digest(&crashed))).encode(), ) .unwrap(); @@ -1501,6 +1930,15 @@ mod tests { ); assert_eq!(head(&restarted, &realm), revision); assert_eq!(files_at(&realm, &revision), files(&[("crashed.txt", "x")])); + // The repaired record kept the claim's digest. + let replay = commit(&restarted, &realm, crashed.clone()).unwrap(); + assert!(replay.replayed); + let mut different = crashed; + different.changes = vec![write("crashed.txt", "y")]; + assert!(matches!( + commit(&restarted, &realm, different), + Err(CommitError::OperationMismatch { .. }) + )); let next = commit( &restarted, &realm, @@ -1734,4 +2172,211 @@ mod tests { // None of those wrote anything. assert_eq!(head(&service, &realm), root); } + + #[test] + fn reused_operation_id_with_a_different_request_is_refused() { + let realm = Realm::new("op-reuse"); + let service = CommitService::new(); + let root = head(&service, &realm); + let done = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1"), delete("gone")]), + ) + .unwrap(); + + // The same request replays, from a newer parent and in any order. + let replay = commit( + &service, + &realm, + request( + &done.revision, + "op-1", + vec![delete("gone"), write("a", "1")], + ), + ) + .unwrap(); + assert!(replay.replayed); + assert_eq!(replay.revision, done.revision); + + let mut other_message = request(&root, "op-1", vec![write("a", "1"), delete("gone")]); + other_message.message = "Something else".into(); + for reused in [ + request(&root, "op-1", vec![write("a", "2"), delete("gone")]), + request(&root, "op-1", vec![write("a", "1")]), + other_message, + ] { + match commit(&service, &realm, reused) { + Err(CommitError::OperationMismatch { revision, .. }) => { + assert_eq!(revision, done.revision) + } + other => panic!("expected operation-mismatch, got {other:?}"), + } + } + assert_eq!(head(&service, &realm), done.revision); + } + + #[test] + fn unchanged_content_replay_is_checked_too() { + let realm = Realm::new("noop-reuse"); + let service = CommitService::new(); + let root = head(&service, &realm); + let base = commit( + &service, + &realm, + request(&root, "op-1", vec![write("a", "1")]), + ) + .unwrap(); + commit( + &service, + &realm, + request(&base.revision, "op-noop", vec![write("a", "1")]), + ) + .unwrap(); + + let reused = commit( + &service, + &realm, + request(&base.revision, "op-noop", vec![write("a", "2")]), + ); + assert!(matches!(reused, Err(CommitError::OperationMismatch { .. }))); + } + + #[test] + fn spellings_of_one_repo_share_a_gate_that_is_pruned() { + let realm = Realm::new("gate"); + let service = CommitService::new(); + head(&service, &realm); + let alias = std::env::temp_dir().join(format!("deckd-commit-alias-{}", nonce())); + std::os::unix::fs::symlink(&realm.root, &alias).unwrap(); + + let direct = service.gate(&realm.repo()).unwrap(); + let via_alias = service + .gate(&service.resolve_repo_dir(&alias.join(".history")).unwrap()) + .unwrap(); + assert!(Arc::ptr_eq(&direct, &via_alias)); + + drop((direct, via_alias)); + let other = Realm::new("gate-other"); + head(&service, &other); + // Only the gate of the latest call is left, and nothing holds it. + let gates = service.gates.lock().unwrap(); + assert_eq!(gates.len(), 1); + assert!(gates.values().all(|gate| gate.strong_count() == 0)); + drop(gates); + std::fs::remove_file(alias).unwrap(); + } + + #[test] + fn busy_gate_times_out() { + let realm = Realm::new("gate-timeout"); + let service = CommitService::new(); + head(&service, &realm); + let gate = service.gate(&realm.repo()).unwrap(); + let _held = gate.lock().unwrap(); + + let started = Instant::now(); + let error = std::thread::scope(|s| { + s.spawn(|| service.head(&realm.repo()).unwrap_err()) + .join() + .unwrap() + }); + + assert!(matches!(error, CommitError::LockTimeout(_)), "{error:?}"); + assert!(started.elapsed() >= LOCK_TIMEOUT); + } + + #[test] + fn repos_outside_the_root_are_refused() { + let inside = Realm::new("root-inside"); + let outside = Realm::new("root-outside"); + let service = CommitService::with_root(&inside.root).unwrap(); + let escape = inside.root.join("escape"); + std::os::unix::fs::symlink(&outside.root, &escape).unwrap(); + + assert!(service.head(&inside.repo()).is_ok()); + for repo in [outside.repo(), escape, inside.root.clone()] { + assert!( + matches!(service.head(&repo), Err(CommitError::Invalid(_))), + "{repo:?}" + ); + } + assert!(matches!( + service.tree_hash(&outside.repo(), &"ab".repeat(20)), + Err(CommitError::Invalid(_)) + )); + assert!(!outside.repo().exists()); + } + + #[test] + fn unpublished_commit_has_no_tree_hash() { + let realm = Realm::new("orphan"); + let service = CommitService::new(); + let root = head(&service, &realm); + let orphan = unpublished_commit(&realm, &root, "op-crashed"); + + let error = service.tree_hash(&realm.repo(), &orphan).unwrap_err(); + + assert!( + matches!(error, CommitError::RevisionNotFound(_)), + "{error:?}" + ); + assert!(service.tree_hash(&realm.repo(), &root).is_ok()); + } + + #[test] + fn rewriting_an_executable_file_keeps_it_executable() { + let realm = Realm::new("executable"); + let service = CommitService::new(); + let root = head(&service, &realm); + let path = RepoPathBuf::from_internal_string("run.sh".to_owned()).unwrap(); + // Explicit commits write only regular files; imported history may not. + let seeded = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let parent = get_commit(&repo, &root).unwrap(); + let id = repo + .store() + .write_file(&path, &mut &b"#!/bin/sh\n"[..]) + .await + .unwrap(); + let mut builder = MergedTreeBuilder::new(parent.tree()); + builder.set_or_remove( + path.clone(), + Merge::normal(TreeValue::File { + id, + executable: true, + copy_id: CopyId::placeholder(), + }), + ); + let tree = builder.write_tree().await.unwrap(); + let mut tx = repo.start_transaction(); + let commit = tx + .repo_mut() + .new_commit(vec![parent.id().clone()], tree) + .set_description("seed\n") + .write() + .await + .unwrap(); + tx.commit("seed").await.unwrap(); + commit.id().hex() + }); + set_recorded_head(&realm, &seeded); + + let done = commit( + &service, + &realm, + request(&seeded, "op-1", vec![write("run.sh", "#!/bin/sh\necho\n")]), + ) + .unwrap(); + + let executable = pollster::block_on(async { + let repo = load_repo(&realm.repo()).await.unwrap(); + let tree = get_commit(&repo, &done.revision).unwrap().tree(); + match tree.path_value(&path).await.unwrap().into_resolved() { + Ok(Some(TreeValue::File { executable, .. })) => executable, + other => panic!("unexpected {other:?}"), + } + }); + assert!(executable); + } } diff --git a/packages/deckd/src/main.rs b/packages/deckd/src/main.rs index 1da551f..13c8dc2 100644 --- a/packages/deckd/src/main.rs +++ b/packages/deckd/src/main.rs @@ -43,7 +43,11 @@ async fn main() -> Result<(), Box> { }; let history: Arc = Arc::new(jj_history); - let commits = Arc::new(CommitService::new()); + // When set, explicit-commit repos must live under this directory. + let commits = Arc::new(match std::env::var_os("DECKD_REPO_ROOT") { + Some(root) => CommitService::with_root(std::path::Path::new(&root))?, + None => CommitService::new(), + }); #[cfg(feature = "build")] let state = { let jj_bin = std::env::var("JJ_BIN").unwrap_or_else(|_| "jj".into());