From 78b42ae8ad1a47c3083cb34055a4dbe79d1ce221 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 20 Nov 2026 22:32:34 +0100 Subject: [PATCH 1/8] test(transcription): curly contractions and Arabic marks in restored punctuation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both cases fail on 9ef18342: we’ll takes the comma of the Well before it, and Arabic ، ؛ ؟ are dropped. --- src/services/TranscriptionService.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 64bebb4f..73c7ec01 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -1825,6 +1825,18 @@ describe("TranscriptionService", () => { "we'll see Okay", "we'll see. Okay.", ], + [ + "does not give a Qwen contraction with a curly apostrophe the punctuation of a word spelled like it without one", + "Well, we’ll see. ‘Okay.’ Bye.", + "we’ll see Okay Bye", + "we’ll see. ‘Okay.’ Bye.", + ], + [ + "gives Qwen's words Arabic punctuation from its text", + "مرحبا، كيف حالك؟ أنا بخير؛ شكرا.", + "مرحبا كيف حالك أنا بخير شكرا", + "مرحبا، كيف حالك؟ أنا بخير؛ شكرا.", + ], [ "adds nothing to Qwen's words when it lists quotes as entries of their own", '"Hello world." Next!', From 38bc731b4aef3399d2da7fcd5fa074b45ab9a88a Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 20 Nov 2026 22:32:34 +0100 Subject: [PATCH 2/8] fix(transcription): restore punctuation onto curly contractions and from Arabic marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keys read a curly apostrophe as a straight one, so we’ll is not well, and the closing marks include ، ؛ ؟. Budgets rise to the measured 334,117 raw and 109,265 gzip bytes. --- AGENTS.md | 2 +- scripts/size-baseline.md | 11 ++++++++++- scripts/size.mjs | 4 ++-- scripts/size.test.ts | 10 +++++----- src/services/transcription/providers/verboseJson.ts | 7 ++++--- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ee4472f9..e463d575 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ svelte-check's native API path passes when a tsconfig matches no files, which `check:a11y` in temporary workspaces and requires it to fail on that case and when the native compiler cannot start. -After `npm run build`, `npm run size` enforces 334,091 raw bytes and 109,234 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. +After `npm run build`, `npm run size` enforces 334,117 raw bytes and 109,265 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. Before opening a PR or cutting a release, run the CI-equivalent checks locally: diff --git a/scripts/size-baseline.md b/scripts/size-baseline.md index 6fc18ca4..dd838892 100644 --- a/scripts/size-baseline.md +++ b/scripts/size-baseline.md @@ -12,7 +12,7 @@ main.js raw: 307131 bytes; budget 330000 bytes; 22869 bytes remaining main.js gzip (level 9): 97161 bytes; budget 100000 bytes; 2839 bytes remaining ``` -The gzip budget is now 109,234 bytes, and the raw budget 334,091. On 2026-10-14 it rose from 100,000 for +The gzip budget is now 109,265 bytes, and the raw budget 334,117. On 2026-10-14 it rose from 100,000 for the full player's Transcript panel, which the owner approved at about 1.9 KB and which measured 2,576 bytes. The coordinator set 103,000 to keep the panel's Find and Follow chip and to leave room for fixes in flight. The @@ -100,6 +100,15 @@ main.js raw: 334091 bytes; budget 334091 bytes; 0 bytes remaining main.js gzip (level 9): 109234 bytes; budget 109234 bytes; 0 bytes remaining ``` +On 2026-11-20 both rose for punctuation restored onto words with curly +apostrophes and from Arabic marks: from 334,091 raw by 26 bytes, and from +109,234 gzip by 31 bytes. With it: + +```text +main.js raw: 334117 bytes; budget 334117 bytes; 0 bytes remaining +main.js gzip (level 9): 109265 bytes; budget 109265 bytes; 0 bytes remaining +``` + Gzip uses Node's `gzipSync` at level 9, without a filename or timestamp. Budgets use decimal bytes. The brief's 96,533-byte reference is unexplained. This report fixes the compressor instead of comparing compressed sizes across diff --git a/scripts/size.mjs b/scripts/size.mjs index c883ce78..a6378b75 100644 --- a/scripts/size.mjs +++ b/scripts/size.mjs @@ -9,8 +9,8 @@ try { const bundle = readFileSync(path); if (!bundle.length) throw new Error(`${path} is empty. Run npm run build first.`); const measurements = [ - { name: "raw", bytes: bundle.length, budget: 334_091 }, - { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 109_234 }, + { name: "raw", bytes: bundle.length, budget: 334_117 }, + { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 109_265 }, ]; const rows = measurements.map(({ name, bytes, budget }) => { const overage = Math.max(0, bytes - budget); diff --git a/scripts/size.test.ts b/scripts/size.test.ts index 9ecc9e7f..f5c13261 100644 --- a/scripts/size.test.ts +++ b/scripts/size.test.ts @@ -20,19 +20,19 @@ function check(name: string, contents: string | Buffer) { describe("bundle budget CLI", () => { it("allows the raw budget boundary and rejects one extra byte", () => { - expect(check("boundary.js", "a".repeat(334_091)).status).toBe(0); - const result = check("over.js", "a".repeat(334_092)); + expect(check("boundary.js", "a".repeat(334_117)).status).toBe(0); + const result = check("over.js", "a".repeat(334_118)); expect(result.status).toBe(1); - expect(result.stdout).toContain("budget 334091 bytes; OVER by 1 bytes"); + expect(result.stdout).toContain("budget 334117 bytes; OVER by 1 bytes"); }); it("rejects gzip growth even when raw bytes fit", () => { const result = check("gzip.js", randomBytes(110_000)); expect(result.status).toBe(1); expect(result.stdout).toMatch( - /gzip \(level 9\): \d+ bytes; budget 109234 bytes; OVER by \d+ bytes/, + /gzip \(level 9\): \d+ bytes; budget 109265 bytes; OVER by \d+ bytes/, ); - expect(result.stdout).toContain("budget 334091 bytes; 224091 bytes remaining"); + expect(result.stdout).toContain("budget 334117 bytes; 224117 bytes remaining"); }); it("fails on absent or empty artifacts instead of reporting a small build", () => { diff --git a/src/services/transcription/providers/verboseJson.ts b/src/services/transcription/providers/verboseJson.ts index 52ea4e1d..0c25b326 100644 --- a/src/services/transcription/providers/verboseJson.ts +++ b/src/services/transcription/providers/verboseJson.ts @@ -64,7 +64,7 @@ const PLAIN_WORD = /** Opening quotes and brackets a word takes from the start of its token. */ const LEADING = /^['“"‘([¿¡「『《【(]+/u; /** Closing punctuation a word takes from the end of its token, and none of `%` or `#`, which are words' own. */ -const TRAILING = /['.,?!;:…—–”"’)\]。,、?!;:」』》】)।॥]+$/u; +const TRAILING = /['.,?!;:…—–”"’)\]。,、?!;:」』》】)।॥،؛؟]+$/u; /** * Qwen3 ASR, and the servers that run it or Whisper, can list words without punctuation and @@ -85,11 +85,12 @@ function punctuated(words: Word[], text: unknown): Word[] { .replace(/[「『《【(]+/gu, " $&") .split(/\s+/) .filter(Boolean); - // Apostrophes count inside a word, so "we'll" is not "well", and not at its ends, where - // they quote it, as in "'Hello". + // Apostrophes, straight or curly, count inside a word, so "we’ll" is not "well", and not at + // its ends, where they quote it, as in "'Hello". const bare = (token: string) => token .toLowerCase() + .replace(/’/g, "'") .replace(/[^\p{L}\p{M}\p{N}']/gu, "") .replace(/^'+|'+$/g, ""); const keys = tokens.map(bare); From 66ea40be3615f5cb64e109d3d6e218cec286ae18 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 20 Nov 2026 22:32:34 +0100 Subject: [PATCH 3/8] chore(verify): answer Qwen3 ASR with unpunctuated words, as it does --- .claude/skills/verify/features/transcripts.md | 2 +- .claude/skills/verify/net.mjs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.claude/skills/verify/features/transcripts.md b/.claude/skills/verify/features/transcripts.md index a63dda42..30b1f1c3 100644 --- a/.claude/skills/verify/features/transcripts.md +++ b/.claude/skills/verify/features/transcripts.md @@ -56,7 +56,7 @@ Preconditions: - Scribe v2, public. Choose `ElevenLabs Scribe v2`, play `Fixture Episode 1` of `PodNotes Fixture` and run the audio command. `$NET log` shows the catalog `GET`, then one `POST https://openrouter.ai/api/v1/audio/transcriptions` with `contentType` `application/json`, under 1,000 `uploadBytes` and the episode's address as `sourceUrl`, and no request for `/episode-1.wav` after the command. - Grok STT 2.0. On `Recommended (Grok STT 2.0)`, play `Fixtures/Local Fixture.wav` and run the audio command. The `POST` is `application/json` with about 4/3 of the file's bytes, its audio in base64, and no `sourceUrl`. - In parts. Write a 70 min MP3 of two voices, a 300 Hz tone and a 700 Hz tone that take turns every 45 s: `ffmpeg -f lavfi -i "aevalsrc='0.5*sin(2*PI*if(lt(mod(t\,90)\,45)\,300\,700)*t)':s=8000:d=4200" -ac 1 -ar 8000 -b:a 8k "$VAULT/Fixtures/Two Voices.mp3"`. Play it and run the audio command on Grok. `$NET log` shows three `POST https://openrouter.ai/api/v1/audio/transcriptions` entries of about 1.9 MB each, one after the other, then the speaker reel, one of about 82 KB whose `decodedSeconds` is `61.56`: six 9 s clips with 1.5 s of silence between two. The fixture numbers the voices in the order they sound in each request, so the second part, which starts with the high tone, numbers them the other way. The note still numbers the low voice `1` and the high voice `2` throughout, the paragraph at `00:23:15` runs on across the first part boundary, and the progress notice ends with `Transcription completed and saved.` With `ElevenLabs Scribe v2` the parts go as multipart, no reel follows, each part's fixed answer numbers two more speakers, and the notice ends with `OpenRouter transcribed this episode in 3 parts, and speaker labels restart in each, so one person can have several labels.` - - Voxtral Mini Transcribe 3 and Qwen3 ASR 1.7B. With `Speaker labels` on, choose each in `Model`. The `Speaker labels` row reads ` does not label speakers.` Play `Two Voices.mp3` and run the audio command. `$NET log` shows five `multipart/form-data` `POST https://openrouter.ai/api/v1/audio/transcriptions` entries of about 840 KB, one after the other, and no reel. Qwen3 ASR's are about 98 bytes larger, its `timestamp_granularities[]` field. The fixture answers Voxtral with segments, and with a 400 if it asks for word times. The note has `[00:00:00](...) This text came from the fixture provider.` and `[00:00:12](...) A second speaker answers.` for each part, at `00:14:00`, `00:28:00`, `00:42:00` and `00:56:00` with links to `time=840.096` and on, with no `**1:**`. The progress notice ends with `Saved without speaker labels, because does not label speakers. These providers do: ElevenLabs, Deepgram, OpenRouter.` + - Voxtral Mini Transcribe 3 and Qwen3 ASR 1.7B. With `Speaker labels` on, choose each in `Model`. The `Speaker labels` row reads ` does not label speakers.` Play `Two Voices.mp3` and run the audio command. `$NET log` shows five `multipart/form-data` `POST https://openrouter.ai/api/v1/audio/transcriptions` entries of about 840 KB, one after the other, and no reel. Qwen3 ASR's are about 98 bytes larger, its `timestamp_granularities[]` field. The fixture answers Voxtral with segments, and with a 400 if it asks for word times, and Qwen3 ASR with words that carry no punctuation, which PodNotes restores from the answer's text. The note has `[00:00:00](...) This text came from the fixture provider.` and `[00:00:12](...) A second speaker answers.` for each part, at `00:14:00`, `00:28:00`, `00:42:00` and `00:56:00` with links to `time=840.096` and on, with no `**1:**`. The progress notice ends with `Saved without speaker labels, because does not label speakers. These providers do: ElevenLabs, Deepgram, OpenRouter.` - **OpenAI.** Choose `OpenAI` with its fake key. `Untranscribed Episode` downloads its WAV and `$NET log` shows a `POST https://api.openai.com/v1/audio/transcriptions` answered by the fixture provider. The note holds the fixture text with no timestamps. - In parts. Write an MP3 longer than 2,070 s into the vault, for example 70 min in 4 MB: `ffmpeg -f lavfi -i sine=frequency=440:duration=4200 -ac 1 -ar 8000 -b:a 8k "$VAULT/Fixtures/Long Fixture.mp3"`. Play it ([local-files.md](./local-files.md)) and run the audio command. `$NET log` shows three `POST https://api.openai.com/v1/audio/transcriptions` entries of about 1.4 MB each, one after the other, and the note holds the fixture text three times with no timestamps. - Speaker labels. With `Speaker labels` on, as in a library that labelled speakers with OpenAI on 2.24.0, the progress notice ends with `Saved without speaker labels, because GPT Transcribe does not label speakers. These providers do: ElevenLabs, Deepgram, OpenRouter.` and stays until clicked. diff --git a/.claude/skills/verify/net.mjs b/.claude/skills/verify/net.mjs index 25abb0e1..93676f63 100644 --- a/.claude/skills/verify/net.mjs +++ b/.claude/skills/verify/net.mjs @@ -110,7 +110,8 @@ const watchSource = (turns) => `(() => { // OpenRouter's verbose_json words, at the same times, with numbered speakers. const openRouterWords = (sentence, speaker, from) => words(sentence, speaker, from).map(({ punctuated_word, start, end }) => ({ word: punctuated_word, start, end, speaker, speaker_label: "speaker_" + speaker, confidence: 0.99 })); const openrouter = { text: fullText, words: turns.flatMap(([from, speaker, line]) => openRouterWords(line, speaker, from)) }; - // Voxtral times segments and refuses word times; Qwen3 ASR times words and labels no speaker. + // Voxtral times segments and refuses word times. Qwen3 ASR times words, labels no speaker, + // and keeps the punctuation only in its text. const plainOpenRouter = { "mistralai/voxtral-mini-transcribe-3-0": (form) => form.has("timestamp_granularities[]") @@ -122,7 +123,7 @@ const watchSource = (turns) => `(() => { return { id, start: from, end: Math.round(turnWords[turnWords.length - 1].end * 10) / 10, text: " " + line }; }), }, - "qwen/qwen3-asr-1.7b": () => ({ text: openrouter.text, words: openrouter.words.map(({ word, start, end }) => ({ word, start, end })) }), + "qwen/qwen3-asr-1.7b": () => ({ text: openrouter.text, words: openrouter.words.map(({ word, start, end }) => ({ word: word.replace(/^[^\\p{L}\\p{M}\\p{N}]+|[^\\p{L}\\p{M}\\p{N}]+$/gu, ""), start, end })) }), }; const openrouterCatalog = { data: ${jsLiteral(OPENROUTER_MODELS)}.map((id) => ({ id, expiration_date: null })) }; // Grok's MP3 uploads are heard as a diarizing model hears them: a tone under 500 Hz is one From a0578750f531822258eea98b8bf4e4b1f43859fc Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 21 Nov 2026 00:42:06 +0100 Subject: [PATCH 4/8] test(transcription): words split at an apostrophe and numbers grouped with one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both fail on 66ea40be: Rock n roll loses the ! of Rock’n’roll!, and 1000 the period of 1’000. The independent verifier on #552 found them. --- src/services/TranscriptionService.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 73c7ec01..8e3933cb 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -1831,6 +1831,18 @@ describe("TranscriptionService", () => { "we’ll see Okay Bye", "we’ll see. ‘Okay.’ Bye.", ], + [ + "gives the last of the Qwen words split at an apostrophe in its text that token's punctuation", + "Rock’n’roll! Yes, we’ll. Rock'n'roll!", + "Rock n roll Yes we ll Rock n roll", + "Rock n roll! Yes, we ll. Rock n roll!", + ], + [ + "restores punctuation after a Qwen number its text groups with an apostrophe", + "It costs 1’000. Or 2'000. Next!", + "It costs 1000 Or 2000 Next", + "It costs 1000. Or 2000. Next!", + ], [ "gives Qwen's words Arabic punctuation from its text", "مرحبا، كيف حالك؟ أنا بخير؛ شكرا.", From 3e44ed33a0e16ab227e1a69e93ad0b68949c589b Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 21 Nov 2026 00:42:06 +0100 Subject: [PATCH 5/8] fix(transcription): match words split at an apostrophe and numbers grouped with one A word that ends at an apostrophe inside its token moves past it, so we and ll both fall in we'll, and an apostrophe between digits is dropped from the key. Budgets rise to the measured 334,180 raw and 109,299 gzip bytes. --- AGENTS.md | 2 +- scripts/size-baseline.md | 13 +++++++------ scripts/size.mjs | 4 ++-- scripts/size.test.ts | 10 +++++----- src/services/transcription/providers/verboseJson.ts | 8 +++++--- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e463d575..1eabf98e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ svelte-check's native API path passes when a tsconfig matches no files, which `check:a11y` in temporary workspaces and requires it to fail on that case and when the native compiler cannot start. -After `npm run build`, `npm run size` enforces 334,117 raw bytes and 109,265 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. +After `npm run build`, `npm run size` enforces 334,180 raw bytes and 109,299 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. Before opening a PR or cutting a release, run the CI-equivalent checks locally: diff --git a/scripts/size-baseline.md b/scripts/size-baseline.md index dd838892..c999aa0e 100644 --- a/scripts/size-baseline.md +++ b/scripts/size-baseline.md @@ -12,7 +12,7 @@ main.js raw: 307131 bytes; budget 330000 bytes; 22869 bytes remaining main.js gzip (level 9): 97161 bytes; budget 100000 bytes; 2839 bytes remaining ``` -The gzip budget is now 109,265 bytes, and the raw budget 334,117. On 2026-10-14 it rose from 100,000 for +The gzip budget is now 109,299 bytes, and the raw budget 334,180. On 2026-10-14 it rose from 100,000 for the full player's Transcript panel, which the owner approved at about 1.9 KB and which measured 2,576 bytes. The coordinator set 103,000 to keep the panel's Find and Follow chip and to leave room for fixes in flight. The @@ -100,13 +100,14 @@ main.js raw: 334091 bytes; budget 334091 bytes; 0 bytes remaining main.js gzip (level 9): 109234 bytes; budget 109234 bytes; 0 bytes remaining ``` -On 2026-11-20 both rose for punctuation restored onto words with curly -apostrophes and from Arabic marks: from 334,091 raw by 26 bytes, and from -109,234 gzip by 31 bytes. With it: +On 2026-11-21 both rose for punctuation restored onto words with curly +apostrophes, words split at an apostrophe and numbers grouped with one, and +from Arabic marks: from 334,091 raw by 89 bytes, and from 109,234 gzip by 65 +bytes. With it: ```text -main.js raw: 334117 bytes; budget 334117 bytes; 0 bytes remaining -main.js gzip (level 9): 109265 bytes; budget 109265 bytes; 0 bytes remaining +main.js raw: 334180 bytes; budget 334180 bytes; 0 bytes remaining +main.js gzip (level 9): 109299 bytes; budget 109299 bytes; 0 bytes remaining ``` Gzip uses Node's `gzipSync` at level 9, without a filename or timestamp. diff --git a/scripts/size.mjs b/scripts/size.mjs index a6378b75..4acd6cb0 100644 --- a/scripts/size.mjs +++ b/scripts/size.mjs @@ -9,8 +9,8 @@ try { const bundle = readFileSync(path); if (!bundle.length) throw new Error(`${path} is empty. Run npm run build first.`); const measurements = [ - { name: "raw", bytes: bundle.length, budget: 334_117 }, - { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 109_265 }, + { name: "raw", bytes: bundle.length, budget: 334_180 }, + { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 109_299 }, ]; const rows = measurements.map(({ name, bytes, budget }) => { const overage = Math.max(0, bytes - budget); diff --git a/scripts/size.test.ts b/scripts/size.test.ts index f5c13261..989d9605 100644 --- a/scripts/size.test.ts +++ b/scripts/size.test.ts @@ -20,19 +20,19 @@ function check(name: string, contents: string | Buffer) { describe("bundle budget CLI", () => { it("allows the raw budget boundary and rejects one extra byte", () => { - expect(check("boundary.js", "a".repeat(334_117)).status).toBe(0); - const result = check("over.js", "a".repeat(334_118)); + expect(check("boundary.js", "a".repeat(334_180)).status).toBe(0); + const result = check("over.js", "a".repeat(334_181)); expect(result.status).toBe(1); - expect(result.stdout).toContain("budget 334117 bytes; OVER by 1 bytes"); + expect(result.stdout).toContain("budget 334180 bytes; OVER by 1 bytes"); }); it("rejects gzip growth even when raw bytes fit", () => { const result = check("gzip.js", randomBytes(110_000)); expect(result.status).toBe(1); expect(result.stdout).toMatch( - /gzip \(level 9\): \d+ bytes; budget 109265 bytes; OVER by \d+ bytes/, + /gzip \(level 9\): \d+ bytes; budget 109299 bytes; OVER by \d+ bytes/, ); - expect(result.stdout).toContain("budget 334117 bytes; 224117 bytes remaining"); + expect(result.stdout).toContain("budget 334180 bytes; 224180 bytes remaining"); }); it("fails on absent or empty artifacts instead of reporting a small build", () => { diff --git a/src/services/transcription/providers/verboseJson.ts b/src/services/transcription/providers/verboseJson.ts index 0c25b326..b98cb318 100644 --- a/src/services/transcription/providers/verboseJson.ts +++ b/src/services/transcription/providers/verboseJson.ts @@ -86,20 +86,22 @@ function punctuated(words: Word[], text: unknown): Word[] { .split(/\s+/) .filter(Boolean); // Apostrophes, straight or curly, count inside a word, so "we’ll" is not "well", and not at - // its ends, where they quote it, as in "'Hello". + // its ends, where they quote it, as in "'Hello", or between digits, as in "1’000". const bare = (token: string) => token .toLowerCase() .replace(/’/g, "'") .replace(/[^\p{L}\p{M}\p{N}']/gu, "") + .replace(/(\p{N})'(?=\p{N})/gu, "$1") .replace(/^'+|'+$/g, ""); const keys = tokens.map(bare); const wordKeys = words.map((word) => bare(word.text)); - /** Where word `w` ends when it starts at offset `o` of token `t`. */ + /** Where word `w` ends when it starts at offset `o` of token `t`, past an apostrophe after it. */ const step = (w: number, t: number, o: number): [number, number] | undefined => { const key = wordKeys[w]; if (!key || t >= keys.length || !keys[t].startsWith(key, o)) return undefined; - const end = o + key.length; + // The words "we" and "ll" both fall in "we'll". + const end = o + key.length + (keys[t][o + key.length] === "'" ? 1 : 0); return end === keys[t].length ? [t + 1, 0] : [t, end]; }; /** Whether the two words after `w` follow on from where `w` ends. */ From 4aea0f2bbc6b67fb579a51bb148b792ae64c3938 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 21 Nov 2026 00:55:23 +0100 Subject: [PATCH 6/8] test(transcription): words that leave out an apostrophe their text has MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 90s loses the ! of 90’s! on 3e44ed33, which the second independent verifier on #552 found. Two more cases guard against a word spelled like an apostrophe word, its against It's, and a later token pulling a split word out of its own. --- src/services/TranscriptionService.test.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 8e3933cb..ca7f6742 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -1838,10 +1838,22 @@ describe("TranscriptionService", () => { "Rock n roll! Yes, we ll. Rock n roll!", ], [ - "restores punctuation after a Qwen number its text groups with an apostrophe", - "It costs 1’000. Or 2'000. Next!", - "It costs 1000 Or 2000 Next", - "It costs 1000. Or 2000. Next!", + "restores punctuation to Qwen's words that leave out an apostrophe their text has", + "It costs 1’000. Or 2'000. Back in the 90’s! Next.", + "It costs 1000 Or 2000 Back in the 90s Next", + "It costs 1000. Or 2000. Back in the 90s! Next.", + ], + [ + "keeps the first of the Qwen words split from one token of its text in that token when a later one spells it", + "Short-term, short. Done.", + "Short term short Done", + "Short term, short. Done.", + ], + [ + "does not give a Qwen word the punctuation of one that is spelled like it with an apostrophe", + "It’s, its own. Done.", + "its own Done", + "its own. Done.", ], [ "gives Qwen's words Arabic punctuation from its text", From 412f4c1c452c7b8a3a7d524dea0089d5d46ced5e Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 21 Nov 2026 00:55:23 +0100 Subject: [PATCH 7/8] fix(transcription): match words without apostrophes and prefer their exact spelling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Words match the text with apostrophes dropped, so 90s finds 90’s, 1000 finds 1’000 and rock n roll finds rock'n'roll, which replaces the apostrophe skip and the digit rule. A word that matches a whole token only that way, as its does It's, moves to a later token in reach that spells it exactly, which keeps we’ll off a nearby Well,. Budgets rise to the measured 334,318 raw and 109,351 gzip bytes. --- AGENTS.md | 2 +- scripts/size-baseline.md | 14 ++++----- scripts/size.mjs | 4 +-- scripts/size.test.ts | 10 +++--- .../transcription/providers/verboseJson.ts | 31 ++++++++++++++----- 5 files changed, 39 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1eabf98e..2de98529 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ svelte-check's native API path passes when a tsconfig matches no files, which `check:a11y` in temporary workspaces and requires it to fail on that case and when the native compiler cannot start. -After `npm run build`, `npm run size` enforces 334,180 raw bytes and 109,299 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. +After `npm run build`, `npm run size` enforces 334,318 raw bytes and 109,351 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. Before opening a PR or cutting a release, run the CI-equivalent checks locally: diff --git a/scripts/size-baseline.md b/scripts/size-baseline.md index c999aa0e..6df3ed96 100644 --- a/scripts/size-baseline.md +++ b/scripts/size-baseline.md @@ -12,7 +12,7 @@ main.js raw: 307131 bytes; budget 330000 bytes; 22869 bytes remaining main.js gzip (level 9): 97161 bytes; budget 100000 bytes; 2839 bytes remaining ``` -The gzip budget is now 109,299 bytes, and the raw budget 334,180. On 2026-10-14 it rose from 100,000 for +The gzip budget is now 109,351 bytes, and the raw budget 334,318. On 2026-10-14 it rose from 100,000 for the full player's Transcript panel, which the owner approved at about 1.9 KB and which measured 2,576 bytes. The coordinator set 103,000 to keep the panel's Find and Follow chip and to leave room for fixes in flight. The @@ -100,14 +100,14 @@ main.js raw: 334091 bytes; budget 334091 bytes; 0 bytes remaining main.js gzip (level 9): 109234 bytes; budget 109234 bytes; 0 bytes remaining ``` -On 2026-11-21 both rose for punctuation restored onto words with curly -apostrophes, words split at an apostrophe and numbers grouped with one, and -from Arabic marks: from 334,091 raw by 89 bytes, and from 109,234 gzip by 65 -bytes. With it: +On 2026-11-21 both rose for punctuation restored onto words that match their +text without apostrophes, each kept apart from a word spelled like it with one, +and from Arabic marks: from 334,091 raw by 227 bytes, and from 109,234 gzip by +117 bytes. With it: ```text -main.js raw: 334180 bytes; budget 334180 bytes; 0 bytes remaining -main.js gzip (level 9): 109299 bytes; budget 109299 bytes; 0 bytes remaining +main.js raw: 334318 bytes; budget 334318 bytes; 0 bytes remaining +main.js gzip (level 9): 109351 bytes; budget 109351 bytes; 0 bytes remaining ``` Gzip uses Node's `gzipSync` at level 9, without a filename or timestamp. diff --git a/scripts/size.mjs b/scripts/size.mjs index 4acd6cb0..e6d4292a 100644 --- a/scripts/size.mjs +++ b/scripts/size.mjs @@ -9,8 +9,8 @@ try { const bundle = readFileSync(path); if (!bundle.length) throw new Error(`${path} is empty. Run npm run build first.`); const measurements = [ - { name: "raw", bytes: bundle.length, budget: 334_180 }, - { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 109_299 }, + { name: "raw", bytes: bundle.length, budget: 334_318 }, + { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 109_351 }, ]; const rows = measurements.map(({ name, bytes, budget }) => { const overage = Math.max(0, bytes - budget); diff --git a/scripts/size.test.ts b/scripts/size.test.ts index 989d9605..fc23fbdf 100644 --- a/scripts/size.test.ts +++ b/scripts/size.test.ts @@ -20,19 +20,19 @@ function check(name: string, contents: string | Buffer) { describe("bundle budget CLI", () => { it("allows the raw budget boundary and rejects one extra byte", () => { - expect(check("boundary.js", "a".repeat(334_180)).status).toBe(0); - const result = check("over.js", "a".repeat(334_181)); + expect(check("boundary.js", "a".repeat(334_318)).status).toBe(0); + const result = check("over.js", "a".repeat(334_319)); expect(result.status).toBe(1); - expect(result.stdout).toContain("budget 334180 bytes; OVER by 1 bytes"); + expect(result.stdout).toContain("budget 334318 bytes; OVER by 1 bytes"); }); it("rejects gzip growth even when raw bytes fit", () => { const result = check("gzip.js", randomBytes(110_000)); expect(result.status).toBe(1); expect(result.stdout).toMatch( - /gzip \(level 9\): \d+ bytes; budget 109299 bytes; OVER by \d+ bytes/, + /gzip \(level 9\): \d+ bytes; budget 109351 bytes; OVER by \d+ bytes/, ); - expect(result.stdout).toContain("budget 334180 bytes; 224180 bytes remaining"); + expect(result.stdout).toContain("budget 334318 bytes; 224318 bytes remaining"); }); it("fails on absent or empty artifacts instead of reporting a small build", () => { diff --git a/src/services/transcription/providers/verboseJson.ts b/src/services/transcription/providers/verboseJson.ts index b98cb318..98d3eb2d 100644 --- a/src/services/transcription/providers/verboseJson.ts +++ b/src/services/transcription/providers/verboseJson.ts @@ -85,23 +85,25 @@ function punctuated(words: Word[], text: unknown): Word[] { .replace(/[「『《【(]+/gu, " $&") .split(/\s+/) .filter(Boolean); - // Apostrophes, straight or curly, count inside a word, so "we’ll" is not "well", and not at - // its ends, where they quote it, as in "'Hello", or between digits, as in "1’000". - const bare = (token: string) => + // Words match the text without apostrophes, so "90s" finds "90’s", and "rock", "n" and + // "roll" find "rock'n'roll". + const bare = (token: string) => token.toLowerCase().replace(/[^\p{L}\p{M}\p{N}]/gu, ""); + // A spelling keeps the apostrophes inside a word, straight or curly, so "we’ll" is not + // "well", and drops those at its ends, which quote it, as in "'Hello". + const spelled = (token: string) => token .toLowerCase() .replace(/’/g, "'") .replace(/[^\p{L}\p{M}\p{N}']/gu, "") - .replace(/(\p{N})'(?=\p{N})/gu, "$1") .replace(/^'+|'+$/g, ""); const keys = tokens.map(bare); + const spellings = tokens.map(spelled); const wordKeys = words.map((word) => bare(word.text)); - /** Where word `w` ends when it starts at offset `o` of token `t`, past an apostrophe after it. */ + /** Where word `w` ends when it starts at offset `o` of token `t`. */ const step = (w: number, t: number, o: number): [number, number] | undefined => { const key = wordKeys[w]; if (!key || t >= keys.length || !keys[t].startsWith(key, o)) return undefined; - // The words "we" and "ll" both fall in "we'll". - const end = o + key.length + (keys[t][o + key.length] === "'" ? 1 : 0); + const end = o + key.length; return end === keys[t].length ? [t + 1, 0] : [t, end]; }; /** Whether the two words after `w` follow on from where `w` ends. */ @@ -122,6 +124,21 @@ function punctuated(words: Word[], text: unknown): Word[] { for (let p = t; p < Math.min(keys.length, t + 3) && !start; p++) { if (step(w, p, p === t ? o : 0)) start = [p, p === t ? o : 0]; } + // A word that matches a whole token only once apostrophes are dropped, as "its" matches + // "It's,", moves to a later token in reach that it spells exactly, when there is one. + const spelling = spelled(word.text); + if ( + start?.[1] === 0 && + keys[start[0]] === wordKeys[w] && + spellings[start[0]] !== spelling + ) { + for (let p = start[0] + 1; p < Math.min(keys.length, t + 3); p++) { + if (spellings[p] === spelling) { + start = [p, 0]; + break; + } + } + } for (let p = t + 3; p < Math.min(keys.length, t + RESYNC_TOKENS) && !start; p++) { const at = step(w, p, 0); if (at && followed(w, at)) start = [p, 0]; From 3907092705f20cbe14b0ac527e4dcc646279dd1f Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 21 Nov 2026 01:04:10 +0100 Subject: [PATCH 8/8] fix(transcription): keep master's matching and ship only the Arabic marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three independent verifier rounds each found a case where a new apostrophe rule lost or misplaced a mark that master restores: words split at an apostrophe, 90s against 90’s, and 90s twice against 90’s. 90s!. Which token a word with or without an apostrophe belongs to cannot be told without context, and the real Qwen3 answers never hit it. Matching goes back to master's, so a curly we’ll can still take a nearby omitted Well,'s comma, as before. Budgets are the measured 334,097 raw and 109,248 gzip bytes. --- AGENTS.md | 2 +- scripts/size-baseline.md | 12 ++++---- scripts/size.mjs | 4 +-- scripts/size.test.ts | 10 +++---- src/services/TranscriptionService.test.ts | 30 ------------------- .../transcription/providers/verboseJson.ts | 26 ++-------------- 6 files changed, 16 insertions(+), 68 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2de98529..bc6f3869 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ svelte-check's native API path passes when a tsconfig matches no files, which `check:a11y` in temporary workspaces and requires it to fail on that case and when the native compiler cannot start. -After `npm run build`, `npm run size` enforces 334,318 raw bytes and 109,351 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. +After `npm run build`, `npm run size` enforces 334,097 raw bytes and 109,248 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. Before opening a PR or cutting a release, run the CI-equivalent checks locally: diff --git a/scripts/size-baseline.md b/scripts/size-baseline.md index 6df3ed96..604d3b64 100644 --- a/scripts/size-baseline.md +++ b/scripts/size-baseline.md @@ -12,7 +12,7 @@ main.js raw: 307131 bytes; budget 330000 bytes; 22869 bytes remaining main.js gzip (level 9): 97161 bytes; budget 100000 bytes; 2839 bytes remaining ``` -The gzip budget is now 109,351 bytes, and the raw budget 334,318. On 2026-10-14 it rose from 100,000 for +The gzip budget is now 109,248 bytes, and the raw budget 334,097. On 2026-10-14 it rose from 100,000 for the full player's Transcript panel, which the owner approved at about 1.9 KB and which measured 2,576 bytes. The coordinator set 103,000 to keep the panel's Find and Follow chip and to leave room for fixes in flight. The @@ -100,14 +100,12 @@ main.js raw: 334091 bytes; budget 334091 bytes; 0 bytes remaining main.js gzip (level 9): 109234 bytes; budget 109234 bytes; 0 bytes remaining ``` -On 2026-11-21 both rose for punctuation restored onto words that match their -text without apostrophes, each kept apart from a word spelled like it with one, -and from Arabic marks: from 334,091 raw by 227 bytes, and from 109,234 gzip by -117 bytes. With it: +On 2026-11-21 both rose for Arabic marks restored onto unpunctuated words: from +334,091 raw by 6 bytes, and from 109,234 gzip by 14 bytes. With it: ```text -main.js raw: 334318 bytes; budget 334318 bytes; 0 bytes remaining -main.js gzip (level 9): 109351 bytes; budget 109351 bytes; 0 bytes remaining +main.js raw: 334097 bytes; budget 334097 bytes; 0 bytes remaining +main.js gzip (level 9): 109248 bytes; budget 109248 bytes; 0 bytes remaining ``` Gzip uses Node's `gzipSync` at level 9, without a filename or timestamp. diff --git a/scripts/size.mjs b/scripts/size.mjs index e6d4292a..11b08680 100644 --- a/scripts/size.mjs +++ b/scripts/size.mjs @@ -9,8 +9,8 @@ try { const bundle = readFileSync(path); if (!bundle.length) throw new Error(`${path} is empty. Run npm run build first.`); const measurements = [ - { name: "raw", bytes: bundle.length, budget: 334_318 }, - { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 109_351 }, + { name: "raw", bytes: bundle.length, budget: 334_097 }, + { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 109_248 }, ]; const rows = measurements.map(({ name, bytes, budget }) => { const overage = Math.max(0, bytes - budget); diff --git a/scripts/size.test.ts b/scripts/size.test.ts index fc23fbdf..ec4d5129 100644 --- a/scripts/size.test.ts +++ b/scripts/size.test.ts @@ -20,19 +20,19 @@ function check(name: string, contents: string | Buffer) { describe("bundle budget CLI", () => { it("allows the raw budget boundary and rejects one extra byte", () => { - expect(check("boundary.js", "a".repeat(334_318)).status).toBe(0); - const result = check("over.js", "a".repeat(334_319)); + expect(check("boundary.js", "a".repeat(334_097)).status).toBe(0); + const result = check("over.js", "a".repeat(334_098)); expect(result.status).toBe(1); - expect(result.stdout).toContain("budget 334318 bytes; OVER by 1 bytes"); + expect(result.stdout).toContain("budget 334097 bytes; OVER by 1 bytes"); }); it("rejects gzip growth even when raw bytes fit", () => { const result = check("gzip.js", randomBytes(110_000)); expect(result.status).toBe(1); expect(result.stdout).toMatch( - /gzip \(level 9\): \d+ bytes; budget 109351 bytes; OVER by \d+ bytes/, + /gzip \(level 9\): \d+ bytes; budget 109248 bytes; OVER by \d+ bytes/, ); - expect(result.stdout).toContain("budget 334318 bytes; 224318 bytes remaining"); + expect(result.stdout).toContain("budget 334097 bytes; 224097 bytes remaining"); }); it("fails on absent or empty artifacts instead of reporting a small build", () => { diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index ca7f6742..93b6d8bb 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -1825,36 +1825,6 @@ describe("TranscriptionService", () => { "we'll see Okay", "we'll see. Okay.", ], - [ - "does not give a Qwen contraction with a curly apostrophe the punctuation of a word spelled like it without one", - "Well, we’ll see. ‘Okay.’ Bye.", - "we’ll see Okay Bye", - "we’ll see. ‘Okay.’ Bye.", - ], - [ - "gives the last of the Qwen words split at an apostrophe in its text that token's punctuation", - "Rock’n’roll! Yes, we’ll. Rock'n'roll!", - "Rock n roll Yes we ll Rock n roll", - "Rock n roll! Yes, we ll. Rock n roll!", - ], - [ - "restores punctuation to Qwen's words that leave out an apostrophe their text has", - "It costs 1’000. Or 2'000. Back in the 90’s! Next.", - "It costs 1000 Or 2000 Back in the 90s Next", - "It costs 1000. Or 2000. Back in the 90s! Next.", - ], - [ - "keeps the first of the Qwen words split from one token of its text in that token when a later one spells it", - "Short-term, short. Done.", - "Short term short Done", - "Short term, short. Done.", - ], - [ - "does not give a Qwen word the punctuation of one that is spelled like it with an apostrophe", - "It’s, its own. Done.", - "its own Done", - "its own. Done.", - ], [ "gives Qwen's words Arabic punctuation from its text", "مرحبا، كيف حالك؟ أنا بخير؛ شكرا.", diff --git a/src/services/transcription/providers/verboseJson.ts b/src/services/transcription/providers/verboseJson.ts index 98d3eb2d..9fe98c2e 100644 --- a/src/services/transcription/providers/verboseJson.ts +++ b/src/services/transcription/providers/verboseJson.ts @@ -85,19 +85,14 @@ function punctuated(words: Word[], text: unknown): Word[] { .replace(/[「『《【(]+/gu, " $&") .split(/\s+/) .filter(Boolean); - // Words match the text without apostrophes, so "90s" finds "90’s", and "rock", "n" and - // "roll" find "rock'n'roll". - const bare = (token: string) => token.toLowerCase().replace(/[^\p{L}\p{M}\p{N}]/gu, ""); - // A spelling keeps the apostrophes inside a word, straight or curly, so "we’ll" is not - // "well", and drops those at its ends, which quote it, as in "'Hello". - const spelled = (token: string) => + // Apostrophes count inside a word, so "we'll" is not "well", and not at its ends, where + // they quote it, as in "'Hello". + const bare = (token: string) => token .toLowerCase() - .replace(/’/g, "'") .replace(/[^\p{L}\p{M}\p{N}']/gu, "") .replace(/^'+|'+$/g, ""); const keys = tokens.map(bare); - const spellings = tokens.map(spelled); const wordKeys = words.map((word) => bare(word.text)); /** Where word `w` ends when it starts at offset `o` of token `t`. */ const step = (w: number, t: number, o: number): [number, number] | undefined => { @@ -124,21 +119,6 @@ function punctuated(words: Word[], text: unknown): Word[] { for (let p = t; p < Math.min(keys.length, t + 3) && !start; p++) { if (step(w, p, p === t ? o : 0)) start = [p, p === t ? o : 0]; } - // A word that matches a whole token only once apostrophes are dropped, as "its" matches - // "It's,", moves to a later token in reach that it spells exactly, when there is one. - const spelling = spelled(word.text); - if ( - start?.[1] === 0 && - keys[start[0]] === wordKeys[w] && - spellings[start[0]] !== spelling - ) { - for (let p = start[0] + 1; p < Math.min(keys.length, t + 3); p++) { - if (spellings[p] === spelling) { - start = [p, 0]; - break; - } - } - } for (let p = t + 3; p < Math.min(keys.length, t + RESYNC_TOKENS) && !start; p++) { const at = step(w, p, 0); if (at && followed(w, at)) start = [p, 0];