diff --git a/.changeset/atomic-matchstick-download.md b/.changeset/atomic-matchstick-download.md new file mode 100644 index 000000000..10c17242a --- /dev/null +++ b/.changeset/atomic-matchstick-download.md @@ -0,0 +1,5 @@ +--- +"@graphprotocol/graph-cli": patch +--- + +Publish the downloaded Matchstick executable atomically so parallel `graph test` processes cannot execute a partial or still-open binary. diff --git a/packages/cli/src/command-helpers/download-binary.test.ts b/packages/cli/src/command-helpers/download-binary.test.ts new file mode 100644 index 000000000..ac6e038f8 --- /dev/null +++ b/packages/cli/src/command-helpers/download-binary.test.ts @@ -0,0 +1,96 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { downloadBinary } from './download-binary.js'; + +describe('downloadBinary', () => { + let dir: string; + let destination: string; + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), 'graph-download-')); + destination = path.join(dir, 'matchstick'); + }); + afterEach(async () => { + vi.unstubAllGlobals(); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('publishes only a complete executable after the download closes', async () => { + let finish!: () => void; + let started!: () => void; + const writing = new Promise(resolve => { + started = resolve; + }); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('#!/bin/sh\n')); + finish = () => { + controller.enqueue(new TextEncoder().encode('exit 0\n')); + controller.close(); + }; + started(); + }, + }); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(body)), + ); + const pending = downloadBinary('https://example.test/binary', destination); + await writing; + await vi.waitFor(async () => { + const stagingDir = (await fs.readdir(dir)).find(name => name.startsWith('.matchstick-')); + expect(stagingDir).toBeDefined(); + expect(await fs.readFile(path.join(dir, stagingDir!, 'binary'), 'utf8')).toBe('#!/bin/sh\n'); + }); + await expect(fs.access(destination)).rejects.toThrow(); + finish(); + await pending; + expect(await fs.readFile(destination, 'utf8')).toBe('#!/bin/sh\nexit 0\n'); + expect((await fs.stat(destination)).mode & 0o777).toBe(0o755); + expect(await fs.readdir(dir)).toEqual(['matchstick']); + }); + + it('handles competing downloads without overwriting the published binary', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response('complete executable')), + ); + await Promise.all( + Array.from({ length: 8 }, () => downloadBinary('https://example.test/binary', destination)), + ); + expect(await fs.readFile(destination, 'utf8')).toBe('complete executable'); + expect(await fs.readdir(dir)).toEqual(['matchstick']); + }); + + it('does not replace an executable already published by another process', async () => { + await fs.writeFile(destination, 'already running'); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response('new download')), + ); + await downloadBinary('https://example.test/binary', destination); + expect(await fs.readFile(destination, 'utf8')).toBe('already running'); + expect(await fs.readdir(dir)).toEqual(['matchstick']); + }); + + it('cleans failed downloads without creating a cache entry', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.error(new Error('download failed')); + }, + }), + ), + ), + ); + await expect(downloadBinary('https://example.test/binary', destination)).rejects.toThrow( + 'download failed', + ); + expect(await fs.readdir(dir)).toEqual([]); + }); +}); diff --git a/packages/cli/src/command-helpers/download-binary.ts b/packages/cli/src/command-helpers/download-binary.ts new file mode 100644 index 000000000..38a1e3adb --- /dev/null +++ b/packages/cli/src/command-helpers/download-binary.ts @@ -0,0 +1,27 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { pipeline } from 'node:stream/promises'; + +/** Publish a complete executable without exposing partial downloads to other CLI processes. */ +export async function downloadBinary(url: string, destination: string) { + const dir = path.dirname(destination); + await fs.promises.mkdir(dir, { recursive: true }); + const stagingDir = await fs.promises.mkdtemp(path.join(dir, '.matchstick-')); + const stagingPath = path.join(stagingDir, 'binary'); + try { + const response = await fetch(url); + if (!response.ok) throw new Error(`Status: ${response.statusText}`); + if (!response.body) throw new Error('No response body received'); + await pipeline(response.body, fs.createWriteStream(stagingPath, { flags: 'wx' })); + await fs.promises.chmod(stagingPath, '755'); + try { + // A hard link atomically publishes the closed, executable file. Another + // downloader may have won the race; its complete binary is equally usable. + await fs.promises.link(stagingPath, destination); + } catch (error) { + if (error.code !== 'EEXIST') throw error; + } + } finally { + await fs.promises.rm(stagingDir, { recursive: true, force: true }); + } +} diff --git a/packages/cli/src/commands/test.ts b/packages/cli/src/commands/test.ts index 878c3c4e9..b4d14edf4 100644 --- a/packages/cli/src/commands/test.ts +++ b/packages/cli/src/commands/test.ts @@ -3,12 +3,12 @@ import events from 'node:events'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { pipeline } from 'node:stream/promises'; import { fileURLToPath } from 'node:url'; import { filesystem, patching, print, system } from 'gluegun'; import yaml from 'js-yaml'; import semver from 'semver'; import { Args, Command, Flags } from '@oclif/core'; +import { downloadBinary } from '../command-helpers/download-binary.js'; import { GRAPH_CLI_SHARED_HEADERS } from '../constants.js'; export default class TestCommand extends Command { @@ -166,14 +166,7 @@ async function runBinary( if (!fs.existsSync(binPath)) { this.log(`Downloading matchstick binary: ${url}`); - await fs.promises.mkdir(binDir, { recursive: true }); - const response = await fetch(url); - if (!response.ok) throw new Error(`Status: ${response.statusText}`); - if (!response.body) throw new Error('No response body received'); - - const fileStream = fs.createWriteStream(binPath); - await pipeline(response.body, fileStream); - await fs.promises.chmod(binPath, '755'); + await downloadBinary(url, binPath); } } catch (e) { this.error(