[heft] Add shard result contracts and output ownership

apps/heft/src/index.ts

@@ -79,6 +79,8 @@ export {
 
 export type { IScopedLogger } from './pluginFramework/logging/ScopedLogger';
 
+export type { IShardResultManifest, ShardStatus } from './sharding/IShardResultManifest';
+
 // Re-export types required to use custom command-line parameters
 export type {
   CommandLineParameter,

apps/heft/src/schemas/shard-result-manifest.schema.json

@@ -0,0 +1,110 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Heft Shard Result Manifest",
+  "description": "This schema describes the \"shard-manifest.json\" file that each shard writes into its output directory to report what it did.",
+  "type": "object",
+
+  "additionalProperties": false,
+
+  "required": [
+    "version",
+    "shardIndex",
+    "shardCount",
+    "generation",
+    "producerAttempt",
+    "status",
+    "workload",
+    "startedAt",
+    "durationMs",
+    "diagnosticCount",
+    "itemCount"
+  ],
+
+  "properties": {
+    "version": {
+      "title": "Manifest Version",
+      "description": "The version of this manifest format. A collator must refuse a version it does not understand rather than guessing at the shape.",
+      "enum": [1]
+    },
+
+    "shardIndex": {
+      "title": "Shard Index",
+      "description": "The 1-based index of the shard that produced this manifest, matching the numbering used by both Rush and Jest.",
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991
+    },
+
+    "shardCount": {
+      "title": "Shard Count",
+      "description": "The total number of shards the producing shard believed it was part of. The collator compares this against its own shard count to detect a reconfiguration that a cache hit would otherwise hide.",
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991
+    },
+
+    "generation": {
+      "title": "Shard Generation",
+      "description": "An opaque token identifying the sharded run that produced this manifest. The collator requires it to be uniform across every manifest, which is what prevents results from two different runs being merged together.",
+      "type": "string",
+      "minLength": 1
+    },
+
+    "producerAttempt": {
+      "title": "Producer Attempt",
+      "description": "A non-semantic execution-attempt token used for live publication fencing and provenance. It is not part of semantic cache identity.",
+      "type": "string",
+      "minLength": 1
+    },
+
+    "status": {
+      "title": "Shard Status",
+      "description": "\"completed\" means the shard ran its workload to completion, whether or not that workload reported failures. \"harness-failure\" means the shard could not run its workload at all, so its counts carry no information.",
+      "enum": ["completed", "harness-failure"]
+    },
+
+    "workload": {
+      "title": "Workload Name",
+      "description": "The kind of work this shard performed, for example \"jest\" or \"lint\". The neutral core treats this as opaque; only an adapter interprets it.",
+      "type": "string",
+      "minLength": 1
+    },
+
+    "startedAt": {
+      "title": "Started At",
+      "description": "An ISO 8601 timestamp recording when the shard began its workload.",
+      "type": "string",
+      "format": "date-time",
+      "minLength": 1
+    },
+
+    "durationMs": {
+      "title": "Duration In Milliseconds",
+      "description": "How long the shard's workload took, in milliseconds.",
+      "type": "number",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+
+    "diagnosticCount": {
+      "title": "Diagnostic Count",
+      "description": "The number of problems the shard found, for example failing tests or lint errors. The collator sums this across shards to reach its verdict.",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+
+    "itemCount": {
+      "title": "Item Count",
+      "description": "The number of units of work the shard actually processed, for example tests run or files linted. The adapter applies its whole-workload empty-selection policy.",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+
+    "payload": {
+      "title": "Adapter Payload",
+      "description": "Adapter-specific data that the neutral core passes through without inspecting. A Jest adapter uses this to carry the information its merge step needs."
+    }
+  }
+}

apps/heft/src/sharding/IShardResultManifest.ts

@@ -0,0 +1,111 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+/**
+ * Whether a shard was able to run its workload at all.
+ *
+ * @remarks
+ * This is deliberately separate from whether the workload found problems. A shard that runs
+ * every test it was given and observes ten failures is `completed`; a shard whose test runner
+ * crashed before it could start is `harness-failure`. A consumer must not interpret a
+ * `harness-failure` shard's `diagnosticCount` and `itemCount` as completed work.
+ *
+ * @public
+ */
+export type ShardStatus = 'completed' | 'harness-failure';
+
+/**
+ * What a single shard reports about the work it performed.
+ *
+ * @remarks
+ * Producers write this as `shard-manifest.json` in their owned output directory after
+ * publishing all payload files. A successful exit without a manifest is not completion evidence.
+ *
+ * The fields above `payload` are deliberately workload-agnostic: `diagnosticCount` and
+ * `itemCount` are generic enough that validation and verdict logic need no knowledge of what
+ * the shard was actually doing. Anything workload-specific belongs in `payload`.
+ *
+ * @public
+ */
+export interface IShardResultManifest {
+  /**
+   * The version of this manifest format.
+   */
+  version: 1;
+
+  /**
+   * The 1-based index of the shard that produced this manifest, matching the numbering used by
+   * both Rush and Jest.
+   */
+  shardIndex: number;
+
+  /**
+   * The total number of shards the producing shard believed it was part of.
+   *
+   * @remarks
+   * Consumers must compare this against the expected shard count. Without it, changing the shard count
+   * and then hitting the build cache for a stale shard would silently produce a partial result.
+   */
+  shardCount: number;
+
+  /**
+   * An opaque token identifying the sharded run that produced this manifest.
+   *
+   * @remarks
+   * Consumers must compare this with the current invocation's independently supplied generation,
+   * not just with sibling manifests. A uniformly stale set is not a current result.
+   */
+  generation: string;
+
+  /**
+   * A non-semantic token identifying the live execution attempt that published this manifest.
+   *
+   * @remarks
+   * Unlike {@link IShardResultManifest.generation}, this changes for equivalent retries and does
+   * not participate in cache identity. It records which fenced writer performed the final commit.
+   */
+  producerAttempt: string;
+
+  /**
+   * Whether the shard ran its workload to completion.
+   */
+  status: ShardStatus;
+
+  /**
+   * The kind of work this shard performed, for example `jest` or `lint`.
+   */
+  workload: string;
+
+  /**
+   * An ISO 8601 timestamp recording when the shard began its workload.
+   */
+  startedAt: string;
+
+  /**
+   * How long the shard's workload took, in milliseconds.
+   */
+  durationMs: number;
+
+  /**
+   * The number of problems the shard found, for example failing tests or lint errors.
+   */
+  diagnosticCount: number;
+
+  /**
+   * The number of units of work the shard actually processed, for example tests run or files
+   * linted.
+   *
+   * @remarks
+   * The workload consumer is responsible for its empty-workload policy.
+   */
+  itemCount: number;
+
+  /**
+   * Adapter-specific data that the neutral core passes through without inspecting.
+   *
+   * @remarks
+   * The generic schema does not interpret this field. Workload consumers must validate their
+   * own payload format before using it. Producers with no additional data may omit it.
+   */
+  payload?: unknown;
+}

apps/heft/src/sharding/ShardFileSystem.ts

@@ -0,0 +1,141 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import fs from 'node:fs/promises';
+import fsSync, { type BigIntStats } from 'node:fs';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+
+import { FileSystem } from '@rushstack/node-core-library';
+
+/**
+ * Identifies an atomic publication so cancellation cannot invalidate a replacement file.
+ * Callers must retain exclusive destination ownership through their final cancellation check.
+ * @public
+ */
+export interface IShardFilePublication {
+  invalidateIfOwned(): void;
+}
+
+/**
+ * An exclusive live claim that serializes revocation and final promotion.
+ * The callback must execute synchronously and cannot release the claim.
+ * @public
+ */
+export interface IShardFilePublicationGuard {
+  commit(action: () => void): void;
+}
+
+/**
+ * Rejects paths outside the build folder and symbolic links/junctions below it.
+ * The build folder itself may be a symlinked workspace.
+ * @public
+ */
+export async function validateShardPathAsync(
+  buildFolderPath: string,
+  targetPath: string,
+  abortSignal?: AbortSignal
+): Promise<void> {
+  const relative: string = path.relative(buildFolderPath, targetPath);
+  if (!relative || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+    throw new Error(`Shard paths must be below the build folder: "${targetPath}".`);
+  }
+  let current: string = buildFolderPath;
+  for (const segment of relative.split(path.sep)) {
+    abortSignal?.throwIfAborted();
+    current = path.join(current, segment);
+    try {
+      if ((await fs.lstat(current)).isSymbolicLink()) {
+        throw new Error(`Shard paths must not traverse a symbolic link: "${current}".`);
+      }
+    } catch (error) {
+      if (!FileSystem.isNotExistError(error as Error)) throw error;
+    }
+    abortSignal?.throwIfAborted();
+  }
+}
+
+/**
+ * Removes only a known output file, tolerating absence but not other filesystem errors.
+ * @public
+ */
+export async function invalidateShardFileAsync(filename: string): Promise<void> {
+  try {
+    await fs.unlink(filename);
+  } catch (error) {
+    if (!FileSystem.isNotExistError(error as Error)) throw error;
+  }
+}
+
+/**
+ * Publishes a complete file by same-directory atomic rename. Callers must invalidate their
+ * previous output before starting work, and validate the destination before calling this helper.
+ * @public
+ */
+export async function writeShardFileAtomicallyAsync(
+  filename: string,
+  contents: string,
+  abortSignal?: AbortSignal,
+  publicationGuard?: IShardFilePublicationGuard
+): Promise<IShardFilePublication> {
+  abortSignal?.throwIfAborted();
+  const folder: string = path.dirname(filename);
+  await fs.mkdir(folder, { recursive: true });
+  abortSignal?.throwIfAborted();
+  const temporaryPath: string = path.join(folder, `.${path.basename(filename)}.${randomUUID()}.tmp`);
+  const handle: fs.FileHandle = await fs.open(temporaryPath, 'wx');
+  let published: boolean = false;
+  let identity: BigIntStats;
+  try {
+    try {
+      abortSignal?.throwIfAborted();
+      identity = await handle.stat({ bigint: true });
+      await handle.writeFile(contents);
+      await handle.sync();
+      abortSignal?.throwIfAborted();
+    } finally {
+      await handle.close();
+    }
+    abortSignal?.throwIfAborted();
+    const publication: IShardFilePublication = {
+      invalidateIfOwned: () => {
+        let current: BigIntStats;
+        try {
+          current = fsSync.lstatSync(filename, { bigint: true });
+        } catch (error) {
+          if (!FileSystem.isNotExistError(error as Error)) throw error;
+          return;
+        }
+        if (
+          current.isFile() &&
+          current.ino === identity.ino &&
+          // Windows lstat reports dev=0; fstat reports the volume serial number.
+          (process.platform === 'win32' || current.dev === identity.dev)
+        ) {
+          fsSync.unlinkSync(filename);
+        }
+      }
+    };
+    if (publicationGuard) {
+      // No await between the ownership check and rename: revocation/release in this process
+      // cannot interleave, and other processes cannot acquire the exclusively held claim.
+      publicationGuard.commit(() => {
+        abortSignal?.throwIfAborted();
+        fsSync.renameSync(temporaryPath, filename);
+        published = true;
+      });
+    } else {
+      await fs.rename(temporaryPath, filename);
+      published = true;
+    }
+    if (abortSignal?.aborted) {
+      publication.invalidateIfOwned();
+      abortSignal.throwIfAborted();
+    }
+    return publication;
+  } finally {
+    // Rename consumes the staging file. No asynchronous cleanup may follow the final
+    // cancellation check on the success path.
+    if (!published) await FileSystem.deleteFileAsync(temporaryPath);
+  }
+}