Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "Add pure execution planning and export configured commands, explicit environment overrides and per-operation metadata through rush-buildxl-graph-plugin.",
"type": "minor"
}
],
"packageName": "@microsoft/rush"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@rushstack/rush-daemon",
"comment": "Update the operation graph test double for external execution planning.",
"type": "none"
}
],
"packageName": "@rushstack/rush-daemon"
}
4 changes: 4 additions & 0 deletions common/reviews/api/rush-buildxl-graph-plugin.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ export interface IBuildXLRushGraph {
// (undocumented)
repoSettings: {
commonTempFolder: string;
executionContractVersion?: 1;
};
}

Expand All @@ -38,9 +39,12 @@ export interface IGraphNode {
cacheable?: false;
command: string;
dependencies: string[];
environmentVariables?: Record<string, string>;
id: string;
outputFolderNames?: string[];
package: string;
task: string;
warningsAreAllowed?: true;
workingDirectory: string;
}

Expand Down
8 changes: 8 additions & 0 deletions common/reviews/api/rush-lib.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,7 @@ export function getWorkspaceRequestOperationEnvironment(hostEnvironment: Readonl

// @alpha (undocumented)
export interface IBaseOperationExecutionResult {
readonly environmentOverrides?: Readonly<Record<string, string>>;
getStateHash(): string;
getStateHashComponents(): IOperationStateHashComponents;
readonly metadataFolderPath: string;
Expand Down Expand Up @@ -475,6 +476,7 @@ export interface ICobuildLockProvider {
// @alpha
export interface IConfigurableOperation extends IBaseOperationExecutionResult {
enabled: boolean;
environmentOverrides?: Record<string, string>;
shouldRunnerPersist: boolean;
}

Expand Down Expand Up @@ -771,6 +773,7 @@ export interface IOperationGraph {
addTerminalDestination(destination: TerminalWritable): void;
allowOversubscription: boolean;
closeRunnersAsync(operations?: Iterable<Operation>): Promise<void>;
createExecutionPlanAsync(options: IOperationGraphIterationOptions): Promise<ReadonlyMap<Operation, IOperationExecutionResult>>;
debugMode: boolean;
deleteResults?(operations: Iterable<Operation>): void;
discardScheduledIteration(): boolean;
Expand Down Expand Up @@ -891,6 +894,7 @@ export interface IOperationRunner {
cacheable: boolean;
closeAsync?(): Promise<void>;
executeAsync(context: IOperationRunnerContext, lastState?: IOperationLastState): Promise<OperationStatus>;
getCommandForExecution?(): string;
getConfigHash(): string;
readonly isActive?: boolean;
readonly isNoOp?: boolean;
Expand Down Expand Up @@ -1679,6 +1683,10 @@ export class PhasedCommandEngineUsageError extends Error {

// @alpha
export class PhasedCommandHooks {
readonly beforeExecuteGraphAsync: AsyncSeriesBailHook<[
IOperationGraph,
IOperationGraphContext
], true | undefined | void>;
readonly createOperationsAsync: AsyncSeriesWaterfallHook<[
Set<Operation>,
ICreateOperationsContext
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ class TestOperationGraph implements IOperationGraph {
return Promise.resolve();
}

public async createExecutionPlanAsync(): Promise<ReadonlyMap<Operation, IOperationExecutionResult>> {
throw new Error('External execution is not used by this test graph.');
}

public executeScheduledIterationAsync(): Promise<boolean> {
return Promise.resolve(false);
}
Expand Down
23 changes: 23 additions & 0 deletions libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -712,6 +712,9 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
public async createEngineAsync(abortSignal?: AbortSignal): Promise<IPhasedCommandEngine> {
this.validateEngineCommand();
await this.initializePluginsAsync();
if (this.hooks.beforeExecuteGraphAsync.isUsed()) {
throw new Error('External execution hooks require --no-daemon.');
}
let engine: IPhasedCommandEngine | undefined;
await this.#runAsync((result) => {
engine = result;
Expand Down Expand Up @@ -1175,6 +1178,9 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
await hooks.onGraphCreatedAsync.promise(graph, graphContext);
});
if (onEngine) {
if (hooks.beforeExecuteGraphAsync.isUsed()) {
throw new Error('External execution hooks require --no-daemon.');
}
// Before the graph is transferred, so that it is disposed below.
abortSignal?.throwIfAborted();
if (!getGraphInputsSnapshotAsync || !initialSnapshot) {
Expand Down Expand Up @@ -1202,6 +1208,23 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
return;
}

let executeLocally: boolean = false;
try {
executeLocally = !(await hooks.beforeExecuteGraphAsync.promise(graph, graphContext));
} finally {
if (!executeLocally) {
await graph.closeRunnersAsync();
}
}
if (!executeLocally) {
this.sessionAbortController.signal.throwIfAborted();
stopwatch.stop();
if (!this.#ignoreHooksParameter.value) {
this.#doAfterTask();
}
return;
}

const abortPromise: Promise<void> = once(this.sessionAbortController.signal, 'abort').then(async () => {
terminal.writeLine(`Shutting down Rush...`);
return await graph.abortCurrentIterationAsync();
Expand Down
62 changes: 62 additions & 0 deletions libraries/rush-lib/src/cli/test/RushCommandLineParser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import { OperationStatus } from '../../logic/operations/OperationStatus';
import type { IRepositoryLockWait } from '../../logic/RepositoryLockWait';
import type { ITelemetryData } from '../../logic/Telemetry';
import type { RushCommandLineParser as RushCommandLineParserType } from '../RushCommandLineParser';
import type { PhasedScriptAction } from '../scriptActions/PhasedScriptAction';
import {
getCommandLineParserInstanceAsync,
setSpawnMock,
Expand Down Expand Up @@ -844,6 +845,67 @@ describe('RushCommandLineParser', () => {
});
});

describe('external graph execution', () => {
it.each([true, undefined])(
'runs after graph-created taps and handles external execution: %s',
async (handled) => {
const { parser, spawnMock } = await getCommandLineParserInstanceAsync(
'basicAndRunBuildActionRepo',
'build'
);
const calls: string[] = [];
const close = jest.fn(async () => {});
parser.rushSession.hooks.runAnyPhasedCommand.tap('ExternalExecutionTest', (command) => {
command.hooks.beforeExecuteGraphAsync.tapPromise(
'export',
async (graph, context): Promise<true | undefined> => {
calls.push('export');
if (handled) {
const plan = await graph.createExecutionPlanAsync({
inputsSnapshot: context.initialSnapshot
});
expect(plan.size).toBe(2);
expect(
[...plan.values()].every((record) => record.environmentOverrides?.PUBLIC_ID === 'id')
).toBe(true);
}
return handled ? true : undefined;
}
);
command.hooks.onGraphCreatedAsync.tap('configure', (graph) => {
calls.push('graph-created');
graph.hooks.configureIteration.tap('identity', (records) => {
calls.push('configure');
for (const record of records.values()) record.environmentOverrides = { PUBLIC_ID: 'id' };
});
for (const operation of graph.operations) operation.runner!.closeAsync = close;
});
});
await expect(parser.executeAsync()).resolves.toBe(true);
expect(calls.slice(0, 3)).toEqual(['graph-created', 'export', 'configure']);
expect(spawnMock).toHaveBeenCalledTimes(handled ? 0 : 2);
if (handled) expect(close).toHaveBeenCalledTimes(2);
}
);

it('refuses engine transfer rather than ignoring external execution hooks', async () => {
const { parser, spawnMock } = await getCommandLineParserInstanceAsync(
'basicAndRunBuildActionRepo',
'build'
);
const exportGraph = jest.fn();
parser.rushSession.hooks.runAnyPhasedCommand.tap('ExternalExecutionTest', (command) => {
command.hooks.beforeExecuteGraphAsync.tap('export', exportGraph);
});
const action: PhasedScriptAction = parser.getAction('build') as PhasedScriptAction;
await expect(action.createEngineAsync()).rejects.toThrow(
'External execution hooks require --no-daemon.'
);
expect(exportGraph).not.toHaveBeenCalled();
expect(spawnMock).not.toHaveBeenCalled();
});
});

describe('in repo plugin that produces no operations', () => {
it('succeeds when a plugin returns an empty operation set', async () => {
// Regression test: `@rushstack/rush-buildxl-graph-plugin` writes the build graph to disk in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@ export interface IOperationStateHashComponents {
* @alpha
*/
export interface IBaseOperationExecutionResult {
/**
* Explicit, non-secret environment overrides for local and external execution.
* These values may be serialized in graph exports; never include credentials.
* Plugins must account for these values in the operation's input/configuration identity.
*/
readonly environmentOverrides?: Readonly<Record<string, string>>;

/**
* The operation itself
*/
Expand Down Expand Up @@ -63,6 +70,11 @@ export interface IBaseOperationExecutionResult {
* @alpha
*/
export interface IConfigurableOperation extends IBaseOperationExecutionResult {
/**
* Explicit environment overrides, applied after local environment hooks.
*/
environmentOverrides?: Record<string, string>;

/**
* True if the operation should execute in this iteration, false otherwise.
*/
Expand Down
12 changes: 11 additions & 1 deletion libraries/rush-lib/src/logic/operations/IOperationGraph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ export interface IOperationGraphExtensionResult {
readonly reason?: string;

/**
* The operations whose `enabled` state, runner policy or state hash the extension changed. Empty if the iteration
* The operations whose `enabled` state, runner policy, environment overrides or state hash the extension changed. Empty if the iteration
* was not extended.
*/
readonly changedOperations: ReadonlySet<Operation>;
Expand Down Expand Up @@ -286,6 +286,16 @@ export interface IOperationGraph {
*/
closeRunnersAsync(operations?: Iterable<Operation>): Promise<void>;

/**
* Creates configured records without scheduling operations or running execution, cache or environment hooks.
* Runs only `configureIteration`, using the supplied inputs (or the graph's snapshot provider).
* Intended for external executors and graph export. The graph must not have an active or scheduled iteration.
* Planning does not replace execution results or consume an iteration ID.
*/
createExecutionPlanAsync(
options: IOperationGraphIterationOptions
): Promise<ReadonlyMap<Operation, IOperationExecutionResult>>;

/**
* Drops retained results after the host has awaited runner and watcher cleanup.
* Rejects executing/prepared iterations and runners that still report active resources.
Expand Down
7 changes: 7 additions & 0 deletions libraries/rush-lib/src/logic/operations/IOperationRunner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,13 @@ export interface IOperationRunner {
*/
getConfigHash(): string;

/**
* Returns the initial shell command for execution by an external executor.
* Unlike {@link IOperationRunner.getConfigHash}, this includes parameters excluded from cache identity.
* Runners that cannot be represented as a shell command should omit this method.
*/
getCommandForExecution?(): string;

/**
* If this runner performs any background work to optimize future runs, this method will clean it up.
* Must be safe to call again after the resources have been released.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import {
Terminal,
type TerminalWritable
} from '@rushstack/terminal';
import { InternalError, NewlineKind, FileError } from '@rushstack/node-core-library';
import { EnvironmentMap, InternalError, NewlineKind, FileError } from '@rushstack/node-core-library';
import { CollatedTerminal, type CollatedWriter, type StreamCollator } from '@rushstack/stream-collator';

import { coerceParallelism } from './ParseParallelism';
Expand Down Expand Up @@ -127,6 +127,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
*/
public enabled: boolean;

public environmentOverrides?: Record<string, string> = {};

/**
* If true, this operation's runner should remain active after this iteration.
*/
Expand Down Expand Up @@ -274,7 +276,13 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
}

public get environment(): IEnvironment | undefined {
return this.#context.createEnvironment?.(this);
const environment: IEnvironment | undefined = this.#context.createEnvironment?.(this);
if (this.environmentOverrides && Object.keys(this.environmentOverrides).length > 0) {
const merged: EnvironmentMap = new EnvironmentMap(environment ?? process.env);
merged.mergeFromObject(this.environmentOverrides);
return merged.toObject();
}
return environment;
}

public get abortSignal(): AbortSignal | undefined {
Expand Down
Loading
Loading