Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Reproducible 10k-Call Latency Benchmark

on:
push:
branches: [main]
pull_request:
branches: [main]
workflow_dispatch:

permissions: read-all

jobs:
benchmark-10k:
name: 10k-Call Latency Matrix & Regression Gate
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- name: Setup pnpm
uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.1.0
with:
version: 9

- name: Setup Node.js
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 22
cache: 'pnpm'

- name: Install dependencies
run: pnpm install --frozen-lockfile

- name: Run 10,000-Call Reproducible Latency Matrix
run: npx tsx benchmarks/run-10k-matrix.ts --iterations 10000 --out benchmarks/out --max-p99-ms 15

- name: Upload JSON & CSV Benchmark Artifacts
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: agentshield-10k-benchmark-matrix
path: |
benchmarks/out/benchmark-matrix.json
benchmarks/out/benchmark-matrix.csv
retention-days: 30
268 changes: 268 additions & 0 deletions benchmarks/run-10k-matrix.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
#!/usr/bin/env tsx
/**
* Reproducible 10,000-Call Latency & Throughput Benchmark Matrix (Phase 9)
*
* Benchmarks core AgentShield hot paths and outputs machine-readable JSON & CSV
* latency percentiles (min, mean, p50, p95, p99, max) for CI regression gating.
*
* Usage:
* npx tsx benchmarks/run-10k-matrix.ts [--iterations 10000] [--out benchmarks/out] [--max-p99-ms 15]
*/

import { mkdirSync, writeFileSync } from "node:fs";
import { cpus, platform, release } from "node:os";
import { join, resolve } from "node:path";
import { performance } from "node:perf_hooks";
import { fileURLToPath } from "node:url";
import { redactText } from "../src/core-redactor.js";
import { AgentShieldEngine } from "../src/engine.js";
import { AgentCommandSandbox } from "../src/sandbox.js";
import { ZygoteProcessPool } from "../src/zygote-pool.js";

export interface BenchmarkMetric {
scenario: string;
iterations: number;
totalMs: number;
opsPerSec: number;
minMicros: number;
meanMicros: number;
p50Micros: number;
p95Micros: number;
p99Micros: number;
maxMicros: number;
p99Ms: number;
passedGate: boolean;
}

export interface BenchmarkMatrixReport {
schemaVersion: "1.0";
generatedAt: string;
environment: {
nodeVersion: string;
platform: string;
osRelease: string;
cpuModel: string;
cpuCores: number;
};
iterationsPerScenario: number;
maxP99MsGate: number;
allGatesPassed: boolean;
results: BenchmarkMetric[];
}

export interface BenchmarkRunOptions {
iterations?: number;
warmup?: number;
outputDir?: string;
maxP99Ms?: number;
}

function round(n: number, decimals = 2): number {
const factor = 10 ** decimals;
return Math.round(n * factor) / factor;
}

function percentile(sorted: Float64Array, p: number): number {
if (sorted.length === 0) return 0;
const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
return sorted[idx];
}

async function measureScenario(
scenario: string,
iterations: number,
warmup: number,
maxP99Ms: number,
fn: (i: number) => void | Promise<void>,
): Promise<BenchmarkMetric> {
for (let i = 0; i < warmup; i++) {
await fn(i);
}

const samplesMicros = new Float64Array(iterations);
const suiteStart = performance.now();

for (let i = 0; i < iterations; i++) {
const t0 = performance.now();
await fn(i);
samplesMicros[i] = (performance.now() - t0) * 1000;
}

const totalMs = Math.max(0.001, performance.now() - suiteStart);
samplesMicros.sort();

let sum = 0;
for (let i = 0; i < iterations; i++) {
sum += samplesMicros[i];
}

const p99Micros = percentile(samplesMicros, 99);
const p99Ms = p99Micros / 1000;

return {
scenario,
iterations,
totalMs: round(totalMs, 2),
opsPerSec: Math.round((iterations / totalMs) * 1000),
minMicros: round(samplesMicros[0], 2),
meanMicros: round(sum / iterations, 2),
p50Micros: round(percentile(samplesMicros, 50), 2),
p95Micros: round(percentile(samplesMicros, 95), 2),
p99Micros: round(p99Micros, 2),
maxMicros: round(samplesMicros[iterations - 1], 2),
p99Ms: round(p99Ms, 4),
passedGate: p99Ms <= maxP99Ms,
};
}

export function formatBenchmarkCsv(report: BenchmarkMatrixReport): string {
const header =
"scenario,iterations,total_ms,ops_per_sec,min_us,mean_us,p50_us,p95_us,p99_us,max_us,p99_ms,passed_gate";
const rows = report.results.map(
(r) =>
`${r.scenario},${r.iterations},${r.totalMs},${r.opsPerSec},${r.minMicros},${r.meanMicros},${r.p50Micros},${r.p95Micros},${r.p99Micros},${r.maxMicros},${r.p99Ms},${r.passedGate}`,
);
return `${header}\n${rows.join("\n")}\n`;
}

export async function runBenchmarkMatrix(
options: BenchmarkRunOptions = {},
): Promise<BenchmarkMatrixReport> {
const iterations = options.iterations ?? 10_000;
const warmup = options.warmup ?? Math.min(200, Math.max(20, Math.floor(iterations / 20)));
const maxP99Ms = options.maxP99Ms ?? 15;

// Build synthetic runtime payloads with zero high-entropy static literals in source
const syntheticPat = ["ghp", "a".repeat(36)].join("_");
const cleanPayload = "INFO [worker-42] Processed 128 analytics rows in 3.2ms status=200";
const secretPayload = `DEBUG auth_header=Bearer ${syntheticPat} region=us-east-1`;

const engine = new AgentShieldEngine();
const sandbox = new AgentCommandSandbox();
const zygotePool = new ZygoteProcessPool({ minIdle: 4, maxCapacity: 8 });

const results: BenchmarkMetric[] = [];
try {
results.push(
await measureScenario(
"core_redactor_mixed",
iterations,
warmup,
maxP99Ms,
(i) => {
redactText(i % 2 === 0 ? cleanPayload : secretPayload);
},
),
);

results.push(
await measureScenario(
"engine_scan_and_redact",
iterations,
warmup,
maxP99Ms,
(i) => {
engine.scanAndRedact(i % 2 === 0 ? cleanPayload : secretPayload);
},
),
);

results.push(
await measureScenario(
"sandbox_command_inspect",
iterations,
warmup,
maxP99Ms,
(i) => {
sandbox.inspectCommand(i % 2 === 0 ? "git status --short" : "cat .env && printenv");
},
),
);

results.push(
await measureScenario(
"zygote_pool_dispatch",
iterations,
warmup,
maxP99Ms,
async () => {
await zygotePool.dispatch({ action: "inspect", command: "ls -la" });
},
),
);
} finally {
zygotePool.shutdown();
}

const cpuInfo = cpus();
const report: BenchmarkMatrixReport = {
schemaVersion: "1.0",
generatedAt: new Date().toISOString(),
environment: {
nodeVersion: process.version,
platform: platform(),
osRelease: release(),
cpuModel: cpuInfo[0]?.model || "unknown",
cpuCores: cpuInfo.length || 1,
},
iterationsPerScenario: iterations,
maxP99MsGate: maxP99Ms,
allGatesPassed: results.every((r) => r.passedGate),
results,
};

if (options.outputDir) {
const outDir = resolve(options.outputDir);
mkdirSync(outDir, { recursive: true });
writeFileSync(
join(outDir, "benchmark-matrix.json"),
`${JSON.stringify(report, null, 2)}\n`,
"utf8",
);
writeFileSync(join(outDir, "benchmark-matrix.csv"), formatBenchmarkCsv(report), "utf8");
}

return report;
}

const isMain =
process.argv[1] !== undefined && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url));

if (isMain) {
const args = process.argv.slice(2);
const iterIdx = args.indexOf("--iterations");
const outIdx = args.indexOf("--out");
const gateIdx = args.indexOf("--max-p99-ms");

const iterations = iterIdx !== -1 && args[iterIdx + 1] ? Number(args[iterIdx + 1]) : 10_000;
const outputDir = outIdx !== -1 && args[outIdx + 1] ? args[outIdx + 1] : "benchmarks/out";
const maxP99Ms = gateIdx !== -1 && args[gateIdx + 1] ? Number(args[gateIdx + 1]) : 15;

runBenchmarkMatrix({ iterations, outputDir, maxP99Ms })
.then((report) => {
console.log(
`\n📊 AgentShield 10k-Call Reproducible Benchmark Matrix (${report.iterationsPerScenario} iterations/scenario)`,
);
console.table(
report.results.map((r) => ({
scenario: r.scenario,
ops_sec: r.opsPerSec,
p50_us: r.p50Micros,
p95_us: r.p95Micros,
p99_us: r.p99Micros,
gate_ok: r.passedGate,
})),
);
console.log(`✅ Artifacts written to ${resolve(outputDir)}/benchmark-matrix.{json,csv}`);
if (!report.allGatesPassed) {
console.error(
`❌ Benchmark regression gate failed: one or more scenarios exceeded ${report.maxP99MsGate}ms p99.`,
);
process.exit(1);
}
})
.catch((err) => {
console.error("❌ Benchmark runner failed:", err);
process.exit(1);
});
}
49 changes: 49 additions & 0 deletions tests/phase9-benchmark-matrix.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import assert from "node:assert/strict";
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import { formatBenchmarkCsv, runBenchmarkMatrix } from "../benchmarks/run-10k-matrix.js";

test("Phase 9: reproducible latency benchmark matrix emits valid JSON & CSV artifacts and passes p99 gate", async () => {
const outDir = mkdtempSync(join(tmpdir(), "agentshield-bench-"));
try {
const report = await runBenchmarkMatrix({
iterations: 500,
warmup: 25,
outputDir: outDir,
maxP99Ms: 25,
});

assert.equal(report.schemaVersion, "1.0");
assert.equal(report.iterationsPerScenario, 500);
assert.equal(report.allGatesPassed, true);
assert.equal(report.results.length, 4);

for (const metric of report.results) {
assert.ok(metric.opsPerSec > 0, `${metric.scenario} opsPerSec must be > 0`);
assert.ok(metric.p50Micros >= 0);
assert.ok(metric.p95Micros >= metric.p50Micros);
assert.ok(metric.p99Micros >= metric.p95Micros);
assert.equal(metric.passedGate, true);
}

const jsonPath = join(outDir, "benchmark-matrix.json");
const csvPath = join(outDir, "benchmark-matrix.csv");
assert.ok(existsSync(jsonPath));
assert.ok(existsSync(csvPath));

const parsedJson = JSON.parse(readFileSync(jsonPath, "utf8"));
assert.equal(parsedJson.results.length, 4);

const csv = readFileSync(csvPath, "utf8");
assert.equal(csv, formatBenchmarkCsv(report));
assert.match(csv, /^scenario,iterations,total_ms,ops_per_sec/);
assert.match(csv, /core_redactor_mixed,500,/);
assert.match(csv, /engine_scan_and_redact,500,/);
assert.match(csv, /sandbox_command_inspect,500,/);
assert.match(csv, /zygote_pool_dispatch,500,/);
} finally {
rmSync(outDir, { recursive: true, force: true });
}
});
Loading