diff --git a/index.html b/index.html index 7891bb7f7b..1ee236062b 100644 --- a/index.html +++ b/index.html @@ -27,7 +27,7 @@
- +
@@ -37,6 +37,13 @@
+
+ +
+
+
+
+
diff --git a/resources/benchmark-configurator.mjs b/resources/benchmark-configurator.mjs index caac91775f..2e7d6729a6 100644 --- a/resources/benchmark-configurator.mjs +++ b/resources/benchmark-configurator.mjs @@ -56,6 +56,8 @@ export class BenchmarkConfigurator { this.#suites.forEach((suite) => { if (!suite.tags) suite.tags = []; + if (!("measurePrepare" in suite)) + suite.measurePrepare = false; if (suite.url.startsWith("experimental/")) suite.tags.unshift("all", "experimental"); else @@ -115,8 +117,9 @@ export class BenchmarkConfigurator { } _loadSuite(suite) { - suite.tags.forEach((tag) => this.#tags.add(tag)); - this.#suites.push(suite); + suite.tags?.forEach((tag) => this.#tags.add(tag)); + + this.#suites.push({ ...suite, tags: suite.tags ? [...suite.tags] : [] }); } enableSuites(names, tags) { diff --git a/resources/developer-mode.mjs b/resources/developer-mode.mjs index 1a549c8267..c604b9ecc8 100644 --- a/resources/developer-mode.mjs +++ b/resources/developer-mode.mjs @@ -29,6 +29,7 @@ export function createDeveloperModeContainer() { settings.append(createUIForWaitAfterSuite()); settings.append(createUIForAsyncSteps()); settings.append(createUIForLayoutMode()); + settings.append(createUIForPreload()); settings.append(createUIForComplexity()); content.append(document.createElement("hr")); @@ -54,6 +55,12 @@ function createUIForWarmupSuite() { }); } +function createUIForPreload() { + return createCheckboxUI("Use service worker for resource preloading", params.preload, (isChecked) => { + params.preload = isChecked; + }); +} + function createUIForMeasurePrepare() { return createCheckboxUI("Measure Prepare", params.measurePrepare, (isChecked) => { params.measurePrepare = isChecked; diff --git a/resources/main.css b/resources/main.css index a571bd6eb8..0945a2c5eb 100644 --- a/resources/main.css +++ b/resources/main.css @@ -474,7 +474,8 @@ button.show-about { display: none; } -#progress { +#progress, +#preload-progress { position: absolute; bottom: -6px; left: 60px; @@ -484,7 +485,8 @@ button.show-about { border-right: 6px solid var(--background); } -#progress-completed { +#progress-completed, +#preload-progress-completed { position: absolute; top: 0; left: 0; @@ -495,11 +497,13 @@ button.show-about { background-color: var(--inactive-color); } -#progress-completed::-webkit-progress-value { +#progress-completed::-webkit-progress-value, +#preload-progress-completed::-webkit-progress-value { background-color: var(--foreground); } -#progress-completed::-moz-progress-bar { +#progress-completed::-moz-progress-bar, +#preload-progress-completed::-moz-progress-bar { background-color: var(--foreground); } @@ -510,7 +514,9 @@ button.show-about { background-color: var(--background); } -#info { +#info, +#preload-info { + display: none; position: absolute; bottom: -25px; left: 60px; @@ -520,11 +526,13 @@ button.show-about { text-align: center; font-size: 12px; } -#info-label { +#info-label, +#preload-info-label { position: absolute; left: 6px; } -#info-progress { +#info-progress, +#preload-info-progress { position: absolute; right: 6px; text-align: right; @@ -1001,3 +1009,46 @@ section#about .note { color: #fff; stroke: #fff; } + +#preload-progress, +#preload-info { + display: none; +} + +[data-benchmark-state="PRELOADING"] { + cursor: wait; + pointer-events: none; +} + +[data-benchmark-state="PRELOADING"] #preload-progress, +[data-benchmark-state="PRELOADING"] #preload-info { + display: block; +} + +[data-benchmark-state="RUNNING"] #info { + display: block; +} + +[data-benchmark-state="PRELOADING"] .start-tests-button { + color: var(--foreground); + background-image: linear-gradient(-45deg, var(--foreground) 3px, transparent 3px, transparent 50%, var(--foreground) 50%, var(--foreground) calc(50% + 3px), transparent calc(50% + 3px), transparent 100%); + background-size: 20px 20px; + animation: barber-pole 1s linear infinite; + pointer-events: none; +} + +[data-benchmark-state="PRELOADING"] .start-tests-button > div { + background-color: var(--background); + border-radius: 10px; + padding: 0 0.3em; + display: inline-block; +} + +@keyframes barber-pole { + 0% { + background-position: 0 0, 0 0; + } + 100% { + background-position: 20px 0, 0 0; + } +} diff --git a/resources/main.mjs b/resources/main.mjs index b63dc06a17..8feace7012 100644 --- a/resources/main.mjs +++ b/resources/main.mjs @@ -4,8 +4,11 @@ import { renderMetricView } from "./metric-ui.mjs"; import { defaultParams, params } from "./shared/params.mjs"; import { createDeveloperModeContainer } from "./developer-mode.mjs"; +import { ResourcePreloader, PreloadStatusUpdater } from "./preloader.mjs"; + const BENCHMARK_STATE = Object.freeze({ IDLE: "IDLE", + PRELOADING: "PRELOADING", READY: "READY", RUNNING: "RUNNING", DONE: "DONE", @@ -23,30 +26,46 @@ class MainBenchmarkClient { _state = BENCHMARK_STATE.IDLE; _developerModeContainer = null; _metrics = Object.create(null); + _resourcePreloader = new ResourcePreloader(); _steppingPromise = null; _steppingResolver = null; _benchmarkConfiguratorPromise = null; + _isInitialized = false; + + get isInitialized() { + return this._isInitialized; + } constructor() { this._benchmarkConfiguratorPromise = import("./benchmark-configurator.mjs"); this.prepareUI(); this.evaluateParams(); this._showSection(window.location.hash); - - this._benchmarkConfiguratorPromise.then(() => { - window.dispatchEvent(new Event("SpeedometerReady")); - }); } isRunning() { return this._state === BENCHMARK_STATE.RUNNING; } + isPreloading() { + return this._state === BENCHMARK_STATE.PRELOADING; + } + + hasError() { + return this._state === BENCHMARK_STATE.ERROR; + } + hasFinished() { - return this._state === BENCHMARK_STATE.DONE || this._state === BENCHMARK_STATE.ERROR; + return this._state === BENCHMARK_STATE.DONE || this.hasError(); + } + + isDisabled() { + return this._state !== BENCHMARK_STATE.READY && this._state !== BENCHMARK_STATE.DONE && this._state !== BENCHMARK_STATE.IDLE; } async start() { + if (this.isPreloading()) + return; if (this._isStepping()) this._clearStepping(); else if (await this._startBenchmark()) @@ -54,14 +73,18 @@ class MainBenchmarkClient { } async step() { + if (this.isPreloading()) + return; const currentSteppingResolver = this._steppingResolver; this._steppingPromise = new Promise((resolve) => { this._steppingResolver = resolve; }); currentSteppingResolver?.(); if (!this.isRunning()) { - await this._startBenchmark(); - this._showSection("#running"); + if (await this._startBenchmark()) + this._showSection("#running"); + else + this._clearStepping(); } } @@ -82,7 +105,7 @@ class MainBenchmarkClient { } async _startBenchmark() { - if (this.isRunning()) + if (this.isRunning() || this.isPreloading()) return false; const { benchmarkConfigurator } = await this._benchmarkConfiguratorPromise; @@ -103,6 +126,11 @@ class MainBenchmarkClient { } if (!this._isStepping()) this._developerModeContainer?.remove(); + + await this._preloadResources(benchmarkConfigurator); + if (this.hasError()) + return false; + this._progressCompleted = document.getElementById("progress-completed"); if (params.iterationCount < 50) { const progressNode = document.getElementById("progress"); @@ -114,10 +142,10 @@ class MainBenchmarkClient { } this._metrics = Object.create(null); - this.stepCount = params.iterationCount * totalSuitesCount; this._progressCompleted.max = this.stepCount; this.suitesCount = enabledSuites.length; + this._setBenchmarkState(BENCHMARK_STATE.RUNNING); const runner = new BenchmarkRunner(benchmarkConfigurator.suites, this); runner.runMultipleIterations(params.iterationCount); @@ -129,7 +157,7 @@ class MainBenchmarkClient { } _assertNotAborted() { - if (this._state === BENCHMARK_STATE.ERROR) + if (this.hasError()) throw new Error("Benchmark aborted by another process."); } @@ -167,11 +195,13 @@ class MainBenchmarkClient { this._finishedTestCount = 0; } - didFinishLastIteration(metrics) { + async didFinishLastIteration(metrics) { console.assert(this.isRunning()); + this._clearStepping(); this._metrics = metrics; this._setBenchmarkState(BENCHMARK_STATE.DONE); + await this._resourcePreloader.teardown(); const scoreResults = this._computeResults(this._measuredValuesList, "score"); if (scoreResults.isValid) @@ -188,8 +218,9 @@ class MainBenchmarkClient { } handleError(error) { - if (this._state === BENCHMARK_STATE.ERROR) + if (this.hasError()) return; + this._clearStepping(); this._metrics = Object.create(null); this._setBenchmarkState(BENCHMARK_STATE.ERROR); this._populateErrorMessage(error.message); @@ -392,25 +423,67 @@ class MainBenchmarkClient { document.body.append(this._developerModeContainer); } - if (params.startAutomatically) - this.start(); - else + await this._resourcePreloader.setup(); + if (!params.developerMode) + await this._preloadResources(benchmarkConfigurator); + + if (!this.hasError()) { this._setBenchmarkState(BENCHMARK_STATE.READY); + if (params.startAutomatically) + await this.start(); + } + + this._isInitialized = true; + window.dispatchEvent(new Event("SpeedometerReady")); + } + + async _preloadResources(benchmarkConfigurator) { + await this._resourcePreloader.stopPreloading(); + await this._resourcePreloader.setup(); + if (!params.preload) + return; + + const enabledSuites = benchmarkConfigurator.suites.filter((suite) => suite.enabled); + if (this._resourcePreloader.isCached(enabledSuites)) + return; + + const clearCache = !params.isDefault(); + const preloadStatusUpdater = new PreloadStatusUpdater(); + + try { + this._setBenchmarkState(BENCHMARK_STATE.PRELOADING); + preloadStatusUpdater.start(); + await this._resourcePreloader.preloadSuites(enabledSuites, clearCache, preloadStatusUpdater.onProgress.bind(preloadStatusUpdater)); + } catch (error) { + this.handleError(error); + } finally { + preloadStatusUpdater.stop(); + } } - async _setBenchmarkState(state) { + _setBenchmarkState(state) { this._state = state; document.body.setAttribute("data-benchmark-state", state); const startButtons = document.querySelectorAll(".start-tests-button"); - if (state !== BENCHMARK_STATE.RUNNING) { + if (this.isPreloading()) { + if (document.documentElement.getAttribute("data-visible-section") !== "home") + this._showSection("#home"); + startButtons.forEach((btn) => { + btn.disabled = true; + btn.innerHTML = "
Preloading
"; + }); + } else if (this.isRunning()) { + document.body.style.removeProperty("--preload-progress"); + startButtons.forEach((btn) => { + btn.disabled = true; + }); + } else { + document.body.style.removeProperty("--preload-progress"); + const isDisabled = this.isDisabled(); startButtons.forEach((btn) => { btn.innerHTML = "
Start Test
"; + btn.disabled = isDisabled; }); - if (state === BENCHMARK_STATE.READY) { - startButtons.forEach((btn) => { - btn.disabled = false; - }); - } } } @@ -434,7 +507,7 @@ class MainBenchmarkClient { _logoClickHandler(event) { // Prevent any accidental UI changes during benchmark runs. - if (!this.isRunning()) + if (!this.isRunning() && !this.isPreloading()) this._showSection("#home"); event.preventDefault(); return false; diff --git a/resources/preloader.mjs b/resources/preloader.mjs new file mode 100644 index 0000000000..e67d694ba8 --- /dev/null +++ b/resources/preloader.mjs @@ -0,0 +1,364 @@ +import { SW_MESSAGES } from "./shared/sw-messages.mjs"; +import { params } from "./shared/params.mjs"; + +const PRELOAD_TIMEOUT_MS = 60_000; + +export class ResourcePreloader { + constructor(customParams = params) { + this._params = customParams; + this._registration = null; + this._sw = null; + this._preloadParams = null; + this._pendingRequest = null; + this._onPreloadProgress = null; + this._activePreloadPromise = null; + this._activeStopPromise = null; + this._nextRequestId = 1; + this._boundOnMessage = this.onMessage.bind(this); + this._messageListenerAdded = false; + } + + _isServiceWorkerSupported() { + return Boolean(globalThis.navigator?.serviceWorker); + } + + _serializeParams(suitesOrParams = this._params, maybeSuites = undefined) { + const targetParams = typeof suitesOrParams?.toSearchParams === "function" ? suitesOrParams : this._params; + const suites = Array.isArray(suitesOrParams) ? suitesOrParams : maybeSuites; + const searchParamsStr = targetParams.toSearchParams(); + if (!Array.isArray(suites)) + return searchParamsStr; + const suiteNames = suites + .filter((s) => typeof s === "string" ? true : s.enabled !== false) + .map((s) => typeof s === "string" ? s : s.name) + .join(","); + return `${searchParamsStr}|suites:${suiteNames}`; + } + + isCached(enabledSuites) { + if (!this._params.preload || (!this._registration && !this._sw) || !this._preloadParams) + return false; + if (enabledSuites === undefined) + return this._preloadParams.split("|suites:")[0] === this._params.toSearchParams(); + return this._preloadParams === this._serializeParams(this._params, enabledSuites); + } + + async setup() { + if (!this._isServiceWorkerSupported()) + return; + + if (!this._params.preload) { + if (this._registration || this._sw) + await this._unregisterOldServiceWorkers(); + return; + } + + if (this._registration && this._sw) + return; + + await this._registerServiceWorker(); + this._setupMessageListener(); + } + + async _unregisterOldServiceWorkers() { + this._removeMessageListener(); + const activeRegistration = this._registration; + this._registration = null; + this._sw = null; + this._preloadParams = null; + + if (!this._isServiceWorkerSupported()) + return; + + const expectedScope = new URL("./", window.location.href).href; + let unregisteredActive = false; + const existingRegistrations = await navigator.serviceWorker?.getRegistrations?.() ?? []; + for (const existing of existingRegistrations) { + if (existing.scope === expectedScope) { + await existing.unregister?.(); + if (existing === activeRegistration) + unregisteredActive = true; + } + } + if (!unregisteredActive && (!activeRegistration?.scope || activeRegistration.scope === expectedScope)) + await activeRegistration?.unregister?.(); + } + + async _registerServiceWorker() { + if (!this._isServiceWorkerSupported()) + return; + + this._registration = await navigator.serviceWorker?.register("./sw.mjs", { type: "module" }); + const readyRegistration = await navigator.serviceWorker?.ready; + this._sw = this._registration?.active || readyRegistration?.active || navigator.serviceWorker?.controller || null; + } + + _setupMessageListener() { + if (!this._isServiceWorkerSupported() || this._messageListenerAdded) + return; + navigator.serviceWorker?.addEventListener("message", this._boundOnMessage); + this._messageListenerAdded = true; + } + + _removeMessageListener() { + if (!this._isServiceWorkerSupported() || !this._messageListenerAdded) + return; + navigator.serviceWorker?.removeEventListener("message", this._boundOnMessage); + this._messageListenerAdded = false; + } + + _isValidMessage(data) { + if (!this._pendingRequest) + return false; + + if (data.requestId !== undefined && data.requestId !== this._pendingRequest.requestId) + return false; + + if (this._pendingRequest.type !== data.type) + return false; + + return true; + } + + onMessage(event) { + const data = event?.data || {}; + + if (data.type === SW_MESSAGES.FATAL_ERROR) { + globalThis.benchmarkClient?.handleError(new Error(data.message)); + return; + } + + if (!this._isValidMessage(data)) + return; + + if (data.status === SW_MESSAGES.PRELOAD_PROGRESS) { + this._onPreloadProgress?.(data); + return; + } + + const { resolve, reject, timeoutId } = this._pendingRequest; + if (timeoutId) + clearTimeout(timeoutId); + this._pendingRequest = null; + + if (data.status === SW_MESSAGES.ERROR) + reject(new Error(data.message || "Unknown SW Error")); + else + resolve(data); + } + + // Allow preempting in-flight preloads or overriding any pending request with an explicit stop. + _isCooperativePreemption(messageData) { + return this._pendingRequest?.type === SW_MESSAGES.PRELOAD_SUITES || messageData?.type === SW_MESSAGES.STOP_PRELOADING; + } + + isCooperativePreemption(messageData) { + return this._isCooperativePreemption(messageData); + } + + _postMessage(messageData, timeoutMs = 0) { + if (!this._sw) + return Promise.resolve(null); + + if (this._pendingRequest) { + const logMessage = `Already waiting for ${this._pendingRequest.type}, overriding with ${messageData.type}`; + if (this._isCooperativePreemption(messageData)) { + console.warn(logMessage); + this._resolvePendingRequest(null); + } else { + this._rejectPendingRequest(logMessage); + } + } + + const requestId = this._nextRequestId++; + const payload = { ...messageData, requestId }; + + return new Promise((resolve, reject) => { + let timeoutId = null; + + if (timeoutMs > 0) { + timeoutId = setTimeout(() => { + if (this._pendingRequest?.requestId === requestId) + this._rejectPendingRequest(`Service worker message timed out: ${messageData.type}`); + }, timeoutMs); + } + + this._pendingRequest = { type: messageData.type, requestId, resolve, reject, timeoutId }; + this._sw?.postMessage(payload); + }); + } + + _resolvePendingRequest(data = null) { + const pendingRequest = this._pendingRequest; + this._pendingRequest = null; + if (pendingRequest?.timeoutId) + clearTimeout(pendingRequest.timeoutId); + + pendingRequest?.resolve(data); + } + + _rejectPendingRequest(errorMessage) { + const pendingRequest = this._pendingRequest; + this._pendingRequest = null; + if (pendingRequest?.timeoutId) + clearTimeout(pendingRequest.timeoutId); + + console.warn(errorMessage); + pendingRequest?.reject(new Error(errorMessage)); + } + + async teardown() { + this._preloadParams = null; + if (!this._sw && !this._registration) + return; + + await this.stopPreloading(); + + const response = await this.getFailedRequests(); + await this._unregisterOldServiceWorkers(); + if (response?.requests?.length > 0) { + console.warn("The following requests failed during the benchmark and bypassed the cache:"); + console.warn(response.requests.join("\n")); + throw new Error("The benchmark had missing resources that were not cached. Check the console for details."); + } + } + + async stopPreloading() { + if (!this._sw) + return; + + if (this._activeStopPromise) { + await this._activeStopPromise; + return; + } + + if (this._activePreloadPromise || this._pendingRequest?.type === SW_MESSAGES.PRELOAD_SUITES) + this._preloadParams = null; + + const activePreloadPromise = this._activePreloadPromise; + this._activeStopPromise = this._postMessage({ type: SW_MESSAGES.STOP_PRELOADING }); + try { + await this._activeStopPromise; + if (activePreloadPromise) + await activePreloadPromise; + } finally { + this._activeStopPromise = null; + } + } + + async preloadSuites(suites, clearCache = true, onProgress) { + if (!this._isServiceWorkerSupported()) + return undefined; + + if (!this._params.preload) { + if (this._registration || this._sw) + await this._unregisterOldServiceWorkers(); + return undefined; + } + + if (this._activeStopPromise) + await this._activeStopPromise; + + if (!this._sw && !this._registration) + await this.setup(); + + if (!this._sw) + return undefined; + + if (this._activePreloadPromise || this._onPreloadProgress) + throw new Error("Preload already active"); + + this._preloadParams = null; + + const suitesData = suites + .filter((s) => s.resources) + .map((s) => ({ + name: s.name, + url: new URL(s.url, window.location.href).href, + resources: new URL(s.resources, window.location.href).href, + })); + + if (suitesData.length === 0) + return undefined; + + this._onPreloadProgress = onProgress; + const startTime = performance.now(); + try { + this._activePreloadPromise = this._postMessage({ type: SW_MESSAGES.PRELOAD_SUITES, suites: suitesData, clearCache }, PRELOAD_TIMEOUT_MS); + const response = await this._activePreloadPromise; + if (this._activeStopPromise) + await this._activeStopPromise; + + if (response?.status === SW_MESSAGES.PRELOAD_DONE) { + const timeTakenMs = performance.now() - startTime; + const sizeMB = ((response.transferredSize || 0) / (1024 * 1024)).toFixed(2); + const timeSec = (timeTakenMs / 1000).toFixed(2); + console.log(`Preloaded ${response.count} files (${sizeMB} MB transferred) in ${timeSec}s`); + this._preloadParams = this._serializeParams(this._params, suites); + } + return response; + } finally { + this._activePreloadPromise = null; + this._onPreloadProgress = null; + } + } + + async getFailedRequests() { + if (!this._sw) + return { requests: [] }; + + const response = await this._postMessage({ type: SW_MESSAGES.GET_FAILED_REQUESTS }); + return response || { requests: [] }; + } +} + +export class PreloadStatusUpdater { + constructor() { + this._latestData = null; + this._rafId = null; + this._updateFn = this._updateUI.bind(this); + + this._progressCompletedElement = document.getElementById("preload-progress-completed"); + this._infoLabelElement = document.getElementById("preload-info-label"); + this._infoProgressElement = document.getElementById("preload-info-progress"); + } + + start() { + this.stop(); + this._progressCompletedElement.max = 1; + this._progressCompletedElement.value = 0; + this._infoLabelElement.textContent = ""; + this._infoProgressElement.textContent = ""; + document.body.style.setProperty("--preload-progress", "0%"); + } + + onProgress(data) { + this._latestData = data; + if (!this._rafId) + this._rafId = requestAnimationFrame(this._updateFn); + } + + _updateUI() { + this._rafId = null; + const data = this._latestData; + if (!data) + return; + + const { loaded, total, url, suiteName } = data; + document.body.style.setProperty("--preload-progress", `${total > 0 ? (loaded / total) * 100 : 100}%`); + this._progressCompletedElement.max = total > 0 ? total : 1; + this._progressCompletedElement.value = total > 0 ? loaded : 0; + const match = url?.match(/(?:suites(?:-experimental)?|resources)\/(.+)$/); + const filename = match?.[1] ?? ""; + this._infoLabelElement.textContent = `${suiteName}: ${filename}`; + this._infoProgressElement.textContent = `${loaded} / ${total}`; + } + + stop() { + if (this._rafId) { + cancelAnimationFrame(this._rafId); + this._rafId = null; + } + this._latestData = null; + } +} diff --git a/resources/shared/params.mjs b/resources/shared/params.mjs index f9e0a0541a..3acb07d35b 100644 --- a/resources/shared/params.mjs +++ b/resources/shared/params.mjs @@ -43,6 +43,8 @@ export class Params { measurePrepare = false; // External config url to override internal tests. config = ""; + // Use service worker for resource preloading. + preload = false; constructor(searchParams = undefined, warnUnused = false) { if (searchParams) @@ -88,6 +90,7 @@ export class Params { this.layoutMode = this._parseEnumParam(searchParams, "layoutMode", LAYOUT_MODES); this.measurePrepare = this._parseBooleanParam(searchParams, "measurePrepare"); this.config = this._parseConfig(searchParams); + this.preload = this._parseBooleanParam(searchParams, "preload"); if (warnUnused) { const unused = Array.from(searchParams.keys()); @@ -99,8 +102,9 @@ export class Params { _parseBooleanParam(searchParams, paramKey) { if (!searchParams.has(paramKey)) return false; + const value = searchParams.get(paramKey)?.toLowerCase(); searchParams.delete(paramKey); - return true; + return value !== "false" && value !== "0"; } _parserNumberParam(searchParams, paramKey, minValue) { diff --git a/resources/shared/request-limiter.mjs b/resources/shared/request-limiter.mjs new file mode 100644 index 0000000000..44c2c48ddf --- /dev/null +++ b/resources/shared/request-limiter.mjs @@ -0,0 +1,38 @@ +export class RequestLimiter { + constructor(limit) { + this._limit = limit; + this._active = 0; + this._queue = []; + } + + async _processQueue() { + while (this._queue.length > 0) { + const task = this._queue.shift(); + try { + await task(); + } catch (e) { + // Individual task errors are handled by their respective promises + } + } + this._active--; + } + + schedule(fn) { + return new Promise((resolve, reject) => { + const task = () => fn().then(resolve, reject); + task.resolve = resolve; + this._queue.push(task); + + if (this._active < this._limit) { + this._active++; + this._processQueue(); // Intentionally not awaited + } + }); + } + + clear() { + for (const task of this._queue) + task.resolve?.(0); + this._queue.length = 0; + } +} diff --git a/resources/shared/sw-messages.mjs b/resources/shared/sw-messages.mjs new file mode 100644 index 0000000000..7e1cff97f8 --- /dev/null +++ b/resources/shared/sw-messages.mjs @@ -0,0 +1,11 @@ +export const SW_MESSAGES = Object.freeze({ + PRELOAD_SUITES: "PRELOAD_SUITES", + PRELOAD_PROGRESS: "PRELOAD_PROGRESS", + PRELOAD_DONE: "PRELOAD_DONE", + STOP_PRELOADING: "STOP_PRELOADING", + FATAL_ERROR: "FATAL_ERROR", + ERROR: "ERROR", + SUCCESS: "SUCCESS", + GET_FAILED_REQUESTS: "GET_FAILED_REQUESTS", + FAILED_REQUESTS: "FAILED_REQUESTS", +}); diff --git a/resources/suite-runner.mjs b/resources/suite-runner.mjs index ca87f36f3d..eedd795794 100644 --- a/resources/suite-runner.mjs +++ b/resources/suite-runner.mjs @@ -117,7 +117,7 @@ export class SuiteRunner { const suitePrepare = this.#suiteResults.prepare; if (suiteTotal === 0) throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`); - if (this.#params.measurePrepare && suitePrepare === 0) + if ((this.#params.measurePrepare || this.#suite.measurePrepare) && suitePrepare === 0) throw new Error(`Got invalid 0-time prepare time for suite ${this.#suite.name}: ${suitePrepare}`); } diff --git a/suites-experimental/javascript-wc-indexeddb/dist/src/speedometer-utils/params.mjs b/suites-experimental/javascript-wc-indexeddb/dist/src/speedometer-utils/params.mjs index 5e6246320a..4a41279504 100644 --- a/suites-experimental/javascript-wc-indexeddb/dist/src/speedometer-utils/params.mjs +++ b/suites-experimental/javascript-wc-indexeddb/dist/src/speedometer-utils/params.mjs @@ -39,6 +39,8 @@ export class Params { measurePrepare = false; // External config url to override internal tests. config = ""; + // Use service worker for resource preloading. + preload = false; constructor(searchParams = undefined, warnUnused = false) { if (searchParams) @@ -74,6 +76,7 @@ export class Params { this.layoutMode = this._parseEnumParam(searchParams, "layoutMode", LAYOUT_MODES); this.measurePrepare = this._parseBooleanParam(searchParams, "measurePrepare"); this.config = this._parseConfig(searchParams); + this.preload = this._parseBooleanParam(searchParams, "preload"); if (warnUnused) { const unused = Array.from(searchParams.keys()); @@ -85,8 +88,9 @@ export class Params { _parseBooleanParam(searchParams, paramKey) { if (!searchParams.has(paramKey)) return false; + const value = searchParams.get(paramKey)?.toLowerCase(); searchParams.delete(paramKey); - return true; + return value !== "false" && value !== "0"; } _parseIntParam(searchParams, paramKey, minValue) { @@ -211,6 +215,10 @@ export class Params { toSearchParams() { return this.toSearchParamsObject().toString(); } + + isDefault() { + return this === defaultParams; + } } function isValidJsonUrl(url) { diff --git a/suites/default-suites.mjs b/suites/default-suites.mjs index 2e9522bc60..7ffede005f 100644 --- a/suites/default-suites.mjs +++ b/suites/default-suites.mjs @@ -1006,7 +1006,8 @@ export const DefaultSuites = freezeSuites([ }, { name: "Perf-Dashboard", - url: "suites/perf.webkit.org/public/v3/#/charts/?since=1678991819934&paneList=((55-1974-null-null-(5-2.5-500)))", + url: "suites/perf.webkit.org/public/v3/index.html#/charts/?since=1678991819934&paneList=((55-1974-null-null-(5-2.5-500)))", + resources: "suites/perf.webkit.org/resources.txt", tags: ["default", "chart", "webcomponents"], async prepare(page) { await page.waitForElement("#app-is-ready"); diff --git a/suites/perf.webkit.org/resources.txt b/suites/perf.webkit.org/resources.txt new file mode 100644 index 0000000000..661dcb374f --- /dev/null +++ b/suites/perf.webkit.org/resources.txt @@ -0,0 +1,144 @@ +public/api/analysis-tasks-platform=55-metric=1006 +public/api/analysis-tasks-platform=55-metric=1407 +public/api/analysis-tasks-platform=55-metric=1648 +public/api/analysis-tasks-platform=55-metric=1649 +public/api/analysis-tasks-platform=55-metric=1974 +public/api/measurement-set-platform=16-metric=1 +public/api/measurement-set-platform=16-metric=13 +public/api/measurement-set-platform=16-metric=17 +public/api/measurement-set-platform=16-metric=1726 +public/api/measurement-set-platform=16-metric=6 +public/api/measurement-set-platform=16-metric=7 +public/api/measurement-set-platform=55-metric=1 +public/api/measurement-set-platform=55-metric=1006 +public/api/measurement-set-platform=55-metric=13 +public/api/measurement-set-platform=55-metric=1407 +public/api/measurement-set-platform=55-metric=1648 +public/api/measurement-set-platform=55-metric=1649 +public/api/measurement-set-platform=55-metric=17 +public/api/measurement-set-platform=55-metric=1726 +public/api/measurement-set-platform=55-metric=1974 +public/api/measurement-set-platform=55-metric=6 +public/api/measurement-set-platform=55-metric=7 +public/common.css +public/data/manifest.json +public/data/measurement-set-16-1-1682812800000.json +public/data/measurement-set-16-1.json +public/data/measurement-set-16-13-1682812800000.json +public/data/measurement-set-16-13.json +public/data/measurement-set-16-17-1682812800000.json +public/data/measurement-set-16-17.json +public/data/measurement-set-16-1726-1682812800000.json +public/data/measurement-set-16-1726.json +public/data/measurement-set-16-6-1682812800000.json +public/data/measurement-set-16-6.json +public/data/measurement-set-16-7-1682812800000.json +public/data/measurement-set-16-7.json +public/data/measurement-set-55-1-1682812800000.json +public/data/measurement-set-55-1.json +public/data/measurement-set-55-13-1682812800000.json +public/data/measurement-set-55-13.json +public/data/measurement-set-55-1407-1682812800000.json +public/data/measurement-set-55-1407.json +public/data/measurement-set-55-1648-1682812800000.json +public/data/measurement-set-55-1648.json +public/data/measurement-set-55-1649-1682812800000.json +public/data/measurement-set-55-1649.json +public/data/measurement-set-55-17-1682812800000.json +public/data/measurement-set-55-17.json +public/data/measurement-set-55-1726-1682812800000.json +public/data/measurement-set-55-1726.json +public/data/measurement-set-55-1974-1682812800000.json +public/data/measurement-set-55-1974.json +public/data/measurement-set-55-6-1682812800000.json +public/data/measurement-set-55-6.json +public/data/measurement-set-55-7-1682812800000.json +public/data/measurement-set-55-7.json +public/shared/common-component-base.js +public/shared/common-remote.js +public/shared/statistics.js +public/v3/async-task.js +public/v3/bundled-scripts.js +public/v3/commit-set-range-bisector.js +public/v3/components/analysis-results-viewer.js +public/v3/components/analysis-task-bug-list.js +public/v3/components/bar-graph-group.js +public/v3/components/base.js +public/v3/components/button-base.js +public/v3/components/chart-pane-base.js +public/v3/components/chart-revision-range.js +public/v3/components/chart-status-evaluator.js +public/v3/components/chart-styles.js +public/v3/components/close-button.js +public/v3/components/combo-box.js +public/v3/components/commit-log-viewer.js +public/v3/components/custom-analysis-task-configurator.js +public/v3/components/custom-configuration-test-group-form.js +public/v3/components/customizable-test-group-form.js +public/v3/components/dashboard-chart-status-view.js +public/v3/components/editable-text.js +public/v3/components/expand-collapse-button.js +public/v3/components/freshness-indicator.js +public/v3/components/instant-file-uploader.js +public/v3/components/interactive-time-series-chart.js +public/v3/components/minus-button.js +public/v3/components/mutable-list-view.js +public/v3/components/owned-commit-viewer.js +public/v3/components/pane-selector.js +public/v3/components/plus-button.js +public/v3/components/ratio-bar-graph.js +public/v3/components/repetition-type-selection.js +public/v3/components/results-table.js +public/v3/components/spinner-icon.js +public/v3/components/test-group-form.js +public/v3/components/test-group-results-viewer.js +public/v3/components/test-group-revision-table.js +public/v3/components/time-series-chart.js +public/v3/components/warning-icon.js +public/v3/index.html +public/v3/instrumentation.js +public/v3/lazily-evaluated-function.js +public/v3/main.js +public/v3/models/analysis-results.js +public/v3/models/analysis-task.js +public/v3/models/bug-tracker.js +public/v3/models/bug.js +public/v3/models/build-request.js +public/v3/models/builder.js +public/v3/models/commit-log.js +public/v3/models/commit-set.js +public/v3/models/data-model.js +public/v3/models/manifest.js +public/v3/models/measurement-adaptor.js +public/v3/models/measurement-cluster.js +public/v3/models/measurement-set.js +public/v3/models/metric.js +public/v3/models/platform-group.js +public/v3/models/platform.js +public/v3/models/repository.js +public/v3/models/test-group.js +public/v3/models/test.js +public/v3/models/time-series.js +public/v3/models/triggerable.js +public/v3/models/uploaded-file.js +public/v3/pages/analysis-category-page.js +public/v3/pages/analysis-category-toolbar.js +public/v3/pages/analysis-task-page.js +public/v3/pages/build-request-queue-page.js +public/v3/pages/chart-pane-status-view.js +public/v3/pages/chart-pane.js +public/v3/pages/charts-page.js +public/v3/pages/charts-toolbar.js +public/v3/pages/create-analysis-task-page.js +public/v3/pages/dashboard-page.js +public/v3/pages/dashboard-toolbar.js +public/v3/pages/domain-control-toolbar.js +public/v3/pages/heading.js +public/v3/pages/page-router.js +public/v3/pages/page-with-heading.js +public/v3/pages/page.js +public/v3/pages/summary-page.js +public/v3/pages/test-freshness-page.js +public/v3/pages/toolbar.js +public/v3/privileged-api.js +public/v3/remote.js diff --git a/sw.mjs b/sw.mjs new file mode 100644 index 0000000000..d288499ae5 --- /dev/null +++ b/sw.mjs @@ -0,0 +1,239 @@ +import { SW_MESSAGES } from "./resources/shared/sw-messages.mjs"; +import { RequestLimiter } from "./resources/shared/request-limiter.mjs"; + +const CACHE_NAME = "speedometer-cache-v4"; +const MAX_CONCURRENT_REQUESTS = 20; +const WORKER_STATE = { + failedRequests: new Set(), + abortController: null, + requestLimiter: new RequestLimiter(MAX_CONCURRENT_REQUESTS), +}; + +function replyToClient(event, status, msg = {}) { + const type = event.data?.type; + const requestId = event.data?.requestId; + event.source?.postMessage({ type, status, requestId, ...msg }); +} + +function replyError(event, message) { + replyToClient(event, SW_MESSAGES.ERROR, { message }); +} + +async function handleStopPreloadingMessage(event) { + await stopPreloading(event.source?.id); + replyToClient(event, SW_MESSAGES.STOP_PRELOADING); +} + +function abortActivePreload() { + WORKER_STATE.abortController?.abort(); + WORKER_STATE.abortController = null; + WORKER_STATE.requestLimiter.clear(); +} + +async function stopPreloading(currentClientId) { + abortActivePreload(); + await stopAllOtherClients(currentClientId); +} + +async function stopAllOtherClients(currentClientId) { + const allClients = await self.clients.matchAll({ includeUncontrolled: true }); + for (const client of allClients) { + if (currentClientId && client.id !== currentClientId) + client.postMessage({ type: SW_MESSAGES.FATAL_ERROR, message: "Aborted Run: Another tab took over." }); + } + +} + +async function handlePreloadSuitesMessage(event, { suites = [], clearCache = true }) { + abortActivePreload(); + const abortController = new AbortController(); + const signal = abortController.signal; + WORKER_STATE.abortController = abortController; + + try { + await stopAllOtherClients(event.source?.id); + signal.throwIfAborted(); + // debounce on progress callbacks to limit overhead. + const MAX_RATE_MS = 100; + const onProgress = (payload) => { + if (!signal.aborted) + replyToClient(event, SW_MESSAGES.PRELOAD_PROGRESS, payload); + }; + const { transferredSize, count } = await preloadSuites(suites, clearCache, throttle(onProgress, MAX_RATE_MS), signal); + signal.throwIfAborted(); + replyToClient(event, SW_MESSAGES.PRELOAD_DONE, { transferredSize, count }); + } catch (error) { + if (error?.name === "AbortError" || signal.aborted) + return; + console.error("Error during preload:", error); + replyError(event, error?.message || "Failed to preload resources."); + } finally { + if (WORKER_STATE.abortController === abortController) + WORKER_STATE.abortController = null; + } +} + +function throttle(callback, max_rate_ms) { + let lastCallbackTime = 0; + return (...args) => { + const now = Date.now(); + if (now - lastCallbackTime > max_rate_ms) { + lastCallbackTime = now; + callback(...args); + } + }; +} + +async function preloadSuites(suites, clearCache, onProgress, signal) { + WORKER_STATE.failedRequests.clear(); + if (clearCache) { + await caches.delete(CACHE_NAME); + signal.throwIfAborted(); + } + + const cache = await caches.open(CACHE_NAME); + signal.throwIfAborted(); + const urlsToCache = await getUrlsToCache(suites, signal); + signal.throwIfAborted(); + const total = urlsToCache.length; + + let loaded = 0; + let transferredSize = 0; + + const promises = urlsToCache.map(async (item) => { + const size = await fetchAndCache(cache, item.url, signal); + signal.throwIfAborted(); + transferredSize += size; + loaded++; + if (!signal.aborted) + onProgress?.({ loaded, total, url: item.url, suiteName: item.suiteName }); + }); + + await Promise.all(promises); + signal.throwIfAborted(); + return { transferredSize, count: urlsToCache.length }; +} + +async function getUrlsToCache(suites, signal) { + const urlsToCache = []; + for (const suite of suites) { + signal.throwIfAborted(); + if (suite?.resources) + urlsToCache.push(...await parseSuiteResources(suite, signal)); + } + + return urlsToCache; +} + +async function parseSuiteResources(suite, signal) { + const response = await fetch(suite.resources, { signal }); + if (!response.ok) { + const errorMsg = `Failed to load resources for suite ${suite.name}: ${suite.resources} returned ${response.status}`; + console.error(errorMsg); + throw new Error(errorMsg); + } + const text = await response.text(); + signal.throwIfAborted(); + const baseUrl = new URL(suite.resources, self.location.href); + return text + .trim() + .split("\n") + .map((resourceUrl) => resourceUrl.trim()) + .filter(Boolean) + .map((resourceUrl) => ({ + url: new URL(resourceUrl, baseUrl).href, + suiteName: suite.name, + })); +} + +async function fetchAndCache(cache, url, signal) { + signal.throwIfAborted(); + + return WORKER_STATE.requestLimiter.schedule(async () => { + signal.throwIfAborted(); + const request = new Request(url, { cache: "no-cache", signal }); + const existing = await cache.match(request, { ignoreSearch: true }); + signal.throwIfAborted(); + if (existing) + return getResponseSize(existing); + + try { + const response = await fetch(request); + signal.throwIfAborted(); + if (!response.ok) { + console.error(`Cache failed for ${url}: HTTP ${response.status}`); + WORKER_STATE.failedRequests.add(url); + return 0; + } + const size = getResponseSize(response); + await cache.put(request, response); + return size; + } catch (e) { + signal.throwIfAborted(); + console.error("Cache failed for", url, e); + WORKER_STATE.failedRequests.add(url); + return 0; + } + }); +} + +function getResponseSize(response) { + const contentLength = response.headers.get("content-length"); + return contentLength ? parseInt(contentLength, 10) : 0; +} + +async function handleGetFailedRequestsMessage(event) { + replyToClient(event, SW_MESSAGES.FAILED_REQUESTS, { requests: Array.from(WORKER_STATE.failedRequests) }); +} + +const MESSAGE_HANDLERS = Object.freeze({ + [SW_MESSAGES.PRELOAD_SUITES]: handlePreloadSuitesMessage, + [SW_MESSAGES.STOP_PRELOADING]: handleStopPreloadingMessage, + [SW_MESSAGES.GET_FAILED_REQUESTS]: handleGetFailedRequestsMessage, +}); + +self.addEventListener("install", () => { + self.skipWaiting(); +}); + +self.addEventListener("activate", (event) => { + event.waitUntil(self.clients.claim()); +}); + +self.addEventListener("message", (event) => { + const { data } = event; + if (!data) + return; + const handler = MESSAGE_HANDLERS[data.type]; + if (handler) + event.waitUntil(handler(event, data)); + else + console.error("Unknown service worker message type:", data.type); +}); + +self.addEventListener("fetch", (event) => { + if (event.request.cache === "only-if-cached" && event.request.mode !== "same-origin") + return; + + event.respondWith( + (async () => { + let requestToMatch = event.request; + const urlObj = new URL(event.request.url); + if (urlObj.pathname.endsWith("/")) { + urlObj.pathname += "index.html"; + requestToMatch = urlObj.href; + } + + const cachedResponse = await caches.match(requestToMatch, { cacheName: CACHE_NAME, ignoreSearch: true }); + if (cachedResponse) + return cachedResponse; + + try { + return await fetch(event.request); + } catch (error) { + WORKER_STATE.failedRequests.add(event.request.url); + return Response.error(); + } + })() + ); +}); diff --git a/tests/index.html b/tests/index.html index e7d5ad9b7d..5c3abd7f4a 100644 --- a/tests/index.html +++ b/tests/index.html @@ -29,7 +29,10 @@ const cacheBust = Date.now(); await import("./unittests/benchmark-runner.mjs?t=" + cacheBust); + await import("./unittests/benchmark-configurator.mjs?t=" + cacheBust); await import("./unittests/params.mjs?t=" + cacheBust); + await import("./unittests/preloader.mjs?t=" + cacheBust); + await import("./unittests/request-limiter.mjs?t=" + cacheBust); await import("./unittests/suites.mjs?t=" + cacheBust); globalThis.testResults = undefined; diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs index d36b3b18d0..81478e64fc 100644 --- a/tests/run-end2end.mjs +++ b/tests/run-end2end.mjs @@ -17,13 +17,50 @@ async function testPage(url) { console.log(`Testing: ${url}`); await driver.get(`http://localhost:${port}/${url}`); - await driver.executeAsyncScript((callback) => { - if (globalThis.benchmarkClient) - callback(); - else - globalThis.addEventListener("SpeedometerReady", () => callback(), { once: true }); + const initResult = await driver.executeAsyncScript((callback) => { + if (globalThis.benchmarkClient?.isInitialized) { + callback({}); + return; + } + if (globalThis.benchmarkClient?.hasFinished?.()) { + callback({ + error: { + message: document.getElementById("invalid-score-text")?.textContent || "Benchmark failed during initialization", + }, + }); + return; + } + globalThis.addEventListener("SpeedometerReady", () => callback({}), { once: true }); + globalThis.addEventListener("error", (event) => { + callback({ + error: { + message: event.message || "Unknown Runtime Error", + source: event.filename, + lineno: event.lineno, + colno: event.colno, + stack: event.error?.stack, + }, + }); + }); + globalThis.addEventListener("unhandledrejection", (e) => { + const reason = e.reason; + const message = reason instanceof Error ? reason.message : typeof reason === "object" && reason !== null ? JSON.stringify(reason) : String(reason); + callback({ + error: { + message: `Unhandled Rejection: ${message}`, + stack: reason?.stack, + }, + }); + }); }); + if (initResult?.error) { + const { error } = initResult; + const stackTrace = error.stack ? `\nBrowser Stack Trace:\n${error.stack}` : ""; + const location = error.source ? ` at ${error.source}:${error.lineno}:${error.colno}` : ""; + throw new Error(`${error.message}${location}${stackTrace}`); + } + console.log(" - Awaiting Benchmark"); const { error, metrics } = await driver.executeAsyncScript((callback) => { globalThis.addEventListener( @@ -35,24 +72,35 @@ async function testPage(url) { { once: true } ); // Install error handlers to report page errors back to selenium. - globalThis.addEventListener("error", (message, source, lineno, colno, error) => + globalThis.addEventListener("error", (event) => { callback({ - error: { message, source, lineno, colno, error }, - }) - ); + error: { + message: event.message || "Unknown Runtime Error", + source: event.filename, + lineno: event.lineno, + colno: event.colno, + stack: event.error?.stack, + }, + }); + }); globalThis.addEventListener("unhandledrejection", (e) => { + const reason = e.reason; + const message = reason instanceof Error ? reason.message : typeof reason === "object" && reason !== null ? JSON.stringify(reason) : String(reason); callback({ error: { - message: e.reason.toString(), - stack: e.reason?.stack, + message: `Unhandled Rejection: ${message}`, + stack: reason?.stack, }, }); }); globalThis.benchmarkClient.start(); }); - if (error) - throw new Error(error.message + (error?.stack ?? "")); + if (error) { + const stackTrace = error.stack ? `\nBrowser Stack Trace:\n${error.stack}` : ""; + const location = error.source ? ` at ${error.source}:${error.lineno}:${error.colno}` : ""; + throw new Error(`${error.message}${location}${stackTrace}`); + } validateMetrics(metrics); return metrics; @@ -61,6 +109,7 @@ async function testPage(url) { function validateMetrics(metrics) { for (const [name, metric] of Object.entries(metrics)) validateMetric(name, metric); + assert(metrics.Geomean.mean > 0); assert(metrics.Score.mean > 0); } @@ -97,6 +146,18 @@ async function testAll() { assert(metrics.Score.values.length === 1); } +async function testPreloadEnabled() { + const metrics = await testPage("index.html?iterationCount=1&suites=Perf-Dashboard&preload=true"); + assert("Perf-Dashboard" in metrics); + assert(metrics["Perf-Dashboard"].values.length === 1); +} + +async function testPreloadDisabled() { + const metrics = await testPage("index.html?iterationCount=1&suites=Perf-Dashboard&preload=false"); + assert("Perf-Dashboard" in metrics); + assert(metrics["Perf-Dashboard"].values.length === 1); +} + async function testDeveloperMode() { const params = ["developerMode", "iterationCount=1", "warmupBeforeSync=2", "waitBeforeSync=2", "shuffleSeed=123", "suites=Perf-Dashboard"]; const metrics = await testPage(`index.html?${params.join("&")}`); @@ -110,6 +171,115 @@ async function testDeveloperMode() { }); } +async function testDeveloperModePreload() { + const url = "index.html?developerMode=true&iterationCount=1&suites=Perf-Dashboard"; + console.log(`Testing: ${url} (enabling preload via Developer Mode checkbox)`); + await driver.get(`http://localhost:${port}/${url}`); + + const initResult = await driver.executeAsyncScript((callback) => { + if (globalThis.benchmarkClient?.isInitialized) { + callback({}); + return; + } + globalThis.addEventListener("SpeedometerReady", () => callback({}), { once: true }); + globalThis.addEventListener("error", (event) => { + callback({ error: { message: event.message || "Unknown Runtime Error", stack: event.error?.stack } }); + }); + globalThis.addEventListener("unhandledrejection", (e) => { + const reason = e.reason; + callback({ error: { message: `Unhandled Rejection: ${reason?.message || String(reason)}`, stack: reason?.stack } }); + }); + }); + + if (initResult?.error) + throw new Error(`${initResult.error.message}${initResult.error.stack ? `\n${initResult.error.stack}` : ""}`); + + console.log(" - Toggling preload checkbox and awaiting Benchmark"); + const result = await driver.executeAsyncScript((callback) => { + const labels = Array.from(document.querySelectorAll(".developer-mode label")); + const preloadLabel = labels.find((label) => label.textContent.includes("Use service worker for resource preloading")); + const preloadCheckbox = preloadLabel?.querySelector('input[type="checkbox"]'); + if (!preloadCheckbox || preloadCheckbox.checked) { + callback({ error: { message: `Unexpected initial preload checkbox state: ${preloadCheckbox?.checked}` } }); + return; + } + + const preloadProgressEl = document.getElementById("preload-progress"); + const preloadInfoEl = document.getElementById("preload-info"); + const initialProgressHidden = getComputedStyle(preloadProgressEl).display === "none" && getComputedStyle(preloadInfoEl).display === "none"; + + preloadCheckbox.click(); + if (!preloadCheckbox.checked || !window.location.search.includes("preload=true")) { + callback({ error: { message: `Preload checkbox click did not update state or URL: checked=${preloadCheckbox.checked}, search=${window.location.search}` } }); + return; + } + + const seenStates = []; + let preloadProgressVisibleDuringPreload = false; + let startButtonDisabledDuringPreload = false; + const observer = new MutationObserver(() => { + const state = document.body.getAttribute("data-benchmark-state"); + seenStates.push(state); + if (state === "PRELOADING") { + preloadProgressVisibleDuringPreload = getComputedStyle(preloadProgressEl).display !== "none" && getComputedStyle(preloadInfoEl).display !== "none"; + startButtonDisabledDuringPreload = document.querySelector(".start-tests-button")?.disabled === true; + } + }); + observer.observe(document.body, { attributes: true, attributeFilter: ["data-benchmark-state"] }); + + let preloadResult = null; + const preloader = globalThis.benchmarkClient._resourcePreloader; + const origPreloadSuites = preloader.preloadSuites.bind(preloader); + preloader.preloadSuites = async (...args) => { + preloadResult = await origPreloadSuites(...args); + return preloadResult; + }; + + globalThis.addEventListener( + "SpeedometerDone", + () => { + observer.disconnect(); + const preloadProgressHiddenAfterDone = getComputedStyle(preloadProgressEl).display === "none" && getComputedStyle(preloadInfoEl).display === "none"; + callback({ + metrics: globalThis.benchmarkClient.metrics, + initialProgressHidden, + seenStates, + preloadProgressVisibleDuringPreload, + startButtonDisabledDuringPreload, + preloadProgressHiddenAfterDone, + preloadResult, + }); + }, + { once: true } + ); + globalThis.addEventListener("error", (event) => { + observer.disconnect(); + callback({ error: { message: event.message || "Unknown Runtime Error", stack: event.error?.stack } }); + }); + globalThis.addEventListener("unhandledrejection", (e) => { + observer.disconnect(); + const reason = e.reason; + callback({ error: { message: `Unhandled Rejection: ${reason?.message || String(reason)}`, stack: reason?.stack } }); + }); + + globalThis.benchmarkClient.start(); + }); + + if (result?.error) + throw new Error(`${result.error.message}${result.error.stack ? `\n${result.error.stack}` : ""}`); + + assert(result.initialProgressHidden, "Expected #preload-progress and #preload-info to be hidden before preloading"); + assert(result.seenStates.includes("PRELOADING"), `Expected PRELOADING state, saw: ${result.seenStates.join(", ")}`); + assert(result.preloadProgressVisibleDuringPreload, "Expected #preload-progress and #preload-info to be visible during PRELOADING"); + assert(result.startButtonDisabledDuringPreload, "Expected .start-tests-button to be disabled during PRELOADING"); + assert(result.preloadProgressHiddenAfterDone, "Expected #preload-progress and #preload-info to be hidden after DONE"); + assert(result.preloadResult?.status === "PRELOAD_DONE" && result.preloadResult?.count > 0, "Expected Service Worker preloadSuites to finish with PRELOAD_DONE"); + + validateMetrics(result.metrics); + assert("Perf-Dashboard" in result.metrics); + assert(result.metrics["Perf-Dashboard"].values.length === 1); +} + async function test() { try { await testWithRetries(retry); @@ -140,7 +310,10 @@ async function tryTests() { await driver.manage().setTimeouts({ script: 60000 }); await testIterations(); await testAll(); + await testPreloadEnabled(); + await testPreloadDisabled(); await testDeveloperMode(); + await testDeveloperModePreload(); console.log("\nTests complete!"); } diff --git a/tests/unittests/benchmark-configurator.mjs b/tests/unittests/benchmark-configurator.mjs new file mode 100644 index 0000000000..82585db5f1 --- /dev/null +++ b/tests/unittests/benchmark-configurator.mjs @@ -0,0 +1,42 @@ +import { BenchmarkConfigurator } from "../../resources/benchmark-configurator.mjs"; + +describe("BenchmarkConfigurator", () => { + it("should set measurePrepare to false on processed suites if not present", async () => { + const configurator = new BenchmarkConfigurator(); + await configurator.init(); + const suites = configurator.suites; + expect(suites.length).to.be.greaterThan(0); + suites.forEach((suite) => { + expect(suite).to.have.property("measurePrepare"); + expect(suite.measurePrepare).to.be.a("boolean"); + }); + }); + + it("should validate URLs correctly in _isValidUrl", () => { + const configurator = new BenchmarkConfigurator(); + expect(configurator._isValidUrl("http://example.com")).to.be(true); + expect(configurator._isValidUrl("suites/todomvc/index.html")).to.be(true); + expect(configurator._isValidUrl("")).to.be(false); + expect(configurator._isValidUrl(null)).to.be(false); + }); + + it("should enable suites by names", async () => { + const configurator = new BenchmarkConfigurator(); + await configurator.init(); + const firstSuiteName = configurator.suites[0].name; + configurator.enableSuites([firstSuiteName], []); + expect(configurator.suites[0].enabled).to.be(true); + // Assuming there are multiple suites + expect(configurator.suites[1].enabled).to.be(false); + }); + + it("should enable suites by tags", async () => { + const configurator = new BenchmarkConfigurator(); + await configurator.init(); + configurator.enableSuites([], ["todomvc"]); + const todomvcSuites = configurator.suites.filter((suite) => suite.tags.includes("todomvc")); + const otherSuites = configurator.suites.filter((suite) => !suite.tags.includes("todomvc")); + todomvcSuites.forEach((suite) => expect(suite.enabled).to.be(true)); + otherSuites.forEach((suite) => expect(suite.enabled).to.be(false)); + }); +}); diff --git a/tests/unittests/params.mjs b/tests/unittests/params.mjs index 0b3821445c..b236f01e4c 100644 --- a/tests/unittests/params.mjs +++ b/tests/unittests/params.mjs @@ -74,6 +74,22 @@ describe("Params", () => { ); expect(params.toSearchParams()).to.equal("suites="); }); + it("should contain preload when enabled", () => { + const params = new Params( + new URLSearchParams({ + preload: "true", + }) + ); + expect(params.toSearchParams()).to.equal("preload=true"); + }); + it("should not contain preload when disabled", () => { + const params = new Params( + new URLSearchParams({ + preload: "false", + }) + ); + expect(params.toSearchParams()).to.equal(""); + }); }); describe("isDefault", () => { it("should return true for defaultParams", () => { @@ -94,6 +110,14 @@ describe("Params", () => { }); describe("parse input params", () => { + it("should parse preload boolean param", () => { + expect(new Params(new URLSearchParams("preload")).preload).to.be(true); + expect(new Params(new URLSearchParams({ preload: "true" })).preload).to.be(true); + expect(new Params(new URLSearchParams({ preload: "1" })).preload).to.be(true); + expect(new Params(new URLSearchParams({ preload: "false" })).preload).to.be(false); + expect(new Params(new URLSearchParams({ preload: "FALSE" })).preload).to.be(false); + expect(new Params(new URLSearchParams({ preload: "0" })).preload).to.be(false); + }); it("should parse custom viewport", () => { const params = new Params( new URLSearchParams({ diff --git a/tests/unittests/preloader.mjs b/tests/unittests/preloader.mjs new file mode 100644 index 0000000000..37c90975a8 --- /dev/null +++ b/tests/unittests/preloader.mjs @@ -0,0 +1,305 @@ +import { ResourcePreloader } from "../../resources/preloader.mjs"; +import { Params } from "../../resources/shared/params.mjs"; +import { SW_MESSAGES } from "../../resources/shared/sw-messages.mjs"; + +const TEST_SUITES = [ + { + name: "SuiteA", + url: "suites/suite-a/index.html", + resources: "suites/suite-a/resources.txt", + enabled: true, + }, + { + name: "SuiteB", + url: "suites/suite-b/index.html", + resources: "suites/suite-b/resources.txt", + enabled: true, + }, +]; + +function createMutableParams(search = "developerMode=true") { + return new Params(new URLSearchParams(search)); +} + +function createMockServiceWorkerEnv(scopeUrl) { + const listeners = new Map(); + const activeWorker = { + postMessage: sinon.stub(), + }; + const registration = { + scope: scopeUrl, + active: activeWorker, + unregister: sinon.stub().resolves(true), + }; + const serviceWorkerContainer = { + controller: activeWorker, + ready: Promise.resolve(registration), + register: sinon.stub().resolves(registration), + getRegistrations: sinon.stub().resolves([registration]), + addEventListener: sinon.stub().callsFake((type, fn) => { + listeners.set(type, fn); + }), + removeEventListener: sinon.stub().callsFake((type, fn) => { + if (listeners.get(type) === fn) + listeners.delete(type); + }), + emitMessage(data) { + listeners.get("message")?.({ data }); + }, + }; + return { serviceWorkerContainer, registration, activeWorker, listeners }; +} + +describe("ResourcePreloader", () => { + let createdWindow = false; + let hadOwnServiceWorker = false; + let originalServiceWorkerDescriptor; + + function setNavigatorServiceWorker(value) { + Object.defineProperty(globalThis.navigator, "serviceWorker", { + value, + configurable: true, + writable: true, + }); + } + + beforeEach(() => { + if (typeof globalThis.window === "undefined") { + globalThis.window = { + location: { + href: "http://localhost:8080/index.html", + }, + }; + createdWindow = true; + } + hadOwnServiceWorker = Object.prototype.hasOwnProperty.call(globalThis.navigator, "serviceWorker"); + originalServiceWorkerDescriptor = Object.getOwnPropertyDescriptor(globalThis.navigator, "serviceWorker"); + }); + + afterEach(() => { + if (hadOwnServiceWorker && originalServiceWorkerDescriptor) + Object.defineProperty(globalThis.navigator, "serviceWorker", originalServiceWorkerDescriptor); + else + delete globalThis.navigator.serviceWorker; + + if (createdWindow) { + delete globalThis.window; + createdWindow = false; + } + }); + + it("should not throw in non-secure contexts where navigator.serviceWorker is undefined", async () => { + setNavigatorServiceWorker(undefined); + const testParams = createMutableParams("developerMode=true&preload=true"); + const preloader = new ResourcePreloader(testParams); + + await preloader.setup(); + expect(preloader._registration).to.be(null); + expect(preloader._sw).to.be(null); + + const preloadResult = await preloader.preloadSuites(TEST_SUITES); + expect(preloadResult).to.be(undefined); + expect(preloader.isCached(TEST_SUITES)).to.be(false); + + await preloader.teardown(); + }); + + it("should not touch navigator.serviceWorker on setup() when params.preload is false", async () => { + const expectedScope = new URL("./", window.location.href).href; + const { serviceWorkerContainer } = createMockServiceWorkerEnv(expectedScope); + setNavigatorServiceWorker(serviceWorkerContainer); + + const testParams = createMutableParams("developerMode=true&preload=false"); + const preloader = new ResourcePreloader(testParams); + + await preloader.setup(); + expect(serviceWorkerContainer.register.called).to.be(false); + expect(serviceWorkerContainer.getRegistrations.called).to.be(false); + expect(serviceWorkerContainer.addEventListener.called).to.be(false); + }); + + it("should lazily set up Service Worker when params.preload is toggled from false to true, and tear down when toggled back to false", async () => { + const expectedScope = new URL("./", window.location.href).href; + const { serviceWorkerContainer, registration, activeWorker } = createMockServiceWorkerEnv(expectedScope); + setNavigatorServiceWorker(serviceWorkerContainer); + + const testParams = createMutableParams("developerMode=true&preload=false"); + const preloader = new ResourcePreloader(testParams); + + // Initial setup with preload=false does not register SW. + await preloader.setup(); + expect(serviceWorkerContainer.register.called).to.be(false); + + // Toggle preload to true (as in Developer Mode) and call preloadSuites(). + testParams.preload = true; + activeWorker.postMessage.callsFake((msg) => { + if (msg.type === SW_MESSAGES.PRELOAD_SUITES) { + serviceWorkerContainer.emitMessage({ + type: SW_MESSAGES.PRELOAD_SUITES, + status: SW_MESSAGES.PRELOAD_DONE, + requestId: msg.requestId, + transferredSize: 1024, + count: 2, + }); + } + }); + + await preloader.preloadSuites(TEST_SUITES); + expect(serviceWorkerContainer.register.calledOnce).to.be(true); + expect(preloader.isCached(TEST_SUITES)).to.be(true); + + // Toggle preload back to false: preloadSuites() should tear down registration and skip preloading. + testParams.preload = false; + activeWorker.postMessage.resetHistory(); + await preloader.preloadSuites(TEST_SUITES); + + expect(registration.unregister.calledOnce).to.be(true); + expect(activeWorker.postMessage.called).to.be(false); + expect(preloader._registration).to.be(null); + expect(preloader._sw).to.be(null); + expect(preloader.isCached(TEST_SUITES)).to.be(false); + }); + + it("should only unregister Service Worker registrations matching the current scope", async () => { + const expectedScope = new URL("./", window.location.href).href; + const otherScope = new URL("/other-benchmark/", window.location.href).href; + const { serviceWorkerContainer, registration, activeWorker } = createMockServiceWorkerEnv(expectedScope); + + const otherRegistration = { + scope: otherScope, + unregister: sinon.stub().resolves(true), + }; + serviceWorkerContainer.getRegistrations.resolves([otherRegistration, registration]); + setNavigatorServiceWorker(serviceWorkerContainer); + + activeWorker.postMessage.callsFake((msg) => { + if (msg.type === SW_MESSAGES.STOP_PRELOADING) { + serviceWorkerContainer.emitMessage({ + type: SW_MESSAGES.STOP_PRELOADING, + status: SW_MESSAGES.STOP_PRELOADING, + requestId: msg.requestId, + }); + } else if (msg.type === SW_MESSAGES.GET_FAILED_REQUESTS) { + serviceWorkerContainer.emitMessage({ + type: SW_MESSAGES.GET_FAILED_REQUESTS, + status: SW_MESSAGES.FAILED_REQUESTS, + requestId: msg.requestId, + requests: [], + }); + } + }); + + const testParams = createMutableParams("developerMode=true&preload=true"); + const preloader = new ResourcePreloader(testParams); + await preloader.setup(); + await preloader.teardown(); + + expect(registration.unregister.calledOnce).to.be(true); + expect(otherRegistration.unregister.called).to.be(false); + }); + + it("should ignore stale PRELOAD_PROGRESS and PRELOAD_DONE messages from an earlier requestId", async () => { + const expectedScope = new URL("./", window.location.href).href; + const { serviceWorkerContainer, activeWorker } = createMockServiceWorkerEnv(expectedScope); + setNavigatorServiceWorker(serviceWorkerContainer); + + const testParams = createMutableParams("developerMode=true&preload=true"); + const preloader = new ResourcePreloader(testParams); + await preloader.setup(); + + let firstPreloadRequestId = null; + activeWorker.postMessage.callsFake((msg) => { + if (msg.type === SW_MESSAGES.PRELOAD_SUITES && firstPreloadRequestId === null) { + firstPreloadRequestId = msg.requestId; + // Do not reply immediately; simulate in-flight preload that gets stopped. + } else if (msg.type === SW_MESSAGES.STOP_PRELOADING) { + // Before STOP_PRELOADING replies, emit stale PRELOAD_PROGRESS and PRELOAD_DONE from the first request! + serviceWorkerContainer.emitMessage({ + type: SW_MESSAGES.PRELOAD_SUITES, + status: SW_MESSAGES.PRELOAD_PROGRESS, + requestId: firstPreloadRequestId, + loaded: 1, + total: 2, + url: "suites/suite-a/app.js", + suiteName: "SuiteA", + }); + serviceWorkerContainer.emitMessage({ + type: SW_MESSAGES.PRELOAD_SUITES, + status: SW_MESSAGES.PRELOAD_DONE, + requestId: firstPreloadRequestId, + transferredSize: 512, + count: 2, + }); + // Now reply to STOP_PRELOADING with its matching requestId. + serviceWorkerContainer.emitMessage({ + type: SW_MESSAGES.STOP_PRELOADING, + status: SW_MESSAGES.STOP_PRELOADING, + requestId: msg.requestId, + }); + } + }); + + const progressSpy = sinon.spy(); + const preloadPromise = preloader.preloadSuites(TEST_SUITES, true, progressSpy); + const stopPromise = preloader.stopPreloading(); + + const [preloadResult] = await Promise.all([preloadPromise, stopPromise]); + expect(preloadResult).to.be(null); + + // Emit another late message after no request is pending: must not throw. + serviceWorkerContainer.emitMessage({ + type: SW_MESSAGES.PRELOAD_SUITES, + status: SW_MESSAGES.PRELOAD_DONE, + requestId: firstPreloadRequestId, + transferredSize: 512, + count: 2, + }); + }); + + it("should return false from isCached(enabledSuites) when stopped early or when enabled suites change", async () => { + const expectedScope = new URL("./", window.location.href).href; + const { serviceWorkerContainer, activeWorker } = createMockServiceWorkerEnv(expectedScope); + setNavigatorServiceWorker(serviceWorkerContainer); + + const testParams = createMutableParams("developerMode=true&preload=true"); + const preloader = new ResourcePreloader(testParams); + await preloader.setup(); + + // 1. Preload stopped early -> isCached must be false. + activeWorker.postMessage.callsFake((msg) => { + if (msg.type === SW_MESSAGES.STOP_PRELOADING) { + serviceWorkerContainer.emitMessage({ + type: SW_MESSAGES.STOP_PRELOADING, + status: SW_MESSAGES.STOP_PRELOADING, + requestId: msg.requestId, + }); + } + }); + + const abortedPreload = preloader.preloadSuites(TEST_SUITES); + await preloader.stopPreloading(); + const abortedResult = await abortedPreload; + expect(abortedResult).to.be(null); + expect(preloader.isCached(TEST_SUITES)).to.be(false); + + // 2. Full preload succeeds -> isCached is true for the same suites, false if enabled suites change. + activeWorker.postMessage.callsFake((msg) => { + if (msg.type === SW_MESSAGES.PRELOAD_SUITES) { + serviceWorkerContainer.emitMessage({ + type: SW_MESSAGES.PRELOAD_SUITES, + status: SW_MESSAGES.PRELOAD_DONE, + requestId: msg.requestId, + transferredSize: 2048, + count: 2, + }); + } + }); + + await preloader.preloadSuites(TEST_SUITES); + expect(preloader.isCached(TEST_SUITES)).to.be(true); + + // Changing enabled suites (e.g. toggling a suite checkbox in Developer Mode) invalidates cache. + const subsetSuites = [TEST_SUITES[0]]; + expect(preloader.isCached(subsetSuites)).to.be(false); + }); +}); diff --git a/tests/unittests/request-limiter.mjs b/tests/unittests/request-limiter.mjs new file mode 100644 index 0000000000..d8c3cda7aa --- /dev/null +++ b/tests/unittests/request-limiter.mjs @@ -0,0 +1,108 @@ +import { RequestLimiter } from "../../resources/shared/request-limiter.mjs"; + +describe("RequestLimiter", () => { + it("should process requests within the limit concurrently", async () => { + const limiter = new RequestLimiter(2); + let active = 0; + let maxActive = 0; + + const task = async () => { + active++; + maxActive = Math.max(maxActive, active); + await new Promise((resolve) => setTimeout(resolve, 10)); + active--; + }; + + const p1 = limiter.schedule(task); + const p2 = limiter.schedule(task); + const p3 = limiter.schedule(task); + + await Promise.all([p1, p2, p3]); + + expect(maxActive).to.be(2); + expect(limiter._active).to.be(0); + }); + + it("should resolve with the correct value", async () => { + const limiter = new RequestLimiter(1); + const p = limiter.schedule(async () => "result"); + const result = await p; + expect(result).to.be("result"); + }); + + it("should reject if the task throws", async () => { + const limiter = new RequestLimiter(1); + const p = limiter.schedule(async () => { + throw new Error("task failed"); + }); + + let threw = false; + try { + await p; + } catch (e) { + threw = true; + expect(e.message).to.be("task failed"); + } + expect(threw).to.be(true); + // Wait for microtasks to flush so _processQueue can decrement _active + await new Promise((r) => setTimeout(r, 0)); + expect(limiter._active).to.be(0); + }); + + it("should continue processing queue if a task rejects", async () => { + const limiter = new RequestLimiter(1); + + const p1 = limiter.schedule(async () => { + throw new Error("failed"); + }); + const p2 = limiter.schedule(async () => "success"); + + try { + await p1; + } catch (e) { + // expected to throw + } + + const res2 = await p2; + expect(res2).to.be("success"); + }); + + it("should clear the queue and resolve pending requests with 0", async () => { + const limiter = new RequestLimiter(1); + let executed = 0; + + const p1 = limiter.schedule(async () => { + executed++; + await new Promise((resolve) => setTimeout(resolve, 10)); + return 1; + }); + + const p2 = limiter.schedule(async () => { + executed++; + return 2; + }); + + const p3 = limiter.schedule(async () => { + executed++; + return 3; + }); + + limiter.clear(); + + const res1 = await p1; + const res2 = await p2; + const res3 = await p3; + + // p1 already started, so it finishes normally + expect(res1).to.be(1); + // p2 and p3 were cleared, so they resolve with 0 + expect(res2).to.be(0); + expect(res3).to.be(0); + + // only p1 actually ran + expect(executed).to.be(1); + + // eventually active count resets when p1 completes + expect(limiter._active).to.be(0); + }); +});