From 794f96ca31ac44ef1f0d20f8dd85239530e7f95d Mon Sep 17 00:00:00 2001 From: Coding-Dev-Tools <274649000+Coding-Dev-Tools@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:47:14 -0500 Subject: [PATCH 1/2] fix(ci): harden run_preflight.ps1 parity and dependency verification Ensure run_preflight.ps1 verifies required test dependencies via real module imports without stdout coercion, includes sqlite_vec with pip install -e ".[test]" guidance, mirrors CI ENGRAPHIS_INDEX_ROOTS and eval gates, and expands release infrastructure regression coverage. --- run_preflight.ps1 | 275 ++++++++++++++++++++++----- tests/test_release_infrastructure.py | 101 ++++++++++ 2 files changed, 325 insertions(+), 51 deletions(-) diff --git a/run_preflight.ps1 b/run_preflight.ps1 index b1964b27e..925d322f2 100644 --- a/run_preflight.ps1 +++ b/run_preflight.ps1 @@ -1,25 +1,45 @@ -# Engraphis local pre-flight verification. -# Runs the same offline gates as CI before a push, so a broken tree costs no CI time. +<# +.SYNOPSIS + Engraphis local pre-flight verification. +.DESCRIPTION + Mirrors the offline CI gates (ruff, manifest, CSP drift, pyright, pytest, eval/*). + Use -Fast for a Python 3.9+ two-file smoke or -SkipEval to omit eval modules. +.PARAMETER Fast + Run lint, manifest/asset checks, and the two-file smoke suite (skips pyright and eval). +.PARAMETER SkipEval + Run the full offline pytest suite without the eval/* verification gates. +.PARAMETER Workers + Optional pytest-xdist worker count (or 'auto') for the offline unit suite. +#> param( [switch]$Fast, - [switch]$SkipEval + [switch]$SkipEval, + [ValidatePattern('^$|^(auto|[1-9][0-9]*)$')] + [string]$Workers = "" ) $ErrorActionPreference = "Stop" +$modeLabel = if ($Fast) { 'Fast smoke' } elseif ($SkipEval) { 'Offline unit suite without eval' } else { 'Complete primary gate' } +$isPartialRun = $Fast -or $SkipEval + Write-Host "================================================================================" -ForegroundColor Cyan Write-Host " ENGRAPHIS LOCAL PRE-FLIGHT VERIFICATION" -ForegroundColor Cyan -Write-Host " Mode: $(if ($Fast) {'Fast smoke'} elseif ($SkipEval) {'Full offline (no eval)'} else {'Complete primary gate'})" -ForegroundColor Cyan +Write-Host " Mode: $modeLabel" -ForegroundColor Cyan Write-Host "================================================================================" -ForegroundColor Cyan $startTime = Get-Date -# Locate the appropriate python executable (.venv preferred) +# Locate the appropriate python executable (.venv preferred, Windows or POSIX layout) $repoRoot = $PSScriptRoot if (-not $repoRoot) { $repoRoot = (Get-Location).Path } -$venvPython = Join-Path $repoRoot ".venv\Scripts\python.exe" -if (Test-Path $venvPython) { +$pythonCandidates = @( + (Join-Path $repoRoot ".venv\Scripts\python.exe"), + (Join-Path $repoRoot ".venv/bin/python") +) +$venvPython = $pythonCandidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1 +if ($venvPython) { $pythonExe = $venvPython Write-Host "[Python] Using local venv: $pythonExe" -ForegroundColor Gray } else { @@ -27,75 +47,228 @@ if (Test-Path $venvPython) { Write-Host "[Python] Using system python / uv" -ForegroundColor Gray } +$ruffCandidates = @( + (Join-Path $repoRoot ".venv\Scripts\ruff.exe"), + (Join-Path $repoRoot ".venv/bin/ruff") +) +$venvRuff = $ruffCandidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1 + +function Test-PythonModule { + param([string]$ModuleName) + # A real import, not find_spec: a broken wheel or a missing DLL surfaces here. Seed + # LASTEXITCODE so an unlaunchable interpreter cannot inherit 0 and return True. + $ErrorActionPreference = 'Continue' + $global:LASTEXITCODE = 1 + try { + $null = & $pythonExe -c "import importlib, sys; importlib.import_module(sys.argv[1])" $ModuleName 2>$null + return ($LASTEXITCODE -eq 0) + } catch { + return $false + } +} + +# Locate pyright: the venv console script first, then PATH, then python -m pyright. +# [tool.pyright] declares no venvPath/venv, so the gate pins the interpreter it should +# analyse explicitly with --pythonpath; the branches below only choose how pyright +# itself is launched. +$pyrightCandidates = @( + (Join-Path $repoRoot ".venv\Scripts\pyright.exe"), + (Join-Path $repoRoot ".venv/bin/pyright") +) +$venvPyright = $pyrightCandidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1 +$pyrightMode = $null +$pyrightExe = $null +if ($venvPyright) { + $pyrightMode = "exe" + $pyrightExe = $venvPyright +} elseif (Get-Command pyright -ErrorAction SilentlyContinue) { + $pyrightMode = "exe" + $pyrightExe = (Get-Command pyright).Source +} elseif (Test-PythonModule "pyright") { + $pyrightMode = "module" +} +if ($pyrightMode -eq "exe") { + Write-Host "[Pyright] Using: $pyrightExe" -ForegroundColor Gray +} elseif ($pyrightMode -eq "module") { + Write-Host "[Pyright] Using: $pythonExe -m pyright" -ForegroundColor Gray +} + +# pytest's own CI runs always supply a private runner-owned --basetemp; mirror that here +# so a stale temp dir from an earlier run cannot poison this one. +$pytestBaseTemp = Join-Path ([IO.Path]::GetTempPath()) ("engraphis-preflight-pytest-" + [Guid]::NewGuid().ToString("N")) + +$script:lastStepPassed = $false + function Invoke-GateStep { param( [string]$Name, [scriptblock]$Action ) + $script:lastStepPassed = $false Write-Host "`n[STEP] Starting: $Name" -ForegroundColor Yellow $stepStart = Get-Date + $global:LASTEXITCODE = 0 try { & $Action if ($LASTEXITCODE -ne 0 -and $LASTEXITCODE -ne $null) { Write-Host "[FAIL] $Name exited with code $LASTEXITCODE" -ForegroundColor Red - return $false + return } $duration = ((Get-Date) - $stepStart).TotalSeconds Write-Host ("[PASS] {0} ({1:N1}s)" -f $Name, $duration) -ForegroundColor Green - return $true + $script:lastStepPassed = $true } catch { Write-Host "[ERROR] $Name failed: $_" -ForegroundColor Red - return $false } } -# 1. Ruff linting -$step1 = Invoke-GateStep "Ruff Linting" { - if (Test-Path (Join-Path $repoRoot ".venv\Scripts\ruff.exe")) { - & (Join-Path $repoRoot ".venv\Scripts\ruff.exe") check . - } elseif (Get-Command uv -ErrorAction SilentlyContinue) { - uv run --with ruff ruff check . - } else { - & $pythonExe -m ruff check . +$locationPushed = $false +try { + # Run every gate from the repository root so `./run_preflight.ps1` behaves the same + # when it is invoked from anywhere else. + Push-Location -Path $repoRoot + $locationPushed = $true + + # 1. Ruff linting + Invoke-GateStep "Ruff Linting" { + if ($venvRuff) { + & $venvRuff check . + } elseif (Get-Command uv -ErrorAction SilentlyContinue) { + uv run --with ruff ruff check . + } else { + & $pythonExe -m ruff check . + } } -} -if (-not $step1) { exit 1 } + if (-not $script:lastStepPassed) { exit 1 } -# 2. Manifest check -$step2 = Invoke-GateStep "Commercial Manifest Boundary" { - & $pythonExe scripts/check_commercial_manifest.py -} -if (-not $step2) { exit 1 } + # 2. Manifest check + Invoke-GateStep "Commercial Manifest Boundary" { + & $pythonExe scripts/check_commercial_manifest.py + } + if (-not $script:lastStepPassed) { exit 1 } -# 3. Asset drift check -$step3 = Invoke-GateStep "Strict-CSP Asset Drift Check" { - & $pythonExe scripts/externalize_dashboard_assets.py -} -if (-not $step3) { exit 1 } + # 3. Asset drift check + Invoke-GateStep "Strict-CSP Asset Drift Check" { + & $pythonExe scripts/externalize_dashboard_assets.py + } + if (-not $script:lastStepPassed) { exit 1 } -# 4. Pytest suite -$step4 = Invoke-GateStep "Offline Unit Test Suite" { - if ($Fast) { - & $pythonExe -m pytest tests/test_adaptive_context.py tests/test_v2_service_binding.py -q - } else { - & $pythonExe -m pytest tests/ -q + # 4. Pyright type check (core + backends; skipped in -Fast so Python 3.9 numpy-only smoke works) + if (-not $Fast) { + Invoke-GateStep "Pyright Type Check" { + $pyrightArgs = @() + if ($venvPython) { + $pyrightArgs += @("--pythonpath", $pythonExe) + } + if ($pyrightMode -eq "exe") { + & $pyrightExe @pyrightArgs + } elseif ($pyrightMode -eq "module") { + & $pythonExe -m pyright @pyrightArgs + } else { + throw "pyright not found. Install the dev/test extras or add pyright to PATH." + } + } + if (-not $script:lastStepPassed) { exit 1 } } -} -if (-not $step4) { exit 1 } -if (-not $Fast -and -not $SkipEval) { - # 5. Eval Gates if eval datasets are present - if (Test-Path "eval\datasets\sample.jsonl") { - $step5 = Invoke-GateStep "Retrieval Eval Gate (sample)" { - & $pythonExe -m eval.harness --dataset eval/datasets/sample.jsonl --k 5 + # 5. Pytest suite + Invoke-GateStep "Offline Unit Test Suite" { + if ($Fast -and -not [string]::IsNullOrWhiteSpace($Workers)) { + Write-Host "[pytest] -Workers ignored in -Fast mode" -ForegroundColor Gray + } + if (-not $Fast) { + # [test] extras require Python 3.10+; read version without Select-Object so $LASTEXITCODE stays 0. + $pyVer = (& $pythonExe -c "import sys; print('%d.%d' % sys.version_info[:2])" 2>$null | Out-String).Trim() + if ($LASTEXITCODE -eq 0 -and $pyVer -match '^\d+\.\d+$' -and ([Version]$pyVer -lt [Version]'3.10')) { + throw ("Full offline preflight requires Python 3.10+ for [test] extras (found Python $pyVer). Run .\run_preflight.ps1 -Fast on Python 3.9, matching CI's numpy-only job.") + } + # A stripped venv would let the extras-gated server/MCP tests skip silently and + # report green. Fail loudly instead: the full offline suite needs them. + $requiredModules = @("fastapi", "httpx", "mcp", "multipart", "pydantic", "sqlite_vec", "uvicorn") + $missingModules = @($requiredModules | Where-Object { -not (Test-PythonModule $_) }) + if ($missingModules.Count -gt 0) { + throw ('Missing full-stack test dependencies: {0}. Install them with: pip install -e ".[test]" (Python 3.10+)' -f ($missingModules -join ", ")) + } + $pytestArgs = @("-o", "addopts=", "tests/", "-q", "-rs", "--basetemp", $pytestBaseTemp) + if (Test-PythonModule "xdist") { + if (-not [string]::IsNullOrWhiteSpace($Workers)) { + Write-Host "[pytest] pytest-xdist detected; running with -n $Workers" -ForegroundColor Gray + $pytestArgs += @("-n", $Workers) + } elseif ($env:OS -eq 'Windows_NT') { + Write-Host "[pytest] Windows host detected; running serially to avoid subprocess timeout contention (pass -Workers auto to opt in)" -ForegroundColor Gray + } else { + Write-Host "[pytest] pytest-xdist detected; running with -n auto" -ForegroundColor Gray + $pytestArgs += @("-n", "auto") + } + } else { + Write-Host "[pytest] pytest-xdist not installed; running serially" -ForegroundColor Gray + } + } else { + $pytestArgs = @("-o", "addopts=", "tests/test_adaptive_context.py", "tests/test_v2_service_binding.py", "-q", "-rs", "--basetemp", $pytestBaseTemp) + } + # CI scopes ENGRAPHIS_INDEX_ROOTS to pytest; include Python's tempfile.gettempdir(). + $previousIndexRoots = $env:ENGRAPHIS_INDEX_ROOTS + $pyTemp = (& $pythonExe -c "import tempfile; print(tempfile.gettempdir())" 2>$null | Out-String).Trim() + $indexRoots = @($repoRoot, [IO.Path]::GetTempPath().TrimEnd('\', '/'), $pytestBaseTemp) + if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace($pyTemp)) { $indexRoots += $pyTemp } + $env:ENGRAPHIS_INDEX_ROOTS = (($indexRoots | Select-Object -Unique) -join [IO.Path]::PathSeparator) + try { + & $pythonExe -m pytest @pytestArgs + } finally { + $env:ENGRAPHIS_INDEX_ROOTS = $previousIndexRoots + } + } + if (-not $script:lastStepPassed) { exit 1 } + + if (-not $Fast -and -not $SkipEval) { + # 6. Eval Gates matching the offline CI suite + $evalGates = @( + @{ Name = "Retrieval Eval Gate (sample)"; Dataset = "eval/datasets/sample.jsonl"; Args = @("-m", "eval.harness", "--dataset", "eval/datasets/sample.jsonl", "--k", "5") }, + @{ Name = "Retrieval Eval Gate (codemem)"; Dataset = "eval/datasets/codemem.jsonl"; Args = @("-m", "eval.harness", "--dataset", "eval/datasets/codemem.jsonl", "--k", "5") }, + @{ Name = "Ablation Eval Gate"; Args = @("-m", "eval.ablation") }, + @{ Name = "Reinforcement Eval Gate"; Args = @("-m", "eval.reinforcement") }, + @{ Name = "Adversarial Memory Security Gate"; Args = @("-m", "eval.adversarial_memory_security") }, + @{ Name = "Grounded Recall Eval Gate"; Args = @("-m", "eval.grounded") }, + @{ Name = "Code-Agent Arm Gate"; Args = @("-m", "eval.code_arm") }, + @{ Name = "Spec Crawl Eval Gate"; Args = @("-m", "eval.spec_crawl") }, + @{ Name = "Evidence Contracts Gate"; Args = @("-m", "eval.evidence_contracts") } + ) + foreach ($gate in $evalGates) { + if ($gate.ContainsKey("Dataset")) { + $datasetPath = Join-Path $repoRoot ($gate.Dataset -replace '/', [IO.Path]::DirectorySeparatorChar) + if (-not (Test-Path -LiteralPath $datasetPath)) { + Write-Host "[FAIL] Missing required eval dataset '$($gate.Dataset)' (expected at $datasetPath)." -ForegroundColor Red + Write-Host "[FAIL] The pre-flight gate must never silently skip a missing dataset; restore it and re-run." -ForegroundColor Red + exit 1 + } + } + $evalArgs = $gate.Args + Invoke-GateStep $gate.Name { + & $pythonExe @evalArgs + } + if (-not $script:lastStepPassed) { exit 1 } } - if (-not $step5) { exit 1 } } -} -$totalDuration = ((Get-Date) - $startTime).TotalSeconds -Write-Host "`n================================================================================" -ForegroundColor Cyan -Write-Host ("[PRE-FLIGHT] ALL GATES GREEN in {0:N1}s." -f $totalDuration) -ForegroundColor Green -Write-Host "Safe to commit and push." -Write-Host "================================================================================" -ForegroundColor Cyan -exit 0 + $totalDuration = ((Get-Date) - $startTime).TotalSeconds + Write-Host "`n================================================================================" -ForegroundColor Cyan + if ($isPartialRun) { + Write-Host ("[PRE-FLIGHT] LIMITED GATES PASSED ({0}) in {1:N1}s." -f $modeLabel, $totalDuration) -ForegroundColor Yellow + Write-Host "Only the '$modeLabel' gates ran; the full offline CI gate did not." -ForegroundColor Yellow + Write-Host "Run the complete .\run_preflight.ps1 before push." -ForegroundColor Yellow + } else { + Write-Host ("[PRE-FLIGHT] ALL GATES GREEN in {0:N1}s." -f $totalDuration) -ForegroundColor Green + Write-Host "Safe to commit and push." + } + Write-Host "================================================================================" -ForegroundColor Cyan + exit 0 +} finally { + if ($locationPushed) { Pop-Location } + if (Test-Path -LiteralPath $pytestBaseTemp) { + $tempRootPrefix = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\', '/') + [IO.Path]::DirectorySeparatorChar + $resolvedBaseTemp = [IO.Path]::GetFullPath($pytestBaseTemp) + if ($resolvedBaseTemp.StartsWith($tempRootPrefix, [StringComparison]::OrdinalIgnoreCase)) { + Remove-Item -LiteralPath $resolvedBaseTemp -Recurse -Force -ErrorAction SilentlyContinue + } + } +} diff --git a/tests/test_release_infrastructure.py b/tests/test_release_infrastructure.py index 85b337229..05e817319 100644 --- a/tests/test_release_infrastructure.py +++ b/tests/test_release_infrastructure.py @@ -3,8 +3,12 @@ import json import re +import shutil +import subprocess from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] @@ -818,3 +822,100 @@ def test_release_workflow_permissions_are_least_privilege(): assert "contents: write" not in body, job assert "id-token: write" not in body, job + +def test_run_preflight_ps1_aligns_with_ci_gates_and_checks_dependencies(): + """The local pre-flight must run the offline CI gates, not a comfortable subset.""" + script = _text("run_preflight.ps1") + + for command in ( + "ruff check .", + "scripts/check_commercial_manifest.py", + "scripts/externalize_dashboard_assets.py", + "pyright", + "pytest", + ): + assert command in script, command + + # Every eval module the offline CI job runs, and both JSONL datasets it feeds the + # harness. A dropped module silently shrinks the gate instead of failing it. + assert set(re.findall(r'"eval\.([a-z_]+)"', script)) == { + "harness", + "ablation", + "reinforcement", + "adversarial_memory_security", + "grounded", + "code_arm", + "spec_crawl", + "evidence_contracts", + } + assert len(re.findall(r'"eval\.harness"', script)) == 2 + assert '"eval/datasets/sample.jsonl"' in script + assert '"eval/datasets/codemem.jsonl"' in script + + assert 'Invoke-GateStep "Pyright Type Check"' in script + assert 'Invoke-GateStep "Offline Unit Test Suite"' in script + + # A missing dataset is a hard failure resolved against the repository root; it must + # never be folded into `continue`, which would report green after skipping the gate. + assert re.search(r"-not\s*\(\s*Test-Path[^\n]*\bcontinue\b", script) is None + dataset_check = script.split("$datasetPath = Join-Path $repoRoot", 1)[1].split( + "Invoke-GateStep", 1 + )[0] + assert "exit 1" in dataset_check + assert "continue" not in dataset_check + assert "Push-Location" in script + assert "Join-Path $repoRoot" in script + + # A real import catches a broken native wheel or a missing DLL; find_spec() alone would + # report those as installed. stdout is discarded so a stray line cannot become the + # PowerShell return value. + assert "importlib.import_module(sys.argv[1])" in script + assert "$null = & $pythonExe -c" in script + + # CI runs pytest with ENGRAPHIS_INDEX_ROOTS covering the checkout and its temp dirs; + # the pre-flight mirrors that and restores the caller's value afterwards. + assert "$env:ENGRAPHIS_INDEX_ROOTS" in script + assert "$previousIndexRoots = $env:ENGRAPHIS_INDEX_ROOTS" in script + assert "[pytest] -Workers ignored in -Fast mode" in script + + # CI runs pytest with the ini-file addopts neutralised and skip reasons visible. The + # full suite line has to carry all three so a local pyproject/pytest override cannot + # change what the pre-flight measured. + full_suite = [ + line for line in script.splitlines() if "$pytestArgs = " in line and '"tests/"' in line + ] + assert len(full_suite) == 1 + line = full_suite[0] + assert '"-o", "addopts="' in line + assert '"-q"' in line + assert '"-rs"' in line + assert '"--basetemp"' in line + + # A venv without the full-stack extras would skip the server/MCP tests silently, and + # ".[all,dev]" would drag in torch and the transcription/document extras that this gate does not need. + assert '"fastapi", "httpx", "mcp", "multipart", "pydantic", "sqlite_vec", "uvicorn"' in script + assert 'pip install -e ".[test]"' in script + + # Derive the parity list from the CI job itself so a new offline gate cannot be added + # to CI and silently left out of the local pre-flight. + for job in ("test", "core-py39"): + block = _job_block(_text(".github/workflows/ci.yml"), job) + for dataset in re.findall(r"eval/datasets/[\w.-]+\.jsonl", block): + assert f'"{dataset}"' in script, dataset + for module in re.findall(r"python -m (eval\.[a-z_]+)", block): + assert f'"{module}"' in script, module + + +def test_run_preflight_ps1_parses_without_powershell_syntax_errors(): + shell = shutil.which("pwsh") or shutil.which("powershell") + if not shell: + pytest.skip("pwsh/powershell is not installed") + script_path = str(ROOT / "run_preflight.ps1").replace("'", "''") + cmd = ( + "$errs = $null; [void][System.Management.Automation.Language.Parser]::ParseFile(" + f"'{script_path}', $null, [ref]$errs); Write-Output $errs.Count" + ) + proc = subprocess.run([shell, "-NoProfile", "-NonInteractive", "-Command", cmd], + capture_output=True, text=True, check=True) + assert proc.stdout.strip().splitlines()[-1] == "0" + From c0b0e68168de8469942ccc53947e6915eacd30db Mon Sep 17 00:00:00 2001 From: Coding-Dev-Tools <274649000+Coding-Dev-Tools@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:49:21 -0500 Subject: [PATCH 2/2] fix(ci): harden run_preflight.ps1 stderr handling, tool resolution, and execution tests Run Python probes under Continue via Invoke-PythonStdout so stderr warnings do not raise NativeCommandError in PowerShell 5.1, check Python 3.10+ before Pyright, skip Pyright discovery in -Fast, prefer local/PATH/module ruff before bounded uv --no-project fallback, isolate PYTEST_ADDOPTS alongside ENGRAPHIS_INDEX_ROOTS, and add cross-shell subprocess behavioral regression tests. --- run_preflight.ps1 | 136 ++++++++---- tests/test_release_infrastructure.py | 306 +++++++++++++++++++++++---- 2 files changed, 359 insertions(+), 83 deletions(-) diff --git a/run_preflight.ps1 b/run_preflight.ps1 index 925d322f2..5eafbfc57 100644 --- a/run_preflight.ps1 +++ b/run_preflight.ps1 @@ -41,10 +41,18 @@ $pythonCandidates = @( $venvPython = $pythonCandidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1 if ($venvPython) { $pythonExe = $venvPython + $hasResolvedPython = $true Write-Host "[Python] Using local venv: $pythonExe" -ForegroundColor Gray } else { - $pythonExe = "python" - Write-Host "[Python] Using system python / uv" -ForegroundColor Gray + $pythonCmd = Get-Command python, python3 -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($pythonCmd) { + $pythonExe = $pythonCmd.Source + $hasResolvedPython = $true + } else { + $pythonExe = "python" + $hasResolvedPython = $false + } + Write-Host "[Python] Using system python / uv ($pythonExe)" -ForegroundColor Gray } $ruffCandidates = @( @@ -53,44 +61,62 @@ $ruffCandidates = @( ) $venvRuff = $ruffCandidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1 -function Test-PythonModule { - param([string]$ModuleName) - # A real import, not find_spec: a broken wheel or a missing DLL surfaces here. Seed - # LASTEXITCODE so an unlaunchable interpreter cannot inherit 0 and return True. +function Invoke-PythonStdout { + param([string[]]$Arguments) + # Run Python probes under Continue so stderr noise (e.g. PYTHONWARNINGS=default, + # PYTHONPROFILEIMPORTTIME=1) cannot raise NativeCommandError in PowerShell 5.1. $ErrorActionPreference = 'Continue' + $prevExitCode = $global:LASTEXITCODE $global:LASTEXITCODE = 1 try { - $null = & $pythonExe -c "import importlib, sys; importlib.import_module(sys.argv[1])" $ModuleName 2>$null - return ($LASTEXITCODE -eq 0) + $output = (& $pythonExe @Arguments 2>$null | Out-String).Trim() + return @{ + Ok = ($LASTEXITCODE -eq 0) + Output = $output + } } catch { - return $false + return @{ + Ok = $false + Output = "" + } + } finally { + $global:LASTEXITCODE = $prevExitCode } } -# Locate pyright: the venv console script first, then PATH, then python -m pyright. +function Test-PythonModule { + param([string]$ModuleName) + # A real import, not find_spec: a broken wheel or a missing DLL surfaces here. + $probe = Invoke-PythonStdout -Arguments @("-c", "import importlib, sys; importlib.import_module(sys.argv[1])", $ModuleName) + return $probe.Ok +} + +# Locate pyright only when full gates run (-Fast skips Pyright Type Check). # [tool.pyright] declares no venvPath/venv, so the gate pins the interpreter it should # analyse explicitly with --pythonpath; the branches below only choose how pyright # itself is launched. -$pyrightCandidates = @( - (Join-Path $repoRoot ".venv\Scripts\pyright.exe"), - (Join-Path $repoRoot ".venv/bin/pyright") -) -$venvPyright = $pyrightCandidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1 $pyrightMode = $null $pyrightExe = $null -if ($venvPyright) { - $pyrightMode = "exe" - $pyrightExe = $venvPyright -} elseif (Get-Command pyright -ErrorAction SilentlyContinue) { - $pyrightMode = "exe" - $pyrightExe = (Get-Command pyright).Source -} elseif (Test-PythonModule "pyright") { - $pyrightMode = "module" -} -if ($pyrightMode -eq "exe") { - Write-Host "[Pyright] Using: $pyrightExe" -ForegroundColor Gray -} elseif ($pyrightMode -eq "module") { - Write-Host "[Pyright] Using: $pythonExe -m pyright" -ForegroundColor Gray +if (-not $Fast) { + $pyrightCandidates = @( + (Join-Path $repoRoot ".venv\Scripts\pyright.exe"), + (Join-Path $repoRoot ".venv/bin/pyright") + ) + $venvPyright = $pyrightCandidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1 + if ($venvPyright) { + $pyrightMode = "exe" + $pyrightExe = $venvPyright + } elseif (Get-Command pyright -ErrorAction SilentlyContinue) { + $pyrightMode = "exe" + $pyrightExe = (Get-Command pyright).Source + } elseif (Test-PythonModule "pyright") { + $pyrightMode = "module" + } + if ($pyrightMode -eq "exe") { + Write-Host "[Pyright] Using: $pyrightExe" -ForegroundColor Gray + } elseif ($pyrightMode -eq "module") { + Write-Host "[Pyright] Using: $pythonExe -m pyright" -ForegroundColor Gray + } } # pytest's own CI runs always supply a private runner-owned --basetemp; mirror that here @@ -110,7 +136,7 @@ function Invoke-GateStep { $global:LASTEXITCODE = 0 try { & $Action - if ($LASTEXITCODE -ne 0 -and $LASTEXITCODE -ne $null) { + if ($null -ne $LASTEXITCODE -and $LASTEXITCODE -ne 0) { Write-Host "[FAIL] $Name exited with code $LASTEXITCODE" -ForegroundColor Red return } @@ -126,17 +152,21 @@ $locationPushed = $false try { # Run every gate from the repository root so `./run_preflight.ps1` behaves the same # when it is invoked from anywhere else. - Push-Location -Path $repoRoot + Push-Location -LiteralPath $repoRoot $locationPushed = $true # 1. Ruff linting Invoke-GateStep "Ruff Linting" { if ($venvRuff) { & $venvRuff check . + } elseif (Get-Command ruff -ErrorAction SilentlyContinue) { + ruff check . + } elseif (Test-PythonModule "ruff") { + & $pythonExe -m ruff check . } elseif (Get-Command uv -ErrorAction SilentlyContinue) { - uv run --with ruff ruff check . + uv run --no-project --with "ruff>=0.15.22,<0.18" ruff check . } else { - & $pythonExe -m ruff check . + throw "ruff not found. Install dev/test extras, add ruff to PATH, or install uv." } } if (-not $script:lastStepPassed) { exit 1 } @@ -155,9 +185,14 @@ try { # 4. Pyright type check (core + backends; skipped in -Fast so Python 3.9 numpy-only smoke works) if (-not $Fast) { + $pyVerProbe = Invoke-PythonStdout -Arguments @("-c", "import sys; print('%d.%d' % sys.version_info[:2])") + if ($pyVerProbe.Ok -and $pyVerProbe.Output -match '^\d+\.\d+$' -and ([Version]$pyVerProbe.Output -lt [Version]'3.10')) { + Write-Host ("[FAIL] Full offline preflight requires Python 3.10+ for [test] extras (found Python {0}). Run .\run_preflight.ps1 -Fast on Python 3.9, matching CI's numpy-only job." -f $pyVerProbe.Output) -ForegroundColor Red + exit 1 + } Invoke-GateStep "Pyright Type Check" { $pyrightArgs = @() - if ($venvPython) { + if ($hasResolvedPython) { $pyrightArgs += @("--pythonpath", $pythonExe) } if ($pyrightMode -eq "exe") { @@ -177,11 +212,6 @@ try { Write-Host "[pytest] -Workers ignored in -Fast mode" -ForegroundColor Gray } if (-not $Fast) { - # [test] extras require Python 3.10+; read version without Select-Object so $LASTEXITCODE stays 0. - $pyVer = (& $pythonExe -c "import sys; print('%d.%d' % sys.version_info[:2])" 2>$null | Out-String).Trim() - if ($LASTEXITCODE -eq 0 -and $pyVer -match '^\d+\.\d+$' -and ([Version]$pyVer -lt [Version]'3.10')) { - throw ("Full offline preflight requires Python 3.10+ for [test] extras (found Python $pyVer). Run .\run_preflight.ps1 -Fast on Python 3.9, matching CI's numpy-only job.") - } # A stripped venv would let the extras-gated server/MCP tests skip silently and # report green. Fail loudly instead: the full offline suite needs them. $requiredModules = @("fastapi", "httpx", "mcp", "multipart", "pydantic", "sqlite_vec", "uvicorn") @@ -201,21 +231,41 @@ try { $pytestArgs += @("-n", "auto") } } else { - Write-Host "[pytest] pytest-xdist not installed; running serially" -ForegroundColor Gray + if (-not [string]::IsNullOrWhiteSpace($Workers)) { + Write-Host "[pytest] -Workers ignored because pytest-xdist is not installed" -ForegroundColor Gray + } else { + Write-Host "[pytest] pytest-xdist not installed; running serially" -ForegroundColor Gray + } } } else { $pytestArgs = @("-o", "addopts=", "tests/test_adaptive_context.py", "tests/test_v2_service_binding.py", "-q", "-rs", "--basetemp", $pytestBaseTemp) } - # CI scopes ENGRAPHIS_INDEX_ROOTS to pytest; include Python's tempfile.gettempdir(). + # CI scopes ENGRAPHIS_INDEX_ROOTS to pytest; include Python's tempfile.gettempdir(), + # and clear caller PYTEST_ADDOPTS so local pytest flags cannot skew pre-flight. + $hadIndexRoots = Test-Path Env:ENGRAPHIS_INDEX_ROOTS $previousIndexRoots = $env:ENGRAPHIS_INDEX_ROOTS - $pyTemp = (& $pythonExe -c "import tempfile; print(tempfile.gettempdir())" 2>$null | Out-String).Trim() + $hadPytestAddopts = Test-Path Env:PYTEST_ADDOPTS + $previousPytestAddopts = $env:PYTEST_ADDOPTS + $pyTempProbe = Invoke-PythonStdout -Arguments @("-c", "import tempfile; print(tempfile.gettempdir())") $indexRoots = @($repoRoot, [IO.Path]::GetTempPath().TrimEnd('\', '/'), $pytestBaseTemp) - if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace($pyTemp)) { $indexRoots += $pyTemp } + if ($pyTempProbe.Ok -and -not [string]::IsNullOrWhiteSpace($pyTempProbe.Output)) { + $indexRoots += $pyTempProbe.Output + } $env:ENGRAPHIS_INDEX_ROOTS = (($indexRoots | Select-Object -Unique) -join [IO.Path]::PathSeparator) + if ($hadPytestAddopts) { Remove-Item Env:PYTEST_ADDOPTS -ErrorAction SilentlyContinue } try { & $pythonExe -m pytest @pytestArgs } finally { - $env:ENGRAPHIS_INDEX_ROOTS = $previousIndexRoots + if ($hadIndexRoots) { + $env:ENGRAPHIS_INDEX_ROOTS = $previousIndexRoots + } else { + Remove-Item Env:ENGRAPHIS_INDEX_ROOTS -ErrorAction SilentlyContinue + } + if ($hadPytestAddopts) { + $env:PYTEST_ADDOPTS = $previousPytestAddopts + } else { + Remove-Item Env:PYTEST_ADDOPTS -ErrorAction SilentlyContinue + } } } if (-not $script:lastStepPassed) { exit 1 } diff --git a/tests/test_release_infrastructure.py b/tests/test_release_infrastructure.py index 05e817319..c91a5d0fc 100644 --- a/tests/test_release_infrastructure.py +++ b/tests/test_release_infrastructure.py @@ -2,10 +2,13 @@ from __future__ import annotations import json +import os import re import shutil import subprocess +import sys from pathlib import Path +from typing import Any import pytest @@ -823,6 +826,163 @@ def test_release_workflow_permissions_are_least_privilege(): assert "id-token: write" not in body, job +def _powershell_shells() -> list[str]: + """Return installed PowerShell executables (`pwsh`, `powershell`), deduplicated.""" + shells: list[str] = [] + seen: set[str] = set() + for name in ("pwsh", "powershell"): + exe = shutil.which(name) + if not exe: + continue + key = str(Path(exe).resolve()).lower() + if key not in seen: + seen.add(key) + shells.append(exe) + return shells + + +def _write_preflight_fixture(tmp_path: Path, *, include_datasets: bool = True) -> dict[str, Path]: + """Create an isolated repo + cross-platform CLI shims to run `run_preflight.ps1`.""" + repo = tmp_path / "repo" + (repo / "scripts").mkdir(parents=True) + (repo / "tests").mkdir(parents=True) + shutil.copy2(ROOT / "run_preflight.ps1", repo / "run_preflight.ps1") + for script_name in ("check_commercial_manifest.py", "externalize_dashboard_assets.py"): + (repo / "scripts" / script_name).write_text("raise SystemExit(0)\n", encoding="utf-8") + for test_name in ("test_adaptive_context.py", "test_v2_service_binding.py"): + (repo / "tests" / test_name).write_text("", encoding="utf-8") + if include_datasets: + (repo / "eval" / "datasets").mkdir(parents=True) + for dataset_name in ("sample.jsonl", "codemem.jsonl"): + (repo / "eval" / "datasets" / dataset_name).write_text("{}\n", encoding="utf-8") + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + outside_dir = tmp_path / "outside" + outside_dir.mkdir() + trace_file = tmp_path / "trace.jsonl" + driver = tmp_path / "shim_driver.py" + driver.write_text( + """import json +import os +import sys +from pathlib import Path + +tool = sys.argv[1] +args = sys.argv[2:] +trace_path = Path(os.environ["PREFLIGHT_TRACE_FILE"]) +with trace_path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps({ + "tool": tool, + "args": args, + "cwd": os.getcwd(), + "ENGRAPHIS_INDEX_ROOTS": os.environ.get("ENGRAPHIS_INDEX_ROOTS"), + "PYTEST_ADDOPTS": os.environ.get("PYTEST_ADDOPTS"), + }) + "\\n") + +if os.environ.get("PREFLIGHT_STDERR_NOISE"): + sys.stderr.write("import time: simulated stderr noise\\n") + +if tool == "python" and len(args) >= 2 and args[0] == "-c": + code = args[1] + if "version_info" in code: + sys.stdout.write(os.environ.get("PREFLIGHT_PY_VERSION", "3.11") + "\\n") + raise SystemExit(0) + if "gettempdir" in code: + sys.stdout.write(os.environ.get("PREFLIGHT_PY_TEMPDIR", str(trace_path.parent)) + "\\n") + raise SystemExit(0) + if "import_module" in code and len(args) >= 3: + missing = {m for m in os.environ.get("PREFLIGHT_MISSING_MODULES", "").split(",") if m} + raise SystemExit(1 if args[2] in missing else 0) + +if tool == "python" and len(args) >= 2 and args[0] == "-m" and args[1] == "pytest": + rc = int(os.environ.get("PREFLIGHT_PYTEST_EXIT", "0")) + if rc != 0: + sys.stdout.write("stdout noise before failure\\n") + raise SystemExit(rc) + +raise SystemExit(0) +""", + encoding="utf-8", + ) + + for tool in ("python", "ruff", "pyright"): + if os.name == "nt": + shim = bin_dir / f"{tool}.cmd" + shim.write_text( + f'@echo off\r\n"{sys.executable}" "{driver}" {tool} %*\r\n', + encoding="utf-8", + ) + else: + shim = bin_dir / tool + shim.write_text( + f'#!/bin/sh\nexec "{sys.executable}" "{driver}" {tool} "$@"\n', + encoding="utf-8", + ) + shim.chmod(0o755) + + return { + "repo": repo, + "bin_dir": bin_dir, + "outside_dir": outside_dir, + "trace_file": trace_file, + } + + +def _invoke_preflight( + shell: str, + fixture: dict[str, Path], + ps_args: str, + extra_env: dict[str, str], +) -> tuple[subprocess.CompletedProcess[str], list[dict[str, Any]], dict[str, Any]]: + """Run `run_preflight.ps1` via PowerShell and return result, trace, and caller state.""" + if fixture["trace_file"].exists(): + fixture["trace_file"].unlink() + env = os.environ.copy() + env["PATH"] = os.pathsep.join([str(fixture["bin_dir"]), env.get("PATH", "")]) + env["PREFLIGHT_TRACE_FILE"] = str(fixture["trace_file"]) + for key in ( + "PREFLIGHT_STDERR_NOISE", + "PREFLIGHT_PY_VERSION", + "PREFLIGHT_MISSING_MODULES", + "PREFLIGHT_PYTEST_EXIT", + ): + env.pop(key, None) + env.update(extra_env) + + script_escaped = str(fixture["repo"] / "run_preflight.ps1").replace("'", "''") + cmd = ( + f"& '{script_escaped}' {ps_args}; " + "$exitCode = $LASTEXITCODE; " + "Write-Output ('__CALLER_STATE__=' + ((@{ " + "cwd = (Get-Location).Path; " + "index = $env:ENGRAPHIS_INDEX_ROOTS; " + "addopts = $env:PYTEST_ADDOPTS; " + "exit_code = $exitCode " + "}) | ConvertTo-Json -Compress))" + ) + proc = subprocess.run( + [shell, "-NoProfile", "-NonInteractive", "-Command", cmd], + cwd=str(fixture["outside_dir"]), + env=env, + capture_output=True, + text=True, + timeout=60, + ) + traces: list[dict[str, Any]] = [] + if fixture["trace_file"].exists(): + traces = [ + json.loads(line) + for line in fixture["trace_file"].read_text(encoding="utf-8").splitlines() + if line.strip() + ] + state: dict[str, Any] = {} + for line in proc.stdout.splitlines(): + if line.startswith("__CALLER_STATE__="): + state = json.loads(line.split("=", 1)[1]) + return proc, traces, state + + def test_run_preflight_ps1_aligns_with_ci_gates_and_checks_dependencies(): """The local pre-flight must run the offline CI gates, not a comfortable subset.""" script = _text("run_preflight.ps1") @@ -855,44 +1015,18 @@ def test_run_preflight_ps1_aligns_with_ci_gates_and_checks_dependencies(): assert 'Invoke-GateStep "Pyright Type Check"' in script assert 'Invoke-GateStep "Offline Unit Test Suite"' in script - # A missing dataset is a hard failure resolved against the repository root; it must - # never be folded into `continue`, which would report green after skipping the gate. + # A missing dataset must never be folded into `continue`, which would skip green. assert re.search(r"-not\s*\(\s*Test-Path[^\n]*\bcontinue\b", script) is None - dataset_check = script.split("$datasetPath = Join-Path $repoRoot", 1)[1].split( - "Invoke-GateStep", 1 - )[0] - assert "exit 1" in dataset_check - assert "continue" not in dataset_check - assert "Push-Location" in script - assert "Join-Path $repoRoot" in script - - # A real import catches a broken native wheel or a missing DLL; find_spec() alone would - # report those as installed. stdout is discarded so a stray line cannot become the - # PowerShell return value. assert "importlib.import_module(sys.argv[1])" in script - assert "$null = & $pythonExe -c" in script - - # CI runs pytest with ENGRAPHIS_INDEX_ROOTS covering the checkout and its temp dirs; - # the pre-flight mirrors that and restores the caller's value afterwards. assert "$env:ENGRAPHIS_INDEX_ROOTS" in script - assert "$previousIndexRoots = $env:ENGRAPHIS_INDEX_ROOTS" in script - assert "[pytest] -Workers ignored in -Fast mode" in script - - # CI runs pytest with the ini-file addopts neutralised and skip reasons visible. The - # full suite line has to carry all three so a local pyproject/pytest override cannot - # change what the pre-flight measured. - full_suite = [ - line for line in script.splitlines() if "$pytestArgs = " in line and '"tests/"' in line - ] - assert len(full_suite) == 1 - line = full_suite[0] - assert '"-o", "addopts="' in line - assert '"-q"' in line - assert '"-rs"' in line - assert '"--basetemp"' in line - - # A venv without the full-stack extras would skip the server/MCP tests silently, and - # ".[all,dev]" would drag in torch and the transcription/document extras that this gate does not need. + assert "$env:PYTEST_ADDOPTS" in script + + # CI neutralizes ini addopts, keeps skip reasons visible, and isolates basetemp. + for flag in ('"-o"', '"addopts="', '"-q"', '"-rs"', '"--basetemp"'): + assert flag in script, flag + + # A venv without the full-stack extras would skip server/MCP tests silently, while + # ".[all,dev]" drags in heavy optional ML extras that the offline gate does not need. assert '"fastapi", "httpx", "mcp", "multipart", "pydantic", "sqlite_vec", "uvicorn"' in script assert 'pip install -e ".[test]"' in script @@ -907,15 +1041,107 @@ def test_run_preflight_ps1_aligns_with_ci_gates_and_checks_dependencies(): def test_run_preflight_ps1_parses_without_powershell_syntax_errors(): - shell = shutil.which("pwsh") or shutil.which("powershell") - if not shell: + shells = _powershell_shells() + if not shells: pytest.skip("pwsh/powershell is not installed") script_path = str(ROOT / "run_preflight.ps1").replace("'", "''") cmd = ( "$errs = $null; [void][System.Management.Automation.Language.Parser]::ParseFile(" f"'{script_path}', $null, [ref]$errs); Write-Output $errs.Count" ) - proc = subprocess.run([shell, "-NoProfile", "-NonInteractive", "-Command", cmd], - capture_output=True, text=True, check=True) - assert proc.stdout.strip().splitlines()[-1] == "0" + for shell in shells: + proc = subprocess.run( + [shell, "-NoProfile", "-NonInteractive", "-Command", cmd], + capture_output=True, + text=True, + check=True, + ) + assert proc.stdout.strip().splitlines()[-1] == "0", shell + + +def test_run_preflight_ps1_tolerates_stderr_restores_caller_state_and_fails_on_noisy_gate( + tmp_path: Path, +): + shells = _powershell_shells() + if not shells: + pytest.skip("pwsh/powershell is not installed") + shell = shells[0] + fixture = _write_preflight_fixture(tmp_path) + base_env = { + "PREFLIGHT_STDERR_NOISE": "1", + "PREFLIGHT_PY_VERSION": "3.11", + "ENGRAPHIS_INDEX_ROOTS": "caller-sentinel", + "PYTEST_ADDOPTS": "--caller-addopt", + } + + # Passing -Fast run with noisy Python stderr and caller env vars set. + proc_ok, traces_ok, state_ok = _invoke_preflight(shell, fixture, "-Fast", base_env) + assert state_ok["exit_code"] == 0, proc_ok.stdout + proc_ok.stderr + assert Path(state_ok["cwd"]).resolve() == fixture["outside_dir"].resolve() + assert state_ok["index"] == "caller-sentinel" + assert state_ok["addopts"] == "--caller-addopt" + + pytest_calls = [t for t in traces_ok if t["tool"] == "python" and t["args"][:2] == ["-m", "pytest"]] + assert len(pytest_calls) == 1 + assert pytest_calls[0]["PYTEST_ADDOPTS"] is None + assert str(fixture["repo"].resolve()) in str(pytest_calls[0]["ENGRAPHIS_INDEX_ROOTS"]) + assert not any(t["tool"] == "pyright" for t in traces_ok) + + # Regression for Invoke-GateStep false-green: stdout + exit 3 must fail with exit 1 + # and still restore caller CWD / env variables. + fail_env = dict(base_env, PREFLIGHT_PYTEST_EXIT="3") + proc_fail, _, state_fail = _invoke_preflight(shell, fixture, "-Fast", fail_env) + assert state_fail["exit_code"] == 1 + assert "[FAIL] Offline Unit Test Suite exited with code 3" in proc_fail.stdout + assert Path(state_fail["cwd"]).resolve() == fixture["outside_dir"].resolve() + assert state_fail["index"] == "caller-sentinel" + assert state_fail["addopts"] == "--caller-addopt" + + +def test_run_preflight_ps1_enforces_python_version_extras_and_datasets(tmp_path: Path): + shells = _powershell_shells() + if not shells: + pytest.skip("pwsh/powershell is not installed") + shell = shells[0] + fixture = _write_preflight_fixture(tmp_path, include_datasets=False) + + # Python 3.9 without -Fast must explain -Fast before failing on missing Pyright/extras. + proc_py39, traces_py39, state_py39 = _invoke_preflight( + shell, fixture, "-SkipEval", {"PREFLIGHT_PY_VERSION": "3.9"} + ) + assert state_py39["exit_code"] == 1 + assert "found Python 3.9" in proc_py39.stdout + assert "-Fast" in proc_py39.stdout + assert not any(t["tool"] == "pyright" for t in traces_py39) + + # Python 3.11 missing a [test] extra passes Pyright (with --pythonpath) and fails Step 5 loudly. + proc_missing_mod, traces_mod, state_mod = _invoke_preflight( + shell, + fixture, + "-SkipEval -Workers 2", + {"PREFLIGHT_PY_VERSION": "3.11", "PREFLIGHT_MISSING_MODULES": "fastapi,xdist"}, + ) + assert state_mod["exit_code"] == 1 + assert "fastapi" in proc_missing_mod.stdout + assert 'pip install -e ".[test]"' in proc_missing_mod.stdout + pyright_calls = [t for t in traces_mod if t["tool"] == "pyright"] + assert len(pyright_calls) == 1 + assert "--pythonpath" in pyright_calls[0]["args"] + + # Python 3.11 with all [test] extras but missing xdist logs -Workers fallback and passes -SkipEval. + proc_no_xdist, _, state_no_xdist = _invoke_preflight( + shell, + fixture, + "-SkipEval -Workers 2", + {"PREFLIGHT_PY_VERSION": "3.11", "PREFLIGHT_MISSING_MODULES": "xdist"}, + ) + assert state_no_xdist["exit_code"] == 0, proc_no_xdist.stdout + proc_no_xdist.stderr + assert "[pytest] -Workers ignored because pytest-xdist is not installed" in proc_no_xdist.stdout + + # Missing required eval dataset in full mode must fail with exit 1 instead of skipping. + proc_missing_ds, _, state_ds = _invoke_preflight( + shell, fixture, "", {"PREFLIGHT_PY_VERSION": "3.11"} + ) + assert state_ds["exit_code"] == 1 + assert "Missing required eval dataset 'eval/datasets/sample.jsonl'" in proc_missing_ds.stdout