diff --git a/.containerignore b/.containerignore new file mode 100644 index 00000000..a361da89 --- /dev/null +++ b/.containerignore @@ -0,0 +1,16 @@ +# Exclude entire build context, then add back only doc-core pack inputs (see Containerfile COPY . .) +** + +!package.json +!package-lock.json +!astro.config.mjs +!tsconfig.json +!wrangler.jsonc +!cli +!cli/** +!src +!src/** +!public +!public/** +!textContent +!textContent/** diff --git a/.gitignore b/.gitignore index 5efbdc5a..a1ed0244 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,5 @@ coverage/ .wrangler/ -temp \ No newline at end of file +temp +tests/container/output \ No newline at end of file diff --git a/README.mdx b/README.mdx index 9d5cfc87..30104710 100644 --- a/README.mdx +++ b/README.mdx @@ -49,4 +49,7 @@ All commands are run from the root of the project, from a terminal: | `npm run build:cli` | Create a JS build of the documentation core CLI | | `npm run build:cli:watch` | Run the CLI builder in watch mode | | `npm run build:props` | Create a json file of your TsDoc compatible in code documentation | +| `npm run test:api:integration` | Build local doc-core, resolve the latest PatternFly Org commit once, then run two builds against the reviewed API snapshot | +| `ORG_REF= npm run test:api:integration` | Run that audit against a specific PatternFly Org commit | +| `npm run test:api:integration-dev` | Run the paired audit and update the reviewed SHA snapshot | | `patternfly-doc-core generate-package-props` | Create a package metadata artifact at `schema/props.json` containing its format version, package identity, and components | diff --git a/cli/__tests__/precomputeExamplesByPath.test.ts b/cli/__tests__/precomputeExamplesByPath.test.ts new file mode 100644 index 00000000..9514cb02 --- /dev/null +++ b/cli/__tests__/precomputeExamplesByPath.test.ts @@ -0,0 +1,120 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' + +import { + precomputeExamplesByPath, + writePrecomputedExamplesCache, +} from '../precomputeExamplesByPath.js' +import type { DocsConfig } from '../getConfig.js' + +describe('precomputeExamplesByPath', () => { + let tempRoot = '' + let configDir = '' + + beforeEach(async () => { + tempRoot = await mkdtemp(join(tmpdir(), 'pf-precompute-')) + configDir = join(tempRoot, 'site') + await mkdir(join(configDir, 'src', 'content', 'components', 'alert'), { + recursive: true, + }) + }) + + afterEach(async () => { + await rm(tempRoot, { recursive: true, force: true }) + }) + + it('reads the generated MDX for a normal Markdown content glob', async () => { + await writeFile( + join(configDir, 'src/content/components/alert/Alert.md'), + '---\nsection: components\nid: Alert\n---\n', + ) + const mdxPath = join(configDir, 'src/content/components/alert/Alert.mdx') + await writeFile( + mdxPath, + `--- +section: components +id: Alert +--- +### Default + +`, + ) + + const config: DocsConfig = { + outputDir: 'dist', + propsGlobs: [], + content: [ + { + name: 'content', + base: 'src/content', + pattern: '**/*.md', + version: 'v6', + }, + ], + } + + const byPath = await precomputeExamplesByPath(config, configDir, configDir) + + expect(byPath['src/content/components/alert/Alert.mdx']).toEqual([ + { exampleName: 'AlertBasic', title: 'Default' }, + ]) + }) + + it('uses generated MDX for a brace glob', async () => { + const base = join(configDir, 'src/content/components/button') + await mkdir(base, { recursive: true }) + await writeFile( + join(base, 'Button.md'), + `--- +section: components +id: Button +--- +### Should ignore + +`, + ) + await writeFile( + join(base, 'Button.mdx'), + `--- +section: components +id: Button +--- +### From MDX + +`, + ) + + const config: DocsConfig = { + outputDir: 'dist', + propsGlobs: [], + content: [ + { + name: 'content', + base: 'src/content', + pattern: '**/*.{md,mdx}', + version: 'v6', + }, + ], + } + + const byPath = await precomputeExamplesByPath(config, configDir, configDir) + const examples = byPath['src/content/components/button/Button.mdx'] ?? [] + expect(examples.map((item) => item.exampleName)).toEqual(['ButtonBasic']) + }) + + it('writes precomputed cache', async () => { + const outDir = join(tempRoot, 'dist') + const sample = { + 'src/content/components/alert/Alert.mdx': [ + { exampleName: 'AlertBasic', title: 'Default' }, + ], + } + await writePrecomputedExamplesCache(outDir, sample) + const raw = await readFile( + join(outDir, '.precomputed-examples.json'), + 'utf8', + ) + expect(JSON.parse(raw)).toEqual(sample) + }) +}) diff --git a/cli/cli.ts b/cli/cli.ts index 9db0448f..cfccfe93 100755 --- a/cli/cli.ts +++ b/cli/cli.ts @@ -2,6 +2,10 @@ /* eslint-disable no-console */ import { Command } from 'commander' import { build, dev, preview, sync } from 'astro' +import { + precomputeExamplesByPath, + writePrecomputedExamplesCache, +} from './precomputeExamplesByPath.js' import { dirname, join, resolve } from 'path' import { createCollectionContent, @@ -20,7 +24,6 @@ import { hasFile } from './hasFile.js' import { convertToMDX } from './convertToMDX.js' import { mkdir, copyFile } from 'fs/promises' import { fileURLToPath } from 'url' -import { fileExists } from './fileExists.js' const currentDir = process.cwd() const config = await getConfig(`${currentDir}/pf-docs.config.mjs`) @@ -120,30 +123,6 @@ async function transformMDContentToMDX(verbose = false) { ) } -async function initializeApiIndex(program: Command) { - const { verbose } = program.opts() - const templateIndexPath = join(astroRoot, 'cli', 'templates', 'apiIndex.json') - const targetIndexPath = join(absoluteOutputDir, 'apiIndex.json') - const indexExists = await fileExists(targetIndexPath) - - // early return if the file exists from a previous build - if (indexExists) { - if (verbose) { - console.log('apiIndex.json already exists, skipping initialization') - } - return - } - - try { - await copyFile(templateIndexPath, targetIndexPath) - if (verbose) { - console.log('Initialized apiIndex.json') - } - } catch (e: any) { - console.error('Error copying apiIndex.json template:', e) - } -} - async function buildProject(program: Command): Promise { const { verbose, apiOnly } = program.opts() @@ -167,7 +146,13 @@ async function buildProject(program: Command): Promise { await updateContent(program) await transformMDContentToMDX(verbose) await generateProps(program, true) - await initializeApiIndex(program) + + process.env.PF_DOCS_ROOT = currentDir + process.env.PF_DOCS_REQUIRE_PRECOMPUTED_EXAMPLES = 'true' + await writePrecomputedExamplesCache( + absoluteOutputDir, + await precomputeExamplesByPath(config, currentDir, astroRoot), + ) const docsOutputDir = join(absoluteOutputDir, 'docs') @@ -175,6 +160,10 @@ async function buildProject(program: Command): Promise { root: astroRoot, outDir: docsOutputDir, }) + await copyFile( + join(docsOutputDir, 'apiIndex.json'), + join(absoluteOutputDir, 'apiIndex.json'), + ) return config } @@ -239,9 +228,9 @@ program.command('start').action(async () => { if (apiOnly) { process.env.PF_API_ONLY = 'true' } + process.env.PF_DOCS_ROOT = currentDir await updateContent(program) await transformMDContentToMDX(verbose) - await initializeApiIndex(program) // if a props file hasn't been generated yet, but the consumer has propsData, it will cause a runtime error so to // prevent that we're just creating a props file regardless of what they say if one doesn't exist yet diff --git a/cli/precomputeExamplesByPath.ts b/cli/precomputeExamplesByPath.ts new file mode 100644 index 00000000..4481aaf5 --- /dev/null +++ b/cli/precomputeExamplesByPath.ts @@ -0,0 +1,81 @@ +import { mkdir, readFile, writeFile } from 'fs/promises' +import { join, relative, resolve } from 'path' +import { glob } from 'glob' +import type { DocsConfig } from './getConfig.js' +import { resolveContentEntryGlob } from './createCollectionContent.js' + +const LIVE_EXAMPLE_REGEX = /]*\ssrc=\{\s*(\w+)\s*\}[^>]*\/?>/g + +export type ExampleEntry = { exampleName: string; title: string | null } + +export type PrecomputedExamplesByPath = Record + +function extractExamplesWithTitles(body: string): ExampleEntry[] { + const examples: ExampleEntry[] = [] + const seen = new Set() + let match + + while ((match = LIVE_EXAMPLE_REGEX.exec(body)) !== null) { + const exampleName = match[1] + if (exampleName && !seen.has(exampleName)) { + seen.add(exampleName) + const textBeforeExample = body.substring(0, match.index) + const h3Matches = [...textBeforeExample.matchAll(/^### (.+)$/gm)] + const title = + h3Matches.length > 0 ? h3Matches[h3Matches.length - 1][1].trim() : null + examples.push({ exampleName, title }) + } + } + + return examples +} + +export async function precomputeExamplesByPath( + config: DocsConfig, + configDir: string, + astroRoot: string, +): Promise { + const repoRootDir = config.repoRoot + ? resolve(configDir, config.repoRoot) + : configDir + const byPath: PrecomputedExamplesByPath = {} + + for (const contentObj of config.content ?? []) { + const globPath = resolveContentEntryGlob( + contentObj, + configDir, + repoRootDir, + false, + ) + if (!globPath) { + continue + } + + const mdxGlobPath = globPath.replace(/\.(?:md|\{md,mdx\})$/, '.mdx') + const files = (await glob(mdxGlobPath)) + .filter((filePath) => filePath.endsWith('.mdx')) + .sort() + + for (const filePath of files) { + const examples = extractExamplesWithTitles( + await readFile(filePath, 'utf8'), + ) + if (examples.length > 0) { + byPath[relative(astroRoot, resolve(filePath)).replace(/\\/g, '/')] = + examples + } + } + } + + return byPath +} + +export async function writePrecomputedExamplesCache( + outputDir: string, + byPath: PrecomputedExamplesByPath, +): Promise { + const cachePath = join(outputDir, '.precomputed-examples.json') + await mkdir(outputDir, { recursive: true }) + await writeFile(cachePath, `${JSON.stringify(byPath, null, 2)}\n`) + return cachePath +} diff --git a/cli/templates/apiIndex.json b/cli/templates/apiIndex.json deleted file mode 100644 index 3e881bc1..00000000 --- a/cli/templates/apiIndex.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "versions": [], - "sections": {}, - "pages": {}, - "tabs": {} -} diff --git a/jest.org-audit.config.ts b/jest.org-audit.config.ts new file mode 100644 index 00000000..48c80fa3 --- /dev/null +++ b/jest.org-audit.config.ts @@ -0,0 +1,10 @@ +import type { Config } from 'jest' + +const config: Config = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: ['/tests/container'], + testMatch: ['/tests/container/**/*.test.ts'], +} + +export default config diff --git a/package.json b/package.json index a4d82e06..ac185031 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,9 @@ "lint": "eslint . --cache --cache-strategy content", "test": "jest", "test:watch": "jest --watch", + "test:api:integration-dev": "ORG_REF=${ORG_REF:-latest} tests/container/run.sh && npm run test:api:integration-jest -- --testPathPattern=apiIntegration -u", + "test:api:integration": "ORG_REF=${ORG_REF:-latest} tests/container/run.sh && npm run test:api:integration-jest -- --testPathPattern=apiIntegration", + "test:api:integration-jest": "jest -c jest.org-audit.config.ts", "semantic-release": "semantic-release", "cf-typegen": "wrangler types", "clean": "rm -rf dist .astro .wrangler" diff --git a/src/__tests__/pages/api/__tests__/[version]/[section]/[page]/[tab]/examples/[example].test.ts b/src/__tests__/pages/api/__tests__/[version]/[section]/[page]/[tab]/examples/[example].test.ts index 05f68bdf..4a9cfaff 100644 --- a/src/__tests__/pages/api/__tests__/[version]/[section]/[page]/[tab]/examples/[example].test.ts +++ b/src/__tests__/pages/api/__tests__/[version]/[section]/[page]/[tab]/examples/[example].test.ts @@ -63,7 +63,7 @@ jest.mock('../../../../../../../../../utils', () => ({ })) jest.mock('../../../../../../../../../utils/apiIndex/generate', () => ({ - generateAndWriteApiIndex: jest.fn().mockResolvedValue({ + generateApiIndex: jest.fn().mockResolvedValue({ versions: ['v6'], sections: { v6: ['components'] }, pages: { 'v6::components': ['alert'] }, diff --git a/src/content.config.ts b/src/content.config.ts index da7b224d..b1837e13 100644 --- a/src/content.config.ts +++ b/src/content.config.ts @@ -19,8 +19,8 @@ function defineContent(contentObj: CollectionDefinition) { 'core-component-docs': 'html', } - // Conversion is completed by the CLI before Astro loads collections. - const mdxPattern = pattern.replace(/\.md$/, '.mdx') + // Load only the generated form: source Markdown and generated MDX have the same IDs. + const mdxPattern = pattern.replace(/\.(?:md|\{md,mdx\})$/, '.mdx') const hasExternalFrontmatter = !!(frontmatterDefaults || frontmatterMapping) diff --git a/src/pages/api/[version]/[section]/[page]/[tab]/examples/[example].ts b/src/pages/api/[version]/[section]/[page]/[tab]/examples/[example].ts index a3d36b9a..d08a2037 100644 --- a/src/pages/api/[version]/[section]/[page]/[tab]/examples/[example].ts +++ b/src/pages/api/[version]/[section]/[page]/[tab]/examples/[example].ts @@ -3,7 +3,7 @@ import type { APIRoute, GetStaticPaths } from 'astro' import { access, readFile } from 'fs/promises' import { isAbsolute, resolve } from 'path' import { createJsonResponse, createTextResponse } from '../../../../../../../utils/apiHelpers' -import { generateAndWriteApiIndex } from '../../../../../../../utils/apiIndex/generate' +import { generateApiIndex } from '../../../../../../../utils/apiIndex/generate' import { getEnrichedCollections } from '../../../../../../../utils/apiRoutes/collections' import { findContentEntryFilePath } from '../../../../../../../utils/apiRoutes/contentMatching' import { extractImports, extractExampleFilePath } from '../../../../../../../utils/apiRoutes/exampleParsing' @@ -11,8 +11,8 @@ import { extractImports, extractExampleFilePath } from '../../../../../../../uti export const prerender = true export const getStaticPaths: GetStaticPaths = async () => { - // Generate index file (will be cached if already generated) - const index = await generateAndWriteApiIndex() + // Generate the build-time index. + const index = await generateApiIndex() const paths: { params: { @@ -153,4 +153,4 @@ export const GET: APIRoute = async ({ params }) => { 500 ) } -} \ No newline at end of file +} diff --git a/src/pages/api/[version]/[section]/[page]/[tab]/text.ts b/src/pages/api/[version]/[section]/[page]/[tab]/text.ts index 219dd2e4..8b10344b 100644 --- a/src/pages/api/[version]/[section]/[page]/[tab]/text.ts +++ b/src/pages/api/[version]/[section]/[page]/[tab]/text.ts @@ -1,7 +1,6 @@ /* eslint-disable no-console */ import type { APIRoute, GetStaticPaths } from 'astro' -import { generateAndWriteApiIndex } from '../../../../../../utils/apiIndex/generate' -import { getApiIndex } from '../../../../../../utils/apiIndex/get' +import { generateApiIndex } from '../../../../../../utils/apiIndex/generate' import { createJsonResponse, createTextResponse, @@ -12,9 +11,8 @@ import { findContentEntry } from '../../../../../../utils/apiRoutes/contentMatch export const prerender = true export const getStaticPaths: GetStaticPaths = async () => { - // Generate index file for server-side routes to use - // This runs once during build when getCollection() is available - const index = await generateAndWriteApiIndex() + // Build static paths from the current collections. + const index = await generateApiIndex() const paths: { params: { version: string; section: string; page: string; tab: string } @@ -61,8 +59,8 @@ export const GET: APIRoute = async ({ params }) => { ) } - // Validate using index first (fast path for 404s) - const index = await getApiIndex() + // Generate the current index while prerendering. + const index = await generateApiIndex() // Check if version exists if (!index.versions.includes(version)) { diff --git a/src/pages/api/component-index.json.ts b/src/pages/api/component-index.json.ts index 063638e6..fae16a34 100644 --- a/src/pages/api/component-index.json.ts +++ b/src/pages/api/component-index.json.ts @@ -33,8 +33,7 @@ function pageToPascalCase(page: string): string { export const GET: APIRoute = async () => { try { - // This route can prerender before the routes that write apiIndex.json. - // Generate from current collections rather than a previous build's index. + // Generate from current collections and the prepared examples snapshot. const index = await generateApiIndex() // Frontmatter may document both component and interface props records. diff --git a/src/pages/api/index.ts b/src/pages/api/index.ts index 95bfbfa0..a5cd309c 100644 --- a/src/pages/api/index.ts +++ b/src/pages/api/index.ts @@ -2,15 +2,15 @@ * PatternFly Documentation API * * Architecture: - * - Build-time: Static API index generated from content collections (data.ts) + * - Build-time: Astro emits the static API index from content collections * - Runtime: Routes fetch /apiIndex.json (prerendered static file) via HTTP * - Workers-compatible: No Node.js filesystem APIs in SSR handlers * - Optimized bundle: Runtime handlers don't bundle the 500KB+ index data * - Flattened structure: Subsections are kebab-cased and encoded into page names with underscores (e.g., "forms_checkbox") * * Data Flow: - * 1. Build: generateApiIndex() → data.ts + apiIndex.json - * 2. Static paths: getStaticPaths() uses getApiIndex() (build-time import) + * 1. Build: prepare examples, then Astro emits /apiIndex.json + * 2. Static paths: generateApiIndex() reads the prepared snapshot * 3. SSR handlers: GET handlers use fetchApiIndex(url) (runtime fetch) */ import type { APIRoute } from 'astro' @@ -510,7 +510,7 @@ export const GET: APIRoute = async () => 'GET /api/v6/components/forms_checkbox/react/examples/CheckboxBasic → (example code)', ], architecture: { - buildTime: 'Static index generated to data.ts and apiIndex.json', + buildTime: 'Astro emits the static apiIndex.json from prepared content', runtime: 'SSR routes fetch /apiIndex.json to avoid bundling data into Workers', optimization: 'Workers bundle is ~110K instead of 500KB+ with embedded data', flattenedStructure: 'Subsections are kebab-cased and encoded into page names with underscores (e.g., "forms_checkbox")', diff --git a/src/utils/__tests__/apiIndex.test.ts b/src/utils/__tests__/apiIndex.test.ts index c96de6aa..41ecf016 100644 --- a/src/utils/__tests__/apiIndex.test.ts +++ b/src/utils/__tests__/apiIndex.test.ts @@ -2,7 +2,7 @@ * Tests for API Index utilities * * Architecture: - * - Build-time: getApiIndex() imports static data from data.ts + * - Build-time: getApiIndex() reads the emitted index file * - Runtime: fetchApiIndex() fetches /apiIndex.json via HTTP */ import { getApiIndex, getVersions, getSections, getPages, getTabs, getExamples } from '../apiIndex/get' diff --git a/src/utils/__tests__/componentIndex.test.ts b/src/utils/__tests__/componentIndex.test.ts index 46f139cf..7045b918 100644 --- a/src/utils/__tests__/componentIndex.test.ts +++ b/src/utils/__tests__/componentIndex.test.ts @@ -1,41 +1,55 @@ -import { readFile, writeFile } from 'fs/promises' +import { readFile } from 'fs/promises' import { getCollection } from 'astro:content' import { GET } from '../../pages/api/component-index.json' -import { - generateAndWriteApiIndex, - generateApiIndex, -} from '../apiIndex/generate' +import { generateApiIndex } from '../apiIndex/generate' jest.mock('fs/promises', () => ({ readFile: jest.fn(), - writeFile: jest.fn(), - mkdir: jest.fn(), })) jest.mock('astro:content', () => ({ getCollection: jest.fn() })) jest.mock('../../content', () => ({ content: [{ name: 'react-component-docs', version: 'v6' }] })) jest.mock('../getOutputDir', () => ({ getOutputDir: jest.fn().mockResolvedValue('/output') })) jest.mock('../extractReactTokens', () => ({ extractReactTokens: jest.fn().mockResolvedValue([]) })) +const precomputedExamples = { + 'layouts/Flex.mdx': [{ exampleName: 'FlexBasic', title: 'Basic flex' }], +} + +const propsData = Object.fromEntries( + ['Nav', 'NavItem', 'FileUpload', 'FileUploadField', 'ActionGroup', 'Form', 'Flex', 'FlexItem', 'Chip-deprecated', 'ChipGroup-deprecated', 'Table', 'DataViewTable'].map((name) => [name, { name, props: [] }]), +) +const originalRequirePrecomputed = process.env.PF_DOCS_REQUIRE_PRECOMPUTED_EXAMPLES + beforeEach(() => { jest.clearAllMocks() jest.spyOn(console, 'log').mockImplementation(() => {}) + process.env.PF_DOCS_REQUIRE_PRECOMPUTED_EXAMPLES = 'true' ;(getCollection as jest.Mock).mockResolvedValue([ - { data: { section: 'components', id: 'Navigation', tab: 'react', propComponents: ['Nav', 'NavItem', 'NavMissingProps'] }, body: '', filePath: '/components/Nav.md' }, - { data: { section: 'components', subsection: 'file-upload', id: 'Simple file upload', tab: 'react', propComponents: ['FileUpload', 'FileUploadField'] }, body: '', filePath: '/components/FileUpload.md' }, - { data: { section: 'components', id: 'Form', tab: 'react', propComponents: ['ActionGroup', 'Form'] }, body: '', filePath: '/components/Form.md' }, - { data: { section: 'components', id: 'Navigation', tab: 'html' }, body: '', filePath: '/html/Nav.md' }, - { data: { section: 'components', id: 'Navigation', tab: 'react', propComponents: ['OldNav'] }, body: '', filePath: '/deprecated/components/Nav.md' }, - { data: { section: 'components', id: 'Chip', tab: 'react-deprecated', propComponents: ['Chip', 'ChipGroup'] }, body: '', filePath: '/deprecated/components/Chip.md' }, - { data: { section: 'components', id: 'Table', tab: 'react', propComponents: ['Table'] }, body: '', filePath: '/components/Table.md' }, - { data: { section: 'extensions', subsection: 'data-view', id: 'Table', tab: 'react', propComponents: ['DataViewTable'] }, body: '', filePath: '/extensions/DataViewTable.md' }, - { data: { section: 'foundations-and-styles', subsection: 'layouts', id: 'Flex', tab: 'react', propComponents: ['Flex', 'FlexItem'] }, body: '### Basic flex\n', filePath: '/layouts/Flex.md' }, + { data: { section: 'components', id: 'Navigation', tab: 'react', propComponents: ['Nav', 'NavItem', 'NavMissingProps'] }, body: '', filePath: 'components/Nav.mdx' }, + { data: { section: 'components', subsection: 'file-upload', id: 'Simple file upload', tab: 'react', propComponents: ['FileUpload', 'FileUploadField'] }, body: '', filePath: 'components/FileUpload.mdx' }, + { data: { section: 'components', id: 'Form', tab: 'react', propComponents: ['ActionGroup', 'Form'] }, body: '', filePath: 'components/Form.mdx' }, + { data: { section: 'components', id: 'Navigation', tab: 'html' }, body: '', filePath: 'html/Nav.mdx' }, + { data: { section: 'components', id: 'Navigation', tab: 'react', propComponents: ['OldNav'] }, body: '', filePath: 'deprecated/components/Nav.mdx' }, + { data: { section: 'components', id: 'Chip', tab: 'react-deprecated', propComponents: ['Chip', 'ChipGroup'] }, body: '', filePath: 'deprecated/components/Chip.mdx' }, + { data: { section: 'components', id: 'Table', tab: 'react', propComponents: ['Table'] }, body: '', filePath: 'components/Table.mdx' }, + { data: { section: 'extensions', subsection: 'data-view', id: 'Table', tab: 'react', propComponents: ['DataViewTable'] }, body: '', filePath: 'extensions/DataViewTable.mdx' }, + { data: { section: 'foundations-and-styles', subsection: 'layouts', id: 'Flex', tab: 'react', propComponents: ['Flex', 'FlexItem'] }, body: '', filePath: 'layouts/Flex.mdx' }, ]) - ;(readFile as jest.Mock).mockResolvedValue(JSON.stringify(Object.fromEntries( - ['Nav', 'NavItem', 'FileUpload', 'FileUploadField', 'ActionGroup', 'Form', 'Flex', 'FlexItem', 'Chip-deprecated', 'ChipGroup-deprecated', 'Table', 'DataViewTable'].map((name) => [name, { name, props: [] }]), + ;(readFile as jest.Mock).mockImplementation((filePath: string) => Promise.resolve(JSON.stringify( + filePath === '/output/.precomputed-examples.json' + ? precomputedExamples + : propsData, ))) }) -afterEach(() => jest.restoreAllMocks()) +afterEach(() => { + if (originalRequirePrecomputed === undefined) { + delete process.env.PF_DOCS_REQUIRE_PRECOMPUTED_EXAMPLES + } else { + process.env.PF_DOCS_REQUIRE_PRECOMPUTED_EXAMPLES = originalRequirePrecomputed + } + jest.restoreAllMocks() +}) it('retains active frontmatter mappings without HTML or deprecated entries overwriting them', async () => { const index = await generateApiIndex() @@ -62,26 +76,27 @@ it('indexes current collections, preserving page keys and exposing documented me expect(components.DataViewTable).toMatchObject({ page: 'data-view_table', component: 'DataViewTable', hasProps: true }) expect(components.Flex).toMatchObject({ page: 'layouts_flex', tabs: ['react'], exampleCount: 1 }) expect(components.Form).toMatchObject({ hasProps: true }) - // Only props are read from disk: an old apiIndex.json cannot seed the new index. - expect(readFile).toHaveBeenCalledTimes(1) + expect(readFile).toHaveBeenCalledTimes(2) + expect(readFile).toHaveBeenCalledWith('/output/.precomputed-examples.json', 'utf8') expect(readFile).toHaveBeenCalledWith('/output/props.json', 'utf-8') }) -it('coalesces concurrent API index writes', async () => { - const [firstIndex, secondIndex] = await Promise.all([ - generateAndWriteApiIndex(), - generateAndWriteApiIndex(), - ]) +it('requires the prepared snapshot during a CLI build', async () => { + const missingCache = Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) + ;(readFile as jest.Mock).mockRejectedValueOnce(missingCache) - expect(firstIndex).toEqual(secondIndex) - expect(writeFile).toHaveBeenCalledTimes(1) + await expect(generateApiIndex()).rejects.toMatchObject({ code: 'ENOENT' }) }) it('only advertises deprecated props when the suffixed record exists', async () => { - ;(readFile as jest.Mock).mockResolvedValue(JSON.stringify({ - Chip: { name: 'Chip', props: [] }, - 'ChipGroup-deprecated': { name: 'ChipGroup', props: [] }, - })) + ;(readFile as jest.Mock).mockImplementation((filePath: string) => Promise.resolve(JSON.stringify( + filePath === '/output/.precomputed-examples.json' + ? precomputedExamples + : { + Chip: { name: 'Chip', props: [] }, + 'ChipGroup-deprecated': { name: 'ChipGroup', props: [] }, + }, + ))) const response = await GET({} as any) const { components } = await response.json() diff --git a/src/utils/__tests__/getOutputDir.test.ts b/src/utils/__tests__/getOutputDir.test.ts new file mode 100644 index 00000000..3af66455 --- /dev/null +++ b/src/utils/__tests__/getOutputDir.test.ts @@ -0,0 +1,47 @@ +/** + * @jest-environment node + */ +import { mkdtemp, rm, writeFile } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' + +describe('getOutputDir', () => { + const originalDocsRoot = process.env.PF_DOCS_ROOT + let siteRoot = '' + let otherCwd = '' + + afterEach(async () => { + if (originalDocsRoot === undefined) { + delete process.env.PF_DOCS_ROOT + } else { + process.env.PF_DOCS_ROOT = originalDocsRoot + } + if (siteRoot) { + await rm(siteRoot, { recursive: true, force: true }) + } + if (otherCwd) { + await rm(otherCwd, { recursive: true, force: true }) + } + }) + + it('resolves outputDir from PF_DOCS_ROOT instead of process.cwd()', async () => { + siteRoot = await mkdtemp(join(tmpdir(), 'pf-docs-site-')) + otherCwd = await mkdtemp(join(tmpdir(), 'pf-docs-other-')) + + await writeFile( + join(siteRoot, 'pf-docs.config.mjs'), + "export const config = { outputDir: 'dist', content: [] }\n", + ) + + process.env.PF_DOCS_ROOT = siteRoot + const previousCwd = process.cwd() + process.chdir(otherCwd) + + try { + const { getOutputDir } = await import('../getOutputDir') + await expect(getOutputDir()).resolves.toBe(join(siteRoot, 'dist')) + } finally { + process.chdir(previousCwd) + } + }) +}) diff --git a/src/utils/apiIndex/generate.ts b/src/utils/apiIndex/generate.ts index a393858e..741333cc 100644 --- a/src/utils/apiIndex/generate.ts +++ b/src/utils/apiIndex/generate.ts @@ -1,6 +1,6 @@ /* eslint-disable no-console */ import { join } from 'path' -import { writeFile, mkdir } from 'fs/promises' +import { readFile } from 'fs/promises' import { getCollection } from 'astro:content' import type { CollectionKey } from 'astro:content' import { content } from '../../content' @@ -106,16 +106,29 @@ function extractExamplesWithTitles( return examples } +type PrecomputedExamplesByPath = Record< + string, + { exampleName: string; title: string | null }[] +> + +async function readPrecomputedExamplesByPath( + outputDir: string, +): Promise { + const cachePath = join(outputDir, '.precomputed-examples.json') + return JSON.parse(await readFile(cachePath, 'utf8')) as PrecomputedExamplesByPath +} + /** - * Generates API index by analyzing all content collections - * Extracts versions, sections, pages, and tabs into a hierarchical structure - * This runs during build time when getCollection() is available - * - * @returns Promise resolving to complete API index structure + * Builds API index by analyzing all content collections. */ export async function generateApiIndex(): Promise { console.log('Generating API index from content collections...') + const precomputedExamplesByPath = + process.env.PF_DOCS_REQUIRE_PRECOMPUTED_EXAMPLES === 'true' + ? await readPrecomputedExamplesByPath(await getOutputDir()) + : null + const index: ApiIndex = { versions: [], sections: {}, @@ -141,11 +154,9 @@ export async function generateApiIndex(): Promise { .filter((entry: any) => entry.version === version) .map((entry: any) => entry.name as CollectionKey) - const collections = await Promise.all( - collectionsToFetch.map((name) => getCollection(name)), - ) - + const collections = await Promise.all(collectionsToFetch.map((name) => getCollection(name))) const flatEntries = collections.flat() + flatEntries.sort((a, b) => String(a.filePath).localeCompare(String(b.filePath))) // Collect sections, pages, and tabs in a single pass const sections = new Set() @@ -181,8 +192,7 @@ export async function generateApiIndex(): Promise { sectionPages[pageKey].add(page) // Collect tab - const entryTab = - entry.data.tab || entry.data.source || getDefaultTabForApi(entry.filePath) + const entryTab = entry.data.tab || entry.data.source || getDefaultTabForApi(entry.filePath) const tab = addDemosOrDeprecated(entryTab, entry.filePath) if (!pageTabs[tabKey]) { pageTabs[tabKey] = new Set() @@ -196,9 +206,10 @@ export async function generateApiIndex(): Promise { deprecatedPropComponents[tabKey] = [...new Set(entry.data.propComponents)] } - // Collect examples for this tab const exampleKey = `${tabKey}::${tab}` - const examplesWithTitles = extractExamplesWithTitles(entry.body || '') + const examplesWithTitles = precomputedExamplesByPath + ? precomputedExamplesByPath[entry.filePath || ''] || [] + : extractExamplesWithTitles(entry.body || '') if (examplesWithTitles.length > 0) { tabExamples[exampleKey] = examplesWithTitles } @@ -242,9 +253,11 @@ export async function generateApiIndex(): Promise { index.tabs[key] = Array.from(tabs).sort(sortSources) }) - Object.entries(tabExamples).forEach(([key, examples]) => { - index.examples[key] = examples - }) + Object.entries(tabExamples) + .sort(([a], [b]) => a.localeCompare(b)) + .forEach(([key, examples]) => { + index.examples[key] = examples + }) // Add CSS token objects to index Object.entries(pageCss).forEach(([key, tokens]) => { @@ -254,48 +267,3 @@ export async function generateApiIndex(): Promise { return index } - -/** - * Writes API index to a apiIndex.json file in the user defined output directory - * This file is used by server-side API routes to avoid runtime getCollection() calls - * - * @param index - The API index structure to write - */ -export async function writeApiIndex(index: ApiIndex): Promise { - const outputDir = await getOutputDir() - const indexPath = join(outputDir, 'apiIndex.json') - - try { - await mkdir(outputDir, { recursive: true }) - await writeFile(indexPath, JSON.stringify(index, null, 2)) - console.log(`✓ Generated API index with ${index.versions.length} versions`) - } catch (error) { - console.error('Could not write API index file:', error) - throw error - } -} - -/** - * Generates and writes API index in a single operation - * Called during build in getStaticPaths to ensure index exists for server routes - * - * @returns Promise resolving to the generated API index - */ -let indexWriteInFlight: Promise | null = null - -export async function generateAndWriteApiIndex(): Promise { - if (indexWriteInFlight) { - return indexWriteInFlight - } - - indexWriteInFlight = generateApiIndex() - .then(async (index) => { - await writeApiIndex(index) - return index - }) - .finally(() => { - indexWriteInFlight = null - }) - - return indexWriteInFlight -} diff --git a/src/utils/apiIndex/index.ts b/src/utils/apiIndex/index.ts index e338f619..53b20815 100644 --- a/src/utils/apiIndex/index.ts +++ b/src/utils/apiIndex/index.ts @@ -3,10 +3,8 @@ * * This module provides two different ways to access the API index: * - * 1. Build-time (getApiIndex): Imports static data from data.ts - * - Used in getStaticPaths() and other build-time code - * - Has access to Node.js filesystem APIs - * - Bundles data into the module + * 1. Prerender (generateApiIndex): Builds from content collections and the + * precomputed examples snapshot. Astro emits /apiIndex.json from this result. * * 2. Runtime (fetchApiIndex): Fetches /apiIndex.json via HTTP * - Used in SSR route handlers (Cloudflare Workers) @@ -17,8 +15,6 @@ export { generateApiIndex, - writeApiIndex, - generateAndWriteApiIndex, type ApiIndex, } from './generate' diff --git a/src/utils/getOutputDir.ts b/src/utils/getOutputDir.ts index 698950c6..d89bd2b6 100644 --- a/src/utils/getOutputDir.ts +++ b/src/utils/getOutputDir.ts @@ -1,8 +1,9 @@ -import { join } from 'path' +import { join, resolve } from 'path' import { getConfig } from '../../cli/getConfig' export async function getOutputDir(): Promise { - const config = await getConfig(join(process.cwd(), 'pf-docs.config.mjs')) + const docsRoot = process.env.PF_DOCS_ROOT || process.cwd() + const config = await getConfig(join(docsRoot, 'pf-docs.config.mjs')) if (!config) { throw new Error( 'No config found, please run the `setup` command or manually create a pf-docs.config.mjs file', @@ -15,5 +16,5 @@ export async function getOutputDir(): Promise { ) } - return config.outputDir + return resolve(docsRoot, config.outputDir) } diff --git a/tests/container/Containerfile b/tests/container/Containerfile new file mode 100644 index 00000000..83f4015f --- /dev/null +++ b/tests/container/Containerfile @@ -0,0 +1,49 @@ +# syntax=docker/dockerfile:1 + +# --- Stage 1: Build local doc-core from source --- +FROM node:22-bookworm AS doc-core-builder +WORKDIR /app + +COPY package*.json ./ +RUN npm ci + +COPY . . +RUN npm run build:cli +RUN npm pack --pack-destination=/tmp + +# --- Stage 2: Cache patternfly-org setup --- +FROM node:22-bookworm AS org-base + +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* +RUN corepack enable + +WORKDIR /patternfly-org + +ARG ORG_REPO=https://github.com/patternfly/patternfly-org.git +ARG ORG_REF + +# run.sh resolves `latest` once and passes its commit SHA here. +RUN git init . \ + && git remote add origin "${ORG_REPO}" \ + && git fetch --depth 1 origin "${ORG_REF}" \ + && git checkout --detach FETCH_HEAD + +RUN --mount=type=cache,target=/usr/local/share/.cache/yarn/v6 \ + yarn install --immutable + +# --- Stage 3: Prepare a fresh-runtime org build with local doc-core --- +FROM org-base AS audit-runner + +WORKDIR /patternfly-org +ENV NODE_OPTIONS="--max-old-space-size=8192" + +COPY --from=doc-core-builder /tmp/patternfly-patternfly-doc-core-*.tgz /tmp/patternfly-doc-core.tgz + +RUN yarn workspace site add @patternfly/patternfly-doc-core@file:/tmp/patternfly-doc-core.tgz \ + && yarn workspace site run init:docs + +ENTRYPOINT [ \ + "/bin/sh", \ + "-ec", \ + "cd /patternfly-org; dist=packages/site/dist; yarn ci:build:doc-core; test -s \"${dist}/apiIndex.json\"; test -s \"${dist}/docs/apiIndex.json\"; mkdir -p /output; cp \"${dist}/apiIndex.json\" /output/apiIndex.dist-root.json; cp \"${dist}/docs/apiIndex.json\" /output/apiIndex.docs.json" \ +] diff --git a/tests/container/__snapshots__/apiIntegration.test.ts.snap b/tests/container/__snapshots__/apiIntegration.test.ts.snap new file mode 100644 index 00000000..645f9961 --- /dev/null +++ b/tests/container/__snapshots__/apiIntegration.test.ts.snap @@ -0,0 +1,3 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`PatternFly Org apiIndex integration matches canonical SHA-1 snapshot: canonical-docs-sha1 1`] = `"90929da8f5e957b031296e8420e44a530a5a4426"`; diff --git a/tests/container/apiIntegration.test.ts b/tests/container/apiIntegration.test.ts new file mode 100644 index 00000000..e7698ec3 --- /dev/null +++ b/tests/container/apiIntegration.test.ts @@ -0,0 +1,65 @@ +/** + * @jest-environment node + */ +import { existsSync, readFileSync } from 'fs' +import { join } from 'path' +import crypto from 'crypto' + +const outputDir = join(__dirname, 'output') +const run1Docs = join(outputDir, 'run1', 'apiIndex.docs.json') +const run2Docs = join(outputDir, 'run2', 'apiIndex.docs.json') +const run1Root = join(outputDir, 'run1', 'apiIndex.dist-root.json') +const run2Root = join(outputDir, 'run2', 'apiIndex.dist-root.json') + +function canonicalizeJson(value: any): any { + if (Array.isArray(value)) { + return value.map(canonicalizeJson) + } + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonicalizeJson(value[key])]) + ) + } + return value +} + +function getCanonicalSha1(filePath: string): string { + const raw = readFileSync(filePath, 'utf-8') + const parsed = JSON.parse(raw) + const canonical = JSON.stringify(canonicalizeJson(parsed)) + + return crypto.createHash('sha1').update(canonical).digest('hex') +} + +describe('PatternFly Org apiIndex integration', () => { + beforeAll(() => { + for (const filePath of [run1Docs, run2Docs, run1Root, run2Root]) { + if (!existsSync(filePath)) { + throw new Error( + `Missing ${filePath}. Run tests/container/run.sh first.`, + ) + } + } + }) + + it('matches canonical SHA-1 snapshot', () => { + expect(getCanonicalSha1(run1Docs)).toMatchSnapshot('canonical-docs-sha1') + }) + + it('emits examples and identical docs indexes in consecutive builds', () => { + const firstDocs = readFileSync(run1Docs, 'utf8') + const firstIndex = JSON.parse(firstDocs) as { + examples: Record + } + + expect(Object.keys(firstIndex.examples)).not.toHaveLength(0) + expect(readFileSync(run2Docs, 'utf8')).toBe(firstDocs) + }) + + it("copies Astro's completed index to the root output", () => { + expect(readFileSync(run1Root, 'utf8')).toBe(readFileSync(run1Docs, 'utf8')) + expect(readFileSync(run2Root, 'utf8')).toBe(readFileSync(run2Docs, 'utf8')) + }) +}) diff --git a/tests/container/run.sh b/tests/container/run.sh new file mode 100755 index 00000000..55af8359 --- /dev/null +++ b/tests/container/run.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Resolve one PatternFly Org revision, then run two fresh builds with local doc-core. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +OUTPUT_DIR="${ROOT_DIR}/tests/container/output" +ORG_REF="${ORG_REF:-latest}" +ORG_REPO="${ORG_REPO:-https://github.com/patternfly/patternfly-org.git}" + +if [[ "${ORG_REF}" == "latest" ]]; then + ORG_REF="$(git ls-remote "${ORG_REPO}" HEAD | awk 'NR == 1 { print $1 }')" + if [[ -z "${ORG_REF}" ]]; then + echo "Error: Could not resolve PF-Org HEAD for ${ORG_REPO}." >&2 + exit 1 + fi +fi + +if command -v podman >/dev/null 2>&1; then + CONTAINER_ENGINE="podman" + VOL_FLAG=":Z" +elif command -v docker >/dev/null 2>&1; then + CONTAINER_ENGINE="docker" + VOL_FLAG="" +else + echo "Error: Neither podman nor docker found." >&2 + exit 1 +fi + +rm -rf "${OUTPUT_DIR}" +mkdir -p "${OUTPUT_DIR}/run1" "${OUTPUT_DIR}/run2" + +cd "${ROOT_DIR}" + +echo "==> preparing org apiIndex audit image (${CONTAINER_ENGINE}, ORG_REF=${ORG_REF})" +${CONTAINER_ENGINE} build \ + --build-arg ORG_REF="${ORG_REF}" \ + --build-arg ORG_REPO="${ORG_REPO}" \ + -f tests/container/Containerfile \ + -t pf-doc-core-audit:latest \ + . + +for run_id in run1 run2; do + echo "==> org apiIndex build ${run_id} (${CONTAINER_ENGINE})" + ${CONTAINER_ENGINE} run --rm \ + -v "${OUTPUT_DIR}/${run_id}:/output${VOL_FLAG}" \ + pf-doc-core-audit:latest +done + +echo "==> Artifacts under ${OUTPUT_DIR}"