From 48752749dbf4247b35e7a691bce5d71b9ed62925 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Thu, 5 Nov 2026 09:58:27 +0700 Subject: [PATCH] feat: export Backstage TechDocs projects (#145) Add `codewiki generate --techdocs [--techdocs-dir DIR]` and a standalone `codewiki techdocs [DOCS_DIR] [OUT_DIR]` (no LLM calls) that turn CodeWiki output into an MkDocs project `techdocs-cli generate` builds as is: mkdocs.yml (techdocs-core, nav from the module tree, mermaid superfences) plus docs/ with the overview as index.md. Pages get front matter, a provenance banner and a collapsible list of the documented components linked to the source at the documented commit. JSON files and temp/ are not published, the source docs are never modified, and only a docs/ folder created by an earlier export is replaced. Based on the generator proposed in the issue, with the double docs/docs folder on re-runs and the nav membership check fixed. --- .gitignore | 1 + CHANGELOG.md | 9 + README.md | 8 + codewiki/cli/adapters/doc_generator.py | 40 +++ codewiki/cli/commands/generate.py | 30 ++ codewiki/cli/commands/techdocs.py | 75 +++++ codewiki/cli/main.py | 2 + codewiki/cli/techdocs_generator.py | 439 +++++++++++++++++++++++++ codewiki/cli/utils/instructions.py | 10 + guides/README.md | 1 + guides/backstage-techdocs.md | 107 ++++++ guides/cli-reference.md | 24 ++ tests/test_techdocs.py | 236 +++++++++++++ 13 files changed, 982 insertions(+) create mode 100644 codewiki/cli/commands/techdocs.py create mode 100644 codewiki/cli/techdocs_generator.py create mode 100644 guides/backstage-techdocs.md create mode 100644 tests/test_techdocs.py diff --git a/.gitignore b/.gitignore index 9390c427..aed831bd 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ tests/* !tests/test_min_module_size.py !tests/test_doc_write_guard.py !tests/test_overview_generation.py +!tests/test_techdocs.py # Jupyter *.ipynb diff --git a/CHANGELOG.md b/CHANGELOG.md index 525dace2..47d174ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ uses [Semantic Versioning](https://semver.org/). ### Added +- **Backstage TechDocs output.** `codewiki generate --techdocs` (or + `codewiki techdocs DOCS_DIR OUT_DIR` on existing docs, with no LLM calls) + writes an MkDocs project that `techdocs-cli generate` builds as is: + `mkdocs.yml` with the `techdocs-core` plugin and the module tree as the nav, + plus `docs/` with the overview as `index.md`. Each page gets front matter, a + provenance banner (commit, date, model) and a collapsible list of the + components it documents, linked to the source at the documented commit. + JSON files and `temp/` are left out + ([#145](https://github.com/FSoft-AI4Code/CodeWiki/issues/145)). - **Google provider.** `google` (Gemini API key) and `google-vertex` (Vertex AI with Google credentials: ADC or Workload Identity Federation, plus `--google-project` and `--google-location`) run on Gemini without a base diff --git a/README.md b/README.md index fac564af..594eeb31 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Atlas Cloud, and Codex are also supported. See [Providers and models](./guides/p cd /path/to/your/project codewiki generate # writes ./docs/ codewiki generate --github-pages # also writes an HTML viewer +codewiki generate --techdocs # also exports a Backstage TechDocs site to ./techdocs codewiki generate --update # later: refresh after code changes ``` @@ -181,6 +182,12 @@ its sub-modules (`auth.md`, `auth/login.md`). `--flat` puts every page in `./docs` instead, which helps small models that get relative links wrong. `--update` keeps the layout the docs were generated with. +`--techdocs` (or `codewiki techdocs` on existing docs) also writes a +[Backstage TechDocs](https://backstage.io/docs/features/techdocs/) project, +`./techdocs/mkdocs.yml` plus `./techdocs/docs/`, with `overview.md` as +`index.md` and the module tree as the navigation. See +[Backstage TechDocs](./guides/backstage-techdocs.md). + This repository's own output is checked in under [`./docs/`](./docs/). ## Benchmark results @@ -244,6 +251,7 @@ JavaScript, TypeScript). Artifacts lift the build and testing topics, which | [Artifact-aware generation](./guides/artifact-aware-generation.md) | what gets documented beyond code, and how to tune it | | [Incremental updates](./guides/incremental-updates.md) | how `--update` works, thresholds, the update record | | [MCP / IDE-driven mode](./guides/mcp-ide-mode.md) | Cursor, Claude Desktop, Claude Code, CodeBuddy | +| [Backstage TechDocs](./guides/backstage-techdocs.md) | publish the docs in a Backstage developer portal | | [Development guide](./guides/development.md) | layout, pipeline, adding a language, tests, releasing | | [Docker setup](./guides/docker.md) | the web application in a container | | [CodeWikiBench](https://github.com/FSoft-AI4Code/CodeWikiBench) | the benchmark | diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py index e514eabe..2a012d3a 100644 --- a/codewiki/cli/adapters/doc_generator.py +++ b/codewiki/cli/adapters/doc_generator.py @@ -48,6 +48,7 @@ def __init__( verbose: bool = False, generate_html: bool = False, commit_id: str | None = None, + techdocs_dir: Path | None = None, ): """ Initialize the CLI documentation generator. @@ -58,6 +59,7 @@ def __init__( config: LLM configuration verbose: Enable verbose output generate_html: Whether to generate HTML viewer + techdocs_dir: Export a Backstage TechDocs project here (None: don't) commit_id: Git commit SHA for incremental update tracking """ self.repo_path = repo_path @@ -65,6 +67,8 @@ def __init__( self.config = config self.verbose = verbose self.generate_html = generate_html + self.techdocs_dir = techdocs_dir + self.techdocs_result = None self.commit_id = commit_id self.progress_tracker = ProgressTracker(total_stages=5, verbose=verbose) self.job = DocumentationJob() @@ -181,6 +185,9 @@ def generate(self) -> DocumentationJob: if self.generate_html: self._run_html_generation() + if self.techdocs_dir is not None: + self._run_techdocs_generation() + # Stage 5: Finalization (metadata already created by backend) self._finalize_job() @@ -583,6 +590,39 @@ def _run_html_generation(self): self.progress_tracker.complete_stage() + def _run_techdocs_generation(self): + """Export the docs as a Backstage TechDocs project (no LLM calls).""" + self.progress_tracker.start_stage(4, "TechDocs Export") + + from codewiki.cli.techdocs_generator import detect_repo_url, generate_techdocs + + try: + self.techdocs_result = generate_techdocs( + docs_dir=self.output_dir, + out_dir=self.techdocs_dir, + site_name=self.repo_path.name, + repo_url=detect_repo_url(str(self.repo_path)), + ) + except (OSError, ValueError) as e: + # The docs are already written; a failed export must not fail the run + import click + + click.secho( + f"⚠ TechDocs export failed: {e}\n" + f" Retry with: codewiki techdocs {self.output_dir} {self.techdocs_dir}", + fg="yellow", + err=True, + ) + self.progress_tracker.complete_stage() + return + + if self.verbose: + self.progress_tracker.update_stage( + 1.0, f"Wrote {self.techdocs_result.pages} pages to {self.techdocs_dir}" + ) + + self.progress_tracker.complete_stage() + def _record_usage_statistics(self): """Copy the run's token and cost totals from the usage ledger into the job.""" if self._usage_ledger is None: diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py index c9578786..1dfd07f0 100644 --- a/codewiki/cli/commands/generate.py +++ b/codewiki/cli/commands/generate.py @@ -237,6 +237,18 @@ def _find_affected(tree, parent_names=None): is_flag=True, help="Generate index.html for GitHub Pages deployment", ) +@click.option( + "--techdocs", + is_flag=True, + help="Also export a Backstage TechDocs project (mkdocs.yml + docs/) to --techdocs-dir", +) +@click.option( + "--techdocs-dir", + type=click.Path(file_okay=False), + default="techdocs", + show_default=True, + help="Where --techdocs writes the TechDocs project (outside the --output directory)", +) @click.option( "--no-cache", is_flag=True, @@ -459,6 +471,8 @@ def generate_command( output: str, create_branch: bool, github_pages: bool, + techdocs: bool, + techdocs_dir: str, no_cache: bool, include: str | None, exclude: str | None, @@ -512,6 +526,10 @@ def generate_command( # With git branch creation and GitHub Pages $ codewiki generate --create-branch --github-pages + \b + # Also export a Backstage TechDocs project to ./techdocs + $ codewiki generate --techdocs + \b # Force full regeneration $ codewiki generate --no-cache @@ -606,6 +624,16 @@ def generate_command( # Validate output directory output_dir = Path(output).expanduser().resolve() check_writable_output(output_dir.parent) + techdocs_out = None + if techdocs: + from codewiki.cli.techdocs_generator import check_output_dir + + techdocs_out = Path(techdocs_dir).expanduser().resolve() + # Fail before the LLM run, not after it + try: + check_output_dir(techdocs_out, output_dir) + except ValueError as e: + raise ConfigurationError(f"--techdocs-dir: {e}") from e logger.success(f"Output directory: {output_dir}") @@ -854,6 +882,7 @@ def generate_command( }, verbose=verbose, generate_html=github_pages, + techdocs_dir=techdocs_out, commit_id=commit_id, ) @@ -883,6 +912,7 @@ def generate_command( repo_url=repo_url, branch_name=branch_name, github_pages=github_pages, + techdocs=generator.techdocs_result, files_generated=job.files_generated, statistics={ "module_count": job.module_count, diff --git a/codewiki/cli/commands/techdocs.py b/codewiki/cli/commands/techdocs.py new file mode 100644 index 00000000..be1524a8 --- /dev/null +++ b/codewiki/cli/commands/techdocs.py @@ -0,0 +1,75 @@ +""" +`codewiki techdocs`: export generated docs as a Backstage TechDocs source tree. +""" + +import sys +from pathlib import Path + +import click + +from codewiki.cli.techdocs_generator import TechDocsResult, generate_techdocs + + +def echo_techdocs_next_steps(result: TechDocsResult) -> None: + """How to build and register the exported TechDocs site.""" + click.echo(" Build it:") + etag = f" --etag {result.commit}" if result.commit else "" + click.secho(f" techdocs-cli generate --source-dir {result.out_dir}{etag}", fg="yellow") + click.echo(" Or point the catalog entity at it (catalog-info.yaml):") + click.secho(f" backstage.io/techdocs-ref: dir:./{result.out_dir.name}", fg="yellow") + + +@click.command(name="techdocs") +@click.argument( + "docs_dir", + type=click.Path(file_okay=False, path_type=Path), + default="docs", +) +@click.argument( + "out_dir", + type=click.Path(file_okay=False, path_type=Path), + default="techdocs", +) +@click.option("--site-name", default=None, help="Site name (default: repository folder name)") +@click.option("--repo-url", default=None, help="https repository URL (default: git origin)") +@click.option( + "--source-url-template", + default=None, + help="Source link format with {repo}, {ref} and {path}, e.g. for Bitbucket Server: " + "'https://bb.example.com/projects/P/repos/R/browse/{path}?at={ref}'. " + "GitHub, GitLab and bitbucket.org are detected automatically.", +) +def techdocs_command( + docs_dir: Path, + out_dir: Path, + site_name: str | None, + repo_url: str | None, + source_url_template: str | None, +): + """Export CodeWiki docs as a Backstage TechDocs (MkDocs) project. + + Reads the pages in DOCS_DIR (default: ./docs) and writes mkdocs.yml and + docs/ to OUT_DIR (default: ./techdocs). No LLM is called, so run it again + after every `codewiki generate --update`. The source docs are not changed. + + \b + Examples: + $ codewiki techdocs + $ codewiki techdocs docs techdocs --site-name "Payments API" + """ + docs_dir = docs_dir.expanduser().resolve() + out_dir = out_dir.expanduser().resolve() + try: + result = generate_techdocs( + docs_dir, + out_dir, + site_name=site_name, + repo_url=repo_url, + source_template=source_url_template, + ) + except (FileNotFoundError, ValueError) as e: + click.secho(f"✗ {e}", fg="red", err=True) + sys.exit(1) + + click.secho(f"✓ Wrote {result.pages} TechDocs pages to {result.out_dir}", fg="green") + echo_techdocs_next_steps(result) diff --git a/codewiki/cli/main.py b/codewiki/cli/main.py index a846f2d8..a58dc172 100644 --- a/codewiki/cli/main.py +++ b/codewiki/cli/main.py @@ -8,6 +8,7 @@ from codewiki import __version__ from codewiki.cli.commands.config import config_group from codewiki.cli.commands.generate import generate_command +from codewiki.cli.commands.techdocs import techdocs_command from codewiki.cli.utils.branding import print_banner @@ -43,6 +44,7 @@ def version(): # Register command groups cli.add_command(config_group) cli.add_command(generate_command, name="generate") +cli.add_command(techdocs_command, name="techdocs") @cli.command(name="mcp") diff --git a/codewiki/cli/techdocs_generator.py b/codewiki/cli/techdocs_generator.py new file mode 100644 index 00000000..78700569 --- /dev/null +++ b/codewiki/cli/techdocs_generator.py @@ -0,0 +1,439 @@ +""" +TechDocs source generator for Backstage. + +Turns CodeWiki output into an MkDocs project that ``techdocs-cli generate`` +builds as is:: + + / + ├── mkdocs.yml # site name, repo link, nav from the module tree + └── docs/ + ├── index.md # the repository overview + └── ... # module pages, in the same layout as the source + +Only pages and images are copied: ``module_tree.json``, ``metadata.json``, +``update_record.json`` and ``temp/`` (the dependency graph) stay out of the +published site. Each page gets front matter, a provenance banner and a +collapsible list of the components it documents, linked to the source at the +documented commit. The source docs are not modified, so the export can be +re-run after every ``--update``. + +Based on the generator proposed in +https://github.com/FSoft-AI4Code/CodeWiki/issues/145. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional +from urllib.parse import quote + +import yaml + +from codewiki.src.be.doc_layout import ( + METADATA_FILENAME, + OVERVIEW_STEM, + TEMP_DIR, + list_doc_files, + load_module_tree, + rewrite_page_links, +) +from codewiki.src.be.module_naming import _first_h1, extract_page_titles + +INDEX_FILENAME = "index.md" +DOCS_SUBDIR = "docs" +MKDOCS_FILENAME = "mkdocs.yml" +# Written into the generated docs folder; only a folder carrying it is wiped on re-runs +MARKER_FILENAME = ".codewiki-techdocs" +ASSET_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"} +MAX_LISTED_COMPONENTS = 50 +MAX_DESCRIPTION_CHARS = 160 + +# Source links per host; {repo} is the https repository URL +SOURCE_URL_TEMPLATES = { + "github": "{repo}/blob/{ref}/{path}", + "gitlab": "{repo}/-/blob/{ref}/{path}", + "bitbucket.org": "{repo}/src/{ref}/{path}", +} +COMMIT_URL_TEMPLATES = { + "github": "{repo}/commit/{ref}", + "gitlab": "{repo}/-/commit/{ref}", + "bitbucket.org": "{repo}/commits/{ref}", +} + +# Fenced ```mermaid blocks keep a "mermaid" class, so the TechDocs Mermaid +# addon detects every diagram (including ones with front matter) +MERMAID_FENCES = """markdown_extensions: + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format +""" + +_FENCE_RE = re.compile(r"^\s*(```|~~~)") +_LINK_TEXT_RE = re.compile(r"\[([^\]\n]+)\]\(\s*#]+\.md)") +_MD_LINK_RE = re.compile(r"!?\[([^\]]*)\]\([^)]*\)") +# Lines that are not prose: headings, tables, quotes, list items, images, HTML, rules +_NON_PROSE_RE = re.compile(r"^(#|\||>|[-*+]\s|\d+[.)]\s|!\[|<|---|\*\*\*|___)") + + +@dataclass +class TechDocsResult: + out_dir: Path + pages: int + commit: Optional[str] + + +def _load_metadata(docs_dir: Path) -> dict[str, Any]: + try: + with open(docs_dir / METADATA_FILENAME, encoding="utf-8") as f: + data = json.load(f) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def _humanize(name: str) -> str: + return name.replace("_", " ") + + +def _nav_titles(titles: dict[str, str]) -> dict[str, str]: + """Page titles without inline code marks, for the navigation.""" + return {stem: title.replace("`", "") for stem, title in titles.items()} + + +def _out_relpath(stem: str, rel: str) -> str: + return INDEX_FILENAME if stem == OVERVIEW_STEM else rel + + +def normalize_repo_url(url: Optional[str]) -> Optional[str]: + """https form of a git remote URL (SSH converted, credentials dropped), or None.""" + if not url: + return None + url = url.strip() + # git@host:owner/repo.git -> https://host/owner/repo + scp = re.match(r"^[\w.-]+@([\w.-]+):(?!//)(.+)$", url) + if scp: + url = f"https://{scp.group(1)}/{scp.group(2)}" + # ssh://git@host[:port]/owner/repo.git -> https://host/owner/repo + ssh = re.match(r"^ssh://(?:[^@/]+@)?([\w.-]+)(?::\d+)?/(.+)$", url) + if ssh: + url = f"https://{ssh.group(1)}/{ssh.group(2)}" + if not url.startswith("https://"): + return None + url = re.sub(r"^https://[^@/]+@", "https://", url) # drop credentials + return re.sub(r"\.git$", "", url.rstrip("/")) + + +def detect_repo_url(repo_path: Optional[str]) -> Optional[str]: + """https URL of the ``origin`` remote of ``repo_path``, or None.""" + if not repo_path or not os.path.isdir(repo_path): + return None + try: + import git + + url = git.Repo(repo_path, search_parent_directories=True).remotes.origin.url + except Exception: # noqa: BLE001 — the remote URL is optional + return None + return normalize_repo_url(url) + + +def _host_key(repo_url: Optional[str], templates: dict[str, str]) -> Optional[str]: + if not repo_url: + return None + return next((host for host in templates if host in repo_url), None) + + +def source_url_template(repo_url: Optional[str]) -> Optional[str]: + host = _host_key(repo_url, SOURCE_URL_TEMPLATES) + return SOURCE_URL_TEMPLATES[host] if host else None + + +def build_nav(level: dict[str, Any], pages: dict[str, str], titles: dict[str, str]) -> list[Any]: + """MkDocs nav for one level of the module tree. + + A module with children becomes a section whose first entry is the + module's own page, so ``auth.md`` and ``auth/`` show up as one item. + """ + nav: list[Any] = [] + for name, info in level.items(): + title = titles.get(name) or _humanize(name) + rel = pages.get(name) + children = info.get("children") if isinstance(info, dict) else None + child_nav = build_nav(children, pages, titles) if isinstance(children, dict) else [] + if child_nav: + nav.append({title: ([{title: rel}] if rel else []) + child_nav}) + elif rel: + nav.append({title: rel}) + return nav + + +def _nav_pages(nav: Any) -> set[str]: + """Every page path referenced by a nav.""" + if isinstance(nav, str): + return {nav} + if isinstance(nav, list): + return set().union(*(_nav_pages(item) for item in nav)) if nav else set() + if isinstance(nav, dict): + return set().union(*(_nav_pages(value) for value in nav.values())) if nav else set() + return set() + + +def _module_components(module_tree: dict[str, Any]) -> dict[str, list[str]]: + components: dict[str, list[str]] = {} + stack = [module_tree] + while stack: + level = stack.pop() + for name, info in level.items(): + if not isinstance(info, dict): + continue + components[name] = [c for c in info.get("components") or [] if isinstance(c, str)] + if isinstance(info.get("children"), dict): + stack.append(info["children"]) + return components + + +def _titled_link_text(text: str, titles: dict[str, str]) -> str: + """Show page titles instead of module names in links (``[CLI_Utilities](...)``).""" + + def fix(m: re.Match) -> str: + stem = os.path.basename(m.group(2))[: -len(".md")] + label = m.group(1).strip("` ") + if label in (stem, f"{stem}.md") and stem in titles: + return m.group(0).replace(f"[{m.group(1)}]", f"[{titles[stem]}]", 1) + return m.group(0) + + out = [] + in_fence = False + for line in text.splitlines(keepends=True): + if _FENCE_RE.match(line): + in_fence = not in_fence + out.append(line if in_fence else _LINK_TEXT_RE.sub(fix, line)) + return "".join(out) + + +def _description(text: str) -> Optional[str]: + """First prose paragraph after the H1, as plain text.""" + seen_h1 = False + in_fence = False + paragraph: list[str] = [] + for line in text.splitlines(): + stripped = line.strip() + if _FENCE_RE.match(line): + in_fence = not in_fence + if paragraph: + break + continue + if in_fence: + continue + if not seen_h1: + seen_h1 = stripped.startswith("# ") + continue + if not stripped: + if paragraph: + break + continue + if _NON_PROSE_RE.match(stripped): + if paragraph: + break + continue + paragraph.append(stripped) + if not paragraph: + return None + plain = _MD_LINK_RE.sub(r"\1", " ".join(paragraph)) + plain = re.sub(r"[*_`]", "", plain) + if len(plain) > MAX_DESCRIPTION_CHARS: + plain = plain[: MAX_DESCRIPTION_CHARS - 1].rsplit(" ", 1)[0] + "…" + return plain + + +def _banner(info: dict[str, Any], repo_url: Optional[str]) -> str: + commit = info.get("commit_id") + date = str(info.get("timestamp") or "")[:10] + model = info.get("main_model") + parts = ["This page was written by CodeWiki from the source code"] + if commit: + short = f"`{commit[:8]}`" + host = _host_key(repo_url, COMMIT_URL_TEMPLATES) + if host: + short = f"[{short}]({COMMIT_URL_TEMPLATES[host].format(repo=repo_url, ref=commit)})" + parts.append(f"at commit {short}") + if date: + parts.append(f"on {date}") + if model: + parts.append(f"with `{model}`") + return ( + '!!! info "Generated documentation"\n' + f" {' '.join(parts)}. Check the code before relying on details.\n" + ) + + +def _components_block( + components: list[str], template: Optional[str], repo_url: Optional[str], commit: Optional[str] +) -> str: + if not components: + return "" + lines = [f'??? abstract "Documented components ({len(components)})"'] + for component_id in components[:MAX_LISTED_COMPONENTS]: + path, _, symbol = component_id.partition("::") + label = f"`{symbol or os.path.basename(path)}`" + if template and repo_url and commit: + url = template.format(repo=repo_url, ref=commit, path=quote(path)) + lines.append(f" - [{label}]({url}) — `{path}`") + else: + lines.append(f" - {label} — `{path}`") + if len(components) > MAX_LISTED_COMPONENTS: + lines.append(f" - … and {len(components) - MAX_LISTED_COMPONENTS} more") + return "\n\n" + "\n".join(lines) + "\n" + + +def _insert_after_h1(text: str, block: str) -> str: + lines = text.splitlines(keepends=True) + in_fence = False + for i, line in enumerate(lines): + if _FENCE_RE.match(line): + in_fence = not in_fence + elif not in_fence and line.startswith("# "): + return "".join(lines[: i + 1]) + "\n" + block + "".join(lines[i + 1 :]) + return block + "\n" + text + + +def _front_matter(title: str, description: Optional[str]) -> str: + data = {"title": title} + if description: + data["description"] = description + return ( + "---\n" + yaml.safe_dump(data, allow_unicode=True, sort_keys=False, width=1000) + "---\n\n" + ) + + +def check_output_dir(out_dir: Path, docs_dir: Path) -> None: + """Raise ValueError unless an export to ``out_dir`` is safe; changes nothing. + + The output must not overlap the CodeWiki docs, and an existing + ``/docs`` must carry the marker of a previous export. + """ + out_dir, docs_dir = Path(out_dir), Path(docs_dir) + out, source = out_dir.resolve(), docs_dir.resolve() + if out.is_relative_to(source) or source.is_relative_to(out): + raise ValueError( + f"TechDocs output {out_dir} must be outside the CodeWiki docs {docs_dir} " + "(and must not contain them)" + ) + out_docs = out_dir / DOCS_SUBDIR + if out_docs.exists() and not (out_docs / MARKER_FILENAME).exists(): + raise ValueError( + f"{out_docs} exists and was not generated by CodeWiki; refusing to replace it" + ) + + +def _prepare_out_docs(out_dir: Path, docs_dir: Path) -> Path: + """Create an empty ``/docs``, refusing to touch anything CodeWiki didn't write.""" + check_output_dir(out_dir, docs_dir) + out_docs = out_dir / DOCS_SUBDIR + if out_docs.exists(): + shutil.rmtree(out_docs) + out_docs.mkdir(parents=True) + (out_docs / MARKER_FILENAME).write_text("Generated by codewiki; replaced on every run.\n") + return out_docs + + +def generate_techdocs( + docs_dir: Path, + out_dir: Path, + site_name: Optional[str] = None, + repo_url: Optional[str] = None, + source_template: Optional[str] = None, +) -> TechDocsResult: + """Write an MkDocs/TechDocs project for the CodeWiki docs in ``docs_dir``. + + Raises: + FileNotFoundError: ``docs_dir`` holds no CodeWiki pages. + ValueError: ``out_dir`` overlaps ``docs_dir``, or its ``docs/`` folder + was not written by a previous export. + """ + docs_dir, out_dir = Path(docs_dir), Path(out_dir) + module_tree = load_module_tree(str(docs_dir)) + pages = list_doc_files(str(docs_dir), module_tree) + if not pages: + raise FileNotFoundError(f"No CodeWiki pages found in {docs_dir}") + + metadata = _load_metadata(docs_dir) + info = metadata.get("generation_info") or {} + commit = info.get("commit_id") + repo_url = ( + normalize_repo_url(repo_url) + or repo_url + or detect_repo_url(info.get("repo_path")) + or detect_repo_url(str(docs_dir)) # docs checked in to the repository + ) + source_template = source_template or source_url_template(repo_url) + site_name = ( + site_name or os.path.basename(str(info.get("repo_path") or "").rstrip("/")) or "Code Wiki" + ) + + out_pages = {stem: _out_relpath(stem, rel) for stem, rel in pages.items()} + titles = extract_page_titles(str(docs_dir), module_tree) + for stem, rel in pages.items(): # pages outside the module tree + if stem not in titles and (title := _first_h1(str(docs_dir / rel))): + titles[stem] = title + components = _module_components(module_tree) + banner = _banner(info, repo_url) + + out_docs = _prepare_out_docs(out_dir, docs_dir) + + overview_description = None + for stem, rel in pages.items(): + out_rel = out_pages[stem] + text = (docs_dir / rel).read_text(encoding="utf-8") + text = _titled_link_text(text, titles) + text = rewrite_page_links(text, out_rel, out_pages) + description = _description(text) + if stem == OVERVIEW_STEM: + overview_description = description + text = _insert_after_h1(text, banner) + text = text.rstrip("\n") + _components_block( + components.get(stem, []), source_template, repo_url, commit + ) + title = titles.get(stem) or _humanize(stem) + target = out_docs / out_rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(_front_matter(title, description) + text + "\n", encoding="utf-8") + + for root, dirs, files in os.walk(docs_dir): + dirs[:] = [d for d in dirs if d != TEMP_DIR and not d.startswith(".")] + for name in files: + if Path(name).suffix.lower() in ASSET_EXTENSIONS: + source = Path(root) / name + target = out_docs / source.relative_to(docs_dir) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + + nav_titles = _nav_titles(titles) + nav: list[Any] = [] + if OVERVIEW_STEM in out_pages: + nav.append({nav_titles.get(OVERVIEW_STEM) or "Overview": INDEX_FILENAME}) + nav += build_nav(module_tree, out_pages, nav_titles) + in_nav = _nav_pages(nav) + for stem, rel in sorted(out_pages.items()): + if rel not in in_nav: + nav.append({nav_titles.get(stem) or _humanize(stem): rel}) + + config: dict[str, Any] = {"site_name": site_name} + if overview_description: + config["site_description"] = overview_description + if repo_url: + config["repo_url"] = repo_url + config["edit_uri"] = "" # generated pages are not edited by hand + config["docs_dir"] = DOCS_SUBDIR + config["nav"] = nav + config["plugins"] = ["techdocs-core"] + mkdocs_yml = yaml.safe_dump(config, allow_unicode=True, sort_keys=False, width=1000) + (out_dir / MKDOCS_FILENAME).write_text(mkdocs_yml + MERMAID_FENCES, encoding="utf-8") + + return TechDocsResult(out_dir=out_dir, pages=len(pages), commit=commit) diff --git a/codewiki/cli/utils/instructions.py b/codewiki/cli/utils/instructions.py index c9bacc98..127ecca8 100644 --- a/codewiki/cli/utils/instructions.py +++ b/codewiki/cli/utils/instructions.py @@ -166,6 +166,7 @@ def display_post_generation_instructions( github_pages: bool = False, files_generated: list = None, statistics: dict = None, + techdocs=None, ): """ Display post-generation instructions. @@ -176,6 +177,7 @@ def display_post_generation_instructions( repo_url: GitHub repository URL (optional) branch_name: Git branch name (optional) github_pages: Whether GitHub Pages HTML was generated + techdocs: TechDocsResult of the --techdocs export, if one was written files_generated: List of generated files statistics: Generation statistics """ @@ -211,6 +213,14 @@ def display_post_generation_instructions( _echo_usage(statistics) click.echo() + if techdocs is not None: + from codewiki.cli.commands.techdocs import echo_techdocs_next_steps + + click.secho("Backstage TechDocs:", fg="cyan", bold=True) + click.echo(f" {techdocs.pages} pages in {techdocs.out_dir}") + echo_techdocs_next_steps(techdocs) + click.echo() + # Next steps click.secho("Next steps:", fg="cyan", bold=True) click.echo() diff --git a/guides/README.md b/guides/README.md index d56f123a..c034664b 100644 --- a/guides/README.md +++ b/guides/README.md @@ -14,6 +14,7 @@ CodeWiki's own generated documentation and is not edited by hand. | [Artifact-aware generation](artifact-aware-generation.md) | Understand how build, CI, container, and config files get documented, and how to tune or turn it off | | [Incremental updates](incremental-updates.md) | Keep documentation current with `--update` and understand what it will and will not touch | | [MCP / IDE-driven mode](mcp-ide-mode.md) | Run CodeWiki as an MCP server inside Cursor, Claude Desktop, Claude Code, or CodeBuddy, with no LLM configuration | +| [Backstage TechDocs](backstage-techdocs.md) | Publish the generated docs in Backstage with `--techdocs` or `codewiki techdocs` | | [Development guide](development.md) | Change CodeWiki: project layout, pipeline, adding a language, tests, releasing | | [Docker setup](docker.md) | Run the web application in a container | diff --git a/guides/backstage-techdocs.md b/guides/backstage-techdocs.md new file mode 100644 index 00000000..2ed70142 --- /dev/null +++ b/guides/backstage-techdocs.md @@ -0,0 +1,107 @@ +# Backstage TechDocs + +CodeWiki can publish its documentation in a +[Backstage](https://backstage.io/docs/features/techdocs/) developer portal. +The export turns the generated `docs/` folder into an MkDocs project that +TechDocs builds as is: + +``` +./techdocs/ +├── mkdocs.yml # site name, repo link, techdocs-core, nav from the module tree +└── docs/ + ├── index.md # the repository overview (overview.md) + ├── .md ... + ├── /.md ... + └── *.png, *.svg ... # images found in the CodeWiki docs +``` + +Only pages and images are copied. `module_tree.json`, `metadata.json`, +`update_record.json` and `temp/` stay out of the published site. The CodeWiki +docs themselves are never modified. + +## Exporting + +During generation: + +```bash +codewiki generate --techdocs # writes ./techdocs +codewiki generate --techdocs --techdocs-dir site/td # somewhere else +``` + +From docs you already have, with no LLM calls (for example after +`codewiki generate --update`): + +```bash +codewiki techdocs # ./docs -> ./techdocs +codewiki techdocs docs techdocs --site-name "Payments API" +``` + +Every run replaces `techdocs/docs/` completely. A marker file inside it proves +that an earlier export created it, and CodeWiki refuses to replace a +`docs/` folder without that marker. The output directory must be outside +the CodeWiki docs directory. + +## What each page gets + +- **Front matter:** the page title (its first heading) and a short + description (its first paragraph). +- **A provenance banner:** an info admonition with the commit, date and model + the page was written with, linked to the commit on GitHub, GitLab or + bitbucket.org. +- **A "Documented components" list:** a collapsed block naming the classes and + functions the module covers, each linked to its file at the documented + commit (up to 50 entries). +- **Fixed links:** links between pages are rewritten for the new layout + (links to `overview.md` now point at `index.md`), and a link labelled with a + module name shows that page's title instead. + +The repository URL comes from the git `origin` remote (SSH remotes are +converted to https and credentials are dropped). Override it with `--repo-url`. +For hosts other than GitHub, GitLab and bitbucket.org, such as Bitbucket +Server, give the source link format yourself: + +```bash +codewiki techdocs --repo-url https://bb.example.com/projects/P/repos/R \ + --source-url-template 'https://bb.example.com/projects/P/repos/R/browse/{path}?at={ref}' +``` + +## Publishing in Backstage + +Point the component's `catalog-info.yaml` at the exported folder: + +```yaml +metadata: + annotations: + backstage.io/techdocs-ref: dir:./techdocs +``` + +Or build it yourself, for example in CI with the "recommended" external +build setup: + +```bash +npx @techdocs/cli generate --source-dir ./techdocs --output-dir ./site --etag +npx @techdocs/cli publish --publisher-type --storage-name --entity +``` + +`techdocs-cli generate` runs MkDocs in a Docker image by default. To build +locally instead, run `pip install mkdocs-techdocs-core` and pass `--no-docker`. + +### Mermaid diagrams + +CodeWiki pages contain many Mermaid diagrams. The generated `mkdocs.yml` keeps +every ```` ```mermaid ```` block as `
`, but Backstage
+renders them only when the Mermaid TechDocs addon
+([`backstage-plugin-techdocs-addon-mermaid`](https://www.npmjs.com/package/backstage-plugin-techdocs-addon-mermaid))
+is installed in your Backstage app. Without it, diagrams show as their source
+text.
+
+## Keeping it current
+
+The export is cheap, so run it after every update. In CI:
+
+```bash
+codewiki generate --update --techdocs
+```
+
+`--update` reuses the existing docs and `--techdocs` re-exports all of them.
+To re-export without running generation, use `codewiki techdocs`.
diff --git a/guides/cli-reference.md b/guides/cli-reference.md
index 03284f3f..04252e5b 100644
--- a/guides/cli-reference.md
+++ b/guides/cli-reference.md
@@ -30,6 +30,8 @@ the repository first.
 | `--output`, `-o PATH` | `docs` | Where the documentation is written |
 | `--create-branch` | off | Create a git branch for the documentation changes |
 | `--github-pages` | off | Also write `index.html`, a static viewer for GitHub Pages |
+| `--techdocs` | off | Also export a Backstage TechDocs project (`mkdocs.yml` + `docs/`) after generation. See [`codewiki techdocs`](#codewiki-techdocs) |
+| `--techdocs-dir PATH` | `techdocs` | Where `--techdocs` writes. Must be outside `--output` |
 | `--no-cache` | off | Ignore cached results and rebuild everything |
 | `--verbose`, `-v` | off | Show progress details and debug output |
 | `--prompt-caching` / `--no-prompt-caching` | enabled | Add prompt-cache breakpoints to agent calls. Falls back to normal calls if the provider rejects them |
@@ -160,6 +162,7 @@ apply these rules throughout an existing tree.
 ```bash
 codewiki generate                                   # plain build into ./docs
 codewiki generate --github-pages --create-branch    # with viewer, on a new branch
+codewiki generate --techdocs                        # also export ./techdocs for Backstage
 codewiki generate --update                          # refresh after code changes
 codewiki generate --compare-to abc1234              # refresh relative to a known commit
 codewiki generate --no-artifacts                    # code only, 1.x behaviour
@@ -173,6 +176,27 @@ codewiki generate --max-tokens 16384 --max-depth 3
 
 ---
 
+## `codewiki techdocs`
+
+```bash
+codewiki techdocs [DOCS_DIR] [OUT_DIR]   # defaults: docs techdocs
+```
+
+Exports existing CodeWiki docs as a Backstage TechDocs (MkDocs) project. No
+LLM is called, so run it again after `codewiki generate --update`. The source
+docs are never modified, and the export only ever replaces an `OUT_DIR/docs`
+folder that an earlier export created.
+
+| Flag | Default | Meaning |
+| --- | --- | --- |
+| `--site-name TEXT` | repository folder name | `site_name` in `mkdocs.yml` |
+| `--repo-url URL` | the git `origin` remote | Repository link, and the base for source and commit links |
+| `--source-url-template TEXT` | detected for GitHub, GitLab, bitbucket.org | Source link format with `{repo}`, `{ref}` and `{path}`, e.g. `https://bb.example.com/projects/P/repos/R/browse/{path}?at={ref}` |
+
+See [Backstage TechDocs](backstage-techdocs.md).
+
+---
+
 ## `codewiki config set`
 
 Stores provider and model settings in `~/.codewiki/config.json`. Only the
diff --git a/tests/test_techdocs.py b/tests/test_techdocs.py
new file mode 100644
index 00000000..f51ce7bc
--- /dev/null
+++ b/tests/test_techdocs.py
@@ -0,0 +1,236 @@
+"""Backstage TechDocs export (issue #145): CodeWiki docs -> mkdocs.yml + docs/."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+import yaml
+from click.testing import CliRunner
+
+from codewiki.cli.commands.techdocs import techdocs_command
+from codewiki.cli.techdocs_generator import (
+    MARKER_FILENAME,
+    check_output_dir,
+    generate_techdocs,
+    normalize_repo_url,
+)
+
+COMMIT = "0123456789abcdef0123456789abcdef01234567"
+REPO = "https://github.com/acme/shop"
+TREE = {
+    "auth": {
+        "components": ["src/auth.py::Auth", "src/auth.py::login"],
+        "children": {
+            "login": {"components": ["src/login.py::LoginForm"], "children": {}},
+        },
+    },
+    "billing": {"components": ["src/billing.py"], "children": {}},
+}
+
+
+class _MkDocsLoader(yaml.SafeLoader):
+    """Safe loader that accepts the ``!!python/name`` tag used for mermaid fences."""
+
+
+_MkDocsLoader.add_multi_constructor(
+    "tag:yaml.org,2002:python/name:", lambda loader, suffix, node: suffix
+)
+
+
+def _write(path: Path, text: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+
+
+def _make_docs(root: Path, flat: bool = False) -> Path:
+    docs = root / "docs"
+    login_rel = "login.md" if flat else "auth/login.md"
+    to_overview = "overview.md" if flat else "../overview.md"
+    _write(
+        docs / "overview.md",
+        "# Shop\n\n**Shop** sells things. See [auth](auth.md) and [billing](billing.md).\n\n"
+        "```mermaid\ngraph TD\n  A-->B\n```\n\n![diagram](arch.png)\n",
+    )
+    _write(
+        docs / "auth.md",
+        f"# Authentication\n\n- a bullet\n\nHandles users. Sub-page: [login]({login_rel}).\n"
+        "Back to [overview](overview.md).\n",
+    )
+    _write(docs / login_rel, f"# Login\n\nThe login form. [Overview]({to_overview})\n")
+    _write(docs / "billing.md", "# Billing\n\nCharges cards.\n")
+    _write(docs / "module_tree.json", json.dumps(TREE))
+    _write(
+        docs / "metadata.json",
+        json.dumps(
+            {
+                "generation_info": {
+                    "timestamp": "2026-11-01T10:00:00+00:00",
+                    "main_model": "test-model",
+                    "repo_path": str(root / "shop"),
+                    "commit_id": COMMIT,
+                    "layout": "flat" if flat else "hierarchical",
+                }
+            }
+        ),
+    )
+    _write(docs / "temp" / "dependency_graphs" / "g.json", "{}")
+    (docs / "arch.png").write_bytes(b"\x89PNG")
+    return docs
+
+
+def _load_config(out: Path) -> dict:
+    return yaml.load((out / "mkdocs.yml").read_text(encoding="utf-8"), Loader=_MkDocsLoader)
+
+
+def test_hierarchical_export(tmp_path):
+    docs = _make_docs(tmp_path)
+    out = tmp_path / "techdocs"
+
+    result = generate_techdocs(docs, out, repo_url=REPO + ".git")
+
+    assert result.pages == 4
+    assert result.commit == COMMIT
+    config = _load_config(out)
+    assert config["plugins"] == ["techdocs-core"]
+    assert config["docs_dir"] == "docs"
+    assert config["repo_url"] == REPO
+    assert config["site_name"] == "shop"
+    assert config["site_description"] == "Shop sells things. See Authentication and Billing."
+    assert config["nav"] == [
+        {"Shop": "index.md"},
+        {"Authentication": [{"Authentication": "auth.md"}, {"Login": "auth/login.md"}]},
+        {"Billing": "billing.md"},
+    ]
+    fences = config["markdown_extensions"][0]["pymdownx.superfences"]["custom_fences"]
+    assert fences[0]["class"] == "mermaid"
+
+    out_docs = out / "docs"
+    assert (out_docs / "index.md").is_file()
+    assert not (out_docs / "overview.md").exists()
+    assert (out_docs / "arch.png").read_bytes() == b"\x89PNG"
+    assert not (out_docs / "module_tree.json").exists()
+    assert not (out_docs / "metadata.json").exists()
+    assert not (out_docs / "temp").exists()
+    assert (out_docs / MARKER_FILENAME).is_file()
+
+    # Links to the overview point at index.md, relative to each page
+    assert "[Overview](../index.md)" in (out_docs / "auth/login.md").read_text()
+    auth = (out_docs / "auth.md").read_text()
+    assert "[Shop](index.md)" in auth
+    # Links labelled with the module name show the page title
+    assert "[Login](auth/login.md)" in auth
+
+    # Front matter, provenance banner and component list
+    assert auth.startswith(
+        "---\ntitle: Authentication\ndescription: 'Handles users. Sub-page: Login."
+    )
+    assert f"[`{COMMIT[:8]}`]({REPO}/commit/{COMMIT})" in auth
+    assert "on 2026-11-01 with `test-model`" in auth
+    assert '??? abstract "Documented components (2)"' in auth
+    assert f"[`login`]({REPO}/blob/{COMMIT}/src/auth.py) — `src/auth.py`" in auth
+    billing = (out_docs / "billing.md").read_text()
+    assert f"[`billing.py`]({REPO}/blob/{COMMIT}/src/billing.py)" in billing
+    # The overview documents no components
+    assert "Documented components" not in (out_docs / "index.md").read_text()
+
+    # The source docs are untouched
+    assert (docs / "overview.md").read_text().startswith("# Shop\n")
+
+
+def test_flat_export(tmp_path):
+    docs = _make_docs(tmp_path, flat=True)
+    out = tmp_path / "techdocs"
+
+    generate_techdocs(docs, out)
+
+    config = _load_config(out)
+    assert config["nav"][1] == {
+        "Authentication": [{"Authentication": "auth.md"}, {"Login": "login.md"}]
+    }
+    assert "[Overview](index.md)" in (out / "docs/login.md").read_text()
+    # No repository URL: components are listed without links
+    assert "- `LoginForm` — `src/login.py`" in (out / "docs/login.md").read_text()
+    assert "repo_url" not in config
+
+
+def test_pages_outside_the_tree_are_added_to_nav(tmp_path):
+    docs = _make_docs(tmp_path)
+    _write(docs / "extra.md", "# Extra\n\nNot in the tree.\n")
+
+    generate_techdocs(docs, tmp_path / "techdocs")
+
+    assert _load_config(tmp_path / "techdocs")["nav"][-1] == {"Extra": "extra.md"}
+
+
+def test_rerun_replaces_previous_export(tmp_path):
+    docs = _make_docs(tmp_path)
+    out = tmp_path / "techdocs"
+    generate_techdocs(docs, out)
+    (docs / "billing.md").unlink()
+
+    generate_techdocs(docs, out)
+
+    assert not (out / "docs/billing.md").exists()
+    assert not (out / "docs/docs").exists()
+    assert (out / "docs" / MARKER_FILENAME).is_file()
+
+
+def test_refuses_foreign_docs_folder(tmp_path):
+    docs = _make_docs(tmp_path)
+    out = tmp_path / "techdocs"
+    _write(out / "docs" / "handwritten.md", "# Mine\n")
+
+    with pytest.raises(ValueError, match="not generated by CodeWiki"):
+        generate_techdocs(docs, out)
+    assert (out / "docs/handwritten.md").is_file()
+
+
+@pytest.mark.parametrize("out", ["docs/techdocs", "."])
+def test_refuses_overlapping_output(tmp_path, out):
+    docs = _make_docs(tmp_path)
+    with pytest.raises(ValueError, match="must be outside"):
+        check_output_dir(tmp_path / out, docs)
+
+
+def test_empty_docs_dir(tmp_path):
+    (tmp_path / "docs").mkdir()
+    with pytest.raises(FileNotFoundError):
+        generate_techdocs(tmp_path / "docs", tmp_path / "techdocs")
+    assert not (tmp_path / "techdocs").exists()
+
+
+@pytest.mark.parametrize(
+    "url, expected",
+    [
+        ("git@github.com:acme/shop.git", "https://github.com/acme/shop"),
+        ("ssh://git@gitlab.example.com:2222/team/shop.git", "https://gitlab.example.com/team/shop"),
+        ("https://user:token@github.com/acme/shop.git", "https://github.com/acme/shop"),
+        ("https://github.com/acme/shop/", "https://github.com/acme/shop"),
+        ("/srv/git/shop.git", None),
+        (None, None),
+    ],
+)
+def test_normalize_repo_url(url, expected):
+    assert normalize_repo_url(url) == expected
+
+
+def test_cli_command(tmp_path):
+    docs = _make_docs(tmp_path)
+    out = tmp_path / "td"
+    runner = CliRunner()
+
+    result = runner.invoke(
+        techdocs_command,
+        [str(docs), str(out), "--site-name", "Shop", "--repo-url", REPO],
+    )
+
+    assert result.exit_code == 0, result.output
+    assert "Wrote 4 TechDocs pages" in result.output
+    assert f"--etag {COMMIT}" in result.output
+    assert _load_config(out)["site_name"] == "Shop"
+
+    result = runner.invoke(techdocs_command, [str(docs), str(docs / "sub")])
+    assert result.exit_code == 1
+    assert "must be outside" in result.output