From 817188f7cc910830a134ac2d0195af9059f04b69 Mon Sep 17 00:00:00 2001 From: adssib <75389300+adssib@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:00:26 -0500 Subject: [PATCH] read sources version by version: list, read at, extract A versioned source now has three parts next to Run: ListVersions (every version, oldest first), ReadAt (the source at one version) and Extract (the changes between two snapshots, with previous_version set). Run keeps working and is built from them. core/versions lists versions from GitHub releases, tags and commits, PyPI, npm and the Web Archive at a date, with one version ordering. openapi: every adapter whose LatestTag was releases/latest or the newest commit on its spec can list all of them (Stripe, Datadog, Vercel, Flagsmith, Supabase, Neon, Convex, PlanetScale); the rest list their newest. changelog, migrationguide and removedmembers no longer pin a tag: releases come from PyPI and each is read at its tag; the pinned releases stay as the goldens' cases. Every golden is byte-identical. Co-Authored-By: Claude Opus 5.5 Signed-off-by: adssib <75389300+adssib@users.noreply.github.com> --- AGENTS.md | 3 +- cmd/new-plugin/formats.go | 16 +- core/plugin/versions.go | 70 +++++ core/versions/versions.go | 287 ++++++++++++++++++ core/versions/versions_test.go | 170 +++++++++++ plugins/changelog/changelog.go | 161 +++++++--- plugins/changelog/testdata/README.md | 6 + .../testdata/pypi-urllib3-excerpt.json | 6 + plugins/changelog/versions_test.go | 73 +++++ plugins/migrationguide/migrationguide.go | 106 +++++-- plugins/migrationguide/sdkguide/sdkguide.go | 19 +- plugins/migrationguide/testdata/README.md | 6 + .../testdata/pypi-anthropic-excerpt.json | 6 + plugins/migrationguide/versions_test.go | 62 ++++ plugins/openapi/adapter/adapter.go | 14 +- plugins/openapi/adapter/versions.go | 75 +++++ plugins/openapi/openapi.go | 135 ++++---- plugins/openapi/versions_test.go | 118 +++++++ plugins/removedmembers/newrelic.go | 2 +- plugins/removedmembers/posthog.go | 20 +- plugins/removedmembers/removedmembers.go | 107 +++++-- plugins/removedmembers/testdata/README.md | 1 + .../testdata/pypi-numpy-excerpt.json | 6 + plugins/removedmembers/versions_test.go | 61 ++++ providers/anthropic.md | 2 +- providers/textual.md | 4 +- providers/urllib3.md | 4 +- 27 files changed, 1365 insertions(+), 175 deletions(-) create mode 100644 core/plugin/versions.go create mode 100644 core/versions/versions.go create mode 100644 core/versions/versions_test.go create mode 100644 plugins/changelog/testdata/pypi-urllib3-excerpt.json create mode 100644 plugins/changelog/versions_test.go create mode 100644 plugins/migrationguide/testdata/pypi-anthropic-excerpt.json create mode 100644 plugins/migrationguide/versions_test.go create mode 100644 plugins/openapi/adapter/versions.go create mode 100644 plugins/openapi/versions_test.go create mode 100644 plugins/removedmembers/testdata/pypi-numpy-excerpt.json create mode 100644 plugins/removedmembers/versions_test.go diff --git a/AGENTS.md b/AGENTS.md index 7d832cb..24b5996 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,9 +49,10 @@ One target at a time: `make tidy`, `make build`, `make fmt`, `make vet`, `make i Go module `github.com/patch-it-dev/patch-it-plugins`; the only dependency is `gopkg.in/yaml.v3`. ``` -core/plugin/ the Plugin interface, the registry of providers and their sources (metadata), Env, Result, Spec +core/plugin/ the Plugin interface (Run; ListVersions, ReadAt, Extract), the registry of providers and sources, Env, Result core/fetch/ the HTTP fetch with retries and the ETag / Last-Modified cache core/github/ the newest release / commit on GitHub, and a commit's SHA and date (one place) +core/versions/ every version of a source: GitHub releases, tags and commits, PyPI, npm, the Web Archive at a date core/kit/ core/kit/kittest/ Unchanged, Cite, Wayback; the test kit (Mem, Pages, Ingest, Golden) core/event/ core/migrate/ the change-event types from patch-it's contract (v1 + v2), v1 -> v2 core/diff/ core/normalize/ core/hint/ the OpenAPI diff engine and its helpers diff --git a/cmd/new-plugin/formats.go b/cmd/new-plugin/formats.go index 673586b..7c88291 100644 --- a/cmd/new-plugin/formats.go +++ b/cmd/new-plugin/formats.go @@ -298,15 +298,19 @@ func {{.Lower}}Ingest(t *testing.T, file string) []event.EventV2 { defDir: "plugins/changelog", def: `package changelog -// {{.Ident}} is {{.Provider}}'s changelog at one release. +// {{.Ident}} is {{.Provider}}'s changelog, read release by release (its +// releases listed from the ecosystem's registry, each at TagPrefix+version). // -// ` + todo + `: the ecosystem, the import name, the repo, the release's tag, -// the release as the changelog heads it, and the changelog's path. Say what -// you checked, with the date, as changelog.go does for Textual and urllib3. +// ` + todo + `: the ecosystem, the import name, the repo, the tag prefix and +// the changelog's path. Say what you checked, with the date, as changelog.go +// does for Textual and urllib3. var {{.Ident}} = File{ Package: "{{.ID}}", Ecosystem: "pypi", Module: "{{.ID}}", - Repo: "TODO-owner/TODO-repo", Tag: "v1.0.0", Release: "1.0.0", Path: "CHANGELOG.md", + Repo: "TODO-owner/TODO-repo", TagPrefix: "v", Path: "CHANGELOG.md", } + +// {{.Ident}}Golden is the release the golden test pins. +var {{.Ident}}Golden = {{.Ident}}.At("1.0.0") `, register: `plugin.Register(Plugin{File: {{.Ident}}})`, pluginOf: `Plugin{File: {{.Ident}}}`, @@ -324,7 +328,7 @@ var {{.Ident}} = File{ ingest: `// {{.Lower}}Ingest runs the plugin over a saved copy of the changelog. func {{.Lower}}Ingest(t *testing.T, file string) []event.EventV2 { t.Helper() - return kittest.Ingest(t, Plugin{File: {{.Ident}}}, kittest.Pages{ {{.Ident}}.URL(): file}) + return kittest.Ingest(t, Plugin{File: {{.Ident}}Golden}, kittest.Pages{ {{.Ident}}Golden.URL(): file}) } `, imports: []string{"github.com/patch-it-dev/patch-it-plugins/core/event"}, diff --git a/core/plugin/versions.go b/core/plugin/versions.go new file mode 100644 index 0000000..5971490 --- /dev/null +++ b/core/plugin/versions.go @@ -0,0 +1,70 @@ +package plugin + +import ( + "context" + "time" +) + +// A source that has versions (a spec repo's tags, a package's releases on +// PyPI or npm, a guide per major) is read in three parts, so whoever runs it +// can cover every version, not only the newest: +// +// versions, _ := s.ListVersions(ctx, env) // oldest first +// a, _ := s.ReadAt(ctx, env, versions[i]) +// b, _ := s.ReadAt(ctx, env, versions[i+1]) +// result, _ := s.Extract(ctx, env, a, b) // what changed from a to b +// +// Every consecutive pair is one Extract; a released version never changes, so +// a pair is done once. Run keeps working: it reads the newest version and +// extracts against the last baseline, as before. +// +// A source with no versions of its own (a deprecations page, a feed) has only +// Run: patch-it's hosted ingestion service reads it as a living page. + +// Version is one version of a source. +type Version struct { + // ID is the version as events name it: "2.0.0", "2026-11-19", "2025-03-31.basil". + ID string + // Released is when it came out; zero when the source doesn't say. + Released time.Time + // Ref is what to read it at: a tag, a commit SHA, a branch, a date. + Ref string + // URL is where a person sees this version (a release page, a commit). + URL string +} + +// Snapshot is a source as it was at one version: what ReadAt read. +type Snapshot struct { + Version Version + Content []byte + SourceURL string // the exact document read, pinned to the version +} + +// IsZero is true for the snapshot before the first version: nothing. +func (s Snapshot) IsZero() bool { return s.Version.ID == "" && s.Content == nil } + +// VersionLister lists every version a source has, oldest first. +type VersionLister interface { + ListVersions(ctx context.Context, env Env) ([]Version, error) +} + +// Reader reads a source at one version. +type Reader interface { + ReadAt(ctx context.Context, env Env, v Version) (Snapshot, error) +} + +// Extractor turns two snapshots into the changes between them. from may be +// zero (no earlier version read). The events carry to's version and, when from +// isn't zero, previous_version set to from's. It stores nothing: Result has no +// Baselines (Run adds its own). +type Extractor interface { + Extract(ctx context.Context, env Env, from, to Snapshot) (Result, error) +} + +// Versioned is a plugin read version by version. +type Versioned interface { + Plugin + VersionLister + Reader + Extractor +} diff --git a/core/versions/versions.go b/core/versions/versions.go new file mode 100644 index 0000000..77ebddf --- /dev/null +++ b/core/versions/versions.go @@ -0,0 +1,287 @@ +// Package versions lists a source's versions from where they're published: +// a GitHub repo's releases, tags or the commits that changed a file, a +// package's releases on PyPI or npm, and the Web Archive's capture of a page +// at a date. Every list comes back oldest first, as plugin.Version. +// +// Each function takes how to fetch (a plugin's env.Get, or github.Get for the +// GitHub API with its token), so tests serve saved responses. +package versions + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/patch-it-dev/patch-it-plugins/core/plugin" +) + +// Getter fetches one URL: plugin.Env.Get, or github.Get. +type Getter func(ctx context.Context, url string) ([]byte, error) + +// The services' base URLs; variables only so a test can point them elsewhere. +var ( + GitHubAPI = "https://api.github.com" + PyPI = "https://pypi.org" + NPM = "https://registry.npmjs.org" + Wayback = "https://archive.org" +) + +// MaxPages caps how many pages of 100 a GitHub list reads (1,000 entries). +const MaxPages = 10 + +// pages reads a GitHub list endpoint page by page until a short page. +func pages[T any](ctx context.Context, get Getter, base string) ([]T, error) { + sep := "?" + if strings.Contains(base, "?") { + sep = "&" + } + var all []T + for page := 1; page <= MaxPages; page++ { + body, err := get(ctx, fmt.Sprintf("%s%sper_page=100&page=%d", base, sep, page)) + if err != nil { + return nil, err + } + var got []T + if err := json.Unmarshal(body, &got); err != nil { + return nil, fmt.Errorf("parsing %s: %w", base, err) + } + all = append(all, got...) + if len(got) < 100 { + break + } + } + return all, nil +} + +// GitHubReleases are a repo's published releases ("owner/name"), drafts and +// prereleases left out, oldest first by publication date. keep, when set, +// keeps only the tags it accepts. +func GitHubReleases(ctx context.Context, get Getter, repo string, keep func(tag string) bool) ([]plugin.Version, error) { + rs, err := pages[struct { + Tag string `json:"tag_name"` + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` + Published time.Time `json:"published_at"` + URL string `json:"html_url"` + }](ctx, get, GitHubAPI+"/repos/"+repo+"/releases") + if err != nil { + return nil, fmt.Errorf("listing %s's releases: %w", repo, err) + } + var out []plugin.Version + for _, r := range rs { + if r.Draft || r.Prerelease || r.Tag == "" || (keep != nil && !keep(r.Tag)) { + continue + } + out = append(out, plugin.Version{ID: r.Tag, Released: r.Published, Ref: r.Tag, URL: r.URL}) + } + slices.SortStableFunc(out, func(a, b plugin.Version) int { return a.Released.Compare(b.Released) }) + return out, nil +} + +// GitHubTags are a repo's tags that match pattern; its first submatch, when +// it has one, is the version's ID ("^v(\d+\.\d+\.\d+)$" makes "v1.2.3" 1.2.3). +// GitHub's tag list has no dates, so they're sorted by Compare on the ID. +func GitHubTags(ctx context.Context, get Getter, repo string, pattern *regexp.Regexp) ([]plugin.Version, error) { + ts, err := pages[struct { + Name string `json:"name"` + }](ctx, get, GitHubAPI+"/repos/"+repo+"/tags") + if err != nil { + return nil, fmt.Errorf("listing %s's tags: %w", repo, err) + } + var out []plugin.Version + for _, t := range ts { + m := pattern.FindStringSubmatch(t.Name) + if m == nil { + continue + } + id := t.Name + if len(m) > 1 && m[1] != "" { + id = m[1] + } + out = append(out, plugin.Version{ID: id, Ref: t.Name, URL: "https://github.com/" + repo + "/releases/tag/" + url.PathEscape(t.Name)}) + } + slices.SortStableFunc(out, func(a, b plugin.Version) int { return Compare(a.ID, b.ID) }) + return out, nil +} + +// GitHubCommits are the commits on the default branch that changed path, +// oldest first: a version per change, for a file the repo never tags. The ID +// is the commit's date (YYYY-MM-DD), the Ref its SHA. +func GitHubCommits(ctx context.Context, get Getter, repo, path string) ([]plugin.Version, error) { + cs, err := pages[struct { + SHA string `json:"sha"` + URL string `json:"html_url"` + Commit struct { + Committer struct { + Date time.Time `json:"date"` + } `json:"committer"` + } `json:"commit"` + }](ctx, get, GitHubAPI+"/repos/"+repo+"/commits?path="+url.QueryEscape(path)) + if err != nil { + return nil, fmt.Errorf("listing the commits that changed %s in %s: %w", path, repo, err) + } + var out []plugin.Version + for _, c := range cs { + if c.SHA == "" || c.Commit.Committer.Date.IsZero() { + continue + } + at := c.Commit.Committer.Date.UTC() + out = append(out, plugin.Version{ID: at.Format("2006-01-02"), Released: at, Ref: c.SHA, URL: c.URL}) + } + slices.SortStableFunc(out, func(a, b plugin.Version) int { return a.Released.Compare(b.Released) }) + return out, nil +} + +// PyPIReleases are a package's final releases on PyPI (no pre-, dev- or +// yanked releases, none without files), oldest first by version. The Ref is +// the version; a plugin that reads a repo at a tag maps it. +func PyPIReleases(ctx context.Context, get Getter, pkg string) ([]plugin.Version, error) { + body, err := get(ctx, PyPI+"/pypi/"+url.PathEscape(pkg)+"/json") + if err != nil { + return nil, fmt.Errorf("listing %s's releases on PyPI: %w", pkg, err) + } + var doc struct { + Releases map[string][]struct { + Uploaded time.Time `json:"upload_time_iso_8601"` + Yanked bool `json:"yanked"` + } `json:"releases"` + } + if err := json.Unmarshal(body, &doc); err != nil { + return nil, fmt.Errorf("parsing PyPI's %s: %w", pkg, err) + } + var out []plugin.Version + for v, files := range doc.Releases { + if len(files) == 0 || !Final(v) { + continue + } + var first time.Time + yanked := true + for _, f := range files { + yanked = yanked && f.Yanked + if first.IsZero() || f.Uploaded.Before(first) { + first = f.Uploaded + } + } + if yanked { + continue + } + out = append(out, plugin.Version{ID: v, Released: first.UTC(), Ref: v, URL: "https://pypi.org/project/" + pkg + "/" + v + "/"}) + } + slices.SortStableFunc(out, func(a, b plugin.Version) int { return Compare(a.ID, b.ID) }) + return out, nil +} + +// NPMReleases are a package's published versions on npm, prereleases and +// deprecated versions left out, oldest first by version. +func NPMReleases(ctx context.Context, get Getter, pkg string) ([]plugin.Version, error) { + body, err := get(ctx, NPM+"/"+strings.Replace(url.PathEscape(pkg), "%40", "@", 1)) + if err != nil { + return nil, fmt.Errorf("listing %s's versions on npm: %w", pkg, err) + } + var doc struct { + Versions map[string]struct { + Deprecated any `json:"deprecated"` + } `json:"versions"` + Time map[string]time.Time `json:"time"` + } + if err := json.Unmarshal(body, &doc); err != nil { + return nil, fmt.Errorf("parsing npm's %s: %w", pkg, err) + } + var out []plugin.Version + for v, meta := range doc.Versions { + if !Final(v) || meta.Deprecated != nil { + continue + } + out = append(out, plugin.Version{ID: v, Released: doc.Time[v].UTC(), Ref: v, URL: "https://www.npmjs.com/package/" + pkg + "/v/" + v}) + } + slices.SortStableFunc(out, func(a, b plugin.Version) int { return Compare(a.ID, b.ID) }) + return out, nil +} + +// WaybackAt is the Web Archive's capture of page closest to at: the version +// of a page that only shows its newest state, as it was on a release's date. +// The Ref is the capture's URL (web.archive.org/web//). +func WaybackAt(ctx context.Context, get Getter, page string, at time.Time) (plugin.Version, error) { + q := url.Values{"url": {page}, "timestamp": {at.UTC().Format("20060102150405")}} + body, err := get(ctx, Wayback+"/wayback/available?"+q.Encode()) + if err != nil { + return plugin.Version{}, fmt.Errorf("asking the Web Archive for %s at %s: %w", page, at.Format(time.DateOnly), err) + } + var doc struct { + Snapshots struct { + Closest struct { + Available bool `json:"available"` + URL string `json:"url"` + Timestamp string `json:"timestamp"` + Status string `json:"status"` + } `json:"closest"` + } `json:"archived_snapshots"` + } + if err := json.Unmarshal(body, &doc); err != nil { + return plugin.Version{}, fmt.Errorf("parsing the Web Archive's answer for %s: %w", page, err) + } + c := doc.Snapshots.Closest + if !c.Available || c.URL == "" || (c.Status != "" && c.Status != "200") { + return plugin.Version{}, fmt.Errorf("the Web Archive has no capture of %s near %s", page, at.Format(time.DateOnly)) + } + taken, err := time.Parse("20060102150405", c.Timestamp) + if err != nil { + return plugin.Version{}, fmt.Errorf("the Web Archive's capture of %s has timestamp %q", page, c.Timestamp) + } + return plugin.Version{ID: taken.Format(time.DateOnly), Released: taken, Ref: c.URL, URL: c.URL}, nil +} + +var ( + final = regexp.MustCompile(`^v?\d+(\.\d+)*$`) + number = regexp.MustCompile(`\d+`) +) + +// Final is true for a plain release number ("2.0.0", "v0.86.0", "11.0"), false +// for a pre-release, dev, post or local version ("2.0.0rc1", "1.0.0-beta.2"). +func Final(v string) bool { return final.MatchString(v) } + +// Compare orders two version IDs: their numbers part by part ("1.10" after +// "1.9"), dates as numbers too ("2026-01-02" after "2025-12-31"), then as +// strings when the numbers tie. +func Compare(a, b string) int { + na, nb := number.FindAllString(a, -1), number.FindAllString(b, -1) + for i := 0; i < len(na) && i < len(nb); i++ { + x, _ := strconv.Atoi(na[i]) + y, _ := strconv.Atoi(nb[i]) + if c := cmp.Compare(x, y); c != 0 { + return c + } + } + if c := cmp.Compare(len(na), len(nb)); c != 0 { + return c + } + return strings.Compare(a, b) +} + +// Major is a version's first number ("2" for "2.0.0", "8" for "posthog-v8.1"), +// or -1. +func Major(v string) int { + m := number.FindString(v) + if m == "" { + return -1 + } + n, _ := strconv.Atoi(m) + return n +} + +// Pairs are the consecutive pairs of vs, oldest first: what a backfill +// extracts, one pair at a time. +func Pairs(vs []plugin.Version) [][2]plugin.Version { + var out [][2]plugin.Version + for i := 1; i < len(vs); i++ { + out = append(out, [2]plugin.Version{vs[i-1], vs[i]}) + } + return out +} diff --git a/core/versions/versions_test.go b/core/versions/versions_test.go new file mode 100644 index 0000000..ba1981a --- /dev/null +++ b/core/versions/versions_test.go @@ -0,0 +1,170 @@ +package versions + +import ( + "context" + "fmt" + "regexp" + "strings" + "testing" + "time" + + "github.com/patch-it-dev/patch-it-plugins/core/plugin" +) + +// serve answers saved (here: written out) responses by URL; anything else +// fails the test. +func serve(t *testing.T, pages map[string]string) Getter { + return func(_ context.Context, u string) ([]byte, error) { + b, ok := pages[u] + if !ok { + t.Errorf("fetched %s, which the test didn't save", u) + return nil, fmt.Errorf("no page %s", u) + } + return []byte(b), nil + } +} + +func ids(vs []plugin.Version) string { + var out []string + for _, v := range vs { + out = append(out, v.ID+"@"+v.Ref) + } + return strings.Join(out, " ") +} + +func TestGitHubReleasesSkipsDraftsAndPrereleasesOldestFirst(t *testing.T) { + get := serve(t, map[string]string{ + "https://api.github.com/repos/acme/spec/releases?per_page=100&page=1": `[ + {"tag_name":"v3","published_at":"2026-03-01T00:00:00Z","html_url":"https://github.com/acme/spec/releases/tag/v3"}, + {"tag_name":"v3-rc","prerelease":true,"published_at":"2026-02-20T00:00:00Z"}, + {"tag_name":"v4","draft":true}, + {"tag_name":"v2","published_at":"2026-02-01T00:00:00Z"}, + {"tag_name":"canary-1","published_at":"2026-02-15T00:00:00Z"} + ]`, + }) + vs, err := GitHubReleases(context.Background(), get, "acme/spec", func(tag string) bool { return !strings.HasPrefix(tag, "canary") }) + if err != nil { + t.Fatal(err) + } + if got := ids(vs); got != "v2@v2 v3@v3" { + t.Errorf("got %s", got) + } +} + +func TestGitHubReleasesReadsEveryPage(t *testing.T) { + var full []string + for i := 0; i < 100; i++ { + full = append(full, fmt.Sprintf(`{"tag_name":"v%d","published_at":"2025-01-01T00:00:%02dZ"}`, i, i%60)) + } + get := serve(t, map[string]string{ + "https://api.github.com/repos/acme/spec/releases?per_page=100&page=1": "[" + strings.Join(full, ",") + "]", + "https://api.github.com/repos/acme/spec/releases?per_page=100&page=2": `[{"tag_name":"v100","published_at":"2026-01-01T00:00:00Z"}]`, + }) + vs, err := GitHubReleases(context.Background(), get, "acme/spec", nil) + if err != nil || len(vs) != 101 || vs[100].ID != "v100" { + t.Fatalf("got %d versions, last %v, err %v", len(vs), vs[len(vs)-1], err) + } +} + +func TestGitHubTagsMatchesAndSortsByVersion(t *testing.T) { + get := serve(t, map[string]string{ + "https://api.github.com/repos/acme/sdk/tags?per_page=100&page=1": `[{"name":"v1.10.0"},{"name":"v1.9.0"},{"name":"v2.0.0rc1"},{"name":"nightly"},{"name":"v1.2.0"}]`, + }) + vs, err := GitHubTags(context.Background(), get, "acme/sdk", regexp.MustCompile(`^v(\d+\.\d+\.\d+)$`)) + if err != nil { + t.Fatal(err) + } + if got := ids(vs); got != "1.2.0@v1.2.0 1.9.0@v1.9.0 1.10.0@v1.10.0" { + t.Errorf("got %s", got) + } +} + +func TestGitHubCommitsAreDatedVersions(t *testing.T) { + get := serve(t, map[string]string{ + "https://api.github.com/repos/acme/spec/commits?path=spec%2Fopenapi.json&per_page=100&page=1": `[ + {"sha":"bbb","commit":{"committer":{"date":"2026-11-19T10:00:00Z"}}}, + {"sha":"aaa","commit":{"committer":{"date":"2026-10-01T09:00:00Z"}}} + ]`, + }) + vs, err := GitHubCommits(context.Background(), get, "acme/spec", "spec/openapi.json") + if err != nil { + t.Fatal(err) + } + if got := ids(vs); got != "2026-10-01@aaa 2026-11-19@bbb" { + t.Errorf("got %s", got) + } +} + +func TestPyPIReleasesAreFinalAndNotYanked(t *testing.T) { + get := serve(t, map[string]string{ + "https://pypi.org/pypi/acme/json": `{"releases":{ + "1.0.0":[{"upload_time_iso_8601":"2024-01-01T00:00:00Z"}], + "1.10.0":[{"upload_time_iso_8601":"2025-01-01T00:00:00Z"}], + "1.9.0":[{"upload_time_iso_8601":"2024-06-01T00:00:00Z"}], + "2.0.0rc1":[{"upload_time_iso_8601":"2025-02-01T00:00:00Z"}], + "1.9.1":[{"upload_time_iso_8601":"2024-07-01T00:00:00Z","yanked":true}], + "0.9":[] + }}`, + }) + vs, err := PyPIReleases(context.Background(), get, "acme") + if err != nil { + t.Fatal(err) + } + if got := ids(vs); got != "1.0.0@1.0.0 1.9.0@1.9.0 1.10.0@1.10.0" { + t.Errorf("got %s", got) + } + if !vs[0].Released.Equal(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) { + t.Errorf("released %v", vs[0].Released) + } +} + +func TestNPMReleases(t *testing.T) { + get := serve(t, map[string]string{ + "https://registry.npmjs.org/@acme%2Fsdk": `{"versions":{"1.0.0":{},"1.1.0":{"deprecated":"use 1.1.1"},"1.1.1":{},"2.0.0-beta.1":{}}, + "time":{"1.0.0":"2024-01-01T00:00:00Z","1.1.1":"2024-02-01T00:00:00Z"}}`, + }) + vs, err := NPMReleases(context.Background(), get, "@acme/sdk") + if err != nil { + t.Fatal(err) + } + if got := ids(vs); got != "1.0.0@1.0.0 1.1.1@1.1.1" { + t.Errorf("got %s", got) + } +} + +func TestWaybackAt(t *testing.T) { + get := serve(t, map[string]string{ + "https://archive.org/wayback/available?timestamp=20250301000000&url=https%3A%2F%2Facme.example%2Fdeprecations": `{"archived_snapshots":{"closest":{"available":true,"status":"200","timestamp":"20250228120000","url":"http://web.archive.org/web/20250228120000/https://acme.example/deprecations"}}}`, + "https://archive.org/wayback/available?timestamp=20000101000000&url=https%3A%2F%2Facme.example%2Fdeprecations": `{"archived_snapshots":{}}`, + }) + v, err := WaybackAt(context.Background(), get, "https://acme.example/deprecations", time.Date(2025, 3, 1, 0, 0, 0, 0, time.UTC)) + if err != nil || v.ID != "2025-02-28" || !strings.Contains(v.Ref, "/web/20250228120000/") { + t.Errorf("got %+v, %v", v, err) + } + if _, err := WaybackAt(context.Background(), get, "https://acme.example/deprecations", time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)); err == nil { + t.Error("no capture gave no error") + } +} + +func TestCompareFinalMajorPairs(t *testing.T) { + for _, c := range []struct { + a, b string + want int + }{{"1.9", "1.10", -1}, {"2.0.0", "2.0.0", 0}, {"2026-01-02", "2025-12-31", 1}, {"v2", "v10", -1}, {"1.0", "1.0.1", -1}} { + if got := Compare(c.a, c.b); got != c.want { + t.Errorf("Compare(%q, %q) = %d, want %d", c.a, c.b, got, c.want) + } + } + for v, want := range map[string]bool{"2.0.0": true, "v0.86.0": true, "2.0.0rc1": false, "1.0.0-beta.1": false, "1.0.post1": false} { + if Final(v) != want { + t.Errorf("Final(%q) = %v", v, !want) + } + } + if Major("posthog-v8.1.0") != 8 || Major("none") != -1 { + t.Error("Major") + } + ps := Pairs([]plugin.Version{{ID: "1"}, {ID: "2"}, {ID: "3"}}) + if len(ps) != 2 || ps[0][0].ID != "1" || ps[1][1].ID != "3" { + t.Errorf("Pairs = %v", ps) + } +} diff --git a/plugins/changelog/changelog.go b/plugins/changelog/changelog.go index f5eadea..25bea03 100644 --- a/plugins/changelog/changelog.go +++ b/plugins/changelog/changelog.go @@ -22,6 +22,7 @@ import ( "github.com/patch-it-dev/patch-it-plugins/core/event" "github.com/patch-it-dev/patch-it-plugins/core/plugin" + "github.com/patch-it-dev/patch-it-plugins/core/versions" ) func init() { @@ -29,49 +30,65 @@ func init() { ID: "textual", Name: "Textual", Category: plugin.CategoryPackages, Homepage: "https://textual.textualize.io", Sources: []plugin.Source{{ ID: "textual", Layer: plugin.LayerPackage, Page: "providers/textual.md", - Watches: "Removals and breaking changes in release 0.86.0", - URLs: []string{"https://github.com/Textualize/textual/blob/v0.86.0/CHANGELOG.md"}, - Plugin: Plugin{File: Textual0860}, + Watches: "Names removed, renamed or deprecated in each release, from the release's section of CHANGELOG.md at its tag", + URLs: []string{"https://github.com/Textualize/textual/blob/main/CHANGELOG.md", "https://pypi.org/project/textual/"}, + Plugin: Plugin{File: Textual}, }}, }) plugin.RegisterProvider(plugin.Provider{ ID: "urllib3", Name: "urllib3", Category: plugin.CategoryPackages, Homepage: "https://urllib3.readthedocs.io", Sources: []plugin.Source{{ ID: "urllib3", Layer: plugin.LayerPackage, Page: "providers/urllib3.md", - Watches: "Removals and deprecations in release 2.0.0", - URLs: []string{"https://github.com/urllib3/urllib3/blob/2.0.0/CHANGES.rst"}, - Plugin: Plugin{File: Urllib3200}, + Watches: "Names removed or deprecated in each release, from the release's section of CHANGES.rst at its tag", + URLs: []string{"https://github.com/urllib3/urllib3/blob/main/CHANGES.rst", "https://pypi.org/project/urllib3/"}, + Plugin: Plugin{File: Urllib3}, }}, }) } -// File is one package's changelog at one release. +// File is one package's changelog. Unpinned (Tag and Release empty), it is +// read release by release: every version on the package registry, the file at +// that release's tag (TagPrefix + version). Pinned (At), it is one release. type File struct { Package string // the installed name and the events' api: "urllib3" - Ecosystem string // "pypi" + Ecosystem string // where its releases are listed: "pypi" Module string // the import name, stripped from a symbol's front: "urllib3" Repo string // owner/name on GitHub - Tag string // the release's tag: "2.0.0", "v0.86.0" - Release string // the release as the changelog heads it: "2.0.0" + TagPrefix string // a release's git tag is TagPrefix + its version: "v" for v0.86.0 Path string // the changelog's path in the repo + Tag string // pinned: the release's tag: "2.0.0", "v0.86.0" + Release string // pinned: the release as the changelog heads it: "2.0.0" } -// Textual0860 is Textual's Keep a Changelog CHANGELOG.md at v0.86.0 (checked -// 2026-11-14: "## [0.86.0] - 2024-11-16", "### Removed", and "Breaking -// change:" bullets under "### Changed"). -var Textual0860 = File{ +// At is f pinned to one release. +func (f File) At(release string) File { + f.Release, f.Tag = release, f.TagPrefix+release + return f +} + +// Textual is Textual's Keep a Changelog CHANGELOG.md (checked 2026-11-14 at +// v0.86.0: "## [0.86.0] - 2024-11-16", "### Removed", and "Breaking change:" +// bullets under "### Changed"), tagged v. +var Textual = File{ Package: "textual", Ecosystem: "pypi", Module: "textual", - Repo: "Textualize/textual", Tag: "v0.86.0", Release: "0.86.0", Path: "CHANGELOG.md", + Repo: "Textualize/textual", TagPrefix: "v", Path: "CHANGELOG.md", } -// Urllib3200 is urllib3's CHANGES.rst at 2.0.0 (checked 2026-11-14: "2.0.0 -// (2023-04-26)" over "=====", "Removed" and "Deprecated" over "-----"). -var Urllib3200 = File{ +// Urllib3 is urllib3's CHANGES.rst (checked 2026-11-14 at 2.0.0: "2.0.0 +// (2023-04-26)" over "=====", "Removed" and "Deprecated" over "-----"), +// tagged . +var Urllib3 = File{ Package: "urllib3", Ecosystem: "pypi", Module: "urllib3", - Repo: "urllib3/urllib3", Tag: "2.0.0", Release: "2.0.0", Path: "CHANGES.rst", + Repo: "urllib3/urllib3", Path: "CHANGES.rst", } -// URL is the changelog as it was at the tag: its own provenance. +// The releases the golden files pin. +var ( + Textual0860 = Textual.At("0.86.0") + Urllib3200 = Urllib3.At("2.0.0") +) + +// URL is the changelog as it was at the (pinned) tag: its own provenance. func (f File) URL() string { return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s", f.Repo, f.Tag, f.Path) } @@ -130,20 +147,86 @@ func (f File) added(sections []Section) map[string]bool { return out } -// Plugin reads one changelog at one release. +// Plugin reads a package's changelog: release by release (ListVersions, +// ReadAt, Extract), or, with a pinned File, that one release. type Plugin struct { File File } func (p Plugin) Name() string { return p.File.Package } +// Run reads the pinned release, or the newest one on the registry, and +// stores its events. A release's tag never changes, so a rerun stores nothing +// new: the fingerprints dedupe. func (p Plugin) Run(ctx context.Context, env plugin.Env) (plugin.Result, error) { + v := plugin.Version{ID: p.File.Release, Ref: p.File.Tag} + if p.File.Tag == "" { + vs, err := p.ListVersions(ctx, env) + if err != nil { + return plugin.Result{}, err + } + if len(vs) == 0 { + return plugin.Result{}, fmt.Errorf("%s: no release listed", p.File.Package) + } + v = vs[len(vs)-1] + } + to, err := p.ReadAt(ctx, env, v) + if err != nil { + return plugin.Result{}, err + } + res, err := p.Extract(ctx, env, plugin.Snapshot{}, to) + if err != nil { + return plugin.Result{}, err + } + res.Baselines = []plugin.Spec{plugin.Baseline(p.File.Package, v.ID, v.Ref, to.SourceURL, to.Content, false, env.Now().UTC())} + return res, nil +} + +// ListVersions is every final release of the package on its registry, oldest +// first, each read at TagPrefix + version; a pinned File has its one release. +func (p Plugin) ListVersions(ctx context.Context, env plugin.Env) ([]plugin.Version, error) { f := p.File + if f.Tag != "" { + return []plugin.Version{{ID: f.Release, Ref: f.Tag}}, nil + } + var vs []plugin.Version + var err error + switch f.Ecosystem { + case "pypi": + vs, err = versions.PyPIReleases(ctx, env.Get, f.Package) + case "npm": + vs, err = versions.NPMReleases(ctx, env.Get, f.Package) + default: + return nil, fmt.Errorf("%s: no registry to list %q releases from", f.Package, f.Ecosystem) + } + if err != nil { + return nil, err + } + for i := range vs { + vs[i].Ref = f.TagPrefix + vs[i].ID + } + return vs, nil +} + +// ReadAt fetches the changelog at the release's tag. +func (p Plugin) ReadAt(ctx context.Context, env plugin.Env, v plugin.Version) (plugin.Snapshot, error) { + f := p.File + f.Release, f.Tag = v.ID, v.Ref page, err := env.Get(ctx, f.URL()) if err != nil { - return plugin.Result{}, fmt.Errorf("fetching %s: %w", f.URL(), err) + return plugin.Snapshot{}, fmt.Errorf("fetching %s: %w", f.URL(), err) } - heading, findings, err := f.Read(page) + return plugin.Snapshot{Version: v, Content: page, SourceURL: f.URL()}, nil +} + +// Extract is the to release's section of the changelog: an event per bullet +// that plainly removes, renames or deprecates a name, with previous_version +// from's release. Releases are extracted pair by pair, so from is the release +// right before to and to's section is everything between them. +func (p Plugin) Extract(_ context.Context, env plugin.Env, from, to plugin.Snapshot) (plugin.Result, error) { + f := p.File + f.Release, f.Tag = to.Version.ID, to.Version.Ref + heading, findings, err := f.Read(to.Content) if err != nil { return plugin.Result{}, fmt.Errorf("reading %s: %w", f.URL(), err) } @@ -161,17 +244,18 @@ func (p Plugin) Run(ctx context.Context, env plugin.Env) (plugin.Result, error) Change: c.Change, Symbols: c.Symbols, Replacement: c.Replacement, } e := event.EventV2{ - API: f.Package, - Version: f.Release, - ChangeType: c.ChangeType, - Breaking: c.ChangeType == event.ChangeTypeBreaking, - Subject: subject, - Description: describe(f, c), - MigrationHint: cap2000(fmt.Sprintf("%s (%s's changelog for %s)", fd.Bullet.Text, f.Package, f.Release)), - SourceURL: f.URL(), - SourceExcerpt: cap2000(heading + "\n" + fd.Section + "\n" + fd.Bullet.Raw), - Confidence: 1.0, - DetectedAt: now, + API: f.Package, + Version: f.Release, + PreviousVersion: from.Version.ID, + ChangeType: c.ChangeType, + Breaking: c.ChangeType == event.ChangeTypeBreaking, + Subject: subject, + Description: describe(f, c), + MigrationHint: cap2000(fmt.Sprintf("%s (%s's changelog for %s)", fd.Bullet.Text, f.Package, f.Release)), + SourceURL: f.URL(), + SourceExcerpt: cap2000(heading + "\n" + fd.Section + "\n" + fd.Bullet.Raw), + Confidence: 1.0, + DetectedAt: now, } e.Fingerprint = event.FingerprintV2(e.API, e.Version, e.ChangeType, subject, "") if err := e.Validate(); err != nil { @@ -180,14 +264,11 @@ func (p Plugin) Run(ctx context.Context, env plugin.Env) (plugin.Result, error) } events = append(events, e) } - return plugin.Result{ - Label: f.URL(), - Events: events, - Rejected: rejected, - Baselines: []plugin.Spec{plugin.Baseline(f.Package, f.Release, f.Tag, f.URL(), page, false, now)}, - }, nil + return plugin.Result{Label: f.URL(), Events: events, Rejected: rejected}, nil } +var _ plugin.Versioned = Plugin{} + func describe(f File, c Change) string { names := "`" + strings.Join(c.Symbols, "`, `") + "`" verb := "was" diff --git a/plugins/changelog/testdata/README.md b/plugins/changelog/testdata/README.md index 0f33ed9..8196285 100644 --- a/plugins/changelog/testdata/README.md +++ b/plugins/changelog/testdata/README.md @@ -8,3 +8,9 @@ golden files under `golden/` are what the plugin stores from them |---|---|---|---|---|---| | `textual-v0.86.0-CHANGELOG.md` | https://raw.githubusercontent.com/Textualize/textual/v0.86.0/CHANGELOG.md | `5a738fa` | 2026-11-14 08:36 UTC | MIT (Textual), kept whole | "Breaking change: Removed `App.dark` reactive attribute" | | `urllib3-2.0.0-CHANGES.rst` | https://raw.githubusercontent.com/urllib3/urllib3/2.0.0/CHANGES.rst | `303ba81` | 2026-11-14 08:35 UTC | MIT (urllib3), kept whole | "Removed support for Python 2.7, 3.5, and 3.6" | + +## Version lists + +| File | URL | Saved | License | Use | +|---|---|---|---|---| +| `pypi-urllib3-excerpt.json` | https://pypi.org/pypi/urllib3/json (shape only) | written for the test, 2026-11-20 | not a saved copy: four releases in the shape of PyPI's JSON API (`releases` → files with `upload_time_iso_8601`, `yanked`), dates approximate, a pre-release among them | for the version-listing test only (ListVersions, consecutive pairs); no golden reads it | diff --git a/plugins/changelog/testdata/pypi-urllib3-excerpt.json b/plugins/changelog/testdata/pypi-urllib3-excerpt.json new file mode 100644 index 0000000..43e63ae --- /dev/null +++ b/plugins/changelog/testdata/pypi-urllib3-excerpt.json @@ -0,0 +1,6 @@ +{"releases":{ + "1.26.15":[{"upload_time_iso_8601":"2023-03-10T18:31:28.000000Z","yanked":false}], + "2.0.0a1":[{"upload_time_iso_8601":"2022-11-15T19:43:59.000000Z","yanked":false}], + "2.0.0":[{"upload_time_iso_8601":"2023-04-26T16:43:58.000000Z","yanked":false}], + "2.0.1":[{"upload_time_iso_8601":"2023-04-30T18:42:25.000000Z","yanked":false}] +}} diff --git a/plugins/changelog/versions_test.go b/plugins/changelog/versions_test.go new file mode 100644 index 0000000..803fc34 --- /dev/null +++ b/plugins/changelog/versions_test.go @@ -0,0 +1,73 @@ +package changelog + +import ( + "context" + "strings" + "testing" + + "github.com/patch-it-dev/patch-it-plugins/core/kit/kittest" + "github.com/patch-it-dev/patch-it-plugins/core/plugin" + "github.com/patch-it-dev/patch-it-plugins/core/versions" +) + +// Unpinned, a changelog's versions are the package's final releases on PyPI, +// each read at its tag; the 1.26.15 -> 2.0.0 pair gives the same events as +// the pinned golden, with previous_version set. +func TestReleasesAreListedAndExtractedPairByPair(t *testing.T) { + pages := kittest.Pages{ + "https://pypi.org/pypi/urllib3/json": "testdata/pypi-urllib3-excerpt.json", + Urllib3.At("1.26.15").URL(): "testdata/urllib3-2.0.0-CHANGES.rst", // only 2.0.0's section is read + Urllib3200.URL(): "testdata/urllib3-2.0.0-CHANGES.rst", + } + env := kittest.Env(t, pages) + p := Plugin{File: Urllib3} + ctx := context.Background() + vs, err := p.ListVersions(ctx, env) + if err != nil { + t.Fatal(err) + } + var ids []string + for _, v := range vs { + ids = append(ids, v.ID+"@"+v.Ref) + } + if got := strings.Join(ids, " "); got != "1.26.15@1.26.15 2.0.0@2.0.0 2.0.1@2.0.1" { + t.Fatalf("versions %s", got) + } + pair := versions.Pairs(vs)[0] + from, err := p.ReadAt(ctx, env, pair[0]) + if err != nil { + t.Fatal(err) + } + to, err := p.ReadAt(ctx, env, pair[1]) + if err != nil { + t.Fatal(err) + } + res, err := p.Extract(ctx, env, from, to) + if err != nil || len(res.Rejected) > 0 { + t.Fatalf("%v %v", err, res.Rejected) + } + pinned := kittest.Ingest(t, Plugin{File: Urllib3200}, kittest.Pages{Urllib3200.URL(): "testdata/urllib3-2.0.0-CHANGES.rst"}) + if len(res.Events) == 0 || len(res.Events) != len(pinned) { + t.Fatalf("pair gave %d events, pinned run %d", len(res.Events), len(pinned)) + } + for i, e := range res.Events { + if e.PreviousVersion != "1.26.15" || e.Version != "2.0.0" || e.Fingerprint != pinned[i].Fingerprint { + t.Errorf("event %d: %s<-%s %s, pinned %s", i, e.Version, e.PreviousVersion, e.Fingerprint, pinned[i].Fingerprint) + } + } +} + +// Run on an unpinned file reads the newest release on PyPI. +func TestRunReadsTheNewestRelease(t *testing.T) { + pages := kittest.Pages{ + "https://pypi.org/pypi/urllib3/json": "testdata/pypi-urllib3-excerpt.json", + Urllib3.At("2.0.1").URL(): "testdata/urllib3-2.0.0-CHANGES.rst", + } + _, err := Plugin{File: Urllib3}.Run(context.Background(), kittest.Env(t, pages)) + // The saved file is 2.0.0's, which has no 2.0.1 section: the run says so + // rather than inventing events. + if err == nil || !strings.Contains(err.Error(), "2.0.1") { + t.Errorf("err = %v, want one naming 2.0.1", err) + } + var _ plugin.Versioned = Plugin{} +} diff --git a/plugins/migrationguide/migrationguide.go b/plugins/migrationguide/migrationguide.go index 24b023a..825a8b4 100644 --- a/plugins/migrationguide/migrationguide.go +++ b/plugins/migrationguide/migrationguide.go @@ -1,7 +1,7 @@ // Package migrationguide is the source plugin for an SDK's migration guide, -// read at the release tag of the major it describes: an event per request -// parameter the release removed (sdkguide/). The tag never changes, so -// a rerun stores nothing new: the fingerprints dedupe. +// read at the release tag of each major it describes: an event per request +// parameter the release removed (sdkguide/). A tag never changes, so a rerun +// stores nothing new: the fingerprints dedupe. package migrationguide import ( @@ -10,6 +10,7 @@ import ( "github.com/patch-it-dev/patch-it-plugins/core/migrate" "github.com/patch-it-dev/patch-it-plugins/core/plugin" + "github.com/patch-it-dev/patch-it-plugins/core/versions" "github.com/patch-it-dev/patch-it-plugins/plugins/migrationguide/sdkguide" ) @@ -18,37 +19,108 @@ func init() { ID: "anthropic", Name: "Anthropic", Category: plugin.CategoryAI, Homepage: "https://www.anthropic.com", Sources: []plugin.Source{{ ID: "python-sdk", Layer: plugin.LayerSDK, Page: "providers/anthropic.md", - Watches: "Parameters removed from `messages.create()` and friends in the Python SDK 1.0", - URLs: []string{"https://github.com/anthropics/anthropic-sdk-python/blob/v1.0.0/MIGRATION.md"}, - Plugin: Plugin{Guide: sdkguide.AnthropicV1}, + Watches: "Request parameters each new major of the Python SDK removes from `messages.create()` and friends, from MIGRATION.md at the major's tag", + URLs: []string{"https://github.com/anthropics/anthropic-sdk-python/blob/main/MIGRATION.md", "https://pypi.org/project/anthropic/"}, + Plugin: Plugin{Guide: sdkguide.Anthropic}, }}, }) } -// Plugin reads one SDK's guide to one major release. +// Plugin reads an SDK's migration guide: at the first release of each major +// (ListVersions, ReadAt, Extract), or, with a pinned Guide, that one major. type Plugin struct { Guide sdkguide.Guide } func (p Plugin) Name() string { return p.Guide.API } +// Run reads the pinned major's guide, or the newest major's. func (p Plugin) Run(ctx context.Context, env plugin.Env) (plugin.Result, error) { g := p.Guide + v := plugin.Version{ID: g.Version(), Ref: "v" + g.Version()} + if g.Major == 0 { + vs, err := p.ListVersions(ctx, env) + if err != nil { + return plugin.Result{}, err + } + v = plugin.Version{} + for _, c := range vs { + if firstOfMajor(c.ID) { + v = c + } + } + if v.ID == "" { + return plugin.Result{}, fmt.Errorf("%s: no major release listed", g.API) + } + } + to, err := p.ReadAt(ctx, env, v) + if err != nil { + return plugin.Result{}, err + } + res, err := p.Extract(ctx, env, plugin.Snapshot{}, to) + if err != nil { + return plugin.Result{}, err + } + res.Baselines = []plugin.Spec{plugin.Baseline(g.API, v.ID, v.Ref, to.SourceURL, to.Content, false, env.Now().UTC())} + return res, nil +} + +// ListVersions is every final release of the SDK on PyPI, oldest first; a +// pinned Guide has its one major. +func (p Plugin) ListVersions(ctx context.Context, env plugin.Env) ([]plugin.Version, error) { + g := p.Guide + if g.Major != 0 { + return []plugin.Version{{ID: g.Version(), Ref: "v" + g.Version()}}, nil + } + vs, err := versions.PyPIReleases(ctx, env.Get, g.API) + if err != nil { + return nil, err + } + for i := range vs { + vs[i].Ref = "v" + vs[i].ID + } + return vs, nil +} + +// ReadAt fetches the guide at the release's tag, when the release is the +// first of its major (X.0.0): only a new major has a guide to read. Any other +// release reads as nothing (a snapshot with no content). +func (p Plugin) ReadAt(ctx context.Context, env plugin.Env, v plugin.Version) (plugin.Snapshot, error) { + if !firstOfMajor(v.ID) { + return plugin.Snapshot{Version: v}, nil + } + g := p.Guide.At(versions.Major(v.ID)) page, err := env.Get(ctx, g.URL()) if err != nil { - return plugin.Result{}, fmt.Errorf("fetching %s: %w", g.URL(), err) + return plugin.Snapshot{}, fmt.Errorf("fetching %s: %w", g.URL(), err) } - removals, err := sdkguide.Parse(page, g) + return plugin.Snapshot{Version: v, Content: page, SourceURL: g.URL()}, nil +} + +// Extract is an event per request parameter the to major removed, read from +// its guide, with previous_version from's release. A release that isn't a new +// major has no guide and no events. +func (p Plugin) Extract(_ context.Context, env plugin.Env, from, to plugin.Snapshot) (plugin.Result, error) { + if to.Content == nil { + return plugin.Result{Label: to.Version.ID}, nil + } + g := p.Guide.At(versions.Major(to.Version.ID)) + removals, err := sdkguide.Parse(to.Content, g) if err != nil { return plugin.Result{}, fmt.Errorf("reading %s: %w", g.URL(), err) } - now := env.Now().UTC() - events, errs := sdkguide.Events(g, removals, now) + events, errs := sdkguide.Events(g, removals, env.Now().UTC()) + for i := range events { + events[i].PreviousVersion = from.Version.ID + } converted, invalid := migrate.Fresh(events) - return plugin.Result{ - Label: g.URL(), - Events: converted, - Rejected: append(errs, invalid...), - Baselines: []plugin.Spec{plugin.Baseline(g.API, g.Version(), "v"+g.Version(), g.URL(), page, false, now)}, - }, nil + return plugin.Result{Label: g.URL(), Events: converted, Rejected: append(errs, invalid...)}, nil } + +// firstOfMajor is true for X.0.0, the release a major's guide is tagged at. +func firstOfMajor(v string) bool { + m := versions.Major(v) + return m > 0 && v == fmt.Sprintf("%d.0.0", m) +} + +var _ plugin.Versioned = Plugin{} diff --git a/plugins/migrationguide/sdkguide/sdkguide.go b/plugins/migrationguide/sdkguide/sdkguide.go index 75170f5..903d194 100644 --- a/plugins/migrationguide/sdkguide/sdkguide.go +++ b/plugins/migrationguide/sdkguide/sdkguide.go @@ -22,9 +22,9 @@ import ( // Guide is one SDK's migration guide for one major release. type Guide struct { - API string // the SDK's slug in change events, e.g. "anthropic" + API string // the SDK's slug in change events and its PyPI name, e.g. "anthropic" Repo string // its GitHub repository, owner/name - Major int // the release the guide describes: 1 for "Migrating to v1" + Major int // pinned: the release the guide describes, 1 for "Migrating to v1"; 0 lists them Operation string // the API operation the removed parameters were sent to // Methods is what a table row's method cell must name for its parameters to // count: the plain call, so a row about beta-only methods (where the same @@ -37,16 +37,25 @@ type Guide struct { Symbols []string } -// AnthropicV1 is anthropic-sdk-python's guide to 1.0.0. -var AnthropicV1 = Guide{ +// Anthropic is anthropic-sdk-python's MIGRATION.md, unpinned: read at the +// first release of each major (Major 0; At pins it). +var Anthropic = Guide{ API: "anthropic", Repo: "anthropics/anthropic-sdk-python", - Major: 1, Operation: "POST /v1/messages", Methods: "`messages.create()`", Symbols: []string{"messages.create", "messages.stream", "messages.parse"}, } +// AnthropicV1 is anthropic-sdk-python's guide to 1.0.0: what the golden pins. +var AnthropicV1 = Anthropic.At(1) + +// At is g pinned to one major. +func (g Guide) At(major int) Guide { + g.Major = major + return g +} + // Version is the release that removed the parameters. func (g Guide) Version() string { return fmt.Sprintf("%d.0.0", g.Major) } diff --git a/plugins/migrationguide/testdata/README.md b/plugins/migrationguide/testdata/README.md index 7adb466..7646c57 100644 --- a/plugins/migrationguide/testdata/README.md +++ b/plugins/migrationguide/testdata/README.md @@ -3,3 +3,9 @@ `golden/anthropic-1.0.0.json` is what the `anthropic` plugin stores from anthropic-sdk-python's MIGRATION.md at v1.0.0. The saved guide itself is shared with the `sdkguide` package: see [`sdkguide/testdata/README.md`](../../../sdkguide/testdata/README.md) for its URL, date and license. Rewrite it after an intended change with `go test ./plugins/migrationguide -update`, and say in the PR why the new output is right. + +## Version lists + +| File | URL | Saved | License | Use | +|---|---|---|---|---| +| `pypi-anthropic-excerpt.json` | https://pypi.org/pypi/anthropic/json (shape only) | written for the test, 2026-11-20 | not a saved copy: four releases in the shape of PyPI's JSON API (`releases` → files with `upload_time_iso_8601`, `yanked`), dates approximate, a pre-release among them | for the version-listing test only (ListVersions, consecutive pairs); no golden reads it | diff --git a/plugins/migrationguide/testdata/pypi-anthropic-excerpt.json b/plugins/migrationguide/testdata/pypi-anthropic-excerpt.json new file mode 100644 index 0000000..85fa85b --- /dev/null +++ b/plugins/migrationguide/testdata/pypi-anthropic-excerpt.json @@ -0,0 +1,6 @@ +{"releases":{ + "0.75.0":[{"upload_time_iso_8601":"2026-08-01T00:00:00Z","yanked":false}], + "1.0.0":[{"upload_time_iso_8601":"2026-08-20T00:00:00Z","yanked":false}], + "1.0.1":[{"upload_time_iso_8601":"2026-08-27T00:00:00Z","yanked":false}], + "1.1.0b1":[{"upload_time_iso_8601":"2026-09-01T00:00:00Z","yanked":false}] +}} diff --git a/plugins/migrationguide/versions_test.go b/plugins/migrationguide/versions_test.go new file mode 100644 index 0000000..9aa8f0a --- /dev/null +++ b/plugins/migrationguide/versions_test.go @@ -0,0 +1,62 @@ +package migrationguide + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/patch-it-dev/patch-it-plugins/core/kit/kittest" + "github.com/patch-it-dev/patch-it-plugins/core/versions" + "github.com/patch-it-dev/patch-it-plugins/plugins/migrationguide/sdkguide" +) + +// Unpinned, the SDK's versions are its releases on PyPI; only the pair that +// crosses into a new major reads a guide (at that major's tag) and has +// events, with previous_version set; the others read nothing. +func TestOnlyANewMajorHasAGuide(t *testing.T) { + env := kittest.Env(t, kittest.Pages{ + "https://pypi.org/pypi/anthropic/json": "testdata/pypi-anthropic-excerpt.json", + sdkguide.AnthropicV1.URL(): "sdkguide/testdata/anthropic-MIGRATION-v1.0.0.md", + }) + p := Plugin{Guide: sdkguide.Anthropic} + ctx := context.Background() + vs, err := p.ListVersions(ctx, env) + if err != nil { + t.Fatal(err) + } + var got []string + for _, pair := range versions.Pairs(vs) { + from, err := p.ReadAt(ctx, env, pair[0]) + if err != nil { + t.Fatal(err) + } + to, err := p.ReadAt(ctx, env, pair[1]) + if err != nil { + t.Fatal(err) + } + res, err := p.Extract(ctx, env, from, to) + if err != nil || len(res.Rejected) > 0 { + t.Fatalf("%v %v", err, res.Rejected) + } + var params []string + for _, e := range res.Events { + params = append(params, e.PreviousVersion+"->"+e.Version+":"+e.Subject.Parameter) + } + got = append(got, fmt.Sprintf("%s->%s [%s]", pair[0].ID, pair[1].ID, strings.Join(params, " "))) + } + want := "0.75.0->1.0.0 [0.75.0->1.0.0:temperature 0.75.0->1.0.0:top_p 0.75.0->1.0.0:top_k] | 1.0.0->1.0.1 []" + if strings.Join(got, " | ") != want { + t.Errorf("got %s\nwant %s", strings.Join(got, " | "), want) + } +} + +// Run on the unpinned guide reads the newest major's: the same events as the +// pinned golden. +func TestRunReadsTheNewestMajor(t *testing.T) { + got := kittest.Ingest(t, Plugin{Guide: sdkguide.Anthropic}, kittest.Pages{ + "https://pypi.org/pypi/anthropic/json": "testdata/pypi-anthropic-excerpt.json", + sdkguide.AnthropicV1.URL(): "sdkguide/testdata/anthropic-MIGRATION-v1.0.0.md", + }) + kittest.CompareGolden(t, "testdata/golden/anthropic-1.0.0.json", got) +} diff --git a/plugins/openapi/adapter/adapter.go b/plugins/openapi/adapter/adapter.go index 9131d6b..871d23e 100644 --- a/plugins/openapi/adapter/adapter.go +++ b/plugins/openapi/adapter/adapter.go @@ -4,7 +4,11 @@ // fetch/, and diffing/normalizing/storing are shared, not adapter code. package adapter -import "context" +import ( + "context" + + "github.com/patch-it-dev/patch-it-plugins/core/plugin" +) // Spec is one fetched spec: its raw bytes, the API version it declares, and // the pinned URL it came from. It is not yet parsed or stored — that is @@ -31,3 +35,11 @@ type Adapter interface { type Reshaper interface { SourcePointer(content []byte) func(ptr string) string } + +// Lister is an adapter that can list every version of its spec, oldest first +// (a version's Ref is what Fetch takes): what lets a spec be diffed release by +// release, not only newest against last. One without it is read at LatestTag +// only. +type Lister interface { + Versions(ctx context.Context) ([]plugin.Version, error) +} diff --git a/plugins/openapi/adapter/versions.go b/plugins/openapi/adapter/versions.go new file mode 100644 index 0000000..9e6238e --- /dev/null +++ b/plugins/openapi/adapter/versions.go @@ -0,0 +1,75 @@ +package adapter + +import ( + "context" + "strings" + + "github.com/patch-it-dev/patch-it-plugins/core/github" + "github.com/patch-it-dev/patch-it-plugins/core/plugin" + "github.com/patch-it-dev/patch-it-plugins/core/versions" +) + +// Every version of each spec that can be listed, oldest first (Lister): the +// repo's releases for the adapters whose LatestTag is releases/latest, the +// commits that changed the spec for those that follow a file. The others +// (GitHub's dated versions, GitLab, Elasticsearch, Sentry, Algolia, Appwrite) +// are read at LatestTag only, for now. + +func releases(ctx context.Context, repo string, keep func(string) bool) ([]plugin.Version, error) { + return versions.GitHubReleases(ctx, github.Get, repo, keep) +} + +func commits(ctx context.Context, repo, path string) ([]plugin.Version, error) { + return versions.GitHubCommits(ctx, github.Get, repo, path) +} + +// Versions are stripe/openapi's releases. +func (Stripe) Versions(ctx context.Context) ([]plugin.Version, error) { + return releases(ctx, "stripe/openapi", nil) +} + +// Versions are datadog-api-client-python's releases. +func (Datadog) Versions(ctx context.Context) ([]plugin.Version, error) { + return releases(ctx, "DataDog/datadog-api-client-python", nil) +} + +// Versions are vercel/sdk's releases, canaries left out (as LatestTag does). +func (Vercel) Versions(ctx context.Context) ([]plugin.Version, error) { + return releases(ctx, "vercel/sdk", func(tag string) bool { return !strings.Contains(tag, "canary") }) +} + +// Versions are Flagsmith's releases. +func (Flagsmith) Versions(ctx context.Context) ([]plugin.Version, error) { + return releases(ctx, "Flagsmith/flagsmith", nil) +} + +// Versions are the commits that changed the Management API spec. +func (Supabase) Versions(ctx context.Context) ([]plugin.Version, error) { + return commits(ctx, "supabase/supabase", "apps/docs/spec/api_v1_openapi.json") +} + +// Versions are the commits that changed Neon's spec. +func (Neon) Versions(ctx context.Context) ([]plugin.Version, error) { + return commits(ctx, "neondatabase/neon-pkgs", "packages/sdk/spec/neon-openapi.json") +} + +// Versions are the commits that changed Convex's deployment API spec. +func (Convex) Versions(ctx context.Context) ([]plugin.Version, error) { + return commits(ctx, "get-convex/convex-backend", "npm-packages/@convex-dev/platform/deployment-openapi.json") +} + +// Versions are the commits that changed PlanetScale's spec. +func (PlanetScale) Versions(ctx context.Context) ([]plugin.Version, error) { + return commits(ctx, planetscaleRepo, planetscaleSpecPath) +} + +var ( + _ Lister = Stripe{} + _ Lister = Datadog{} + _ Lister = Vercel{} + _ Lister = Flagsmith{} + _ Lister = Supabase{} + _ Lister = Neon{} + _ Lister = Convex{} + _ Lister = PlanetScale{} +) diff --git a/plugins/openapi/openapi.go b/plugins/openapi/openapi.go index 19bcded..61eeda4 100644 --- a/plugins/openapi/openapi.go +++ b/plugins/openapi/openapi.go @@ -214,106 +214,117 @@ type Plugin struct { func (p Plugin) Name() string { return p.Adapter.API() } -// Run diffs the newest release against whatever was last stored. +// Run diffs the newest release against whatever was last stored: the newest +// version (LatestTag), ReadAt it, and Extract against the last baseline. func (p Plugin) Run(ctx context.Context, env plugin.Env) (plugin.Result, error) { a := p.Adapter tag, err := a.LatestTag(ctx) if err != nil { return plugin.Result{}, fmt.Errorf("finding latest tag: %w", err) } - newSpec, err := a.Fetch(ctx, tag) + to, err := p.ReadAt(ctx, env, plugin.Version{ID: tag, Ref: tag}) if err != nil { - return plugin.Result{}, fmt.Errorf("fetching %s %s: %w", a.API(), tag, err) + return plugin.Result{}, err } last, err := env.Baselines.LastSpec(a.API()) if err != nil { return plugin.Result{}, fmt.Errorf("loading last spec: %w", err) } - if last != nil && last.SHA256 == plugin.SHA256(newSpec.Content) { + if last != nil && last.SHA256 == plugin.SHA256(to.Content) { log.Printf("%s: no change (%s)", a.API(), tag) return plugin.Result{}, nil } - - var oldParsed *diff.Spec - var previousVersion, oldSourceURL string + var from plugin.Snapshot if last != nil { - oldParsed, err = diff.Parse(last.Content) - if err != nil { - return plugin.Result{}, fmt.Errorf("parsing stored spec: %w", err) - } - previousVersion = last.Version - oldSourceURL = last.SourceURL + from = plugin.Snapshot{Version: plugin.Version{ID: last.Version, Ref: last.Tag}, Content: last.Content, SourceURL: last.SourceURL} } - newParsed, err := diff.Parse(newSpec.Content) + res, err := p.Extract(ctx, env, from, to) if err != nil { - return plugin.Result{}, fmt.Errorf("parsing fetched spec: %w", err) + return plugin.Result{}, err } - - var oldContent []byte - if last != nil { - oldContent = last.Content - } - events, errs := p.events(oldParsed, newParsed, oldContent, newSpec.Content, normalize.Source{ - API: a.API(), - Version: newSpec.Version, - PreviousVersion: previousVersion, - NewSpecURL: newSpec.SourceURL, - OldSpecURL: oldSourceURL, - DetectedAt: env.Now().UTC(), - }) - p.attachHints(ctx, newSpec.Version, events) - converted, invalid := migrate.Fresh(events) - return plugin.Result{ - Label: tag, - Events: converted, - Rejected: append(errs, invalid...), - Baselines: []plugin.Spec{plugin.Baseline(a.API(), newSpec.Version, tag, newSpec.SourceURL, newSpec.Content, false, env.Now())}, - }, nil + res.Label = tag + res.Baselines = []plugin.Spec{plugin.Baseline(a.API(), to.Version.ID, tag, to.SourceURL, to.Content, false, env.Now())} + return res, nil } // Replay diffs two given releases. Both specs are saved marked as replays, so // the normal-run baseline never moves; the events are stored like any others: // that's how a historical change becomes something the feed can serve. func (p Plugin) Replay(ctx context.Context, env plugin.Env, from, to string) (plugin.Result, error) { - a := p.Adapter - oldSpec, err := a.Fetch(ctx, from) + old, err := p.ReadAt(ctx, env, plugin.Version{ID: from, Ref: from}) + if err != nil { + return plugin.Result{}, err + } + next, err := p.ReadAt(ctx, env, plugin.Version{ID: to, Ref: to}) + if err != nil { + return plugin.Result{}, err + } + res, err := p.Extract(ctx, env, old, next) if err != nil { - return plugin.Result{}, fmt.Errorf("fetching %s %s: %w", a.API(), from, err) + return plugin.Result{}, err } - newSpec, err := a.Fetch(ctx, to) + now := env.Now() + res.Label = to + res.Baselines = []plugin.Spec{ + plugin.Baseline(p.Adapter.API(), old.Version.ID, from, old.SourceURL, old.Content, true, now), + plugin.Baseline(p.Adapter.API(), next.Version.ID, to, next.SourceURL, next.Content, true, now), + } + return res, nil +} + +// ListVersions is every release of the spec, oldest first, when the adapter +// can list them (adapter.Lister); otherwise only the newest (LatestTag). +func (p Plugin) ListVersions(ctx context.Context, _ plugin.Env) ([]plugin.Version, error) { + if l, ok := p.Adapter.(adapter.Lister); ok { + return l.Versions(ctx) + } + tag, err := p.Adapter.LatestTag(ctx) if err != nil { - return plugin.Result{}, fmt.Errorf("fetching %s %s: %w", a.API(), to, err) + return nil, fmt.Errorf("finding latest tag: %w", err) } - oldParsed, err := diff.Parse(oldSpec.Content) + return []plugin.Version{{ID: tag, Ref: tag}}, nil +} + +// ReadAt fetches the spec at v.Ref. The snapshot's version ID is the version +// the spec itself declares (what events carry), which may differ from the tag. +func (p Plugin) ReadAt(ctx context.Context, _ plugin.Env, v plugin.Version) (plugin.Snapshot, error) { + s, err := p.Adapter.Fetch(ctx, v.Ref) if err != nil { - return plugin.Result{}, fmt.Errorf("parsing %s: %w", from, err) + return plugin.Snapshot{}, fmt.Errorf("fetching %s %s: %w", p.Adapter.API(), v.Ref, err) } - newParsed, err := diff.Parse(newSpec.Content) + v.ID = s.Version + return plugin.Snapshot{Version: v, Content: s.Content, SourceURL: s.SourceURL}, nil +} + +// Extract diffs two snapshots of the spec into events, with to's version and +// from's as previous_version. A zero from is an empty spec before the first. +func (p Plugin) Extract(ctx context.Context, env plugin.Env, from, to plugin.Snapshot) (plugin.Result, error) { + var oldParsed *diff.Spec + if !from.IsZero() { + var err error + if oldParsed, err = diff.Parse(from.Content); err != nil { + return plugin.Result{}, fmt.Errorf("parsing the %s spec at %s: %w", p.Adapter.API(), from.Version.ID, err) + } + } + newParsed, err := diff.Parse(to.Content) if err != nil { - return plugin.Result{}, fmt.Errorf("parsing %s: %w", to, err) + return plugin.Result{}, fmt.Errorf("parsing the %s spec at %s: %w", p.Adapter.API(), to.Version.ID, err) } - events, errs := p.events(oldParsed, newParsed, oldSpec.Content, newSpec.Content, normalize.Source{ - API: a.API(), - Version: newSpec.Version, - PreviousVersion: oldSpec.Version, - NewSpecURL: newSpec.SourceURL, - OldSpecURL: oldSpec.SourceURL, + events, errs := p.events(oldParsed, newParsed, from.Content, to.Content, normalize.Source{ + API: p.Adapter.API(), + Version: to.Version.ID, + PreviousVersion: from.Version.ID, + NewSpecURL: to.SourceURL, + OldSpecURL: from.SourceURL, DetectedAt: env.Now().UTC(), }) - p.attachHints(ctx, newSpec.Version, events) - now := env.Now() + p.attachHints(ctx, to.Version.ID, events) converted, invalid := migrate.Fresh(events) - return plugin.Result{ - Label: to, - Events: converted, - Rejected: append(errs, invalid...), - Baselines: []plugin.Spec{ - plugin.Baseline(a.API(), oldSpec.Version, from, oldSpec.SourceURL, oldSpec.Content, true, now), - plugin.Baseline(a.API(), newSpec.Version, to, newSpec.SourceURL, newSpec.Content, true, now), - }, - }, nil + return plugin.Result{Label: to.Version.Ref, Events: converted, Rejected: append(errs, invalid...)}, nil } +var _ plugin.Versioned = Plugin{} + // events diffs two parsed specs into events, applying Keep, and pointing each // event's source_pointer into the published file when the adapter reshaped it. func (p Plugin) events(oldParsed, newParsed *diff.Spec, oldContent, newContent []byte, src normalize.Source) ([]event.ChangeEvent, []error) { diff --git a/plugins/openapi/versions_test.go b/plugins/openapi/versions_test.go new file mode 100644 index 0000000..e204766 --- /dev/null +++ b/plugins/openapi/versions_test.go @@ -0,0 +1,118 @@ +package openapi + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/patch-it-dev/patch-it-plugins/core/kit/kittest" + "github.com/patch-it-dev/patch-it-plugins/core/plugin" + "github.com/patch-it-dev/patch-it-plugins/core/versions" + "github.com/patch-it-dev/patch-it-plugins/plugins/openapi/adapter" +) + +// listingAdapter is fakeAdapter that can also list its versions. +type listingAdapter struct { + fakeAdapter + list []plugin.Version +} + +func (l listingAdapter) Versions(context.Context) ([]plugin.Version, error) { return l.list, nil } + +var _ adapter.Lister = listingAdapter{} + +// Three releases, read version by version: each consecutive pair is its own +// Extract, with to's version and from's as previous_version, and only what +// changed in that pair. +func TestEveryConsecutivePairIsExtracted(t *testing.T) { + spec := func(v string, props ...string) string { + var ps []string + for _, p := range props { + ps = append(ps, fmt.Sprintf("%q:{\"type\":\"string\"}", p)) + } + return fmt.Sprintf(`{"openapi":"3.0.0","info":{"version":"version-%s"},"paths":{},"components":{"schemas":{"charge":{"type":"object","properties":{%s}}}}}`, v, strings.Join(ps, ",")) + } + a := listingAdapter{ + fakeAdapter: fakeAdapter{latest: "v4", specs: map[string]string{ + "v1": spec("v1", "id", "a", "b", "c"), + "v2": spec("v2", "id", "b", "c"), // a removed + "v3": spec("v3", "id", "c"), // b removed + "v4": spec("v4", "id"), // c removed + }}, + list: []plugin.Version{{ID: "v1", Ref: "v1"}, {ID: "v2", Ref: "v2"}, {ID: "v3", Ref: "v3"}, {ID: "v4", Ref: "v4"}}, + } + p := newPlugin(a, noChangelog) + ctx, e := context.Background(), env(kittest.NewMem()) + vs, err := p.ListVersions(ctx, e) + if err != nil || len(vs) != 4 { + t.Fatalf("ListVersions = %v, %v", vs, err) + } + var got []string + for _, pair := range versions.Pairs(vs) { + from, err := p.ReadAt(ctx, e, pair[0]) + if err != nil { + t.Fatal(err) + } + to, err := p.ReadAt(ctx, e, pair[1]) + if err != nil { + t.Fatal(err) + } + res, err := p.Extract(ctx, e, from, to) + if err != nil || len(res.Rejected) > 0 || len(res.Baselines) > 0 { + t.Fatalf("Extract: %v %v baselines=%d", err, res.Rejected, len(res.Baselines)) + } + for _, ev := range res.Events { + got = append(got, fmt.Sprintf("%s<-%s %s %s", ev.Version, ev.PreviousVersion, ev.ChangeType, ev.Subject.Property)) + } + } + want := "version-v2<-version-v1 breaking a|version-v3<-version-v2 breaking b|version-v4<-version-v3 breaking c" + if strings.Join(got, "|") != want { + t.Errorf("got %s\nwant %s", strings.Join(got, "|"), want) + } +} + +// An adapter that can't list is read at its newest version only. +func TestListVersionsFallsBackToTheNewest(t *testing.T) { + vs, err := newPlugin(fakeAdapter{latest: "v2", specs: specs}, noChangelog).ListVersions(context.Background(), plugin.Env{}) + if err != nil || len(vs) != 1 || vs[0].Ref != "v2" { + t.Errorf("got %v, %v", vs, err) + } +} + +// The real adapters list GitHub releases or the commits that changed their +// spec, through core/versions. +func TestAdaptersListReleasesAndCommits(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/repos/stripe/openapi/releases": + fmt.Fprint(w, `[{"tag_name":"v2","published_at":"2026-02-01T00:00:00Z"},{"tag_name":"v1","published_at":"2026-01-01T00:00:00Z"}]`) + case r.URL.Path == "/repos/supabase/supabase/commits" && r.URL.Query().Get("path") == "apps/docs/spec/api_v1_openapi.json": + fmt.Fprint(w, `[{"sha":"b","commit":{"committer":{"date":"2026-02-01T00:00:00Z"}}},{"sha":"a","commit":{"committer":{"date":"2026-01-01T00:00:00Z"}}}]`) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + old := versions.GitHubAPI + versions.GitHubAPI = srv.URL + defer func() { versions.GitHubAPI = old }() + for _, c := range []struct { + a adapter.Adapter + want string + }{{adapter.Stripe{}, "v1 v2"}, {adapter.Supabase{}, "a b"}} { + vs, err := Plugin{Adapter: c.a}.ListVersions(context.Background(), plugin.Env{}) + if err != nil { + t.Fatalf("%s: %v", c.a.API(), err) + } + var refs []string + for _, v := range vs { + refs = append(refs, v.Ref) + } + if strings.Join(refs, " ") != c.want { + t.Errorf("%s: refs %v, want %s", c.a.API(), refs, c.want) + } + } +} diff --git a/plugins/removedmembers/newrelic.go b/plugins/removedmembers/newrelic.go index 842197d..1fb3cd5 100644 --- a/plugins/removedmembers/newrelic.go +++ b/plugins/removedmembers/newrelic.go @@ -65,7 +65,7 @@ var NewRelic11 = Guide{ Package: "newrelic", Module: "newrelic", Release: "11.0.0", - Tag: "f0b4f817bf9a7b76a78e26609369a8a7ce764c8e", + DocsRef: "f0b4f817bf9a7b76a78e26609369a8a7ce764c8e", Repo: "newrelic/docs-website", Path: "src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-110000.mdx", Section: "Removed features", diff --git a/plugins/removedmembers/posthog.go b/plugins/removedmembers/posthog.go index 7f6b8ce..9250e5c 100644 --- a/plugins/removedmembers/posthog.go +++ b/plugins/removedmembers/posthog.go @@ -56,16 +56,16 @@ package removedmembers // fields, retries, dropped events, options, and "Rename deprecated settings", // whose old names still work) aren't under the heading, so they're never read. var PostHog8 = Guide{ - Package: "posthog", - Module: "posthog", - Release: "8.0.0", - Tag: "posthog-v8.0.0", - Repo: "PostHog/posthog-python", - Path: "docs/migration-7.x-to-8.0.md", - Section: "Fix removed APIs", - Format: FormatMarkdown, - Header: "Removed", - EnvVars: []string{"POSTHOG_CAPTURE_MODE"}, + Package: "posthog", + Module: "posthog", + Release: "8.0.0", + TagPrefix: "posthog-v", + Repo: "PostHog/posthog-python", + Path: "docs/migration-7.x-to-8.0.md", + Section: "Fix removed APIs", + Format: FormatMarkdown, + Header: "Removed", + EnvVars: []string{"POSTHOG_CAPTURE_MODE"}, Paths: map[string]string{ "CaptureV1Error": "capture_v1.CaptureV1Error", "async_batch_post": "_async_request.async_batch_post", diff --git a/plugins/removedmembers/removedmembers.go b/plugins/removedmembers/removedmembers.go index b0ab908..ec78e71 100644 --- a/plugins/removedmembers/removedmembers.go +++ b/plugins/removedmembers/removedmembers.go @@ -29,6 +29,7 @@ import ( "github.com/patch-it-dev/patch-it-plugins/core/event" "github.com/patch-it-dev/patch-it-plugins/core/plugin" + "github.com/patch-it-dev/patch-it-plugins/core/versions" ) func init() { @@ -70,8 +71,13 @@ type Guide struct { Path string // the guide's path in the repo at the release tag Section string // the heading the table is under - // Tag is the release tag; "" is "v" + Release. - Tag string + // TagPrefix makes a release's git tag: TagPrefix + version ("" is "v": + // v2.0.0; PostHog's is "posthog-v"). The guide is read at Release's tag. + TagPrefix string + // DocsRef, when the guide isn't in the package's own repo but in a docs + // repo with no release tags (New Relic's release notes), is the commit + // of Repo the guide is read at. + DocsRef string // Format is how the table is written: FormatRST (the default) or // FormatMarkdown. Format Format @@ -132,11 +138,17 @@ const ( FormatMarkdown Format = "markdown" ) -func (g Guide) tag() string { - if g.Tag != "" { - return g.Tag +func (g Guide) tag() string { return g.tagOf(g.Release) } + +// tagOf is the git ref a release of the package is read at. +func (g Guide) tagOf(release string) string { + if g.DocsRef != "" { + return g.DocsRef + } + if g.TagPrefix != "" { + return g.TagPrefix + release } - return "v" + g.Release + return "v" + release } func (g Guide) doc() string { @@ -263,15 +275,58 @@ type Plugin struct { func (p Plugin) Name() string { return p.Guide.Package } -// Run reads the guide at its tag. The tag never changes, so a rerun stores -// nothing new: the fingerprints dedupe. +// Run reads the guide at its release's tag. The tag never changes, so a +// rerun stores nothing new: the fingerprints dedupe. func (p Plugin) Run(ctx context.Context, env plugin.Env) (plugin.Result, error) { g := p.Guide + v := plugin.Version{ID: g.Release, Ref: g.tag()} + to, err := p.ReadAt(ctx, env, v) + if err != nil { + return plugin.Result{}, err + } + res, err := p.Extract(ctx, env, plugin.Snapshot{}, to) + if err != nil { + return plugin.Result{}, err + } + res.Baselines = []plugin.Spec{plugin.Baseline(g.Package, g.Release, g.tag(), g.URL(), to.Content, false, env.Now().UTC())} + return res, nil +} + +// ListVersions is every final release of the package on PyPI, oldest first, +// each with the ref its guide would be read at. +func (p Plugin) ListVersions(ctx context.Context, env plugin.Env) ([]plugin.Version, error) { + vs, err := versions.PyPIReleases(ctx, env.Get, p.Guide.Package) + if err != nil { + return nil, err + } + for i := range vs { + vs[i].Ref = p.Guide.tagOf(vs[i].ID) + } + return vs, nil +} + +// ReadAt fetches the guide when v is the release it documents; any other +// release has no guide and reads as nothing (a snapshot with no content). +func (p Plugin) ReadAt(ctx context.Context, env plugin.Env, v plugin.Version) (plugin.Snapshot, error) { + g := p.Guide + if v.ID != g.Release { + return plugin.Snapshot{Version: v}, nil + } page, err := env.Get(ctx, g.URL()) if err != nil { - return plugin.Result{}, fmt.Errorf("fetching %s: %w", g.URL(), err) + return plugin.Snapshot{}, fmt.Errorf("fetching %s: %w", g.URL(), err) } - removals, err := Parse(page, g) + return plugin.Snapshot{Version: v, Content: page, SourceURL: g.URL()}, nil +} + +// Extract is an event per removed member in the to release's guide, with +// previous_version from's release; a release with no guide has none. +func (p Plugin) Extract(_ context.Context, env plugin.Env, from, to plugin.Snapshot) (plugin.Result, error) { + g := p.Guide + if to.Content == nil { + return plugin.Result{Label: to.Version.ID}, nil + } + removals, err := Parse(to.Content, g) if err != nil { return plugin.Result{}, fmt.Errorf("reading %s: %w", g.URL(), err) } @@ -295,17 +350,18 @@ func (p Plugin) Run(ctx context.Context, env plugin.Env) (plugin.Result, error) subject.Change, subject.Replacement = event.PackageRenamedSymbol, r.Replacement } e := event.EventV2{ - API: g.Package, - Version: g.Release, - ChangeType: event.ChangeTypeBreaking, - Breaking: true, - Subject: subject, - Description: desc, - MigrationHint: fmt.Sprintf("%s (%s's %s for %s)", g.absoluteLinks(r.Guideline), g.Package, g.doc(), g.Release), - SourceURL: g.URL(), - SourceExcerpt: r.Excerpt, - Confidence: 1.0, - DetectedAt: now, + API: g.Package, + PreviousVersion: from.Version.ID, + Version: g.Release, + ChangeType: event.ChangeTypeBreaking, + Breaking: true, + Subject: subject, + Description: desc, + MigrationHint: fmt.Sprintf("%s (%s's %s for %s)", g.absoluteLinks(r.Guideline), g.Package, g.doc(), g.Release), + SourceURL: g.URL(), + SourceExcerpt: r.Excerpt, + Confidence: 1.0, + DetectedAt: now, } e.Fingerprint = event.FingerprintV2(e.API, e.Version, e.ChangeType, subject, "") if err := e.Validate(); err != nil { @@ -314,14 +370,11 @@ func (p Plugin) Run(ctx context.Context, env plugin.Env) (plugin.Result, error) } events = append(events, e) } - return plugin.Result{ - Label: g.URL(), - Events: events, - Rejected: rejected, - Baselines: []plugin.Spec{plugin.Baseline(g.Package, g.Release, g.tag(), g.URL(), page, false, now)}, - }, nil + return plugin.Result{Label: g.URL(), Events: events, Rejected: rejected}, nil } +var _ plugin.Versioned = Plugin{} + // absoluteLinks prefixes LinkBase to every Markdown link target in s that // starts with "/". func (g Guide) absoluteLinks(s string) string { diff --git a/plugins/removedmembers/testdata/README.md b/plugins/removedmembers/testdata/README.md index eed475a..05e4aa5 100644 --- a/plugins/removedmembers/testdata/README.md +++ b/plugins/removedmembers/testdata/README.md @@ -7,3 +7,4 @@ The golden files under `golden/` are what the plugin stores from them (`go test | `numpy_2_0_migration_guide-v2.0.0.rst` | https://github.com/numpy/numpy/blob/v2.0.0/doc/source/numpy_2_0_migration_guide.rst | 2026-11-04 | BSD-3-Clause (NumPy), kept whole | "The table below provides replacements for the removed members:" | | `migration-7.x-to-8.0-posthog-v8.0.0.md` | https://github.com/PostHog/posthog-python/blob/posthog-v8.0.0/docs/migration-7.x-to-8.0.md (commit 6edff41425188056e48f6119457dc9bc0669ca99) | 2026-11-19 | MIT (the repo's LICENSE is the MIT text, with two copyright lines, so GitHub's detector says NOASSERTION; PyPI says MIT), kept whole, byte for byte | "8.x.x removes these options and APIs:" (Fix removed APIs), with rows such as "\| `CaptureV1Error` \| `CaptureError` \|" | | `python-agent-110000-f0b4f817.mdx` | https://github.com/newrelic/docs-website/blob/f0b4f817bf9a7b76a78e26609369a8a7ce764c8e/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-110000.mdx (the repo has no tags; the file's latest commit) | 2026-11-19 | Excerpt for testing; © New Relic, CC BY-NC-SA 4.0. Kept: the "## Removed features" heading, the bullets that introduce its two tables, the "Legacy API" table and the "Legacy Setting" table (lines 30-31 and 36-66 of 136), every line unchanged. Left out: the front matter, "Notes", the Python 3.7 bullet, and the "Deprecated features", "New features", "Bug fixes" and "Support statement" sections. The plugin finds the same 8 rows and 7 events in the excerpt as in the full file | "The following APIs have been removed:", with rows such as "\| `record_exception` \| [`notice_error()`](/docs/agents/python-agent/python-agent-api/notice_error) \|" | +| `pypi-numpy-excerpt.json` | https://pypi.org/pypi/numpy/json (shape only) | written for the test, 2026-11-20 | not a saved copy: four releases in the shape of PyPI's JSON API (`releases` → files with `upload_time_iso_8601`, `yanked`), dates approximate, a pre-release among them | for the version-listing test only (ListVersions, consecutive pairs); no golden reads it | diff --git a/plugins/removedmembers/testdata/pypi-numpy-excerpt.json b/plugins/removedmembers/testdata/pypi-numpy-excerpt.json new file mode 100644 index 0000000..01ccf4d --- /dev/null +++ b/plugins/removedmembers/testdata/pypi-numpy-excerpt.json @@ -0,0 +1,6 @@ +{"releases":{ + "1.26.4":[{"upload_time_iso_8601":"2024-02-05T00:00:00Z","yanked":false}], + "2.0.0rc2":[{"upload_time_iso_8601":"2024-05-01T00:00:00Z","yanked":false}], + "2.0.0":[{"upload_time_iso_8601":"2024-06-16T00:00:00Z","yanked":false}], + "2.0.1":[{"upload_time_iso_8601":"2024-07-21T00:00:00Z","yanked":false}] +}} diff --git a/plugins/removedmembers/versions_test.go b/plugins/removedmembers/versions_test.go new file mode 100644 index 0000000..a48552f --- /dev/null +++ b/plugins/removedmembers/versions_test.go @@ -0,0 +1,61 @@ +package removedmembers + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/patch-it-dev/patch-it-plugins/core/kit/kittest" + "github.com/patch-it-dev/patch-it-plugins/core/versions" +) + +// The package's releases come from PyPI; only the pair into the release the +// guide documents has events (the golden's, with previous_version set). +func TestOnlyTheGuidesReleaseHasEvents(t *testing.T) { + env := kittest.Env(t, kittest.Pages{ + "https://pypi.org/pypi/numpy/json": "testdata/pypi-numpy-excerpt.json", + NumPy2.URL(): "testdata/numpy_2_0_migration_guide-v2.0.0.rst", + }) + p := Plugin{Guide: NumPy2} + ctx := context.Background() + vs, err := p.ListVersions(ctx, env) + if err != nil { + t.Fatal(err) + } + pinned := kittest.Ingest(t, p, kittest.Pages{NumPy2.URL(): "testdata/numpy_2_0_migration_guide-v2.0.0.rst"}) + var got []string + for _, pair := range versions.Pairs(vs) { + from, err := p.ReadAt(ctx, env, pair[0]) + if err != nil { + t.Fatal(err) + } + to, err := p.ReadAt(ctx, env, pair[1]) + if err != nil { + t.Fatal(err) + } + res, err := p.Extract(ctx, env, from, to) + if err != nil || len(res.Rejected) > 0 { + t.Fatalf("%v %v", err, res.Rejected) + } + for i, e := range res.Events { + if e.PreviousVersion != pair[0].ID || e.Fingerprint != pinned[i].Fingerprint { + t.Errorf("event %d: previous %q, fingerprint differs from the golden's: %v", i, e.PreviousVersion, e.Fingerprint != pinned[i].Fingerprint) + } + } + got = append(got, fmt.Sprintf("%s->%s:%d", pair[0].ID, pair[1].ID, len(res.Events))) + } + want := fmt.Sprintf("1.26.4->2.0.0:%d 2.0.0->2.0.1:0", len(pinned)) + if strings.Join(got, " ") != want || len(pinned) == 0 { + t.Errorf("got %s, want %s", strings.Join(got, " "), want) + } +} + +// Each release's ref follows the guide's tag shape. +func TestTagsFollowTheGuide(t *testing.T) { + for g, want := range map[*Guide]string{&NumPy2: "v2.0.0", &PostHog8: "posthog-v8.0.0", &NewRelic11: "f0b4f817bf9a7b76a78e26609369a8a7ce764c8e"} { + if got := g.tag(); got != want { + t.Errorf("%s: tag %q, want %q", g.Package, got, want) + } + } +} diff --git a/providers/anthropic.md b/providers/anthropic.md index cc9a30e..cac9358 100644 --- a/providers/anthropic.md +++ b/providers/anthropic.md @@ -8,7 +8,7 @@ Format plugin: `migrationguide` (`plugins/migrationguide`), reading with `plugin ## Why this source -- **Source:** [MIGRATION.md at v1.0.0](https://github.com/anthropics/anthropic-sdk-python/blob/v1.0.0/MIGRATION.md), read raw at the tag (`https://raw.githubusercontent.com/anthropics/anthropic-sdk-python/v1.0.0/MIGRATION.md`). Checked on 2026-10-17. +- **Source:** MIGRATION.md at the first release of each major ([at v1.0.0](https://github.com/anthropics/anthropic-sdk-python/blob/v1.0.0/MIGRATION.md)), the majors listed from PyPI; read raw at the tag (`https://raw.githubusercontent.com/anthropics/anthropic-sdk-python/v1.0.0/MIGRATION.md`). Checked on 2026-10-17. - **Why this one:** it's the SDK maintainers' own guide to the major release, with a table of the methods, the removed parameter and what to use instead. A tag never changes, so the URL is its own permanent citation. - **Alternatives considered:** the GitHub release notes and `CHANGELOG.md` (list commits, not "what to use instead"); diffing the SDK's code between versions (finds signature changes but not the maintainers' advice, and is a lot more machinery); Anthropic's model deprecations page (mentions the change, but it's the models page). - **Quote:** "Removed: deprecated request parameters" (section heading), with the row "`temperature`, `top_p`, `top_k` | Remove them." diff --git a/providers/textual.md b/providers/textual.md index dc06f77..4b13089 100644 --- a/providers/textual.md +++ b/providers/textual.md @@ -2,7 +2,7 @@ ## What it watches -Removals and breaking changes in Textual 0.86.0, read from its changelog: symbols removed (`App.dark`, …) or renamed. One `package_release` event per changelog bullet the plugin can read with certainty. Pinned to that release. +Removals and breaking changes in each Textual release, read from that release's section of its changelog: symbols removed (`App.dark`, …) or renamed. One `package_release` event per changelog bullet the plugin can read with certainty. The releases are listed from PyPI (final releases only) and each is read at its tag (`v`); 0.86.0 is the release the golden test pins. Format plugin: `changelog` (`plugins/changelog`). @@ -26,5 +26,5 @@ Format plugin: `changelog` (`plugins/changelog`). ## Known gaps -- Only 0.86.0. Following each new release is planned. +- A release is read at its tag; one whose tag or changelog section is missing fails that pair, it isn't guessed. - Breaking behavior changes written as prose aren't events. diff --git a/providers/urllib3.md b/providers/urllib3.md index c962ac1..df1088c 100644 --- a/providers/urllib3.md +++ b/providers/urllib3.md @@ -2,7 +2,7 @@ ## What it watches -Removals and deprecations in urllib3 2.0.0, read from its changelog: classes, methods and modules removed or renamed. One `package_release` event per bullet the plugin can read with certainty. Pinned to that release. +Removals and deprecations in each urllib3 release, read from that release's section of its changelog: classes, methods and modules removed or renamed. One `package_release` event per bullet the plugin can read with certainty. The releases are listed from PyPI (final releases only) and each is read at its git tag (``); 2.0.0 is the release the golden test pins. Format plugin: `changelog` (`plugins/changelog`). @@ -26,5 +26,5 @@ Format plugin: `changelog` (`plugins/changelog`). ## Known gaps -- Only 2.0.0. +- A release is read at its tag; one whose tag or changelog section is missing fails that pair, it isn't guessed. - Changed defaults (TLS versions, retries) are behavior changes and aren't events.