Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ Go module `github.com/patch-it-dev/patch-it-plugins`; the only dependency is `go
core/plugin/ the Plugin interface (Run; ListVersions, ReadAt, Extract), the registry of providers and sources, Env, Result
core/fetch/ the HTTP fetch with retries and the ETag / Last-Modified cache
core/github/ the newest release / commit on GitHub, and a commit's SHA and date (one place)
core/versions/ every version of a source: GitHub releases, tags and commits, PyPI, npm, the Web Archive at a date
core/versions/ every version of a source: GitHub releases, tags and commits, PyPI, npm, Web Archive captures
core/kit/ core/kit/kittest/ Unchanged, Cite, Wayback, Notices; the test kit (Mem, Pages, Ingest, Golden)
core/extract/ what readers of prose share: RSS/Atom entries, HTML as text, dates, code spans
core/event/ core/migrate/ the change-event types from patch-it's contract (v1 + v2), v1 -> v2
Expand Down
68 changes: 66 additions & 2 deletions core/versions/versions.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Package versions lists a source's versions from where they're published:
// a GitHub repo's releases, tags or the commits that changed a file, a
// package's releases on PyPI or npm, and the Web Archive's capture of a page
// at a date. Every list comes back oldest first, as plugin.Version.
// package's releases on PyPI or npm, and a living page's captures in the Web
// Archive (every distinct one, or the one closest to a date). Every list comes back oldest first, as plugin.Version.
//
// Each function takes how to fetch (a plugin's env.Get, or github.Get for the
// GitHub API with its token), so tests serve saved responses.
Expand Down Expand Up @@ -36,6 +36,13 @@ var (
// MaxPages caps how many pages of 100 a GitHub list reads (1,000 entries).
const MaxPages = 10

// GitHubPages reads a GitHub list endpoint (base, with or without a query)
// page by page, 100 at a time, until a short page or MaxPages: for a list
// the functions here don't cover (a release's assets, another host's base).
func GitHubPages[T any](ctx context.Context, get Getter, base string) ([]T, error) {
return pages[T](ctx, get, base)
}

// pages reads a GitHub list endpoint page by page until a short page.
func pages[T any](ctx context.Context, get Getter, base string) ([]T, error) {
sep := "?"
Expand Down Expand Up @@ -238,6 +245,63 @@ func WaybackAt(ctx context.Context, get Getter, page string, at time.Time) (plug
return plugin.Version{ID: taken.Format(time.DateOnly), Released: taken, Ref: c.URL, URL: c.URL}, nil
}

// WaybackCaptures are the versions of a living page (one that only shows its
// newest state: a deprecations page, a feed) as the Web Archive captured it
// between from and to (zero: no bound), oldest first, from its CDX API: one
// version per distinct content, so captures of an unchanged page collapse
// into the first of them, and a page that changes back is a new version.
// Two distinct captures on one day keep the later one: a version's ID is
// its day (YYYY-MM-DD). The Ref is the capture's original bytes
// (web.archive.org/web/<timestamp>id_/<page>), the URL the capture a person
// sees and an event cites.
//
// One request, made with get: the caller's fetch is what spaces requests
// out and names the client (patch-it's hosted ingestion service fetches one
// at a time per host, with its own User-Agent).
func WaybackCaptures(ctx context.Context, get Getter, page string, from, to time.Time) ([]plugin.Version, error) {
q := url.Values{"url": {page}, "output": {"json"}, "fl": {"timestamp,original,digest"}, "filter": {"statuscode:200"}, "collapse": {"digest"}}
if !from.IsZero() {
q.Set("from", from.UTC().Format("20060102150405"))
}
if !to.IsZero() {
q.Set("to", to.UTC().Format("20060102150405"))
}
body, err := get(ctx, Wayback+"/cdx/search/cdx?"+q.Encode())
if err != nil {
return nil, fmt.Errorf("listing the Web Archive's captures of %s: %w", page, err)
}
var rows [][]string
if len(strings.TrimSpace(string(body))) > 0 {
if err := json.Unmarshal(body, &rows); err != nil {
return nil, fmt.Errorf("parsing the Web Archive's captures of %s: %w", page, err)
}
}
var out []plugin.Version
lastDigest := ""
for i, r := range rows {
if i == 0 || len(r) < 3 { // the header row: ["timestamp","original","digest"]
continue
}
taken, err := time.Parse("20060102150405", r[0])
if err != nil || r[2] == lastDigest {
continue
}
lastDigest = r[2]
v := plugin.Version{
ID: taken.Format(time.DateOnly), Released: taken,
Ref: "https://web.archive.org/web/" + r[0] + "id_/" + r[1],
URL: "https://web.archive.org/web/" + r[0] + "/" + r[1],
}
if n := len(out); n > 0 && out[n-1].ID == v.ID {
out[n-1] = v
continue
}
out = append(out, v)
}
slices.SortStableFunc(out, func(a, b plugin.Version) int { return a.Released.Compare(b.Released) })
return out, nil
}

var (
final = regexp.MustCompile(`^v?\d+(\.\d+)*$`)
number = regexp.MustCompile(`\d+`)
Expand Down
49 changes: 49 additions & 0 deletions core/versions/versions_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,3 +168,52 @@ func TestCompareFinalMajorPairs(t *testing.T) {
t.Errorf("Pairs = %v", ps)
}
}

// The CDX API's answer: a header row, then captures oldest first. The same
// content captured again (same digest) is one version; a change back is a
// new one; two distinct captures on one day keep the later.
func TestWaybackCapturesCollapseDuplicates(t *testing.T) {
const page = "https://example.com/deprecations"
cdx := `[["timestamp","original","digest"],
["20250103120000","https://example.com/deprecations","AAA"],
["20250210080000","https://example.com/deprecations","AAA"],
["20250301090000","https://example.com/deprecations","BBB"],
["20250301170000","https://example.com/deprecations","CCC"],
["20250601000000","https://example.com/deprecations","CCC"],
["20251120000000","https://example.com/deprecations","AAA"]]`
var asked string
get := func(_ context.Context, u string) ([]byte, error) { asked = u; return []byte(cdx), nil }
from, to := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2025, 12, 31, 0, 0, 0, 0, time.UTC)
vs, err := WaybackCaptures(context.Background(), get, page, from, to)
if err != nil {
t.Fatal(err)
}
var got []string
for _, v := range vs {
got = append(got, v.ID+" "+v.Ref)
}
want := []string{
"2025-01-03 https://web.archive.org/web/20250103120000id_/https://example.com/deprecations",
"2025-03-01 https://web.archive.org/web/20250301170000id_/https://example.com/deprecations",
"2025-11-20 https://web.archive.org/web/20251120000000id_/https://example.com/deprecations",
}
if strings.Join(got, "\n") != strings.Join(want, "\n") {
t.Errorf("got\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
for _, p := range []string{"from=20250101000000", "to=20251231000000", "collapse=digest", "filter=statuscode%3A200", "url=https%3A%2F%2Fexample.com%2Fdeprecations"} {
if !strings.Contains(asked, p) {
t.Errorf("asked %s, without %s", asked, p)
}
}
if vs[0].URL != "https://web.archive.org/web/20250103120000/https://example.com/deprecations" {
t.Errorf("URL %s", vs[0].URL)
}
}

// A page the archive never captured has no versions, and no error.
func TestWaybackCapturesOfAnUncapturedPage(t *testing.T) {
get := func(context.Context, string) ([]byte, error) { return []byte("[]"), nil }
if vs, err := WaybackCaptures(context.Background(), get, "https://example.com/x", time.Time{}, time.Time{}); err != nil || len(vs) != 0 {
t.Errorf("%v %v", vs, err)
}
}
4 changes: 2 additions & 2 deletions docs/how-it-works.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ A **provider** (Stripe, OpenAI, GitHub…) has one category and one or more **so

## What a plugin does

0. **List the versions.** A source with versions (a spec's releases, a package on PyPI or npm, a guide per major) lists every one, oldest first (`ListVersions`, from `core/versions`: GitHub releases, tags and commits, PyPI, npm, the Web Archive at a date). Each consecutive pair is read and compared once (`ReadAt`, `Extract`), so a customer several versions behind gets every step of the upgrade. A living page with no versions (a deprecations page, a feed) is read as it is now; each change to it is a dated version.
0. **List the versions.** A source with versions (a spec's releases, a package on PyPI or npm, a guide per major) lists every one, oldest first (`ListVersions`, from `core/versions`: GitHub releases, tags and commits, PyPI, npm, a living page's Web Archive captures). Each consecutive pair is read and compared once (`ReadAt`, `Extract`), so a customer several versions behind gets every step of the upgrade. A living page with no versions (a deprecations page, a feed) is read as it is now; each change to it is a dated version, and its past ones are its Web Archive captures.
1. **Fetch.** The plugin asks its environment for the source (`env.Get`); it never opens connections itself. Pages that change in place are also cited through a Web Archive snapshot, so the link in an event keeps showing what the plugin read.
2. **Compare.** It compares two versions of the source: two OpenAPI specs, two versions of a deprecations page, a release's changelog section against the release before. `Run` does the newest version against the last one it saw (the baseline); `Extract` does any pair.
3. **Emit events.** Each change becomes one event in [patch-it's change-event format](https://github.com/patch-it-dev/patch-it/tree/main/docs/contract): what changed, how bad it is (breaking, deprecation, behavior change, non-breaking), the date it takes effect, and the source URL with a quote. **No source, no event.** When unsure, a plugin emits nothing: a wrong event would become a wrong pull request.
Expand All @@ -20,7 +20,7 @@ There's no list in this repo: `make plugins` prints every provider and source fr
| Path | What's there |
|---|---|
| [`core/plugin/`](../core/plugin/) | The `Plugin` interface (`Run`; `ListVersions`, `ReadAt`, `Extract`), the registry of providers and sources (`RegisterProvider`, `Lookup`, `LookupSource`, `Providers`), `Env`, `Result` |
| [`core/versions/`](../core/versions/) | Every version of a source: GitHub releases, tags and commits, PyPI, npm, the Web Archive at a date |
| [`core/versions/`](../core/versions/) | Every version of a source: GitHub releases, tags and commits, PyPI, npm, a page's Web Archive captures (all, or the one at a date) |
| [`plugins/`](../plugins/) | One package per kind, with its providers; `plugins/all` imports every one |
| [`cmd/new-plugin/`](../cmd/new-plugin/), [`cmd/list-plugins/`](../cmd/list-plugins/) | The starter for a new provider; the list of providers and sources, printed from the registry |
| [`core/kit/`](../core/kit/), [`core/kit/kittest/`](../core/kit/kittest/), [`core/github/`](../core/github/), [`core/extract/`](../core/extract/) | What plugins share (the unchanged-page check, Wayback citing, the newest release or commit on GitHub, RSS/Atom and HTML text helpers) and the test kit (in-memory store, saved pages, golden files) |
Expand Down
4 changes: 2 additions & 2 deletions docs/writing-a-plugin.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,12 +117,12 @@ ReadAt(ctx, env, v Version) (Snapshot, error) // the source at o
Extract(ctx, env, from, to Snapshot) (Result, error) // the changes from one to the next
```

- **List** with [`core/versions`](../core/versions/versions.go): `GitHubReleases`, `GitHubTags`, `GitHubCommits` (the commits that changed a file), `PyPIReleases`, `NPMReleases`, `WaybackAt` (a page as it was on a date). Never pin a tag in code: a version you need for a golden test is a pinned *case* in the test (`changelog.Urllib3.At("2.0.0")`), not the registered source.
- **List** with [`core/versions`](../core/versions/versions.go): `GitHubReleases`, `GitHubTags`, `GitHubCommits` (the commits that changed a file), `PyPIReleases`, `NPMReleases`, `WaybackCaptures` (every distinct Web Archive capture of a living page, dated), `WaybackAt` (a page as it was on a date), `GitHubPages` (any other GitHub list, page by page). An OpenAPI adapter lists its own versions (`adapter.Lister`): every adapter does. Never pin a tag in code: a version you need for a golden test is a pinned *case* in the test (`changelog.Urllib3.At("2.0.0")`), not the registered source.
- **Read at** a version's `Ref` (a tag, a commit SHA, an archive URL) and return the exact URL read as the snapshot's `SourceURL`: it's the events' citation.
- **Extract** sets `previous_version` to `from`'s version; `from` may be zero (nothing before). It returns no baselines.
- `Run` is built from the same parts: the newest version, extracted against the last baseline.

A living page with no versions of its own (a deprecations page, a feed) only has `Run`: it's read as it is now, and each change to it is a dated version.
A living page with no versions of its own (a deprecations page, a feed: a `notices` source) is read as it is now by `Run`, and each change to it is a dated version. Its history comes from the Web Archive: the `notices` kind lists every distinct capture (`WaybackCaptures`), reads each, and extracts what each one added, citing that capture, so a year of it can be backfilled by date with no stored copies.

### Register the provider and its source

Expand Down
66 changes: 66 additions & 0 deletions plugins/notices/notices.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package notices

import (
"cmp"
"context"
"errors"
"fmt"
Expand All @@ -25,6 +26,7 @@ import (
"github.com/patch-it-dev/patch-it-plugins/core/kit"
"github.com/patch-it-dev/patch-it-plugins/core/migrate"
"github.com/patch-it-dev/patch-it-plugins/core/plugin"
"github.com/patch-it-dev/patch-it-plugins/core/versions"
"github.com/patch-it-dev/patch-it-plugins/plugins/notices/eol"
"github.com/patch-it-dev/patch-it-plugins/plugins/notices/feed"
"github.com/patch-it-dev/patch-it-plugins/plugins/notices/retire"
Expand Down Expand Up @@ -199,3 +201,67 @@ func (p Plugin) fetch(ctx context.Context, env plugin.Env, pageURL string) (body
}
return body, archivedURL, nil
}

// A living page has no versions of its own, so its history is the Web
// Archive's: every distinct capture is a version, dated (I19.11). A backfill
// reads the page at each capture and extracts each consecutive pair, the
// same events a run on that day would have stored, each citing its capture.

// ListVersions is every distinct Web Archive capture of the page, oldest
// first (versions.WaybackCaptures); the caller keeps the ones in its window.
func (p Plugin) ListVersions(ctx context.Context, env plugin.Env) ([]plugin.Version, error) {
return versions.WaybackCaptures(ctx, env.Get, p.Page.PageURL(), time.Time{}, time.Time{})
}

// ReadAt reads the page as the capture v holds it: the capture's original
// bytes (v.Ref), cited by its address (v.URL).
func (p Plugin) ReadAt(ctx context.Context, env plugin.Env, v plugin.Version) (plugin.Snapshot, error) {
body, err := env.Get(ctx, v.Ref)
if err != nil {
return plugin.Snapshot{}, fmt.Errorf("fetching %s as captured on %s: %w", p.Page.PageURL(), v.ID, err)
}
return plugin.Snapshot{Version: v, Content: body, SourceURL: cmp.Or(v.URL, v.Ref)}, nil
}

// Extract is an event per notice on the page at to that wasn't at from
// (every one when from is zero), citing to's capture. The events keep the
// notice's own version (its date or API version), so previous_version stays
// empty: from is an earlier day of the page, not an earlier release.
func (p Plugin) Extract(_ context.Context, env plugin.Env, from, to plugin.Snapshot) (plugin.Result, error) {
now := env.Now().UTC()
events, rejected, err := p.events(to, now)
if err != nil {
return plugin.Result{}, err
}
if !from.IsZero() {
before, _, err := p.events(from, now)
if err != nil {
return plugin.Result{}, err
}
had := map[string]bool{}
for _, e := range before {
had[e.Fingerprint] = true
}
kept := events[:0:0]
for _, e := range events {
if !had[e.Fingerprint] {
kept = append(kept, e)
}
}
events = kept
}
return plugin.Result{Label: to.SourceURL, Events: events, Rejected: rejected}, nil
}

// events reads one snapshot's notices into v2 events citing it.
func (p Plugin) events(s plugin.Snapshot, now time.Time) ([]event.EventV2, []error, error) {
found, err := p.Page.Find(s.Content)
if err != nil {
return nil, nil, fmt.Errorf("reading %s: %w", s.SourceURL, err)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking: Run uses ShapeGuard to refuse a read with far fewer notices, but Extract doesn't. Archived captures sometimes hold a challenge or JS-shell page (still HTTP 200), so Find reads 0 notices from them. If one of those falls between two good captures, the next pair sees every notice as new and emits all of them again, citing the later capture. The fingerprints are the same, so a consumer that dedupes by fingerprint will be fine. Still, an option is to treat a from whose read fails the shape guard as unreadable: skip the pair, or compare against the last good snapshot.

}
v1, v2, rejected := p.Page.Events(found, s.SourceURL, now)
converted, invalid := migrate.Fresh(v1)
return append(converted, v2...), append(rejected, invalid...), nil
}

var _ plugin.Versioned = Plugin{}
110 changes: 110 additions & 0 deletions plugins/notices/versions_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package notices

import (
"context"
"fmt"
"os"
"strings"
"testing"
"time"

"github.com/patch-it-dev/patch-it-plugins/core/kit/kittest"
"github.com/patch-it-dev/patch-it-plugins/core/plugin"
"github.com/patch-it-dev/patch-it-plugins/core/versions"
"github.com/patch-it-dev/patch-it-plugins/plugins/notices/retire"
)

// A year of a living page from its Web Archive captures alone: four
// captures, two of them the same content, are three versions; reading each
// and extracting each consecutive pair gives every retirement once, each
// citing the capture where it first appeared. Nothing reaches the network:
// the CDX answer and the captures are served from memory.
func TestBackfillAYearFromCaptures(t *testing.T) {
full, err := os.ReadFile("retire/testdata/openai-deprecations.md")
if err != nil {
t.Fatal(err)
}
lines := strings.SplitAfter(string(full), "\n")
header := strings.Join(lines[:2], "")
// The page as it grew: the oldest capture has the sections from the
// 2026-04-22 announcement down, the next adds three, the newest is today's.
older := header + strings.Join(lines[34:], "")
middle := header + strings.Join(lines[10:], "")
pg := retire.OpenAI
capture := func(ts string) string { return "https://web.archive.org/web/" + ts + "id_/" + pg.URL }
captures := map[string]string{
capture("20250115000000"): older,
capture("20250601000000"): middle,
capture("20251201000000"): string(full),
}
cdx := fmt.Sprintf(`[["timestamp","original","digest"],
["20250115000000",%[1]q,"A"],["20250301000000",%[1]q,"A"],["20250601000000",%[1]q,"B"],["20251201000000",%[1]q,"C"]]`, pg.URL)
env := plugin.Env{
Baselines: kittest.NewMem(),
Now: func() time.Time { return kittest.Clock },
Get: func(_ context.Context, u string) ([]byte, error) {
if strings.HasPrefix(u, versions.Wayback+"/cdx/search/cdx?") {
return []byte(cdx), nil
}
if body, ok := captures[u]; ok {
return []byte(body), nil
}
t.Errorf("fetched %s", u)
return nil, fmt.Errorf("no saved copy of %s", u)
},
}
ctx := context.Background()
p := Plugin{Page: pg}
vs, err := p.ListVersions(ctx, env)
if err != nil {
t.Fatal(err)
}
if len(vs) != 3 || vs[0].ID != "2025-01-15" || vs[2].ID != "2025-12-01" {
t.Fatalf("versions %v", vs)
}
var snaps []plugin.Snapshot
for _, v := range vs {
s, err := p.ReadAt(ctx, env, v)
if err != nil {
t.Fatal(err)
}
snaps = append(snaps, s)
}
firstSeen := map[string]string{} // fingerprint -> the capture it's cited from
for i := range snaps {
var from plugin.Snapshot
if i > 0 {
from = snaps[i-1]
}
res, err := p.Extract(ctx, env, from, snaps[i])
if err != nil || len(res.Rejected) > 0 {
t.Fatalf("%v %v", err, res.Rejected)
}
if len(res.Events) == 0 {
t.Errorf("%s: no new retirement", vs[i].ID)
}
for _, e := range res.Events {
if _, dup := firstSeen[e.Fingerprint]; dup {
t.Errorf("%s: %s again", vs[i].ID, e.Subject.Name)
}
if e.SourceURL != vs[i].URL {
t.Errorf("%s cites %s, want %s", e.Subject.Name, e.SourceURL, vs[i].URL)
}
firstSeen[e.Fingerprint] = e.SourceURL
}
}
// Today's page, read in one go, holds every retirement the history found
// (a retirement can only change its fingerprint, never vanish).
today, err := p.Extract(ctx, env, plugin.Snapshot{}, snaps[2])
if err != nil {
t.Fatal(err)
}
for _, e := range today.Events {
if _, ok := firstSeen[e.Fingerprint]; !ok {
t.Errorf("%s on today's page, never seen in the history", e.Subject.Name)
}
}
if strings.Contains(firstSeen[today.Events[0].Fingerprint], "20250115") {
t.Errorf("today's newest retirement is cited from the oldest capture")
}
}
Loading
Loading