diff --git a/tests/test_archive_save.py b/tests/test_archive_save.py index d99226441c..aa287562a6 100644 --- a/tests/test_archive_save.py +++ b/tests/test_archive_save.py @@ -135,6 +135,13 @@ def test_url_moved_to_another_field_is_not_new(self): def test_no_previous_data(self): assert archive_save.new_urls([], [_entry()]) == ["https://pycon.tz/"] + def test_fragment_is_the_same_page(self): + previous = [_entry()] + current = [_entry(sponsor="https://pycon.tz/#sponsors", coc="https://pycon.tz/#coc")] + + assert archive_save.new_urls(previous, current) == [] + assert archive_save.entry_urls(current[0]) == ["https://pycon.tz/"] + class TestMain: """The command saves new and due URLs once each and reports every result.""" @@ -217,3 +224,53 @@ def test_missing_previous_file_makes_every_url_new(self, tmp_path, monkeypatch): archive_save.main(["--data", str(current), "--previous", str(tmp_path / "missing.yml"), "--delay", "0"]) assert saved == ["https://pycon.tz/"] + + +class TestTurnedAway: + """When the archive turns the run away, the run stops at once and fails.""" + + @pytest.fixture() + def data(self, tmp_path): + current = tmp_path / "conferences.yml" + entries = [ + _entry(), + _entry(conference="PyCon Ghana", link="https://gh.pycon.org/"), + _entry(conference="PyCon Kenya", link="https://pycon.ke/"), + ] + current.write_text(yaml.safe_dump(entries), encoding="utf-8") + return current, tmp_path / "missing.yml", tmp_path / "summary.md" + + def _run(self, current, previous, summary): + args = ["--data", str(current), "--previous", str(previous), "--summary", str(summary)] + return archive_save.main([*args, "--delay", "0"]) + + def test_stops_and_fails_when_turned_away(self, data, monkeypatch): + current, previous, summary = data + calls = [] + + def save(url): + calls.append(url) + ok = url == "https://pycon.tz/" + return wayback.SaveResult(url, ok=ok, detail="HTTP 200" if ok else "error: refused", blocked=not ok) + + monkeypatch.setattr(archive_save.wayback, "save", save) + + assert self._run(current, previous, summary) == 1 + assert calls == ["https://pycon.tz/", "https://gh.pycon.org/"] + report = summary.read_text(encoding="utf-8") + assert "- https://gh.pycon.org/ (error: refused)" in report + assert "### Not tried" in report + assert "- https://pycon.ke/" in report + assert "- Saved: 1 of 2" in report + + def test_page_failure_does_not_stop_the_run(self, data, monkeypatch): + current, previous, summary = data + calls = [] + monkeypatch.setattr( + archive_save.wayback, + "save", + lambda url: calls.append(url) or wayback.SaveResult(url, ok=False, detail="HTTP 523"), + ) + + assert self._run(current, previous, summary) == 0 + assert len(calls) == 3 diff --git a/tests/test_wayback.py b/tests/test_wayback.py index 2d83be1eec..65d45d0fc8 100644 --- a/tests/test_wayback.py +++ b/tests/test_wayback.py @@ -7,6 +7,7 @@ import pytest import requests import responses +from urllib3.exceptions import ReadTimeoutError sys.path.append(str(Path(__file__).parent.parent / "utils")) @@ -147,6 +148,35 @@ def test_refused(self): assert not result.ok assert result.detail == "HTTP 429" + assert result.blocked + + @responses.activate + def test_page_failure_is_not_blocked(self): + responses.add(responses.GET, wayback.SAVE_URL + "https://pycon.tz/", status=523) + + result = wayback.save("https://pycon.tz/") + + assert not result.ok + assert not result.blocked + + @responses.activate + def test_read_timeout_in_body_is_not_blocked(self): + stall = ReadTimeoutError(None, wayback.SAVE_URL, "Read timed out.") + responses.add(responses.GET, wayback.SAVE_URL + "https://pycon.tz/", body=requests.ConnectionError(stall)) + + result = wayback.save("https://pycon.tz/") + + assert not result.ok + assert not result.blocked + + @responses.activate + def test_read_timeout_is_not_blocked(self): + responses.add(responses.GET, wayback.SAVE_URL + "https://pycon.tz/", body=requests.ReadTimeout("slow")) + + result = wayback.save("https://pycon.tz/") + + assert not result.ok + assert not result.blocked @responses.activate def test_connection_error(self): @@ -156,6 +186,7 @@ def test_connection_error(self): assert not result.ok assert result.detail.startswith("error:") + assert result.blocked class TestLinks: diff --git a/utils/archive_save.py b/utils/archive_save.py index 86ffa98aa1..716b004390 100644 --- a/utils/archive_save.py +++ b/utils/archive_save.py @@ -13,8 +13,15 @@ The workflow runs weekly and on pushes to the data, so a point is saved on the first run on or after it: every point after ``--since`` and up to ``--today``. -A CfP with no exact deadline (TBA, None, Cancelled) has no CfP points. Each URL -is saved at most once per run, whichever points it is due for. +A CfP with no exact deadline (TBA, None, Cancelled) has no CfP points. Each page +is saved at most once per run, whichever points it is due for: ``#fragment`` +links to one page count as that page. + +When the archive turns the run away (connection refused, rate limited), the run +stops at once and exits 1, so the workflow run fails and the next run starts +from the last successful one and saves the pages this run could not. Waiting +for a block to lift would cost minutes and could fail all the same. A page the +archive fails to capture is reported and does not fail the run. Only ``_data/conferences.yml`` is read: an edition stays there until well after its last save point. @@ -37,6 +44,7 @@ from datetime import timedelta from datetime import timezone from pathlib import Path +from urllib.parse import urldefrag import yaml @@ -96,7 +104,7 @@ def normalise(url: str) -> str: def entry_urls(entry: Mapping) -> list[str]: - """Return an entry's live URLs: its URL fields that are not archive links. + """Return an entry's live pages: its URL fields that are not archive links. Parameters ---------- @@ -106,14 +114,15 @@ def entry_urls(entry: Mapping) -> list[str]: Returns ------- list[str] - The URLs, in field order + The URLs without ``#fragment``, since the archive saves the page, each + once, in field order """ urls = [] for field in URL_FIELDS: value = entry.get(field) if value and not wayback.is_archived(str(value)): - urls.append(str(value).strip()) - return urls + urls.append(urldefrag(str(value).strip()).url) + return list(dict.fromkeys(urls)) def save_points(entry: Mapping) -> list[date]: @@ -203,7 +212,12 @@ def load(path: Path) -> list[dict]: return [entry for entry in data or [] if isinstance(entry, dict)] -def render_summary(results: list[wayback.SaveResult], new_count: int, due_count: int) -> str: +def render_summary( + results: list[wayback.SaveResult], + new_count: int, + due_count: int, + skipped: list[str] | None = None, +) -> str: """Render the run's report as Markdown for the workflow summary. Parameters @@ -214,6 +228,8 @@ def render_summary(results: list[wayback.SaveResult], new_count: int, due_count: How many of the URLs were new in the data due_count : int How many were due for a save point + skipped : list[str] | None + URLs not tried because the archive turned the run away Returns ------- @@ -231,6 +247,17 @@ def render_summary(results: list[wayback.SaveResult], new_count: int, due_count: if failed: lines.extend(("", "### Not saved", "")) lines.extend(f"- {result.url} ({result.detail})" for result in failed) + if skipped: + lines.extend( + ( + "", + "### Not tried", + "", + "The archive turned the run away, so the run stopped. The next run saves these.", + "", + ), + ) + lines.extend(f"- {url}" for url in skipped) return "\n".join(lines) + "\n" @@ -264,7 +291,8 @@ def main(argv: list[str] | None = None) -> int: Returns ------- int - 0; a page the archive did not save is reported, not an error + 0, also when the archive failed to capture a page (reported); 1 when the + archive turned the run away and the run stopped """ args = build_parser().parse_args(argv) today = args.today or datetime.now(tz=timezone.utc).date() @@ -275,21 +303,30 @@ def main(argv: list[str] | None = None) -> int: urls = list(dict.fromkeys(fresh + due)) print(f"{len(fresh)} new URLs, {len(due)} at a save point, {len(urls)} to save") + if args.dry_run: + for url in urls: + print(f"would save {url}") + return 0 + results = [] + skipped = [] + stopped = False for i, url in enumerate(urls): - if args.dry_run: - print(f"would save {url}") - continue if i: time.sleep(args.delay) result = wayback.save(url) print(f"{'saved' if result.ok else 'not saved'}: {url} ({result.detail})") results.append(result) + if result.blocked: + stopped = True + skipped = urls[i + 1 :] + print(f"The archive turned the run away: stopping, {len(skipped)} URLs not tried") + break - if args.summary and not args.dry_run: + if args.summary: with args.summary.open("a", encoding="utf-8") as f: - f.write(render_summary(results, len(fresh), len(due))) - return 0 + f.write(render_summary(results, len(fresh), len(due), skipped)) + return 1 if stopped else 0 if __name__ == "__main__": diff --git a/utils/wayback.py b/utils/wayback.py index 772d005947..6daae269ea 100644 --- a/utils/wayback.py +++ b/utils/wayback.py @@ -24,11 +24,18 @@ from datetime import timedelta import requests +from urllib3.exceptions import ReadTimeoutError CDX_URL = "https://web.archive.org/cdx/search/cdx" SAVE_URL = "https://web.archive.org/save/" USER_AGENT = "Pythondeadlin.es Archiver/1.0 (https://pythondeadlin.es)" TIMEOUT = 60 +# Seconds to wait for the archive to accept the connection. A refusal arrives at +# once; a long wait here means the archive is not answering at all. +CONNECT_TIMEOUT = 10 +# Statuses with which the archive itself turns the client away (rate limit, +# too many sessions), as opposed to failing to capture one page +BLOCKED_STATUSES = frozenset({429, 509}) # Days after an edition's end in which a capture still shows that edition END_GRACE_DAYS = 14 @@ -52,11 +59,15 @@ class SaveResult: True when the archive accepted the request detail : str The HTTP status, or the error, for the run's report + blocked : bool + True when the archive turned the client away (connection refused or + failed, rate limited) rather than failing on this page """ url: str ok: bool detail: str + blocked: bool = False def is_archived(url: str) -> bool: @@ -169,6 +180,28 @@ def pick_capture(stamps: list[str], end: date, grace_days: int = END_GRACE_DAYS) return max(eligible) if eligible else None +def _is_read_timeout(error: requests.ConnectionError) -> bool: + """Check whether a ConnectionError is a read timeout while the body downloads. + + requests raises ``ReadTimeout`` when the headers are late, but re-raises a + timeout in the body as ``ConnectionError(ReadTimeoutError)``. + + Parameters + ---------- + error : requests.ConnectionError + The error a request raised + + Returns + ------- + bool + True when the error wraps a urllib3 ``ReadTimeoutError`` + """ + return any(isinstance(arg, ReadTimeoutError) for arg in error.args) or isinstance( + error.__context__, + ReadTimeoutError, + ) + + def save(url: str) -> SaveResult: """Ask the Wayback Machine to save a page now. @@ -180,10 +213,21 @@ def save(url: str) -> SaveResult: Returns ------- SaveResult - Whether the archive accepted the request, with the status or error + Whether the archive accepted the request, with the status or error, and + whether the archive turned the client away """ try: - response = requests.get(SAVE_URL + url, headers={"User-Agent": USER_AGENT}, timeout=TIMEOUT) + response = requests.get( + SAVE_URL + url, + headers={"User-Agent": USER_AGENT}, + timeout=(CONNECT_TIMEOUT, TIMEOUT), + ) + except requests.ConnectionError as e: + # Refused, reset or not answering: the archive, not the page. A read + # timeout while the body downloads also arrives as a ConnectionError. + return SaveResult(url, ok=False, detail=f"error: {e}", blocked=not _is_read_timeout(e)) except requests.RequestException as e: + # A read timeout: the capture can still finish on the archive's side return SaveResult(url, ok=False, detail=f"error: {e}") - return SaveResult(url, ok=response.status_code == 200, detail=f"HTTP {response.status_code}") + status = response.status_code + return SaveResult(url, ok=status == 200, detail=f"HTTP {status}", blocked=status in BLOCKED_STATUSES)