Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions tests/test_archive_save.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,13 @@ def test_url_moved_to_another_field_is_not_new(self):
def test_no_previous_data(self):
assert archive_save.new_urls([], [_entry()]) == ["https://pycon.tz/"]

def test_fragment_is_the_same_page(self):
previous = [_entry()]
current = [_entry(sponsor="https://pycon.tz/#sponsors", coc="https://pycon.tz/#coc")]

assert archive_save.new_urls(previous, current) == []
assert archive_save.entry_urls(current[0]) == ["https://pycon.tz/"]


class TestMain:
"""The command saves new and due URLs once each and reports every result."""
Expand Down Expand Up @@ -217,3 +224,53 @@ def test_missing_previous_file_makes_every_url_new(self, tmp_path, monkeypatch):
archive_save.main(["--data", str(current), "--previous", str(tmp_path / "missing.yml"), "--delay", "0"])

assert saved == ["https://pycon.tz/"]


class TestTurnedAway:
"""When the archive turns the run away, the run stops at once and fails."""

@pytest.fixture()
def data(self, tmp_path):
current = tmp_path / "conferences.yml"
entries = [
_entry(),
_entry(conference="PyCon Ghana", link="https://gh.pycon.org/"),
_entry(conference="PyCon Kenya", link="https://pycon.ke/"),
]
current.write_text(yaml.safe_dump(entries), encoding="utf-8")
return current, tmp_path / "missing.yml", tmp_path / "summary.md"

def _run(self, current, previous, summary):
args = ["--data", str(current), "--previous", str(previous), "--summary", str(summary)]
return archive_save.main([*args, "--delay", "0"])

def test_stops_and_fails_when_turned_away(self, data, monkeypatch):
current, previous, summary = data
calls = []

def save(url):
calls.append(url)
ok = url == "https://pycon.tz/"
return wayback.SaveResult(url, ok=ok, detail="HTTP 200" if ok else "error: refused", blocked=not ok)

monkeypatch.setattr(archive_save.wayback, "save", save)

assert self._run(current, previous, summary) == 1
assert calls == ["https://pycon.tz/", "https://gh.pycon.org/"]
report = summary.read_text(encoding="utf-8")
assert "- https://gh.pycon.org/ (error: refused)" in report
assert "### Not tried" in report
assert "- https://pycon.ke/" in report
assert "- Saved: 1 of 2" in report

def test_page_failure_does_not_stop_the_run(self, data, monkeypatch):
current, previous, summary = data
calls = []
monkeypatch.setattr(
archive_save.wayback,
"save",
lambda url: calls.append(url) or wayback.SaveResult(url, ok=False, detail="HTTP 523"),
)

assert self._run(current, previous, summary) == 0
assert len(calls) == 3
31 changes: 31 additions & 0 deletions tests/test_wayback.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import pytest
import requests
import responses
from urllib3.exceptions import ReadTimeoutError

sys.path.append(str(Path(__file__).parent.parent / "utils"))

Expand Down Expand Up @@ -147,6 +148,35 @@ def test_refused(self):

assert not result.ok
assert result.detail == "HTTP 429"
assert result.blocked

@responses.activate
def test_page_failure_is_not_blocked(self):
responses.add(responses.GET, wayback.SAVE_URL + "https://pycon.tz/", status=523)

result = wayback.save("https://pycon.tz/")

assert not result.ok
assert not result.blocked

@responses.activate
def test_read_timeout_in_body_is_not_blocked(self):
stall = ReadTimeoutError(None, wayback.SAVE_URL, "Read timed out.")
responses.add(responses.GET, wayback.SAVE_URL + "https://pycon.tz/", body=requests.ConnectionError(stall))

result = wayback.save("https://pycon.tz/")

assert not result.ok
assert not result.blocked

@responses.activate
def test_read_timeout_is_not_blocked(self):
responses.add(responses.GET, wayback.SAVE_URL + "https://pycon.tz/", body=requests.ReadTimeout("slow"))

result = wayback.save("https://pycon.tz/")

assert not result.ok
assert not result.blocked

@responses.activate
def test_connection_error(self):
Expand All @@ -156,6 +186,7 @@ def test_connection_error(self):

assert not result.ok
assert result.detail.startswith("error:")
assert result.blocked


class TestLinks:
Expand Down
65 changes: 51 additions & 14 deletions utils/archive_save.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,15 @@

The workflow runs weekly and on pushes to the data, so a point is saved on the
first run on or after it: every point after ``--since`` and up to ``--today``.
A CfP with no exact deadline (TBA, None, Cancelled) has no CfP points. Each URL
is saved at most once per run, whichever points it is due for.
A CfP with no exact deadline (TBA, None, Cancelled) has no CfP points. Each page
is saved at most once per run, whichever points it is due for: ``#fragment``
links to one page count as that page.

When the archive turns the run away (connection refused, rate limited), the run
stops at once and exits 1, so the workflow run fails and the next run starts
from the last successful one and saves the pages this run could not. Waiting
for a block to lift would cost minutes and could fail all the same. A page the
archive fails to capture is reported and does not fail the run.

Only ``_data/conferences.yml`` is read: an edition stays there until well after
its last save point.
Expand All @@ -37,6 +44,7 @@
from datetime import timedelta
from datetime import timezone
from pathlib import Path
from urllib.parse import urldefrag

import yaml

Expand Down Expand Up @@ -96,7 +104,7 @@ def normalise(url: str) -> str:


def entry_urls(entry: Mapping) -> list[str]:
"""Return an entry's live URLs: its URL fields that are not archive links.
"""Return an entry's live pages: its URL fields that are not archive links.

Parameters
----------
Expand All @@ -106,14 +114,15 @@ def entry_urls(entry: Mapping) -> list[str]:
Returns
-------
list[str]
The URLs, in field order
The URLs without ``#fragment``, since the archive saves the page, each
once, in field order
"""
urls = []
for field in URL_FIELDS:
value = entry.get(field)
if value and not wayback.is_archived(str(value)):
urls.append(str(value).strip())
return urls
urls.append(urldefrag(str(value).strip()).url)
return list(dict.fromkeys(urls))


def save_points(entry: Mapping) -> list[date]:
Expand Down Expand Up @@ -203,7 +212,12 @@ def load(path: Path) -> list[dict]:
return [entry for entry in data or [] if isinstance(entry, dict)]


def render_summary(results: list[wayback.SaveResult], new_count: int, due_count: int) -> str:
def render_summary(
results: list[wayback.SaveResult],
new_count: int,
due_count: int,
skipped: list[str] | None = None,
) -> str:
"""Render the run's report as Markdown for the workflow summary.

Parameters
Expand All @@ -214,6 +228,8 @@ def render_summary(results: list[wayback.SaveResult], new_count: int, due_count:
How many of the URLs were new in the data
due_count : int
How many were due for a save point
skipped : list[str] | None
URLs not tried because the archive turned the run away

Returns
-------
Expand All @@ -231,6 +247,17 @@ def render_summary(results: list[wayback.SaveResult], new_count: int, due_count:
if failed:
lines.extend(("", "### Not saved", ""))
lines.extend(f"- {result.url} ({result.detail})" for result in failed)
if skipped:
lines.extend(
(
"",
"### Not tried",
"",
"The archive turned the run away, so the run stopped. The next run saves these.",
"",
),
)
lines.extend(f"- {url}" for url in skipped)
return "\n".join(lines) + "\n"


Expand Down Expand Up @@ -264,7 +291,8 @@ def main(argv: list[str] | None = None) -> int:
Returns
-------
int
0; a page the archive did not save is reported, not an error
0, also when the archive failed to capture a page (reported); 1 when the
archive turned the run away and the run stopped
"""
args = build_parser().parse_args(argv)
today = args.today or datetime.now(tz=timezone.utc).date()
Expand All @@ -275,21 +303,30 @@ def main(argv: list[str] | None = None) -> int:
urls = list(dict.fromkeys(fresh + due))
print(f"{len(fresh)} new URLs, {len(due)} at a save point, {len(urls)} to save")

if args.dry_run:
for url in urls:
print(f"would save {url}")
return 0

results = []
skipped = []
stopped = False
for i, url in enumerate(urls):
if args.dry_run:
print(f"would save {url}")
continue
if i:
time.sleep(args.delay)
result = wayback.save(url)
print(f"{'saved' if result.ok else 'not saved'}: {url} ({result.detail})")
results.append(result)
if result.blocked:
stopped = True
skipped = urls[i + 1 :]
print(f"The archive turned the run away: stopping, {len(skipped)} URLs not tried")
break

if args.summary and not args.dry_run:
if args.summary:
with args.summary.open("a", encoding="utf-8") as f:
f.write(render_summary(results, len(fresh), len(due)))
return 0
f.write(render_summary(results, len(fresh), len(due), skipped))
return 1 if stopped else 0


if __name__ == "__main__":
Expand Down
50 changes: 47 additions & 3 deletions utils/wayback.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,18 @@
from datetime import timedelta

import requests
from urllib3.exceptions import ReadTimeoutError

CDX_URL = "https://web.archive.org/cdx/search/cdx"
SAVE_URL = "https://web.archive.org/save/"
USER_AGENT = "Pythondeadlin.es Archiver/1.0 (https://pythondeadlin.es)"
TIMEOUT = 60
# Seconds to wait for the archive to accept the connection. A refusal arrives at
# once; a long wait here means the archive is not answering at all.
CONNECT_TIMEOUT = 10
# Statuses with which the archive itself turns the client away (rate limit,
# too many sessions), as opposed to failing to capture one page
BLOCKED_STATUSES = frozenset({429, 509})
# Days after an edition's end in which a capture still shows that edition
END_GRACE_DAYS = 14

Expand All @@ -52,11 +59,15 @@ class SaveResult:
True when the archive accepted the request
detail : str
The HTTP status, or the error, for the run's report
blocked : bool
True when the archive turned the client away (connection refused or
failed, rate limited) rather than failing on this page
"""

url: str
ok: bool
detail: str
blocked: bool = False


def is_archived(url: str) -> bool:
Expand Down Expand Up @@ -169,6 +180,28 @@ def pick_capture(stamps: list[str], end: date, grace_days: int = END_GRACE_DAYS)
return max(eligible) if eligible else None


def _is_read_timeout(error: requests.ConnectionError) -> bool:
"""Check whether a ConnectionError is a read timeout while the body downloads.

requests raises ``ReadTimeout`` when the headers are late, but re-raises a
timeout in the body as ``ConnectionError(ReadTimeoutError)``.

Parameters
----------
error : requests.ConnectionError
The error a request raised

Returns
-------
bool
True when the error wraps a urllib3 ``ReadTimeoutError``
"""
return any(isinstance(arg, ReadTimeoutError) for arg in error.args) or isinstance(
error.__context__,
ReadTimeoutError,
)


def save(url: str) -> SaveResult:
"""Ask the Wayback Machine to save a page now.

Expand All @@ -180,10 +213,21 @@ def save(url: str) -> SaveResult:
Returns
-------
SaveResult
Whether the archive accepted the request, with the status or error
Whether the archive accepted the request, with the status or error, and
whether the archive turned the client away
"""
try:
response = requests.get(SAVE_URL + url, headers={"User-Agent": USER_AGENT}, timeout=TIMEOUT)
response = requests.get(
SAVE_URL + url,
headers={"User-Agent": USER_AGENT},
timeout=(CONNECT_TIMEOUT, TIMEOUT),
)
except requests.ConnectionError as e:
# Refused, reset or not answering: the archive, not the page. A read
# timeout while the body downloads also arrives as a ConnectionError.
return SaveResult(url, ok=False, detail=f"error: {e}", blocked=not _is_read_timeout(e))
except requests.RequestException as e:
# A read timeout: the capture can still finish on the archive's side
return SaveResult(url, ok=False, detail=f"error: {e}")
return SaveResult(url, ok=response.status_code == 200, detail=f"HTTP {response.status_code}")
status = response.status_code
return SaveResult(url, ok=status == 200, detail=f"HTTP {status}", blocked=status in BLOCKED_STATUSES)
Loading