Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ sensor:
name: Leicester Weather Warnings
feed_url: http://www.metoffice.gov.uk/public/data/PWSCache/WarningsRSS/Region/em
match: Leicester

# A feed that is blocked on this network, fetched through a SOCKS proxy
- platform: feedparser
name: Blocked Feed
feed_url: https://blocked.example.org/rss
proxy: socks5h://192.168.1.10:1080
```

If you wish the integration to look for enclosures in the feed entries, add `image` to the `inclusions` list. Do not use `enclosure`.
Expand Down Expand Up @@ -165,6 +171,7 @@ The integration supports both UI and YAML setup. Most options are the same in bo
| `match` | No | regular expression | none | `Leicester` or `rain\|snow` | Only entries whose title or summary contains a match are kept. Ignores case. |
| `exclude_match` | No | regular expression | none | `sponsored\|advert` | Entries whose title or summary contains a match are removed. Ignores case. |
| `user_agent` | No | string | `Home Assistant Feed-parser Integration <version>` | `Mozilla/5.0 (compatible; MyFeedReader/1.0)` | User-Agent header sent when fetching the feed (and when validating it during UI setup). Leave empty in the UI to use the default. |
| `proxy` | No | URL | none | `socks5h://192.168.1.10:1080` or `http://user:[email protected]:3128` | Proxy used to fetch the feed (and to validate it during UI setup). Schemes: `http`, `https`, `socks4`, `socks4a`, `socks5`, `socks5h`. Leave empty to connect directly. |

### Notes and behavior details

Expand All @@ -173,6 +180,7 @@ The integration supports both UI and YAML setup. Most options are the same in bo
- **Filtering entries with `match` and `exclude_match`**: these choose which *entries* are kept, while `inclusions` and `exclusions` choose which *fields* each entry keeps. They search the entry's `title` and `summary` (the RSS `<description>`), even if those fields are not in `inclusions`. Filtering runs before `show_topn`, so `show_topn` counts matching entries, and the sensor's state is the number of matching entries. To get several filtered sensors from one feed, add the feed more than once with different names and patterns. Patterns are [Python regular expressions](https://docs.python.org/3/library/re.html#regular-expression-syntax) and are found anywhere in the text, so `Leicester` matches "Leicester, Leicestershire". Use a template sensor only if you need to reshape entries, not just choose them.
- **`image` extraction**: adding `image` to `inclusions` enables image URL extraction from image enclosures, Media RSS (`media:content`, `media:thumbnail`), the first image in the summary HTML, or the first image in the full content (`content:encoded`, used by WordPress feeds).
- **`user_agent`**: some servers reject or never answer requests from unknown clients. If a feed works in a browser but fails in Home Assistant, try a browser User-Agent.
- **`proxy`**: use it to reach feeds that are blocked on your network or in your country. Prefer `socks5h://` over `socks5://` for SOCKS proxies: it also looks up the feed's host name through the proxy, which matters when DNS is what blocks the site. A password in the proxy URL is hidden in logs and errors, but it is stored in plain text in Home Assistant's configuration like other settings. Only the feed itself goes through the proxy: images and links in the entries are opened by your browser, which does not use it. Without this option the integration still honors the standard `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY` environment variables if they are set for Home Assistant.
- **UI vs YAML input format**:
- UI uses comma-separated text for `inclusions` and `exclusions`.
- YAML uses proper lists.
Expand Down
37 changes: 30 additions & 7 deletions custom_components/feedparser/config_flow.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
CONF_LOCAL_TIME,
CONF_MANAGED_BY_YAML,
CONF_MATCH,
CONF_PROXY,
CONF_REMOVE_SUMMARY_IMAGE,
CONF_SCAN_INTERVAL,
CONF_SHOW_TOPN,
Expand All @@ -44,7 +45,7 @@
DOMAIN,
ENTRY_VERSION,
)
from .fetch import USER_AGENT, fetch_feed
from .fetch import USER_AGENT, fetch_feed, is_valid_proxy
from .reader import compile_filter
from .yaml_import import (
REASON_MANAGED_IN_UI,
Expand Down Expand Up @@ -174,6 +175,7 @@ def _options_from_input(user_input: Mapping[str, object]) -> dict[str, object]:
CONF_USER_AGENT: _user_agent_from_input(user_input),
CONF_MATCH: _pattern(user_input.get(CONF_MATCH)),
CONF_EXCLUDE_MATCH: _pattern(user_input.get(CONF_EXCLUDE_MATCH)),
CONF_PROXY: str(user_input.get(CONF_PROXY) or "").strip(),
}


Expand All @@ -196,6 +198,15 @@ def _pattern_errors(options: Mapping[str, object]) -> dict[str, str]:
return errors


def _option_errors(options: Mapping[str, object]) -> dict[str, str]:
"""Return form errors for the free-text options (patterns, proxy)."""
errors = _pattern_errors(options)
proxy = str(options.get(CONF_PROXY) or "")
if proxy and not is_valid_proxy(proxy):
errors[CONF_PROXY] = "invalid_proxy"
return errors


def _options_from_import(import_data: Mapping[str, Any]) -> dict[str, object]:
"""Build stored options from a validated YAML sensor configuration."""
user_agent = str(import_data.get(CONF_USER_AGENT) or "").strip()
Expand All @@ -215,6 +226,7 @@ def _options_from_import(import_data: Mapping[str, Any]) -> dict[str, object]:
CONF_USER_AGENT: "" if user_agent == USER_AGENT else user_agent,
CONF_MATCH: _pattern(import_data.get(CONF_MATCH)),
CONF_EXCLUDE_MATCH: _pattern(import_data.get(CONF_EXCLUDE_MATCH)),
CONF_PROXY: str(import_data.get(CONF_PROXY) or "").strip(),
}


Expand All @@ -237,6 +249,7 @@ def _schema_with_defaults( # noqa: PLR0913 - one keyword per form field
user_agent: str = "",
match: str = "",
exclude_match: str = "",
proxy: str = "",
include_feed_identity: bool,
) -> vol.Schema:
"""Build schema for user/options forms."""
Expand Down Expand Up @@ -275,6 +288,7 @@ def _schema_with_defaults( # noqa: PLR0913 - one keyword per form field
vol.Optional(CONF_EXCLUDE_MATCH, default=exclude_match): str,
vol.Optional(CONF_EXCLUSIONS, default=exclusions): str,
vol.Optional(CONF_USER_AGENT, default=user_agent): str,
vol.Optional(CONF_PROXY, default=proxy): str,
}

if include_feed_identity:
Expand Down Expand Up @@ -314,7 +328,7 @@ async def async_step_user(
except vol.Invalid:
errors[CONF_SCAN_INTERVAL] = "scan_interval_too_short"
else:
errors.update(_pattern_errors(options))
errors.update(_option_errors(options))
if not _is_supported_url(feed_url):
errors[CONF_FEED_URL] = "invalid_url"
elif self._is_configured(feed_url, name):
Expand All @@ -326,6 +340,7 @@ async def async_step_user(
errors = await self._async_validate_feed(
feed_url,
str(options[CONF_USER_AGENT]) or USER_AGENT,
str(options[CONF_PROXY]),
)
if not errors:
return cast(
Expand Down Expand Up @@ -433,6 +448,7 @@ async def async_step_reconfigure(
errors = await self._async_validate_feed(
feed_url,
str(entry.options.get(CONF_USER_AGENT) or "") or USER_AGENT,
str(entry.options.get(CONF_PROXY) or ""),
)
if not errors:
# The sensor's unique ID is the entry ID, so the entity ID and
Expand Down Expand Up @@ -508,13 +524,15 @@ async def _async_validate_feed(
self,
feed_url: str,
user_agent: str,
proxy: str = "",
) -> dict[str, str]:
"""Fetch the feed and return form errors (empty if it is usable)."""
try:
await self.hass.async_add_executor_job(
self._validate_feed_url,
feed_url,
user_agent,
proxy,
)
except requests.RequestException:
return {"base": "cannot_connect"}
Expand All @@ -523,13 +541,17 @@ async def _async_validate_feed(
return {}

@staticmethod
def _validate_feed_url(feed_url: str, user_agent: str = USER_AGENT) -> None:
def _validate_feed_url(
feed_url: str,
user_agent: str = USER_AGENT,
proxy: str = "",
) -> None:
"""Validate that the URL can be fetched and parses as a feed.

Uses the same request (User-Agent, timeouts) as the sensor so a feed that
passes here also works at runtime.
Uses the same request (User-Agent, proxy, timeouts) as the sensor so a feed
that passes here also works at runtime.
"""
parsed = feedparser.parse(fetch_feed(feed_url, user_agent))
parsed = feedparser.parse(fetch_feed(feed_url, user_agent, proxy))
# A real feed exposes a version (rss20, atom10, ...) even when it currently
# has no entries; an HTML page or random document does not.
if not parsed.get("version") and not parsed.get("entries"):
Expand All @@ -555,7 +577,7 @@ async def async_step_init(
except vol.Invalid:
errors[CONF_SCAN_INTERVAL] = "scan_interval_too_short"
else:
errors = _pattern_errors(options)
errors = _option_errors(options)
if not errors:
return cast(
"ConfigFlowResult",
Expand All @@ -582,6 +604,7 @@ async def async_step_init(
user_agent=str(merged.get(CONF_USER_AGENT) or ""),
match=str(merged.get(CONF_MATCH) or ""),
exclude_match=str(merged.get(CONF_EXCLUDE_MATCH) or ""),
proxy=str(merged.get(CONF_PROXY) or ""),
)
if user_input is not None:
# keep what the user typed when showing errors
Expand Down
2 changes: 2 additions & 0 deletions custom_components/feedparser/const.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
CONF_USER_AGENT = "user_agent"
CONF_MATCH = "match"
CONF_EXCLUDE_MATCH = "exclude_match"
CONF_PROXY = "proxy"

# Entry data flag: True while the entry follows configuration.yaml, False once the
# user moved it to the UI. Missing on UI entries and on entries imported by 1.0.0.
Expand All @@ -45,4 +46,5 @@
CONF_USER_AGENT,
CONF_MATCH,
CONF_EXCLUDE_MATCH,
CONF_PROXY,
]
8 changes: 6 additions & 2 deletions custom_components/feedparser/coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
CONF_INCLUSIONS,
CONF_LOCAL_TIME,
CONF_MATCH,
CONF_PROXY,
CONF_REMOVE_SUMMARY_IMAGE,
CONF_SCAN_INTERVAL,
CONF_SHOW_TOPN,
Expand All @@ -31,7 +32,7 @@
DEFAULT_TOPN,
DOMAIN,
)
from .fetch import USER_AGENT
from .fetch import USER_AGENT, redact_text
from .reader import FeedReader

if TYPE_CHECKING:
Expand Down Expand Up @@ -81,6 +82,7 @@ def __init__(self, hass: HomeAssistant, entry: FeedparserConfigEntry) -> None:
user_agent=config.get(CONF_USER_AGENT) or USER_AGENT,
match=config.get(CONF_MATCH) or "",
exclude_match=config.get(CONF_EXCLUDE_MATCH) or "",
proxy=config.get(CONF_PROXY) or "",
)
super().__init__(
hass,
Expand All @@ -97,5 +99,7 @@ async def _async_update_data(self) -> FeedEntries:
try:
return await self.hass.async_add_executor_job(self.reader.read)
except requests.RequestException as err:
msg = f"Unable to fetch {self.reader.feed}: {err}"
# the error may quote the proxy URL; never log its password
reason = redact_text(str(err), self.reader.proxy)
msg = f"Unable to fetch {self.reader.feed}: {reason}"
raise UpdateFailed(msg) from err
51 changes: 49 additions & 2 deletions custom_components/feedparser/fetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

from __future__ import annotations

from urllib.parse import urlsplit

import requests
from requests_file import FileAdapter

Expand All @@ -13,18 +15,63 @@
# connection but never answers blocks an executor thread indefinitely.
REQUEST_TIMEOUT = (10, 30)

# Proxy URL schemes requests understands. The socks* schemes need PySocks;
# socks5h / socks4a also resolve the feed's host name through the proxy.
PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h")


def is_valid_proxy(proxy: str) -> bool:
"""Return True if `proxy` looks like scheme://[user:pass@]host:port."""
try:
parts = urlsplit(proxy)
port = parts.port
except ValueError:
return False
return (
parts.scheme.lower() in PROXY_SCHEMES
and bool(parts.hostname)
and port is not None
and parts.path in ("", "/")
and not parts.query
and not parts.fragment
)


def redact_proxy(proxy: str) -> str:
"""Return `proxy` with any password replaced, safe to log."""
try:
parts = urlsplit(proxy)
password = parts.password
except ValueError:
return "<invalid proxy>"
if password is None:
return proxy
netloc = parts.netloc.replace(f":{password}@", ":***@", 1)
return parts._replace(netloc=netloc).geturl()

def fetch_feed(url: str, user_agent: str = USER_AGENT) -> bytes:

def redact_text(text: str, proxy: str) -> str:
"""Remove the proxy password from `text` (e.g. an exception message)."""
if not proxy:
return text
return text.replace(proxy, redact_proxy(proxy))


def fetch_feed(url: str, user_agent: str = USER_AGENT, proxy: str = "") -> bytes:
"""Download a feed and return its raw bytes.

Raw bytes (not ``Response.text``) are returned so that feedparser can detect the
document encoding from the XML prolog instead of relying on requests' guess.

When `proxy` is set, http(s) requests go through it instead of any proxy set
in the environment. ``file://`` feeds never use a proxy.

Raises ``requests.RequestException`` on network or HTTP errors.
"""
with requests.Session() as session:
session.mount("file://", FileAdapter())
session.headers.update({"User-Agent": user_agent})
response = session.get(url, timeout=REQUEST_TIMEOUT)
proxies = {"http": proxy, "https": proxy} if proxy else None
response = session.get(url, timeout=REQUEST_TIMEOUT, proxies=proxies)
response.raise_for_status()
return response.content
1 change: 1 addition & 0 deletions custom_components/feedparser/manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"issue_tracker": "https://github.com/custom-components/feedparser/issues",
"requirements": [
"feedparser>=6.0.12",
"PySocks>=1.7.1",
"python-dateutil",
"requests-file"
],
Expand Down
13 changes: 8 additions & 5 deletions custom_components/feedparser/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from dateutil import parser
from homeassistant.util import dt

from .fetch import USER_AGENT, fetch_feed
from .fetch import USER_AGENT, fetch_feed, redact_proxy

_LOGGER: logging.Logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -93,7 +93,7 @@ def _looks_like_image(url: str) -> bool:
class FeedReader:
"""Download one feed and build its entries according to the sensor options."""

def __init__(
def __init__( # noqa: PLR0913 - one keyword per sensor option
self: FeedReader,
*,
feed: str,
Expand All @@ -107,6 +107,7 @@ def __init__(
user_agent: str = USER_AGENT,
match: str = "",
exclude_match: str = "",
proxy: str = "",
) -> None:
"""Initialize the reader.

Expand All @@ -122,6 +123,7 @@ def __init__(
self._inclusions = inclusions
self._exclusions = exclusions
self._user_agent = user_agent
self.proxy = proxy
self._match = compile_filter(match)
self._exclude_match = compile_filter(exclude_match)

Expand All @@ -136,7 +138,8 @@ def __repr__(self: FeedReader) -> str:
f'user_agent="{self._user_agent}", '
f"match={self._match.pattern if self._match else None!r}, "
f"exclude_match="
f"{self._exclude_match.pattern if self._exclude_match else None!r})"
f"{self._exclude_match.pattern if self._exclude_match else None!r}, "
f"proxy={redact_proxy(self.proxy) if self.proxy else None!r})"
)

def read(self: FeedReader) -> list[dict[str, object]]:
Expand All @@ -150,7 +153,7 @@ def read(self: FeedReader) -> list[dict[str, object]]:
_LOGGER.debug("Feed %s: Polling feed data from %s", self.name, self.feed)
parsed_feed = cast(
"ParsedFeed",
feedparser.parse(fetch_feed(self.feed, self._user_agent)),
feedparser.parse(fetch_feed(self.feed, self._user_agent, self.proxy)),
)
if not parsed_feed.entries:
_LOGGER.warning("Feed %s: No data received.", self.name)
Expand Down Expand Up @@ -319,7 +322,7 @@ def _parse_date_string(date: str) -> datetime | None:
warnings.simplefilter("ignore")
try:
fallback = parser.parse(date)
except (ValueError, OverflowError):
except ValueError, OverflowError:
return None
return fallback if fallback.tzinfo is not None else None

Expand Down
Loading
Loading