diff --git a/rethinkdb/net_trio.py b/rethinkdb/net_trio.py index f1ec6d6..97b9c6d 100644 --- a/rethinkdb/net_trio.py +++ b/rethinkdb/net_trio.py @@ -190,8 +190,15 @@ async def _get_next(self, timeout=None): # pylint: disable=invalid-overridden-m return item async def _parent_continue(self): - """Send a CONTINUE request for this cursor.""" - return await self.conn.parent.resume(self) + """ + Send a CONTINUE request for this cursor. + + A late continue after the socket dies must not fail the caller's nursery. + """ + try: + return await self.conn.parent.resume(self) + except ReqlDriverError: + return None def _maybe_fetch_batch(self): if ( @@ -240,32 +247,43 @@ async def _send(self, data): async with self._stream_lock: try: await self._stream.send_all(data) - except (trio.BrokenResourceError, trio.ClosedResourceError): + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: self._closed = True + raise ReqlDriverError("Connection is closed.") from exc + + async def _receive_some(self, max_bytes): + """Read one chunk, treating EOF and a broken stream as a closed connection.""" + try: + data = await self._stream.receive_some(max_bytes) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + self._closed = True + raise ReqlDriverError("Connection is closed.") from exc + + if not data: + self._closed = True + raise ReqlDriverError("Connection is closed.") + + return data async def _read_until(self, delimiter): - """Naive implementation of reading until a delimiter.""" + """Read from the stream until ``delimiter`` or the connection closes.""" buffer = bytearray() - try: - while True: - data = await self._stream.receive_some(1) - buffer.append(data[0]) - if data == delimiter: - break - except (trio.BrokenResourceError, trio.ClosedResourceError): - self._closed = True + while True: + data = await self._receive_some(1) + buffer.append(data[0]) + if data == delimiter: + break return bytes(buffer) async def _read_exactly(self, num): + """Read exactly ``num`` bytes, or raise if the connection closes.""" data = b"" - try: - while len(data) < num: - data += await self._stream.receive_some(num - len(data)) - return data - except (trio.BrokenResourceError, trio.ClosedResourceError): - self._closed = True + while len(data) < num: + data += await self._receive_some(num - len(data)) + + return data async def connect(self, timeout): try: diff --git a/rethinkdb/net_twisted.py b/rethinkdb/net_twisted.py index 86e80d9..9fbf945 100644 --- a/rethinkdb/net_twisted.py +++ b/rethinkdb/net_twisted.py @@ -74,6 +74,7 @@ def __init__(self, factory: "DatabaseProtoFactory") -> None: self.wait_for_handshake = Deferred() self._open = True + self._handshake_timed_out = False self._timeout_defer: Any | None = None def connectionMade(self) -> None: @@ -90,15 +91,23 @@ def connectionMade(self) -> None: ) def connectionLost(self, reason: Any) -> None: # pylint: disable=signature-differs + """Fail the handshake and notify the connection that the transport is gone.""" self._open = False + if not self.wait_for_handshake.called and not self._handshake_timed_out: + self.wait_for_handshake.errback(ReqlDriverError("Connection is closed.")) + if self.factory.connection_lost is not None: + self.factory.connection_lost(reason) + def _handle_handshake_timeout(self) -> None: # If we are here, we failed to do the handshake before the timeout. # We close the connection and raise an ReqlTimeoutError in the # wait_for_handshake deferred. self._open = False + self._handshake_timed_out = True self.transport.loseConnection() - self.wait_for_handshake.errback(ReqlTimeoutError()) + if not self.wait_for_handshake.called: + self.wait_for_handshake.errback(ReqlTimeoutError()) def _handle_handshake(self, data: bytes) -> None: try: @@ -169,10 +178,17 @@ class DatabaseProtoFactory(ClientFactory): protocol = DatabaseProtocol - def __init__(self, timeout: int, response_handler: Any, handshake: Any) -> None: + def __init__( + self, + timeout: int, + response_handler: Any, + handshake: Any, + connection_lost: Any = None, + ) -> None: self.timeout = timeout self.handshake = handshake self.response_handler = response_handler + self.connection_lost = connection_lost def startedConnecting(self, connector: Any) -> None: """Called when a connection is started.""" @@ -399,7 +415,10 @@ def _connect_timeout( def connect(self, timeout: int) -> Generator: """Establish a connection to the server.""" factory = DatabaseProtoFactory( - timeout, self._handle_response, self._parent.handshake + timeout, + self._handle_response, + self._parent.handshake, + self._on_connection_lost, ) # We connect to the server, and send the handshake payload. @@ -454,6 +473,22 @@ def is_open(self) -> bool: self._connection is not None and self._connection._open # pylint: disable=protected-access ) + def _on_connection_lost(self, reason: Any) -> None: + """ + Fail open cursors and queries when the peer drops the connection. + + ``close`` sets ``_closing`` before it drops the transport, so this + returns immediately when the loss was initiated locally. + """ + if self._closing: + return + + error = getattr(reason, "value", None) + if not isinstance(error, Exception): + error = ReqlDriverError("Connection is closed.") + + self.close(exception=error) + def close( self, noreply_wait: bool = False, diff --git a/tests/integration/test_net_trio.py b/tests/integration/test_net_trio.py index a0c1e87..c9aa2da 100644 --- a/tests/integration/test_net_trio.py +++ b/tests/integration/test_net_trio.py @@ -1,8 +1,26 @@ +# Copyright 2016-2026 RethinkDB +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os +import socket +import threading import pytest +import trio from rethinkdb import net, net_trio, query +from rethinkdb.errors import ReqlRuntimeError from tests.helpers import Scenario @@ -30,6 +48,7 @@ def cleanup_test_dbs(): "test_trio_cursor_db", "test_trio_helpers_db", "test_trio_db", + "test_trio_feed_drop_db", "test_trio_queries_db", ] @@ -352,3 +371,146 @@ async def test_trio_cursor_context_manager(conn): await connection.start(query.db_drop(db_name)) except Exception: pass + + +class _FinProxy: + """Forward one TCP connection, then half-close the client side.""" + + def __init__(self, upstream_host: str, upstream_port: int) -> None: + self._upstream_host = upstream_host + self._upstream_port = upstream_port + self._ready = threading.Event() + self._stop_pump = threading.Event() + self._server: socket.socket | None = None + self._client: socket.socket | None = None + self._upstream: socket.socket | None = None + self.port = 0 + self._thread = threading.Thread(target=self._run, daemon=True) + + def start(self) -> int: + """Listen, and return the local port once it is accepting.""" + self._thread.start() + assert self._ready.wait(5) + return self.port + + def fin(self) -> None: + """Stop forwarding and send a FIN to the driver, leaving the socket open.""" + self._stop_pump.set() + if self._client is None: + return + try: + self._client.shutdown(socket.SHUT_WR) + except OSError: + pass + + def close(self) -> None: + """Close the proxy sockets.""" + self._stop_pump.set() + for sock in (self._client, self._upstream, self._server): + if sock is None: + continue + try: + sock.close() + except OSError: + pass + + def _run(self) -> None: + server = socket.socket() + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + server.listen(1) + self._server = server + self.port = server.getsockname()[1] + self._ready.set() + client, _addr = server.accept() + self._client = client + upstream = socket.create_connection((self._upstream_host, self._upstream_port)) + self._upstream = upstream + + def pump(src: socket.socket, dst: socket.socket) -> None: + try: + while not self._stop_pump.is_set(): + src.settimeout(0.2) + try: + data = src.recv(65536) + except TimeoutError: + continue + if not data: + break + dst.sendall(data) + except OSError: + return + + threading.Thread(target=pump, args=(client, upstream), daemon=True).start() + threading.Thread(target=pump, args=(upstream, client), daemon=True).start() + self._stop_pump.wait() + + +def _sync_run(host: str, port: int, term: object) -> None: + connection = net.make_connection(net.DefaultConnection, host=host, port=port) + try: + term.run(connection) # type: ignore[attr-defined] + finally: + connection.close() + + +@pytest.mark.trio +@pytest.mark.integration +async def test_trio_changefeed_peer_fin_raises(nursery): + """A clean TCP close while waiting for changes raises and the context exits.""" + host = os.getenv("RDB_TEST_HOST", "localhost") + port = int(os.getenv("RDB_TEST_PORT", 28015)) + db_name = "test_trio_feed_drop_db" + table_name = "feed" + try: + _sync_run(host, port, query.db_drop(db_name)) + except Exception: + pass + _sync_run(host, port, query.db_create(db_name)) + proxy = None + try: + _sync_run(host, port, query.db(db_name).table_create(table_name)) + proxy = _FinProxy(host, port) + proxy_port = proxy.start() + + def insert_row() -> None: + _sync_run( + host, + port, + query.db(db_name).table(table_name).insert({"n": 1}), + ) + + with trio.fail_after(8): + connection = await net.make_connection( + net_trio.Connection, + host="127.0.0.1", + port=proxy_port, + db=db_name, + nursery=nursery, + ) + + async def watch_feed() -> None: + async with connection: + cursor = ( + await query.db(db_name) + .table(table_name) + .changes() + .run(connection) + ) + threading.Thread(target=insert_row, daemon=True).start() + saw_change = False + async for _change in cursor: + assert saw_change is False + saw_change = True + proxy.fin() + + with pytest.raises(ReqlRuntimeError, match="Connection is closed"): + await watch_feed() + assert connection.is_open() is False + finally: + if proxy is not None: + proxy.close() + try: + _sync_run(host, port, query.db_drop(db_name)) + except Exception: + pass diff --git a/tests/integration/test_net_twisted.py b/tests/integration/test_net_twisted.py index b00626b..70f1c63 100644 --- a/tests/integration/test_net_twisted.py +++ b/tests/integration/test_net_twisted.py @@ -1,9 +1,24 @@ +# Copyright 2016-2026 RethinkDB +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import pytest -from twisted.internet import defer +from twisted.internet import defer, reactor from rethinkdb import net, net_twisted, query +from rethinkdb.errors import ReqlRuntimeError, ReqlTimeoutError @pytest.fixture @@ -29,6 +44,7 @@ def cleanup_test_dbs(): "test_twisted_cursor_db", "test_twisted_helpers_db", "test_twisted_db", + "test_twisted_feed_drop_db", "test_twisted_queries_db", ] @@ -372,3 +388,60 @@ async def run_test(): pass return defer.ensureDeferred(run_test()) + + +@pytest.mark.twisted +@pytest.mark.integration +def test_twisted_changefeed_connection_lost(conn): + """Dropping the transport while a changefeed is waiting errbacks the next item.""" + + async def run_test(): + connection = await conn + db_name = "test_twisted_feed_drop_db" + table_name = "feed" + try: + try: + await connection.start(query.db_drop(db_name)) + except Exception: + pass + + await connection.start(query.db_create(db_name)) + await connection.start(query.db(db_name).table_create(table_name)) + cursor = await connection.start( + query.db(db_name).table(table_name).changes() + ) + pending = cursor.next() + + def fail_if_still_waiting() -> None: + if not pending.called: + pending.errback(ReqlTimeoutError()) + + timeout = reactor.callLater(5, fail_if_still_waiting) + reactor.callLater( + 0, + connection._instance._connection.transport.loseConnection, # pylint: disable=protected-access + ) + try: + with pytest.raises(ReqlRuntimeError, match="Connection is closed"): + await pending + finally: + if timeout.active(): + timeout.cancel() + assert connection.is_open() is False + finally: + try: + await connection.close(noreply_wait=False) + except Exception: + pass + try: + sync = net.make_connection( + net.DefaultConnection, + host=os.getenv("RDB_TEST_HOST", "localhost"), + port=int(os.getenv("RDB_TEST_PORT", 28015)), + ) + query.db_drop(db_name).run(sync) + sync.close() + except Exception: + pass + + return defer.ensureDeferred(run_test()) diff --git a/tests/test_net_trio.py b/tests/test_net_trio.py new file mode 100644 index 0000000..43131da --- /dev/null +++ b/tests/test_net_trio.py @@ -0,0 +1,147 @@ +# Copyright 2016-2026 RethinkDB +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import trio + +from rethinkdb.errors import ReqlDriverError +from rethinkdb.net_trio import ConnectionInstance, TrioCursor + + +class _ChunkStream: + """Stream that returns scripted chunks from ``receive_some``.""" + + def __init__(self, chunks: list[bytes]) -> None: + self._chunks = list(chunks) + + async def receive_some(self, max_bytes: int) -> bytes: + if not self._chunks: + return b"" + chunk = self._chunks[0] + if not chunk: + self._chunks.pop(0) + return b"" + taken, rest = chunk[:max_bytes], chunk[max_bytes:] + if rest: + self._chunks[0] = rest + else: + self._chunks.pop(0) + return taken + + +class _BrokenStream: + """Stream whose reads and writes fail with ``error``.""" + + def __init__(self, error: Exception) -> None: + self._error = error + + async def receive_some(self, max_bytes: int) -> bytes: + del max_bytes + raise self._error + + async def send_all(self, data: bytes) -> None: + del data + raise self._error + + +class _ClosedParent: + """Parent connection whose ``resume`` reports a closed connection.""" + + async def resume(self, cursor: object) -> None: + del cursor + raise ReqlDriverError("Connection is closed.") + + +class _CursorConn: + """Stand-in for the connection object a cursor points at.""" + + def __init__(self) -> None: + self.parent = _ClosedParent() + + +def _instance(stream: object) -> ConnectionInstance: + instance = ConnectionInstance(parent=None) + instance._stream = stream # pylint: disable=protected-access + return instance + + +@pytest.mark.trio +@pytest.mark.unit +async def test_read_exactly_returns_a_full_chunk() -> None: + """A complete read is returned unchanged, including bytes split across chunks.""" + instance = _instance(_ChunkStream([b"ab", b"cd"])) + + assert await instance._read_exactly(4) == b"abcd" # pylint: disable=protected-access + + +@pytest.mark.trio +@pytest.mark.unit +async def test_read_exactly_empty_chunk_closes_connection() -> None: + """EOF is a closed connection, not another read.""" + instance = _instance(_ChunkStream([b""])) + + with pytest.raises(ReqlDriverError, match="Connection is closed"): + await instance._read_exactly(4) # pylint: disable=protected-access + + assert instance._closed is True # pylint: disable=protected-access + + +@pytest.mark.trio +@pytest.mark.unit +@pytest.mark.parametrize( + "error", + [trio.BrokenResourceError("reset"), trio.ClosedResourceError("closed")], +) +async def test_read_exactly_broken_stream_closes_connection(error: Exception) -> None: + """A broken or closed Trio stream raises a driver error.""" + instance = _instance(_BrokenStream(error)) + + with pytest.raises(ReqlDriverError, match="Connection is closed"): + await instance._read_exactly(4) # pylint: disable=protected-access + + assert instance._closed is True # pylint: disable=protected-access + + +@pytest.mark.trio +@pytest.mark.unit +async def test_read_until_empty_chunk_closes_connection() -> None: + """A handshake read stops with a driver error when the peer sends EOF.""" + instance = _instance(_ChunkStream([b"ok", b""])) + + with pytest.raises(ReqlDriverError, match="Connection is closed"): + await instance._read_until(b"\0") # pylint: disable=protected-access + + assert instance._closed is True # pylint: disable=protected-access + + +@pytest.mark.trio +@pytest.mark.unit +async def test_send_broken_stream_closes_connection() -> None: + """A failed write is reported as a closed connection.""" + instance = _instance(_BrokenStream(trio.BrokenResourceError("reset"))) + + with pytest.raises(ReqlDriverError, match="Connection is closed"): + await instance._send(b"query") # pylint: disable=protected-access + + assert instance._closed is True # pylint: disable=protected-access + + +@pytest.mark.trio +@pytest.mark.unit +async def test_parent_continue_swallows_closed_connection() -> None: + """CONTINUE after the socket dies does not escape into the caller nursery.""" + cursor = TrioCursor.__new__(TrioCursor) + cursor.conn = _CursorConn() # type: ignore[assignment] + + assert await cursor._parent_continue() is None # pylint: disable=protected-access