Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 36 additions & 18 deletions rethinkdb/net_trio.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,15 @@ async def _get_next(self, timeout=None): # pylint: disable=invalid-overridden-m
return item

async def _parent_continue(self):
"""Send a CONTINUE request for this cursor."""
return await self.conn.parent.resume(self)
"""
Send a CONTINUE request for this cursor.

A late continue after the socket dies must not fail the caller's nursery.
"""
try:
return await self.conn.parent.resume(self)
except ReqlDriverError:
return None

def _maybe_fetch_batch(self):
if (
Expand Down Expand Up @@ -240,32 +247,43 @@ async def _send(self, data):
async with self._stream_lock:
try:
await self._stream.send_all(data)
except (trio.BrokenResourceError, trio.ClosedResourceError):
except (trio.BrokenResourceError, trio.ClosedResourceError) as exc:
self._closed = True
raise ReqlDriverError("Connection is closed.") from exc

async def _receive_some(self, max_bytes):
"""Read one chunk, treating EOF and a broken stream as a closed connection."""
try:
data = await self._stream.receive_some(max_bytes)
except (trio.BrokenResourceError, trio.ClosedResourceError) as exc:
self._closed = True
raise ReqlDriverError("Connection is closed.") from exc

if not data:
self._closed = True
raise ReqlDriverError("Connection is closed.")

return data

async def _read_until(self, delimiter):
"""Naive implementation of reading until a delimiter."""
"""Read from the stream until ``delimiter`` or the connection closes."""
buffer = bytearray()

try:
while True:
data = await self._stream.receive_some(1)
buffer.append(data[0])
if data == delimiter:
break
except (trio.BrokenResourceError, trio.ClosedResourceError):
self._closed = True
while True:
data = await self._receive_some(1)
buffer.append(data[0])
if data == delimiter:
break

return bytes(buffer)

async def _read_exactly(self, num):
"""Read exactly ``num`` bytes, or raise if the connection closes."""
data = b""
try:
while len(data) < num:
data += await self._stream.receive_some(num - len(data))
return data
except (trio.BrokenResourceError, trio.ClosedResourceError):
self._closed = True
while len(data) < num:
data += await self._receive_some(num - len(data))

return data

async def connect(self, timeout):
try:
Expand Down
41 changes: 38 additions & 3 deletions rethinkdb/net_twisted.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ def __init__(self, factory: "DatabaseProtoFactory") -> None:
self.wait_for_handshake = Deferred()

self._open = True
self._handshake_timed_out = False
self._timeout_defer: Any | None = None

def connectionMade(self) -> None:
Expand All @@ -90,15 +91,23 @@ def connectionMade(self) -> None:
)

def connectionLost(self, reason: Any) -> None: # pylint: disable=signature-differs
"""Fail the handshake and notify the connection that the transport is gone."""
self._open = False

if not self.wait_for_handshake.called and not self._handshake_timed_out:
self.wait_for_handshake.errback(ReqlDriverError("Connection is closed."))
if self.factory.connection_lost is not None:
self.factory.connection_lost(reason)

def _handle_handshake_timeout(self) -> None:
# If we are here, we failed to do the handshake before the timeout.
# We close the connection and raise an ReqlTimeoutError in the
# wait_for_handshake deferred.
self._open = False
self._handshake_timed_out = True
self.transport.loseConnection()
self.wait_for_handshake.errback(ReqlTimeoutError())
if not self.wait_for_handshake.called:
self.wait_for_handshake.errback(ReqlTimeoutError())

def _handle_handshake(self, data: bytes) -> None:
try:
Expand Down Expand Up @@ -169,10 +178,17 @@ class DatabaseProtoFactory(ClientFactory):

protocol = DatabaseProtocol

def __init__(self, timeout: int, response_handler: Any, handshake: Any) -> None:
def __init__(
self,
timeout: int,
response_handler: Any,
handshake: Any,
connection_lost: Any = None,
) -> None:
self.timeout = timeout
self.handshake = handshake
self.response_handler = response_handler
self.connection_lost = connection_lost

def startedConnecting(self, connector: Any) -> None:
"""Called when a connection is started."""
Expand Down Expand Up @@ -399,7 +415,10 @@ def _connect_timeout(
def connect(self, timeout: int) -> Generator:
"""Establish a connection to the server."""
factory = DatabaseProtoFactory(
timeout, self._handle_response, self._parent.handshake
timeout,
self._handle_response,
self._parent.handshake,
self._on_connection_lost,
)

# We connect to the server, and send the handshake payload.
Expand Down Expand Up @@ -454,6 +473,22 @@ def is_open(self) -> bool:
self._connection is not None and self._connection._open # pylint: disable=protected-access
)

def _on_connection_lost(self, reason: Any) -> None:
"""
Fail open cursors and queries when the peer drops the connection.

``close`` sets ``_closing`` before it drops the transport, so this
returns immediately when the loss was initiated locally.
"""
if self._closing:
return

error = getattr(reason, "value", None)
if not isinstance(error, Exception):
error = ReqlDriverError("Connection is closed.")

self.close(exception=error)

def close(
self,
noreply_wait: bool = False,
Expand Down
162 changes: 162 additions & 0 deletions tests/integration/test_net_trio.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,26 @@
# Copyright 2016-2026 RethinkDB
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import socket
import threading

import pytest
import trio

from rethinkdb import net, net_trio, query
from rethinkdb.errors import ReqlRuntimeError
from tests.helpers import Scenario


Expand Down Expand Up @@ -30,6 +48,7 @@ def cleanup_test_dbs():
"test_trio_cursor_db",
"test_trio_helpers_db",
"test_trio_db",
"test_trio_feed_drop_db",
"test_trio_queries_db",
]

Expand Down Expand Up @@ -352,3 +371,146 @@ async def test_trio_cursor_context_manager(conn):
await connection.start(query.db_drop(db_name))
except Exception:
pass


class _FinProxy:
"""Forward one TCP connection, then half-close the client side."""

def __init__(self, upstream_host: str, upstream_port: int) -> None:
self._upstream_host = upstream_host
self._upstream_port = upstream_port
self._ready = threading.Event()
self._stop_pump = threading.Event()
self._server: socket.socket | None = None
self._client: socket.socket | None = None
self._upstream: socket.socket | None = None
self.port = 0
self._thread = threading.Thread(target=self._run, daemon=True)

def start(self) -> int:
"""Listen, and return the local port once it is accepting."""
self._thread.start()
assert self._ready.wait(5)
return self.port

def fin(self) -> None:
"""Stop forwarding and send a FIN to the driver, leaving the socket open."""
self._stop_pump.set()
if self._client is None:
return
try:
self._client.shutdown(socket.SHUT_WR)
except OSError:
pass

def close(self) -> None:
"""Close the proxy sockets."""
self._stop_pump.set()
for sock in (self._client, self._upstream, self._server):
if sock is None:
continue
try:
sock.close()
except OSError:
pass

def _run(self) -> None:
server = socket.socket()
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("127.0.0.1", 0))
server.listen(1)
self._server = server
self.port = server.getsockname()[1]
self._ready.set()
client, _addr = server.accept()
self._client = client
upstream = socket.create_connection((self._upstream_host, self._upstream_port))
self._upstream = upstream

def pump(src: socket.socket, dst: socket.socket) -> None:
try:
while not self._stop_pump.is_set():
src.settimeout(0.2)
try:
data = src.recv(65536)
except TimeoutError:
continue
if not data:
break
dst.sendall(data)
except OSError:
return

threading.Thread(target=pump, args=(client, upstream), daemon=True).start()
threading.Thread(target=pump, args=(upstream, client), daemon=True).start()
self._stop_pump.wait()


def _sync_run(host: str, port: int, term: object) -> None:
connection = net.make_connection(net.DefaultConnection, host=host, port=port)
try:
term.run(connection) # type: ignore[attr-defined]
finally:
connection.close()


@pytest.mark.trio
@pytest.mark.integration
async def test_trio_changefeed_peer_fin_raises(nursery):
"""A clean TCP close while waiting for changes raises and the context exits."""
host = os.getenv("RDB_TEST_HOST", "localhost")
port = int(os.getenv("RDB_TEST_PORT", 28015))
db_name = "test_trio_feed_drop_db"
table_name = "feed"
try:
_sync_run(host, port, query.db_drop(db_name))
except Exception:
pass
_sync_run(host, port, query.db_create(db_name))
proxy = None
try:
_sync_run(host, port, query.db(db_name).table_create(table_name))
proxy = _FinProxy(host, port)
proxy_port = proxy.start()

def insert_row() -> None:
_sync_run(
host,
port,
query.db(db_name).table(table_name).insert({"n": 1}),
)

with trio.fail_after(8):
connection = await net.make_connection(
net_trio.Connection,
host="127.0.0.1",
port=proxy_port,
db=db_name,
nursery=nursery,
)

async def watch_feed() -> None:
async with connection:
cursor = (
await query.db(db_name)
.table(table_name)
.changes()
.run(connection)
)
threading.Thread(target=insert_row, daemon=True).start()
saw_change = False
async for _change in cursor:
assert saw_change is False
saw_change = True
proxy.fin()

with pytest.raises(ReqlRuntimeError, match="Connection is closed"):
await watch_feed()
assert connection.is_open() is False
finally:
if proxy is not None:
proxy.close()
try:
_sync_run(host, port, query.db_drop(db_name))
except Exception:
pass
Loading
Loading