From 0e691975981dd30cb2c096be781f46e23a7ece4d Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 17 Nov 2026 22:41:00 +0000 Subject: [PATCH] cancel right away on a zero timeout for event listeners and added requests --- docs/06_upgrading/upgrading_to_v2.md | 4 ++++ src/crawlee/_utils/wait.py | 2 +- tests/unit/_utils/test_wait.py | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tests/unit/_utils/test_wait.py diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 1174379fe4..763d139b7f 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -54,6 +54,10 @@ A custom storage backend has more names to update. The `StorageBackend` methods `RequestQueueBackend.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. A subclass that doesn't implement it raises `TypeError` on instantiation. If you have a custom request queue backend, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in backends already implement it. +## A zero timeout cancels right away + +A zero `close_timeout` of an event manager, or a zero `timeout` passed to `EventManager.wait_for_all_listeners_to_complete()`, cancels the running listeners right away. A zero `wait_for_all_requests_to_be_added_timeout` in `add_requests()` does the same for the requests still being added. In v1, a zero timeout meant no limit. To wait without a limit, pass `None`. + ## Keyword-only arguments Secondary arguments of the public API are now keyword-only. The main argument of each call, such as a key, a value, a selector or a path, stays positional. A call that passes any of the following arguments positionally raises `TypeError`, so pass them by name: diff --git a/src/crawlee/_utils/wait.py b/src/crawlee/_utils/wait.py index 9c1e9aa0d9..3a39d4853d 100644 --- a/src/crawlee/_utils/wait.py +++ b/src/crawlee/_utils/wait.py @@ -82,7 +82,7 @@ async def wait_for_all_tasks_to_finish( if not tasks: return - timeout_secs = timeout.total_seconds() if timeout else None + timeout_secs = timeout.total_seconds() if timeout is not None else None try: _, pending = await asyncio.wait(tasks, timeout=timeout_secs) if pending: diff --git a/tests/unit/_utils/test_wait.py b/tests/unit/_utils/test_wait.py new file mode 100644 index 0000000000..65c6af6ca4 --- /dev/null +++ b/tests/unit/_utils/test_wait.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import asyncio +import logging +from datetime import timedelta + +from crawlee._utils.wait import wait_for_all_tasks_to_finish + + +async def test_zero_timeout_cancels_right_away() -> None: + """A zero timeout cancels the unfinished tasks without waiting for them.""" + task = asyncio.create_task(asyncio.Event().wait()) + + # Capped, as a zero timeout taken for no limit would wait for the task forever. + await asyncio.wait_for( + wait_for_all_tasks_to_finish([task], logger=logging.getLogger(__name__), timeout=timedelta(0)), + timeout=5, + ) + + assert task.cancelled()