From 4fd064079b0b4966b2f8b77bf419ac601935c15e Mon Sep 17 00:00:00 2001 From: Laurie O Date: Thu, 1 Sep 2022 18:33:18 +1000 Subject: [PATCH 01/62] Add threading implementation of queue shutdown --- Lib/queue.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Lib/queue.py b/Lib/queue.py index 55f50088460f9e..4d2c1cbe084035 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -25,6 +25,15 @@ class Full(Exception): pass +class ShutDown(Exception): + '''Raised when put/get with shut-down queue.''' + + +_queue_alive = "alive" +_queue_shutdown = "shutdown" +_queue_shutdown_immediate = "shutdown-immediate" + + class Queue: '''Create a queue object with a given maximum size. @@ -54,6 +63,9 @@ def __init__(self, maxsize=0): self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 + # Queue shut-down state + self.shutdown_state = _queue_alive + def task_done(self): '''Indicate that a formerly enqueued task is complete. @@ -87,6 +99,8 @@ def join(self): ''' with self.all_tasks_done: while self.unfinished_tasks: + if self.shutdown_state == _queue_shutdown_immediate: + return self.all_tasks_done.wait() def qsize(self): @@ -130,6 +144,8 @@ def put(self, item, block=True, timeout=None): is immediately available, else raise the Full exception ('timeout' is ignored in that case). ''' + if self.shutdown_state != _queue_alive: + raise ShutDown with self.not_full: if self.maxsize > 0: if not block: @@ -138,6 +154,8 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() + if self.shutdown_state != _queue_alive: + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -147,6 +165,8 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) + if self.shutdown_state != _queue_alive: + raise ShutDown self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() @@ -162,6 +182,8 @@ def get(self, block=True, timeout=None): available, else raise the Empty exception ('timeout' is ignored in that case). ''' + if self.shutdown_state == _queue_shutdown_immediate: + raise ShutDown with self.not_empty: if not block: if not self._qsize(): @@ -169,6 +191,8 @@ def get(self, block=True, timeout=None): elif timeout is None: while not self._qsize(): self.not_empty.wait() + if self.shutdown_state == _queue_shutdown_immediate: + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -178,6 +202,8 @@ def get(self, block=True, timeout=None): if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) + if self.shutdown_state == _queue_shutdown_immediate: + raise ShutDown item = self._get() self.not_full.notify() return item @@ -198,6 +224,26 @@ def get_nowait(self): ''' return self.get(block=False) + def shutdown(self, immediate=False): + '''Shut-down the queue, making queue gets and puts raise. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put(), get() and join() will be + unblocked. The ShutDown exception is raised. + ''' + if immediate: + self.shutdown_state = _queue_shutdown_immediate + with self.not_empty: + self.not_empty.notify_all() + with self.all_tasks_done: + self.all_tasks_done.notify_all() + else: + self.shutdown_state = _queue_shutdown + with self.not_full: + self.not_full.notify_all() + # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held From d942c9e00573442a7a26ce1d9506ea0e94f42ffe Mon Sep 17 00:00:00 2001 From: Laurie O Date: Sun, 18 Sep 2022 18:25:40 +1000 Subject: [PATCH 02/62] Fix up implementation, add unit-tests --- Lib/queue.py | 28 +++++++++++++++++----------- Lib/test/test_queue.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 4d2c1cbe084035..f6af7cb6df5cff 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -187,23 +187,31 @@ def get(self, block=True, timeout=None): with self.not_empty: if not block: if not self._qsize(): + if self.shutdown_state != _queue_alive: + raise ShutDown raise Empty elif timeout is None: while not self._qsize(): + if self.shutdown_state != _queue_alive: + raise ShutDown self.not_empty.wait() - if self.shutdown_state == _queue_shutdown_immediate: + if self.shutdown_state != _queue_alive: raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = time() + timeout while not self._qsize(): + if self.shutdown_state != _queue_alive: + raise ShutDown remaining = endtime - time() if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) - if self.shutdown_state == _queue_shutdown_immediate: + if self.shutdown_state != _queue_alive: raise ShutDown + if self.shutdown_state == _queue_shutdown_immediate: + raise ShutDown item = self._get() self.not_full.notify() return item @@ -230,18 +238,16 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put(), get() and join() will be - unblocked. The ShutDown exception is raised. + All blocked callers of put() will be unblocked, and also get() + and join() if 'immediate'. The ShutDown exception is raised. ''' - if immediate: - self.shutdown_state = _queue_shutdown_immediate - with self.not_empty: + with self.mutex: + if immediate: + self.shutdown_state = _queue_shutdown_immediate self.not_empty.notify_all() - with self.all_tasks_done: self.all_tasks_done.notify_all() - else: - self.shutdown_state = _queue_shutdown - with self.not_full: + else: + self.shutdown_state = _queue_shutdown self.not_full.notify_all() # Override these methods to implement other queue organizations diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 33113a72e6b6a9..354299b9a5b16a 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -241,6 +241,41 @@ def test_shrinking_queue(self): with self.assertRaises(self.queue.Full): q.put_nowait(4) + def test_shutdown_empty(self): + q = self.type2test() + q.shutdown() + try: + q.put("data") + self.fail("Didn't appear to shut-down queue") + except self.queue.ShutDown: + pass + try: + q.get() + self.fail("Didn't appear to shut-down queue") + except self.queue.ShutDown: + pass + + def test_shutdown_nonempty(self): + q = self.type2test() + q.put("data") + q.shutdown() + q.get() + try: + q.get() + self.fail("Didn't appear to shut-down queue") + except self.queue.ShutDown: + pass + + def test_shutdown_immediate(self): + q = self.type2test() + q.put("data") + q.shutdown(immediate=True) + try: + q.get() + self.fail("Didn't appear to shut-down queue") + except self.queue.ShutDown: + pass + class QueueTest(BaseQueueTestMixin): def setUp(self): From f552ac10fa43e5b312092aaeef86076841adf3d1 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Sun, 18 Sep 2022 18:26:02 +1000 Subject: [PATCH 03/62] Implement for asyncio queues --- Lib/asyncio/queues.py | 57 +++++++++++++++++++++++++++- Lib/test/test_asyncio/test_queues.py | 57 ++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index a9656a6df561ba..a869993a1de3fe 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -1,4 +1,11 @@ -__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty') +__all__ = ( + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'QueueFull', + 'QueueEmpty', + 'QueueShutDown', +) import collections import heapq @@ -18,6 +25,16 @@ class QueueFull(Exception): pass +class QueueShutDown(Exception): + """Raised when putting on to or getting from a shut-down Queue.""" + pass + + +_queue_alive = "alive" +_queue_shutdown = "shutdown" +_queue_shutdown_immediate = "shutdown-immediate" + + class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. @@ -41,6 +58,7 @@ def __init__(self, maxsize=0): self._finished = locks.Event() self._finished.set() self._init(maxsize) + self.shutdown_state = _queue_alive # These three are overridable in subclasses. @@ -113,6 +131,8 @@ async def put(self, item): Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. """ + if self.shutdown_state != _queue_alive: + raise QueueShutDown while self.full(): putter = self._get_loop().create_future() self._putters.append(putter) @@ -132,6 +152,8 @@ async def put(self, item): # the call. Wake up the next in line. self._wakeup_next(self._putters) raise + if self.shutdown_state != _queue_alive: + raise QueueShutDown return self.put_nowait(item) def put_nowait(self, item): @@ -139,6 +161,8 @@ def put_nowait(self, item): If no free slot is immediately available, raise QueueFull. """ + if self.shutdown_state != _queue_alive: + raise QueueShutDown if self.full(): raise QueueFull self._put(item) @@ -151,7 +175,11 @@ async def get(self): If queue is empty, wait until an item is available. """ + if self.shutdown_state == _queue_shutdown_immediate: + raise QueueShutDown while self.empty(): + if self.shutdown_state != _queue_alive: + raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) try: @@ -170,6 +198,8 @@ async def get(self): # the call. Wake up the next in line. self._wakeup_next(self._getters) raise + if self.shutdown_state == _queue_shutdown_immediate: + raise QueueShutDown return self.get_nowait() def get_nowait(self): @@ -178,7 +208,11 @@ def get_nowait(self): Return an item if one is immediately available, else raise QueueEmpty. """ if self.empty(): + if self.shutdown_state != _queue_alive: + raise QueueShutDown raise QueueEmpty + elif self.shutdown_state == _queue_shutdown_immediate: + raise QueueShutDown item = self._get() self._wakeup_next(self._putters) return item @@ -214,6 +248,27 @@ async def join(self): if self._unfinished_tasks > 0: await self._finished.wait() + def shutdown(self, immediate=False): + """Shut-down the queue, making queue gets and puts raise. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() will be unblocked, and also get() + and join() if 'immediate'. The QueueShutDown exception is raised. + """ + if immediate: + self.shutdown_state = _queue_shutdown_immediate + while self._getters: + getter = self._getters.popleft() + if not getter.done(): + getter.set_result(None) + else: + self.shutdown_state = _queue_shutdown + while self._putters: + putter = self._putters.popleft() + if not putter.done(): + putter.set_result(None) class PriorityQueue(Queue): """A subclass of Queue; retrieves entries in priority order (lowest first). diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 2d058ccf6a8c72..418c3fe618d89b 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -522,5 +522,62 @@ class PriorityQueueJoinTests(_QueueJoinTestMixin, unittest.IsolatedAsyncioTestCa q_class = asyncio.PriorityQueue +class _QueueShutdownTestMixin: + q_class = None + + async def test_empty(self): + q = self.q_class() + q.shutdown() + try: + await q.put("data") + self.fail("Didn't appear to shut-down queue") + except asyncio.QueueShutDown: + pass + try: + await q.get() + self.fail("Didn't appear to shut-down queue") + except asyncio.QueueShutDown: + pass + + async def test_nonempty(self): + q = self.q_class() + q.put_nowait("data") + q.shutdown() + await q.get() + try: + await q.get() + self.fail("Didn't appear to shut-down queue") + except asyncio.QueueShutDown: + pass + + async def test_immediate(self): + q = self.q_class() + q.put_nowait("data") + q.shutdown(immediate=True) + try: + await q.get() + self.fail("Didn't appear to shut-down queue") + except asyncio.QueueShutDown: + pass + + +class QueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.Queue + + +class LifoQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.LifoQueue + + +class PriorityQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.PriorityQueue + + if __name__ == '__main__': unittest.main() From 78671f9a81c7757563301bb7246863f0d24d09a9 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Sat, 22 Oct 2022 15:39:07 +1000 Subject: [PATCH 04/62] WIP: multiprocessing queue shutdown --- Lib/multiprocessing/queues.py | 26 +++++++++++++++++++++- Lib/test/_test_multiprocessing.py | 36 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index f37f114a968871..26b212d6a50610 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -17,8 +17,9 @@ import types import weakref import errno +import ctypes -from queue import Empty, Full +from queue import Empty, Full, ShutDown import _multiprocessing @@ -28,6 +29,10 @@ from .util import debug, info, Finalize, register_after_fork, is_exiting +_queue_alive = 0 +_queue_shutdown = 1 +_queue_shutdown_immediate = 2 + # # Queue type using a pipe, buffer and thread # @@ -50,6 +55,9 @@ def __init__(self, maxsize=0, *, ctx): # For use by concurrent.futures self._ignore_epipe = False self._reset() + self._shutdown_state = context._default_context.Value( + ctypes.c_uint8, lock=self._rlock + ) if sys.platform != 'win32': register_after_fork(self, Queue._after_fork) @@ -86,20 +94,28 @@ def _reset(self, after_fork=False): def put(self, obj, block=True, timeout=None): if self._closed: raise ValueError(f"Queue {self!r} is closed") + if self._shutdown_state.value != _queue_alive: + raise ShutDown if not self._sem.acquire(block, timeout): raise Full with self._notempty: + if self._shutdown_state.value != _queue_alive: + raise ShutDown if self._thread is None: self._start_thread() self._buffer.append(obj) self._notempty.notify() def get(self, block=True, timeout=None): + if self._shutdown_state.value == _queue_shutdown_immediate: + raise ShutDown if self._closed: raise ValueError(f"Queue {self!r} is closed") if block and timeout is None: with self._rlock: + if self._shutdown_state.value != _queue_alive: + raise ShutDown res = self._recv_bytes() self._sem.release() else: @@ -111,13 +127,19 @@ def get(self, block=True, timeout=None): if block: timeout = deadline - time.monotonic() if not self._poll(timeout): + if self._shutdown_state.value != _queue_alive: + raise ShutDown raise Empty + if self._shutdown_state.value != _queue_alive : + raise ShutDown elif not self._poll(): raise Empty res = self._recv_bytes() self._sem.release() finally: self._rlock.release() + if self._shutdown_state.value == _queue_shutdown: + raise ShutDown # unserialize the data after having released the lock return _ForkingPickler.loads(res) @@ -327,6 +349,8 @@ def task_done(self): def join(self): with self._cond: + if self._shutdown_state.value == _queue_shutdown_immediate: + return if not self._unfinished_tasks._semlock._is_zero(): self._cond.wait() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b78586c560a68a..b25b0dee5eca8d 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -958,6 +958,42 @@ def queue_full(q, maxsize): class _TestQueue(BaseTestCase): + # TODO: add queue shutdown unit-tests + + def test_shutdown_empty(self): + q = self.type2test() + q.shutdown() + try: + q.put("data") + self.fail("Didn't appear to shut-down queue") + except self.queue.ShutDown: + pass + try: + q.get() + self.fail("Didn't appear to shut-down queue") + except self.queue.ShutDown: + pass + + def test_shutdown_nonempty(self): + q = self.type2test() + q.put("data") + q.shutdown() + q.get() + try: + q.get() + self.fail("Didn't appear to shut-down queue") + except self.queue.ShutDown: + pass + + def test_shutdown_immediate(self): + q = self.type2test() + q.put("data") + q.shutdown(immediate=True) + try: + q.get() + self.fail("Didn't appear to shut-down queue") + except self.queue.ShutDown: + pass @classmethod def _test_put(cls, queue, child_can_start, parent_can_continue): From 5f31f8e6be334a0258b656943ea798517cd5159a Mon Sep 17 00:00:00 2001 From: Laurie O Date: Thu, 19 Jan 2023 20:34:24 +1000 Subject: [PATCH 05/62] WIP: multiprocessing queue shutdown --- Lib/test/_test_multiprocessing.py | 71 +++++++++++++++---------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b25b0dee5eca8d..1f9c9aec36b312 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -958,42 +958,6 @@ def queue_full(q, maxsize): class _TestQueue(BaseTestCase): - # TODO: add queue shutdown unit-tests - - def test_shutdown_empty(self): - q = self.type2test() - q.shutdown() - try: - q.put("data") - self.fail("Didn't appear to shut-down queue") - except self.queue.ShutDown: - pass - try: - q.get() - self.fail("Didn't appear to shut-down queue") - except self.queue.ShutDown: - pass - - def test_shutdown_nonempty(self): - q = self.type2test() - q.put("data") - q.shutdown() - q.get() - try: - q.get() - self.fail("Didn't appear to shut-down queue") - except self.queue.ShutDown: - pass - - def test_shutdown_immediate(self): - q = self.type2test() - q.put("data") - q.shutdown(immediate=True) - try: - q.get() - self.fail("Didn't appear to shut-down queue") - except self.queue.ShutDown: - pass @classmethod def _test_put(cls, queue, child_can_start, parent_can_continue): @@ -1313,6 +1277,41 @@ def test_closed_queue_put_get_exceptions(self): q.put('foo') with self.assertRaisesRegex(ValueError, 'is closed'): q.get() + + def test_shutdown_empty(self): + q = multiprocessing.Queue() + q.shutdown() + try: + q.put("data") + self.fail("Didn't appear to shut-down queue") + except pyqueue.ShutDown: + pass + try: + q.get() + self.fail("Didn't appear to shut-down queue") + except pyqueue.ShutDown: + pass + + def test_shutdown_nonempty(self): + q = multiprocessing.Queue() + q.put("data") + q.shutdown() + q.get() + try: + q.get() + self.fail("Didn't appear to shut-down queue") + except pyqueue.ShutDown: + pass + + def test_shutdown_immediate(self): + q = multiprocessing.Queue() + q.put("data") + q.shutdown(immediate=True) + try: + q.get() + self.fail("Didn't appear to shut-down queue") + except pyqueue.ShutDown: + pass # # # From 9bbc5db5b4678be31cd61603880f821632f5a89e Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 10 Feb 2023 17:52:33 +0100 Subject: [PATCH 06/62] change comment --- Lib/queue.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/queue.py b/Lib/queue.py index f6af7cb6df5cff..f08dbd47f188ee 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -245,6 +245,10 @@ def shutdown(self, immediate=False): if immediate: self.shutdown_state = _queue_shutdown_immediate self.not_empty.notify_all() + # set self.unfinished_tasks to 0 + # to break the loop in 'self.join()' + # when quits from `wait()` + self.unfinished_tasks = 0 self.all_tasks_done.notify_all() else: self.shutdown_state = _queue_shutdown From f9f2c0629a5dd283e64f804f88a14d7e4d4655bd Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 10 Feb 2023 17:57:21 +0100 Subject: [PATCH 07/62] call to self._finished.set() in order to release all joined tasks/coros --- Lib/asyncio/queues.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index a869993a1de3fe..5b58f753a0fac7 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -8,6 +8,7 @@ ) import collections +import enum import heapq from types import GenericAlias @@ -30,9 +31,10 @@ class QueueShutDown(Exception): pass -_queue_alive = "alive" -_queue_shutdown = "shutdown" -_queue_shutdown_immediate = "shutdown-immediate" +class _QueueState(enum.Enum): + ALIVE = "alive" + SHUTDOWN = "shutdown" + SHUTDOWN_IMMEDIATE = "shutdown-immediate" class Queue(mixins._LoopBoundMixin): @@ -58,7 +60,7 @@ def __init__(self, maxsize=0): self._finished = locks.Event() self._finished.set() self._init(maxsize) - self.shutdown_state = _queue_alive + self._shutdown_state = _QueueState.ALIVE # These three are overridable in subclasses. @@ -99,6 +101,8 @@ def _format(self): result += f' _putters[{len(self._putters)}]' if self._unfinished_tasks: result += f' tasks={self._unfinished_tasks}' + if self._shutdown_state is not _QueueState.ALIVE: + result += f' shutdown={self._shutdown_state.value}' return result def qsize(self): @@ -131,7 +135,7 @@ async def put(self, item): Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. """ - if self.shutdown_state != _queue_alive: + if self._shutdown_state is not _QueueState.ALIVE: raise QueueShutDown while self.full(): putter = self._get_loop().create_future() @@ -152,7 +156,7 @@ async def put(self, item): # the call. Wake up the next in line. self._wakeup_next(self._putters) raise - if self.shutdown_state != _queue_alive: + if self._shutdown_state is not _QueueState.ALIVE: raise QueueShutDown return self.put_nowait(item) @@ -161,7 +165,7 @@ def put_nowait(self, item): If no free slot is immediately available, raise QueueFull. """ - if self.shutdown_state != _queue_alive: + if self._shutdown_state is not _QueueState.ALIVE: raise QueueShutDown if self.full(): raise QueueFull @@ -175,10 +179,10 @@ async def get(self): If queue is empty, wait until an item is available. """ - if self.shutdown_state == _queue_shutdown_immediate: + if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: raise QueueShutDown while self.empty(): - if self.shutdown_state != _queue_alive: + if self._shutdown_state is not _QueueState.ALIVE: raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) @@ -198,7 +202,7 @@ async def get(self): # the call. Wake up the next in line. self._wakeup_next(self._getters) raise - if self.shutdown_state == _queue_shutdown_immediate: + if self._shutdown_state is not _QueueState.ALIVE: raise QueueShutDown return self.get_nowait() @@ -208,10 +212,10 @@ def get_nowait(self): Return an item if one is immediately available, else raise QueueEmpty. """ if self.empty(): - if self.shutdown_state != _queue_alive: + if self._shutdown_state is not _QueueState.ALIVE: raise QueueShutDown raise QueueEmpty - elif self.shutdown_state == _queue_shutdown_immediate: + elif self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: raise QueueShutDown item = self._get() self._wakeup_next(self._putters) @@ -258,17 +262,20 @@ def shutdown(self, immediate=False): and join() if 'immediate'. The QueueShutDown exception is raised. """ if immediate: - self.shutdown_state = _queue_shutdown_immediate + self._shutdown_state = _QueueState.SHUTDOWN_IMMEDIATE while self._getters: getter = self._getters.popleft() if not getter.done(): getter.set_result(None) else: - self.shutdown_state = _queue_shutdown + self._shutdown_state = _QueueState.SHUTDOWN while self._putters: putter = self._putters.popleft() if not putter.done(): putter.set_result(None) + # Release 'joined' tasks/coros + self._finished.set() + class PriorityQueue(Queue): """A subclass of Queue; retrieves entries in priority order (lowest first). From 7491ef1ff9e36213333e650cd6c5c398174c16df Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 10 Feb 2023 17:59:19 +0100 Subject: [PATCH 08/62] add unitests to `shutdwon` method --- Lib/test/test_asyncio/test_queues.py | 95 ++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 418c3fe618d89b..718b1ef5c77634 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -560,6 +560,101 @@ async def test_immediate(self): except asyncio.QueueShutDown: pass + async def test_shutdown_repr(self): + q = self.q_class() + q.shutdown() + self.assertIn("shutdown", repr(q)) + + q = self.q_class() + q.shutdown(immediate=True) + self.assertIn("shutdown-immediate", repr(q)) + + async def test_get_shutdown_immediate(self): + results = [] + maxsize = 2 + delay = 1e-3 + + async def get_once(q): + try: + msg = await q.get() + results.append(False) + except asyncio.QueueShutDown: + results.append(True) + return True + + async def shutdown(q, delay, immediate): + await asyncio.sleep(delay) + q.shutdown(immediate) + return True + + q = self.q_class(maxsize) + t = [asyncio.create_task(get_once(q)) for _ in range(maxsize)] + t += [asyncio.create_task(shutdown(q, delay, True))] + res = await asyncio.gather(*t) + + self.assertEqual(results, [True]*maxsize) + + + async def test_put_shutdown(self): + maxsize = 2 + results = [] + go = asyncio.Event() + + async def put_twice(q, go, msg): + await q.put(msg) + await go.wait() + try: + await q.put(msg+maxsize) + results.append(False) + except asyncio.QueueShutDown: + results.append(True) + return msg + + async def shutdown(q, go, immediate): + q.shutdown(immediate) + go.set() + + q = self.q_class(maxsize) + t = [asyncio.create_task(put_twice(q, go, i+1)) for i in range(maxsize)] + t += [asyncio.create_task(shutdown(q, go, False))] + res = await asyncio.gather(*t) + + self.assertEqual(results, [True]*maxsize) + + + async def test_put_and_join_shutdown(self): + maxsize = 2 + results = [] + go = asyncio.Event() + + async def put_twice(q, go, msg): + await q.put(msg) + await go.wait() + try: + await q.put(msg+100) + results.append(False) + except asyncio.QueueShutDown: + results.append(True) + return msg + + async def shutdown(q, go, immediate): + q.shutdown(immediate) + go.set() + + async def join(q, delay): + await go.wait() + await q.join() + results.append(True) + return True + + q = self.q_class(maxsize) + t = [asyncio.create_task(put_twice(q, go, i+1)) for i in range(maxsize)] + t += [asyncio.create_task(shutdown(q, go, True)), + asyncio.create_task(join(q, go))] + res = await asyncio.gather(*t) + + self.assertEqual(results, [True]*(maxsize+1)) + class QueueShutdownTests( _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase From dd22c6b53ab99cb2f25f0f4f29e65cb973717198 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 10 Feb 2023 17:59:38 +0100 Subject: [PATCH 09/62] add unitests to `shutdwon` method --- Lib/test/test_queue.py | 187 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 354299b9a5b16a..7b377864308dda 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -276,6 +276,193 @@ def test_shutdown_immediate(self): except self.queue.ShutDown: pass + def test_get_shutdown(self): + q = self.type2test(2) + results = [] + go = threading.Event() + + def get_once(q, go): + try: + go.wait() + msg = q.get() + results.append(False) + except self.queue.ShutDown: + results.append(True) + return True + + tests = ( + (get_once, (q, go)), + (get_once, (q, go)), + ) + threads = [] + for f, params in tests: + thread = threading.Thread(target=f, args=params) + thread.start() + threads.append(thread) + q.shutdown() + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(tests)) + + def test_put_shutdown(self): + q = self.type2test(2) + results = [] + go = threading.Event() + + def put_twice(q, msg, go): + q.put(msg) + go.wait() + try: + q.put(msg) + results.append(False) + except self.queue.ShutDown: + results.append(True) + return msg + + tests = ( + (put_twice, (q, 100, go)), + (put_twice, (q, 200, go)), + ) + threads = [] + for f, params in tests: + thread = threading.Thread(target=f, args=params) + thread.start() + threads.append(thread) + q.shutdown() + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(tests)) + + def _join_shutdown(self, immediate): + q = self.type2test() + results = [] + go = threading.Event() + + def join(q, go): + go.wait() + q.join() + results.append(True) + + tests = ( + (join, (q, go)), + (join, (q, go)), + ) + threads = [] + for f, params in tests: + thread = threading.Thread(target=f, args=params) + thread.start() + threads.append(thread) + go.set() + q.shutdown(immediate) + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(tests)) + + def test_join_shutdown_immediate(self): + return self._join_shutdown(True) + + def test_join_shutdown(self): + return self._join_shutdown(False) + + def _put_and_join_shutdown(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + + def put_twice(q, msg, go): + q.put(msg) + go.wait() + try: + q.put(msg) + results.append(False) + except self.queue.ShutDown: + results.append(True) + return msg + + def join(q, go): + go.wait() + q.join() + results.append(True) + + tests = ( + (put_twice, (q, 100, go)), + (put_twice, (q, 200, go)), + (join, (q, go)), + (join, (q, go)), + ) + threads = [] + for f, params in tests: + thread = threading.Thread(target=f, args=params) + thread.start() + threads.append(thread) + go.set() + q.shutdown(immediate) + if not immediate: + self.assertTrue(q.unfinished_tasks, 2) + for i in range(2): + thread = threading.Thread(target=q.task_done) + thread.start() + threads.append(thread) + + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(tests)) + + def test_put_and_join_shutdown_immediate(self): + return self._put_and_join_shutdown(True) + + def test_put_and_join_shutdown(self): + return self._put_and_join_shutdown(False) + + def _get_and_join_shutdown(self, immediate): + q = self.type2test() + results = [] + go = threading.Event() + + def get_once(q, go): + try: + go.wait() + msg = q.get() + results.append(False) + except self.queue.ShutDown: + results.append(True) + return True + + def join(q, go): + go.wait() + q.join() + results.append(True) + + tests = ( + (get_once, (q, go)), + (get_once, (q, go)), + (join, (q, go)), + (join, (q, go)), + ) + threads = [] + for f, params in tests: + thread = threading.Thread(target=f, args=params) + thread.start() + threads.append(thread) + go.set() + q.shutdown(immediate) + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(tests)) + + def test_get_and_join_shutdown_immediate(self): + return self._get_and_join_shutdown(True) + + def test_get_and_join_shutdown(self): + return self._get_and_join_shutdown(False) + class QueueTest(BaseQueueTestMixin): def setUp(self): From 52393064797dc37ff34939b88165c4118600effb Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 10 Feb 2023 18:17:02 +0100 Subject: [PATCH 10/62] replace global state variable with an enum `_QueueState` --- Lib/queue.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index f08dbd47f188ee..e3c2f01fd57df8 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -1,5 +1,6 @@ '''A multi-producer, multi-consumer queue.''' +import enum import threading import types from collections import deque @@ -29,9 +30,12 @@ class ShutDown(Exception): '''Raised when put/get with shut-down queue.''' -_queue_alive = "alive" -_queue_shutdown = "shutdown" -_queue_shutdown_immediate = "shutdown-immediate" +class _QueueState(enum.Enum): + ALIVE = "alive" + SHUTDOWN = "shutdown" + SHUTDOWN_IMMEDIATE = "shutdown_immediate" + +E class Queue: @@ -64,7 +68,7 @@ def __init__(self, maxsize=0): self.unfinished_tasks = 0 # Queue shut-down state - self.shutdown_state = _queue_alive + self.shutdown_state = _QueueState.ALIVE def task_done(self): '''Indicate that a formerly enqueued task is complete. @@ -99,7 +103,7 @@ def join(self): ''' with self.all_tasks_done: while self.unfinished_tasks: - if self.shutdown_state == _queue_shutdown_immediate: + if self.shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: return self.all_tasks_done.wait() @@ -144,7 +148,7 @@ def put(self, item, block=True, timeout=None): is immediately available, else raise the Full exception ('timeout' is ignored in that case). ''' - if self.shutdown_state != _queue_alive: + if self.shutdown_state is not _QueueState.ALIVE: raise ShutDown with self.not_full: if self.maxsize > 0: @@ -154,7 +158,7 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() - if self.shutdown_state != _queue_alive: + if self.shutdown_state is not _QueueState.ALIVE: raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") @@ -165,7 +169,7 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) - if self.shutdown_state != _queue_alive: + if self.shutdown_state is not _QueueState.ALIVE: raise ShutDown self._put(item) self.unfinished_tasks += 1 @@ -182,35 +186,35 @@ def get(self, block=True, timeout=None): available, else raise the Empty exception ('timeout' is ignored in that case). ''' - if self.shutdown_state == _queue_shutdown_immediate: + if self.shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: raise ShutDown with self.not_empty: if not block: if not self._qsize(): - if self.shutdown_state != _queue_alive: + if self.shutdown_state is not _QueueState.ALIVE: raise ShutDown raise Empty elif timeout is None: while not self._qsize(): - if self.shutdown_state != _queue_alive: + if self.shutdown_state is not _QueueState.ALIVE: raise ShutDown self.not_empty.wait() - if self.shutdown_state != _queue_alive: + if self.shutdown_state is not _QueueState.ALIVE: raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = time() + timeout while not self._qsize(): - if self.shutdown_state != _queue_alive: + if self.shutdown_state is not _QueueState.ALIVE: raise ShutDown remaining = endtime - time() if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) - if self.shutdown_state != _queue_alive: + if self.shutdown_state is not _QueueState.ALIVE: raise ShutDown - if self.shutdown_state == _queue_shutdown_immediate: + if self.shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: raise ShutDown item = self._get() self.not_full.notify() @@ -243,7 +247,7 @@ def shutdown(self, immediate=False): ''' with self.mutex: if immediate: - self.shutdown_state = _queue_shutdown_immediate + self.shutdown_state = _QueueState.SHUTDOWN_IMMEDIATE self.not_empty.notify_all() # set self.unfinished_tasks to 0 # to break the loop in 'self.join()' @@ -251,7 +255,7 @@ def shutdown(self, immediate=False): self.unfinished_tasks = 0 self.all_tasks_done.notify_all() else: - self.shutdown_state = _queue_shutdown + self.shutdown_state = _QueueState.SHUTDOWN self.not_full.notify_all() # Override these methods to implement other queue organizations From 4b127b6376b887a04d975898d24a8284ff0dac44 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 10 Feb 2023 19:21:10 +0100 Subject: [PATCH 11/62] replace global state variable with an enum `_QueueState` - erase E just line below --- Lib/queue.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index e3c2f01fd57df8..9481a91feb4ff6 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -35,8 +35,6 @@ class _QueueState(enum.Enum): SHUTDOWN = "shutdown" SHUTDOWN_IMMEDIATE = "shutdown_immediate" -E - class Queue: '''Create a queue object with a given maximum size. From 6402de7ff7c0a9186d8b4c72a572c2b1af0cefaa Mon Sep 17 00:00:00 2001 From: Duprat Date: Sat, 11 Feb 2023 17:02:37 +0100 Subject: [PATCH 12/62] simplify and unify tests --- Lib/test/test_asyncio/test_queues.py | 91 +++++++++++++++++++--------- 1 file changed, 62 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 718b1ef5c77634..a1a6cd3e0c9c7b 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -570,11 +570,12 @@ async def test_shutdown_repr(self): self.assertIn("shutdown-immediate", repr(q)) async def test_get_shutdown_immediate(self): + q = self.q_class() results = [] - maxsize = 2 - delay = 1e-3 + go = asyncio.Event() - async def get_once(q): + async def get_once(q, go): + await go.wait() try: msg = await q.get() results.append(False) @@ -582,29 +583,36 @@ async def get_once(q): results.append(True) return True - async def shutdown(q, delay, immediate): - await asyncio.sleep(delay) + async def shutdown(q, go, immediate): q.shutdown(immediate) + go.set() return True - q = self.q_class(maxsize) - t = [asyncio.create_task(get_once(q)) for _ in range(maxsize)] - t += [asyncio.create_task(shutdown(q, delay, True))] + tasks = ( + (get_once, (q, go)), + (get_once, (q, go)), + ) + t = [] + for coro, params in tasks: + t.append(asyncio.create_task(coro(*params))) + t.append(asyncio.create_task(shutdown(q, go, True))) res = await asyncio.gather(*t) - self.assertEqual(results, [True]*maxsize) + self.assertEqual(results, [True]*len(tasks)) - async def test_put_shutdown(self): - maxsize = 2 + async def _put_shutdown(self, immediate): + q = self.q_class(2) results = [] go = asyncio.Event() + await q.put("Y") + await q.put("D") + # queue fulled - async def put_twice(q, go, msg): - await q.put(msg) + async def put_once(q, go, msg): await go.wait() try: - await q.put(msg+maxsize) + await q.put(msg) results.append(False) except asyncio.QueueShutDown: results.append(True) @@ -614,24 +622,37 @@ async def shutdown(q, go, immediate): q.shutdown(immediate) go.set() - q = self.q_class(maxsize) - t = [asyncio.create_task(put_twice(q, go, i+1)) for i in range(maxsize)] - t += [asyncio.create_task(shutdown(q, go, False))] + tasks = ( + (put_once, (q, go, 100)), + (put_once, (q, go, 200)), + ) + t = [] + for coro, params in tasks: + t.append(asyncio.create_task(coro(*params))) + t.append(asyncio.create_task(shutdown(q, go, immediate))) res = await asyncio.gather(*t) - self.assertEqual(results, [True]*maxsize) + self.assertEqual(results, [True]*len(tasks)) + async def test_put_shutdown(self): + return await self._put_shutdown(False) - async def test_put_and_join_shutdown(self): - maxsize = 2 + async def test_put_shutdown_immediate(self): + return await self._put_shutdown(True) + + + async def _put_and_join_shutdown(self, immediate): + q = self.q_class(2) results = [] go = asyncio.Event() + await q.put("Y") + await q.put("D") + # queue fulled - async def put_twice(q, go, msg): - await q.put(msg) + async def put_once(q, go, msg): await go.wait() try: - await q.put(msg+100) + await q.put(msg) results.append(False) except asyncio.QueueShutDown: results.append(True) @@ -641,19 +662,31 @@ async def shutdown(q, go, immediate): q.shutdown(immediate) go.set() - async def join(q, delay): + async def join(q, go): await go.wait() await q.join() results.append(True) return True - q = self.q_class(maxsize) - t = [asyncio.create_task(put_twice(q, go, i+1)) for i in range(maxsize)] - t += [asyncio.create_task(shutdown(q, go, True)), - asyncio.create_task(join(q, go))] + tasks = ( + (put_once, (q, go, 'E')), + (put_once, (q, go, 'W')), + (join, (q, go)), + (join, (q, go)), + ) + t = [] + for coro, params in tasks: + t.append(asyncio.create_task(coro(*params))) + t.append(asyncio.create_task(shutdown(q, go, immediate))) res = await asyncio.gather(*t) - self.assertEqual(results, [True]*(maxsize+1)) + self.assertEqual(results, [True]*len(tasks)) + + async def test_put_and_join_shutdown(self): + return await self._put_and_join_shutdown(False) + + async def test_put_and_join_shutdown_immediate(self): + return await self._put_and_join_shutdown(True) class QueueShutdownTests( From be9588bfece569ee5cb8e2932b41495b80bd11c0 Mon Sep 17 00:00:00 2001 From: Duprat Date: Sat, 11 Feb 2023 17:02:53 +0100 Subject: [PATCH 13/62] simplify and unify tests --- Lib/test/test_queue.py | 54 +++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 7b377864308dda..55a02e82e15795 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -282,20 +282,20 @@ def test_get_shutdown(self): go = threading.Event() def get_once(q, go): + go.wait() try: - go.wait() msg = q.get() results.append(False) except self.queue.ShutDown: results.append(True) return True - tests = ( + thrds = ( (get_once, (q, go)), (get_once, (q, go)), ) threads = [] - for f, params in tests: + for f, params in thrds: thread = threading.Thread(target=f, args=params) thread.start() threads.append(thread) @@ -304,15 +304,17 @@ def get_once(q, go): for t in threads: t.join() - self.assertEqual(results, [True]*len(tests)) + self.assertEqual(results, [True]*len(thrds)) def test_put_shutdown(self): q = self.type2test(2) results = [] go = threading.Event() + q.put("Y") + q.put("D") + # queue fulled - def put_twice(q, msg, go): - q.put(msg) + def put_once(q, msg, go): go.wait() try: q.put(msg) @@ -321,12 +323,12 @@ def put_twice(q, msg, go): results.append(True) return msg - tests = ( - (put_twice, (q, 100, go)), - (put_twice, (q, 200, go)), + thrds = ( + (put_once, (q, 100, go)), + (put_once, (q, 200, go)), ) threads = [] - for f, params in tests: + for f, params in thrds: thread = threading.Thread(target=f, args=params) thread.start() threads.append(thread) @@ -335,7 +337,7 @@ def put_twice(q, msg, go): for t in threads: t.join() - self.assertEqual(results, [True]*len(tests)) + self.assertEqual(results, [True]*len(thrds)) def _join_shutdown(self, immediate): q = self.type2test() @@ -347,12 +349,12 @@ def join(q, go): q.join() results.append(True) - tests = ( + thrds = ( (join, (q, go)), (join, (q, go)), ) threads = [] - for f, params in tests: + for f, params in thrds: thread = threading.Thread(target=f, args=params) thread.start() threads.append(thread) @@ -361,7 +363,7 @@ def join(q, go): for t in threads: t.join() - self.assertEqual(results, [True]*len(tests)) + self.assertEqual(results, [True]*len(thrds)) def test_join_shutdown_immediate(self): return self._join_shutdown(True) @@ -373,9 +375,11 @@ def _put_and_join_shutdown(self, immediate): q = self.type2test(2) results = [] go = threading.Event() + q.put("Y") + q.put("D") + # queue fulled - def put_twice(q, msg, go): - q.put(msg) + def put_once(q, msg, go): go.wait() try: q.put(msg) @@ -389,14 +393,14 @@ def join(q, go): q.join() results.append(True) - tests = ( - (put_twice, (q, 100, go)), - (put_twice, (q, 200, go)), + thrds = ( + (put_once, (q, 100, go)), + (put_once, (q, 200, go)), (join, (q, go)), (join, (q, go)), ) threads = [] - for f, params in tests: + for f, params in thrds: thread = threading.Thread(target=f, args=params) thread.start() threads.append(thread) @@ -412,7 +416,7 @@ def join(q, go): for t in threads: t.join() - self.assertEqual(results, [True]*len(tests)) + self.assertEqual(results, [True]*len(thrds)) def test_put_and_join_shutdown_immediate(self): return self._put_and_join_shutdown(True) @@ -426,8 +430,8 @@ def _get_and_join_shutdown(self, immediate): go = threading.Event() def get_once(q, go): + go.wait() try: - go.wait() msg = q.get() results.append(False) except self.queue.ShutDown: @@ -439,14 +443,14 @@ def join(q, go): q.join() results.append(True) - tests = ( + thrds = ( (get_once, (q, go)), (get_once, (q, go)), (join, (q, go)), (join, (q, go)), ) threads = [] - for f, params in tests: + for f, params in thrds: thread = threading.Thread(target=f, args=params) thread.start() threads.append(thread) @@ -455,7 +459,7 @@ def join(q, go): for t in threads: t.join() - self.assertEqual(results, [True]*len(tests)) + self.assertEqual(results, [True]*len(thrds)) def test_get_and_join_shutdown_immediate(self): return self._get_and_join_shutdown(True) From 3613f5d4a2c600ab096d37bf3990a1c2b2fde8d6 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 13 Feb 2023 17:34:55 +0100 Subject: [PATCH 14/62] add `_shutdown_state` to tuples of `__getstate__` and `__setstate__`, add an empty `shutdown` method --- Lib/multiprocessing/queues.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 26b212d6a50610..bfaf81e2dc6616 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -65,11 +65,13 @@ def __init__(self, maxsize=0, *, ctx): def __getstate__(self): context.assert_spawning(self) return (self._ignore_epipe, self._maxsize, self._reader, self._writer, - self._rlock, self._wlock, self._sem, self._opid) + self._rlock, self._wlock, self._sem, self._opid, + self._shutdown_state) def __setstate__(self, state): (self._ignore_epipe, self._maxsize, self._reader, self._writer, - self._rlock, self._wlock, self._sem, self._opid) = state + self._rlock, self._wlock, self._sem, self._opid, + self._shutdown_state) = state self._reset() def _after_fork(self): @@ -159,6 +161,9 @@ def get_nowait(self): def put_nowait(self, obj): return self.put(obj, False) + def shutdown(self, immediate=True): + pass + def close(self): self._closed = True close = self._close From 06775bb94ecff21c3d83a473bc8db592a7f8a0c4 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 15 Feb 2023 15:15:27 +0100 Subject: [PATCH 15/62] Update initial tests with `self.assertRaises` Add an unittest `test_shutdown`transition --- Lib/test/test_asyncio/test_queues.py | 34 +++++++++++++++------------- Lib/test/test_queue.py | 34 +++++++++++++++------------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index a1a6cd3e0c9c7b..7c882f62ef1038 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -528,37 +528,25 @@ class _QueueShutdownTestMixin: async def test_empty(self): q = self.q_class() q.shutdown() - try: + with self.assertRaises(asyncio.QueueShutDown): await q.put("data") - self.fail("Didn't appear to shut-down queue") - except asyncio.QueueShutDown: - pass - try: + with self.assertRaises(asyncio.QueueShutDown): await q.get() - self.fail("Didn't appear to shut-down queue") - except asyncio.QueueShutDown: - pass async def test_nonempty(self): q = self.q_class() q.put_nowait("data") q.shutdown() await q.get() - try: + with self.assertRaises(asyncio.QueueShutDown): await q.get() - self.fail("Didn't appear to shut-down queue") - except asyncio.QueueShutDown: - pass async def test_immediate(self): q = self.q_class() q.put_nowait("data") q.shutdown(immediate=True) - try: + with self.assertRaises(asyncio.QueueShutDown): await q.get() - self.fail("Didn't appear to shut-down queue") - except asyncio.QueueShutDown: - pass async def test_shutdown_repr(self): q = self.q_class() @@ -569,6 +557,20 @@ async def test_shutdown_repr(self): q.shutdown(immediate=True) self.assertIn("shutdown-immediate", repr(q)) + async def test_shutdown_transition(self): + # allowed transitions would be from alive via shutdown to immediate + q = self.q_class() + self.assertEqual("alive", q._shutdown_state.value) + + q.shutdown() + self.assertEqual("shutdown", q._shutdown_state.value) + + q.shutdown(immediate=True) + self.assertEqual("shutdown-immediate", q._shutdown_state.value) + + q.shutdown() + self.assertEqual("shutdown-immediate", q._shutdown_state.value) + async def test_get_shutdown_immediate(self): q = self.q_class() results = [] diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 55a02e82e15795..b25453f9203b5e 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -244,37 +244,39 @@ def test_shrinking_queue(self): def test_shutdown_empty(self): q = self.type2test() q.shutdown() - try: + with self.assertRaises(self.queue.ShutDown): q.put("data") - self.fail("Didn't appear to shut-down queue") - except self.queue.ShutDown: - pass - try: + with self.assertRaises(self.queue.ShutDown): q.get() - self.fail("Didn't appear to shut-down queue") - except self.queue.ShutDown: - pass def test_shutdown_nonempty(self): q = self.type2test() q.put("data") q.shutdown() q.get() - try: + with self.assertRaises(self.queue.ShutDown): q.get() - self.fail("Didn't appear to shut-down queue") - except self.queue.ShutDown: - pass def test_shutdown_immediate(self): q = self.type2test() q.put("data") q.shutdown(immediate=True) - try: + with self.assertRaises(self.queue.ShutDown): q.get() - self.fail("Didn't appear to shut-down queue") - except self.queue.ShutDown: - pass + + def test_shutdown_transition(self): + # allowed transitions would be from alive via shutdown to immediate + q = self.type2test() + self.assertEqual("alive", q.shutdown_state.value) + + q.shutdown() + self.assertEqual("shutdown", q.shutdown_state.value) + + q.shutdown(immediate=True) + self.assertEqual("shutdown-immediate", q.shutdown_state.value) + + q.shutdown(immediate=False) + self.assertEqual("shutdown-immediate", q.shutdown_state.value) def test_get_shutdown(self): q = self.type2test(2) From 6f0101567e493b88d4272fcb3f8e2c55afc63510 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 15 Feb 2023 15:18:11 +0100 Subject: [PATCH 16/62] integration of shudown transition in `shutdown` method Change "shutdown_immediate" to "shutdown-immediate" --- Lib/asyncio/queues.py | 7 +++++-- Lib/queue.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 5b58f753a0fac7..f4e1ba58866e53 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -261,6 +261,9 @@ def shutdown(self, immediate=False): All blocked callers of put() will be unblocked, and also get() and join() if 'immediate'. The QueueShutDown exception is raised. """ + if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + return + if immediate: self._shutdown_state = _QueueState.SHUTDOWN_IMMEDIATE while self._getters: @@ -268,12 +271,12 @@ def shutdown(self, immediate=False): if not getter.done(): getter.set_result(None) else: - self._shutdown_state = _QueueState.SHUTDOWN + self._shutdown_state = _QueueState.SHUTDOWN while self._putters: putter = self._putters.popleft() if not putter.done(): putter.set_result(None) - # Release 'joined' tasks/coros + # Release 'blocked' tasks/coros via `.join()` self._finished.set() diff --git a/Lib/queue.py b/Lib/queue.py index 9481a91feb4ff6..933f1806205072 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -33,7 +33,7 @@ class ShutDown(Exception): class _QueueState(enum.Enum): ALIVE = "alive" SHUTDOWN = "shutdown" - SHUTDOWN_IMMEDIATE = "shutdown_immediate" + SHUTDOWN_IMMEDIATE = "shutdown-immediate" class Queue: @@ -244,6 +244,9 @@ def shutdown(self, immediate=False): and join() if 'immediate'. The ShutDown exception is raised. ''' with self.mutex: + if self.shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + return + if immediate: self.shutdown_state = _QueueState.SHUTDOWN_IMMEDIATE self.not_empty.notify_all() From ff9895dd212a9d51a0d7bcfbca9d223305b69344 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 15 Feb 2023 15:45:50 +0100 Subject: [PATCH 17/62] Set `test_shutdown` prefix to all unittests --- Lib/test/test_asyncio/test_queues.py | 22 +++++++++--------- Lib/test/test_queue.py | 34 ++++++++++++++-------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 7c882f62ef1038..f2c329c94b7997 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -571,7 +571,7 @@ async def test_shutdown_transition(self): q.shutdown() self.assertEqual("shutdown-immediate", q._shutdown_state.value) - async def test_get_shutdown_immediate(self): + async def test_shutdown_immediate_get(self): q = self.q_class() results = [] go = asyncio.Event() @@ -603,7 +603,7 @@ async def shutdown(q, go, immediate): self.assertEqual(results, [True]*len(tasks)) - async def _put_shutdown(self, immediate): + async def _shutdown_put(self, immediate): q = self.q_class(2) results = [] go = asyncio.Event() @@ -636,14 +636,14 @@ async def shutdown(q, go, immediate): self.assertEqual(results, [True]*len(tasks)) - async def test_put_shutdown(self): - return await self._put_shutdown(False) + async def test_shutdown_put(self): + return await self._shutdown_put(False) - async def test_put_shutdown_immediate(self): - return await self._put_shutdown(True) + async def test_shutdown_immediate_put(self): + return await self._shutdown_put(True) - async def _put_and_join_shutdown(self, immediate): + async def _shutdown_put_and_join(self, immediate): q = self.q_class(2) results = [] go = asyncio.Event() @@ -684,11 +684,11 @@ async def join(q, go): self.assertEqual(results, [True]*len(tasks)) - async def test_put_and_join_shutdown(self): - return await self._put_and_join_shutdown(False) + async def test_shutdown_put_and_join(self): + return await self._shutdown_put_and_join(False) - async def test_put_and_join_shutdown_immediate(self): - return await self._put_and_join_shutdown(True) + async def test_shutdown_immediate_put_and_join(self): + return await self._shutdown_put_and_join(True) class QueueShutdownTests( diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index b25453f9203b5e..a2814ac40f2e18 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -278,7 +278,7 @@ def test_shutdown_transition(self): q.shutdown(immediate=False) self.assertEqual("shutdown-immediate", q.shutdown_state.value) - def test_get_shutdown(self): + def test_shutdown_get(self): q = self.type2test(2) results = [] go = threading.Event() @@ -308,7 +308,7 @@ def get_once(q, go): self.assertEqual(results, [True]*len(thrds)) - def test_put_shutdown(self): + def test_shutdown_put(self): q = self.type2test(2) results = [] go = threading.Event() @@ -341,7 +341,7 @@ def put_once(q, msg, go): self.assertEqual(results, [True]*len(thrds)) - def _join_shutdown(self, immediate): + def _shutdown_join(self, immediate): q = self.type2test() results = [] go = threading.Event() @@ -367,13 +367,13 @@ def join(q, go): self.assertEqual(results, [True]*len(thrds)) - def test_join_shutdown_immediate(self): - return self._join_shutdown(True) + def test_shutdown_immediate_join(self): + return self._shutdown_join(True) - def test_join_shutdown(self): - return self._join_shutdown(False) + def test_shutdown_join(self): + return self._shutdown_join(False) - def _put_and_join_shutdown(self, immediate): + def _shutdown_put_and_join(self, immediate): q = self.type2test(2) results = [] go = threading.Event() @@ -420,13 +420,13 @@ def join(q, go): self.assertEqual(results, [True]*len(thrds)) - def test_put_and_join_shutdown_immediate(self): - return self._put_and_join_shutdown(True) + def test_shutdown_immediate_put_and_join(self): + return self._shutdown_put_and_join(True) - def test_put_and_join_shutdown(self): - return self._put_and_join_shutdown(False) + def test_shutdown_put_and_join(self): + return self._shutdown_put_and_join(False) - def _get_and_join_shutdown(self, immediate): + def _shutdown_get_and_join(self, immediate): q = self.type2test() results = [] go = threading.Event() @@ -463,11 +463,11 @@ def join(q, go): self.assertEqual(results, [True]*len(thrds)) - def test_get_and_join_shutdown_immediate(self): - return self._get_and_join_shutdown(True) + def test_shutdown_immediate_get_and_join(self): + return self._shutdown_get_and_join(True) - def test_get_and_join_shutdown(self): - return self._get_and_join_shutdown(False) + def test__shutdown_get_and_join(self): + return self._shutdown_get_and_join(False) class QueueTest(BaseQueueTestMixin): From d42433eed43e86424437936537c59f8eab5d49e2 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 28 Feb 2023 17:45:08 +0100 Subject: [PATCH 18/62] asyncio.queue: refactoring of tests, add new tests, last updates and corrections --- Lib/asyncio/queues.py | 16 +- Lib/test/test_asyncio/test_queues.py | 335 ++++++++++++++++++++------- 2 files changed, 264 insertions(+), 87 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index f4e1ba58866e53..095f8ad0d2c3a7 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -202,7 +202,7 @@ async def get(self): # the call. Wake up the next in line. self._wakeup_next(self._getters) raise - if self._shutdown_state is not _QueueState.ALIVE: + if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: raise QueueShutDown return self.get_nowait() @@ -235,6 +235,8 @@ def task_done(self): Raises ValueError if called more times than there were items placed in the queue. """ + if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + raise QueueShutDown if self._unfinished_tasks <= 0: raise ValueError('task_done() called too many times') self._unfinished_tasks -= 1 @@ -249,8 +251,12 @@ async def join(self): indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. """ + if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + raise QueueShutDown if self._unfinished_tasks > 0: await self._finished.wait() + if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + raise QueueShutDown def shutdown(self, immediate=False): """Shut-down the queue, making queue gets and puts raise. @@ -263,21 +269,21 @@ def shutdown(self, immediate=False): """ if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: return - + # here _shutdown_state is ALIVE or SHUTDOWN if immediate: self._shutdown_state = _QueueState.SHUTDOWN_IMMEDIATE while self._getters: getter = self._getters.popleft() if not getter.done(): getter.set_result(None) - else: + # Release 'blocked' tasks/coros via `.join()` + self._finished.set() + elif self._shutdown_state is _QueueState.ALIVE: # here self._shutdown_state = _QueueState.SHUTDOWN while self._putters: putter = self._putters.popleft() if not putter.done(): putter.set_result(None) - # Release 'blocked' tasks/coros via `.join()` - self._finished.set() class PriorityQueue(Queue): diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index f2c329c94b7997..16b30a5126feda 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -525,12 +525,114 @@ class PriorityQueueJoinTests(_QueueJoinTestMixin, unittest.IsolatedAsyncioTestCa class _QueueShutdownTestMixin: q_class = None + async def _get(self, q, go, results): + await go.wait() + try: + msg = await q.get() + results.append(True) + return msg + except asyncio.QueueShutDown: + results.append(False) + return False + + async def _get_shutdown(self, q, go, results): + await go.wait() + try: + msg = await q.get() + results.append(False) + return msg + except asyncio.QueueShutDown: + results.append(True) + return False + + async def _get_nowait(self, q, go, results): + await go.wait() + try: + msg = q.get_nowait() + results.append(True) + return msg + except asyncio.QueueShutDown: + results.append(False) + return False + + async def _get_task_done(self, q, go, results): + await go.wait() + try: + msg = await q.get() + q.task_done() + results.append(True) + return msg + except asyncio.QueueShutDown: + results.append(False) + return False + + async def _get_nowait_shutdown(self, q, go, results): + await go.wait() + try: + msg = q.get_nowait() + results.append(False) + except asyncio.QueueShutDown: + results.append(True) + return True + + async def _put_shutdown(self, q, go, msg, results): + await go.wait() + try: + await q.put(msg) + results.append(False) + except asyncio.QueueShutDown: + results.append(True) + return msg + + async def _put_nowait_shutdown(self, q, go, msg, results): + await go.wait() + try: + q.put_nowait(msg) + results.append(False) + except asyncio.QueueShutDown: + results.append(True) + return msg + + async def _shutdown(self, q, go, immediate): + await asyncio.sleep(0.001) + q.shutdown(immediate) + await asyncio.sleep(0.001) + go.set() + await asyncio.sleep(0.001) + + async def _join(self, q, go, results): + await go.wait() + try: + await q.join() + results.append(True) + return True + except asyncio.QueueShutDown: + results.append(False) + return False + + async def _join_shutdown(self, q, go, results): + await go.wait() + try: + await q.join() + results.append(False) + return False + except asyncio.QueueShutDown: + results.append(True) + return True + except asyncio.CancelledError: + results.append(True) + raise + async def test_empty(self): q = self.q_class() q.shutdown() - with self.assertRaises(asyncio.QueueShutDown): + with self.assertRaises( + asyncio.QueueShutDown, msg="Didn't appear to shut-down queue" + ): await q.put("data") - with self.assertRaises(asyncio.QueueShutDown): + with self.assertRaises( + asyncio.QueueShutDown, msg="Didn't appear to shut-down queue" + ): await q.get() async def test_nonempty(self): @@ -538,18 +640,24 @@ async def test_nonempty(self): q.put_nowait("data") q.shutdown() await q.get() - with self.assertRaises(asyncio.QueueShutDown): + with self.assertRaises( + asyncio.QueueShutDown, msg="Didn't appear to shut-down queue" + ): await q.get() async def test_immediate(self): q = self.q_class() q.put_nowait("data") q.shutdown(immediate=True) - with self.assertRaises(asyncio.QueueShutDown): + with self.assertRaises( + asyncio.QueueShutDown, msg="Didn't appear to shut-down queue" + ): await q.get() async def test_shutdown_repr(self): q = self.q_class() + self.assertNotIn("alive", repr(q)) + q.shutdown() self.assertIn("shutdown", repr(q)) @@ -557,7 +665,7 @@ async def test_shutdown_repr(self): q.shutdown(immediate=True) self.assertIn("shutdown-immediate", repr(q)) - async def test_shutdown_transition(self): + async def test_shutdown_allowed_transitions(self): # allowed transitions would be from alive via shutdown to immediate q = self.q_class() self.assertEqual("alive", q._shutdown_state.value) @@ -568,73 +676,142 @@ async def test_shutdown_transition(self): q.shutdown(immediate=True) self.assertEqual("shutdown-immediate", q._shutdown_state.value) - q.shutdown() - self.assertEqual("shutdown-immediate", q._shutdown_state.value) + q.shutdown(immediate=False) + self.assertNotEqual("shutdown", q._shutdown_state.value) - async def test_shutdown_immediate_get(self): - q = self.q_class() + async def _shutdown_putters(self, immediate): + delay = 0.001 + q = self.q_class(2) results = [] - go = asyncio.Event() + await q.put("E") + await q.put("W") + # queue full + t = asyncio.create_task(q.put("Y")) + await asyncio.sleep(delay) + self.assertTrue(len(q._putters) == 1) + with self.assertRaises(asyncio.QueueShutDown): + # here `t` raises a QueueShuDown + q.shutdown(immediate) + await t + self.assertTrue(not q._putters) + + async def test_shutdown_putters_deque(self): + return await self._shutdown_putters(False) - async def get_once(q, go): - await go.wait() - try: - msg = await q.get() - results.append(False) - except asyncio.QueueShutDown: - results.append(True) - return True + async def test_shutdown_immediate_putters_deque(self): + return await self._shutdown_putters(True) - async def shutdown(q, go, immediate): + async def _shutdown_getters(self, immediate): + delay = 0.001 + q = self.q_class(1) + results = [] + await q.put("Y") + # queue full + asyncio.create_task(q.get()) + await asyncio.sleep(delay) + t = asyncio.create_task(q.get()) + await asyncio.sleep(delay) + self.assertTrue(len(q._getters) == 1) + if immediate: + # here `t` raises a QueueShuDown + with self.assertRaises(asyncio.QueueShutDown): + q.shutdown(immediate) + await t + self.assertTrue(not q._getters) + else: + # here `t` is always pending q.shutdown(immediate) - go.set() - return True + await asyncio.sleep(delay) + self.assertTrue(q._getters) - tasks = ( - (get_once, (q, go)), - (get_once, (q, go)), - ) + async def test_shutdown_getters_deque(self): + return await self._shutdown_getters(False) + + async def test_shutdown_immediate_getters_deque(self): + return await self._shutdown_getters(True) + + async def _shutdown_get_nowait(self, immediate): + q = self.q_class(2) + results = [] + go = asyncio.Event() + await q.put("Y") + await q.put("D") + nb = q.qsize() + # queue full + + if immediate: + coros = ( + (self._get_nowait_shutdown(q, go, results)), + (self._get_nowait_shutdown(q, go, results)), + ) + else: + coros = ( + (self._get_nowait(q, go, results)), + (self._get_nowait(q, go, results)), + ) t = [] - for coro, params in tasks: - t.append(asyncio.create_task(coro(*params))) - t.append(asyncio.create_task(shutdown(q, go, True))) + for coro in coros: + t.append(asyncio.create_task(coro)) + t.append(asyncio.create_task(self._shutdown(q, go, immediate))) res = await asyncio.gather(*t) - self.assertEqual(results, [True]*len(tasks)) + self.assertEqual(results, [True]*len(coros)) + self.assertEqual(len(q._putters), 0) + if immediate: + self.assertEqual(len(q._getters), 0) + self.assertEqual(q._unfinished_tasks, nb) + async def test_shutdown_get_nowait(self): + return await self._shutdown_get_nowait(False) - async def _shutdown_put(self, immediate): + async def test_shutdown_immediate_get_nowait(self): + return await self._shutdown_get_nowait(True) + + async def test_shutdown_get_task_done_join(self, immediate=False): q = self.q_class(2) results = [] go = asyncio.Event() await q.put("Y") await q.put("D") - # queue fulled + self.assertEqual(q._unfinished_tasks, q.qsize()) - async def put_once(q, go, msg): - await go.wait() - try: - await q.put(msg) - results.append(False) - except asyncio.QueueShutDown: - results.append(True) - return msg + # queue full - async def shutdown(q, go, immediate): - q.shutdown(immediate) - go.set() + coros = ( + (self._get_task_done(q, go, results)), + (self._get_task_done(q, go, results)), + (self._join(q, go, results)), + (self._join(q, go, results)), + ) + t = [] + for coro in coros: + t.append(asyncio.create_task(coro)) + t.append(asyncio.create_task(self._shutdown(q, go, False))) + res = await asyncio.gather(*t) + + self.assertEqual(results, [True]*len(coros)) + self.assertIn(t[0].result(), "YD") + self.assertIn(t[1].result(), "YD") + self.assertNotEqual(t[0].result(), t[1].result()) + self.assertEqual(q._unfinished_tasks, 0) + + async def _shutdown_put(self, immediate): + q = self.q_class() + results = [] + go = asyncio.Event() + # queue not empty - tasks = ( - (put_once, (q, go, 100)), - (put_once, (q, go, 200)), + coros = ( + (self._put_shutdown(q, go, "Y", results)), + (self._put_nowait_shutdown(q, go, "D", results)), ) t = [] - for coro, params in tasks: - t.append(asyncio.create_task(coro(*params))) - t.append(asyncio.create_task(shutdown(q, go, immediate))) + for coro in coros: + t.append(asyncio.create_task(coro)) + t.append(asyncio.create_task(self._shutdown(q, go, immediate))) res = await asyncio.gather(*t) - self.assertEqual(results, [True]*len(tasks)) + self.assertEqual(results, [True]*len(coros)) async def test_shutdown_put(self): return await self._shutdown_put(False) @@ -642,53 +819,47 @@ async def test_shutdown_put(self): async def test_shutdown_immediate_put(self): return await self._shutdown_put(True) - - async def _shutdown_put_and_join(self, immediate): + async def _shutdown_put_join(self, immediate): q = self.q_class(2) results = [] go = asyncio.Event() await q.put("Y") await q.put("D") + nb = q.qsize() # queue fulled - async def put_once(q, go, msg): - await go.wait() - try: - await q.put(msg) - results.append(False) - except asyncio.QueueShutDown: - results.append(True) - return msg - - async def shutdown(q, go, immediate): - q.shutdown(immediate) - go.set() - - async def join(q, go): - await go.wait() - await q.join() - results.append(True) - return True + async def _cancel_join_task(q, delay, t): + await asyncio.sleep(delay) + t.cancel() + await asyncio.sleep(0) + q._finished.set() - tasks = ( - (put_once, (q, go, 'E')), - (put_once, (q, go, 'W')), - (join, (q, go)), - (join, (q, go)), + coros = ( + (self._put_shutdown(q, go, "E", results)), + (self._put_nowait_shutdown(q, go, "W", results)), + (self._join_shutdown(q, go, results)), ) t = [] - for coro, params in tasks: - t.append(asyncio.create_task(coro(*params))) - t.append(asyncio.create_task(shutdown(q, go, immediate))) - res = await asyncio.gather(*t) - - self.assertEqual(results, [True]*len(tasks)) + for coro in coros: + t.append(asyncio.create_task(coro)) + t.append(asyncio.create_task(self._shutdown(q, go, immediate))) + if not immediate: + # Here calls `join` is a blocking operation + # so wait for a delay and cancel this blocked task + t.append(asyncio.create_task(_cancel_join_task(q, 0.01, t[2]))) + with self.assertRaises(asyncio.CancelledError) as e: + await asyncio.gather(*t) + else: + res = await asyncio.gather(*t) + + self.assertEqual(results, [True]*len(coros)) + self.assertTrue(q._finished.is_set()) async def test_shutdown_put_and_join(self): - return await self._shutdown_put_and_join(False) + return await self._shutdown_put_join(False) async def test_shutdown_immediate_put_and_join(self): - return await self._shutdown_put_and_join(True) + return await self._shutdown_put_join(True) class QueueShutdownTests( From 53078bbf9804ac8674c191af604fcd5f6a2d2f08 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 28 Feb 2023 18:28:59 +0100 Subject: [PATCH 19/62] first version working --- Lib/multiprocessing/queues.py | 37 ++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index bfaf81e2dc6616..a629de6125eee0 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -56,7 +56,7 @@ def __init__(self, maxsize=0, *, ctx): self._ignore_epipe = False self._reset() self._shutdown_state = context._default_context.Value( - ctypes.c_uint8, lock=self._rlock + ctypes.c_uint8, _queue_alive, lock=True ) if sys.platform != 'win32': @@ -102,18 +102,16 @@ def put(self, obj, block=True, timeout=None): raise Full with self._notempty: - if self._shutdown_state.value != _queue_alive: - raise ShutDown if self._thread is None: self._start_thread() self._buffer.append(obj) self._notempty.notify() def get(self, block=True, timeout=None): - if self._shutdown_state.value == _queue_shutdown_immediate: - raise ShutDown if self._closed: raise ValueError(f"Queue {self!r} is closed") + if self._shutdown_state.value != _queue_alive: + raise ShutDown if block and timeout is None: with self._rlock: if self._shutdown_state.value != _queue_alive: @@ -132,7 +130,7 @@ def get(self, block=True, timeout=None): if self._shutdown_state.value != _queue_alive: raise ShutDown raise Empty - if self._shutdown_state.value != _queue_alive : + if self._shutdown_state.value != _queue_alive: raise ShutDown elif not self._poll(): raise Empty @@ -162,7 +160,15 @@ def put_nowait(self, obj): return self.put(obj, False) def shutdown(self, immediate=True): - pass + with self._shutdown_state.get_lock(): + if self._shutdown_state.value == _queue_shutdown_immediate: + return + if immediate: + self._shutdown_state.value = _queue_shutdown_immediate + with self._notempty: + self._notempty.notify_all() # cf from @EpicWink + else: + self._shutdown_state.value = _queue_shutdown def close(self): self._closed = True @@ -335,6 +341,8 @@ def __setstate__(self, state): def put(self, obj, block=True, timeout=None): if self._closed: raise ValueError(f"Queue {self!r} is closed") + if self._shutdown_state.value != _queue_alive: + return if not self._sem.acquire(block, timeout): raise Full @@ -347,6 +355,8 @@ def put(self, obj, block=True, timeout=None): def task_done(self): with self._cond: + if self._shutdown_state.value != _queue_alive: + raise ShutDown if not self._unfinished_tasks.acquire(False): raise ValueError('task_done() called too many times') if self._unfinished_tasks._semlock._is_zero(): @@ -354,10 +364,19 @@ def task_done(self): def join(self): with self._cond: - if self._shutdown_state.value == _queue_shutdown_immediate: - return + if self._shutdown_state.value != _queue_alive: + raise ShutDown if not self._unfinished_tasks._semlock._is_zero(): self._cond.wait() + if self._shutdown_state.value == _queue_shutdown_immediate: + raise ShutDown + + def shutdown(self, immediate=True): + initial_shutdown = self._shutdown_state.value + super().shutdown(immediate) + if initial_shutdown == _queue_alive: + with self._cond: + self._cond.notify_all() # here to check YD # # Simplified Queue type -- really just a locked pipe From 0075039bdf0fbff57e5e5ca1263adb1e9f5f9e75 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 28 Feb 2023 19:00:59 +0100 Subject: [PATCH 20/62] Some corrections --- Lib/multiprocessing/queues.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index a629de6125eee0..9d76d8d389c651 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -110,11 +110,11 @@ def put(self, obj, block=True, timeout=None): def get(self, block=True, timeout=None): if self._closed: raise ValueError(f"Queue {self!r} is closed") - if self._shutdown_state.value != _queue_alive: + if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown if block and timeout is None: with self._rlock: - if self._shutdown_state.value != _queue_alive: + if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown res = self._recv_bytes() self._sem.release() @@ -127,10 +127,10 @@ def get(self, block=True, timeout=None): if block: timeout = deadline - time.monotonic() if not self._poll(timeout): - if self._shutdown_state.value != _queue_alive: + if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown raise Empty - if self._shutdown_state.value != _queue_alive: + if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown elif not self._poll(): raise Empty @@ -138,7 +138,7 @@ def get(self, block=True, timeout=None): self._sem.release() finally: self._rlock.release() - if self._shutdown_state.value == _queue_shutdown: + if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown # unserialize the data after having released the lock return _ForkingPickler.loads(res) @@ -342,7 +342,7 @@ def put(self, obj, block=True, timeout=None): if self._closed: raise ValueError(f"Queue {self!r} is closed") if self._shutdown_state.value != _queue_alive: - return + raise ShutDown if not self._sem.acquire(block, timeout): raise Full @@ -355,7 +355,7 @@ def put(self, obj, block=True, timeout=None): def task_done(self): with self._cond: - if self._shutdown_state.value != _queue_alive: + if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown if not self._unfinished_tasks.acquire(False): raise ValueError('task_done() called too many times') From b4a53d292f0de34685b8be145c24076f601db4d2 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Mar 2023 19:18:23 +0100 Subject: [PATCH 21/62] Change Enum to global about `shutdown_state` attr Fixes bugs --- Lib/queue.py | 52 +++++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 933f1806205072..ca1dfeb1ded41c 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -30,11 +30,9 @@ class ShutDown(Exception): '''Raised when put/get with shut-down queue.''' -class _QueueState(enum.Enum): - ALIVE = "alive" - SHUTDOWN = "shutdown" - SHUTDOWN_IMMEDIATE = "shutdown-immediate" - +_queue_alive = "alive" +_queue_shutdown = "shutdown" +_queue_shutdown_immediate = "shutdown-immediate" class Queue: '''Create a queue object with a given maximum size. @@ -66,7 +64,7 @@ def __init__(self, maxsize=0): self.unfinished_tasks = 0 # Queue shut-down state - self.shutdown_state = _QueueState.ALIVE + self.shutdown_state = _queue_alive def task_done(self): '''Indicate that a formerly enqueued task is complete. @@ -83,6 +81,9 @@ def task_done(self): placed in the queue. ''' with self.all_tasks_done: + # here `self.all_task_done` uses `self.mutex` + if self.shutdown_state == _queue_shutdown_immediate: + raise ShutDown unfinished = self.unfinished_tasks - 1 if unfinished <= 0: if unfinished < 0: @@ -100,10 +101,13 @@ def join(self): When the count of unfinished tasks drops to zero, join() unblocks. ''' with self.all_tasks_done: + # here `self.all_task_done` uses `self.mutex` + if self.shutdown_state == _queue_shutdown_immediate: + raise ShutDown while self.unfinished_tasks: - if self.shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: - return self.all_tasks_done.wait() + if self.shutdown_state == _queue_shutdown_immediate: + raise ShutDown def qsize(self): '''Return the approximate size of the queue (not reliable!).''' @@ -146,9 +150,10 @@ def put(self, item, block=True, timeout=None): is immediately available, else raise the Full exception ('timeout' is ignored in that case). ''' - if self.shutdown_state is not _QueueState.ALIVE: - raise ShutDown with self.not_full: + # here `self.not_full` uses `self.mutex`` + if self.shutdown_state != _queue_alive: + raise ShutDown if self.maxsize > 0: if not block: if self._qsize() >= self.maxsize: @@ -156,7 +161,7 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() - if self.shutdown_state is not _QueueState.ALIVE: + if self.shutdown_state != _queue_alive: raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") @@ -167,7 +172,7 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) - if self.shutdown_state is not _QueueState.ALIVE: + if self.shutdown_state != _queue_alive: raise ShutDown self._put(item) self.unfinished_tasks += 1 @@ -184,35 +189,36 @@ def get(self, block=True, timeout=None): available, else raise the Empty exception ('timeout' is ignored in that case). ''' - if self.shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: - raise ShutDown with self.not_empty: + # here `self.not_empty` uses `self.mutex` + if self.shutdown_state == _queue_shutdown_immediate: + raise ShutDown if not block: if not self._qsize(): - if self.shutdown_state is not _QueueState.ALIVE: + if self.shutdown_state != _queue_alive: raise ShutDown raise Empty elif timeout is None: while not self._qsize(): - if self.shutdown_state is not _QueueState.ALIVE: + if self.shutdown_state != _queue_alive: raise ShutDown self.not_empty.wait() - if self.shutdown_state is not _QueueState.ALIVE: + if self.shutdown_state != _queue_alive: raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = time() + timeout while not self._qsize(): - if self.shutdown_state is not _QueueState.ALIVE: + if self.shutdown_state != _queue_alive: raise ShutDown remaining = endtime - time() if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) - if self.shutdown_state is not _QueueState.ALIVE: + if self.shutdown_state != _queue_alive: raise ShutDown - if self.shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + if self.shutdown_state == _queue_shutdown_immediate: raise ShutDown item = self._get() self.not_full.notify() @@ -244,11 +250,11 @@ def shutdown(self, immediate=False): and join() if 'immediate'. The ShutDown exception is raised. ''' with self.mutex: - if self.shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + if self.shutdown_state is _queue_shutdown_immediate: return if immediate: - self.shutdown_state = _QueueState.SHUTDOWN_IMMEDIATE + self.shutdown_state = _queue_shutdown_immediate self.not_empty.notify_all() # set self.unfinished_tasks to 0 # to break the loop in 'self.join()' @@ -256,7 +262,7 @@ def shutdown(self, immediate=False): self.unfinished_tasks = 0 self.all_tasks_done.notify_all() else: - self.shutdown_state = _QueueState.SHUTDOWN + self.shutdown_state = _queue_shutdown self.not_full.notify_all() # Override these methods to implement other queue organizations From dbe207898efa9a2cb9c75f6a9603858116221a37 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Mar 2023 19:19:52 +0100 Subject: [PATCH 22/62] Add new tests Refactoring tests --- Lib/test/test_queue.py | 330 ++++++++++++++++++++++++++--------------- 1 file changed, 209 insertions(+), 121 deletions(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index a2814ac40f2e18..6b65d2a0f1169f 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -264,51 +264,158 @@ def test_shutdown_immediate(self): with self.assertRaises(self.queue.ShutDown): q.get() - def test_shutdown_transition(self): + def test_shutdown_allowed_transitions(self): # allowed transitions would be from alive via shutdown to immediate q = self.type2test() - self.assertEqual("alive", q.shutdown_state.value) + self.assertEqual("alive", q.shutdown_state) q.shutdown() - self.assertEqual("shutdown", q.shutdown_state.value) + self.assertEqual("shutdown", q.shutdown_state) q.shutdown(immediate=True) - self.assertEqual("shutdown-immediate", q.shutdown_state.value) + self.assertEqual("shutdown-immediate", q.shutdown_state) q.shutdown(immediate=False) - self.assertEqual("shutdown-immediate", q.shutdown_state.value) + self.assertNotEqual("shutdown", q.shutdown_state) + @staticmethod + def _wait(): + time.sleep(0.01) - def test_shutdown_get(self): + def _shutdown_all_methods(self, immediate): + q = self.type2test(2) + q.put("L") + self._wait() + q.put_nowait("O") + q.shutdown(immediate) + self._wait() + + with self.assertRaises(self.queue.ShutDown): + q.put("E") + with self.assertRaises(self.queue.ShutDown): + q.put_nowait("W") + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + with self.assertRaises(self.queue.ShutDown): + q.task_done() + with self.assertRaises(self.queue.ShutDown): + q.join() + else: + self.assertIn(q.get(), "LO") + q.task_done() + self._wait() + self.assertIn(q.get(), "LO") + q.task_done() + self._wait() + q.join() + # on shutdown(immediate=False) + # when queue is empty, should raise ShutDown Exception + with self.assertRaises(self.queue.ShutDown): + q.get() # p.get(True) + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() # p.get(False) + with self.assertRaises(self.queue.ShutDown): + q.get(True, 1.0) + + def test_shutdown_all_methods(self): + return self._shutdown_all_methods(False) + + def test_shutdown_immediate_get(self): + return self._shutdown_all_methods(True) + + def _get(self, q, go, results, shutdown=False): + go.wait() + try: + msg = q.get() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _get_shutdown(self, q, go, results): + return self._get(q, go, results, True) + + def _get_task_done(self, q, go, results): + go.wait() + try: + msg = q.get() + q.task_done() + results.append(True) + return msg + except self.queue.ShutDown: + results.append(False) + return False + + def _put(self, q, msg, go, results, shutdown=False): + go.wait() + try: + q.put(msg) + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _put_shutdown(self, q, msg, go, results): + return self._put(q, msg, go, results, True) + + def _join(self, q, results, shutdown=False): + try: + q.join() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _join_shutdown(self, q, results): + return self._join(q, results, True) + + def _shutdown_get(self, immediate): q = self.type2test(2) results = [] go = threading.Event() + q.put("Y") + q.put("D") + # queue full - def get_once(q, go): - go.wait() - try: - msg = q.get() - results.append(False) - except self.queue.ShutDown: - results.append(True) - return True - - thrds = ( - (get_once, (q, go)), - (get_once, (q, go)), - ) + if immediate: + thrds = ( + (self._get_shutdown, (q, go, results)), + (self._get_shutdown, (q, go, results)), + ) + else: + thrds = ( + # on shutdown(immediate=False) + # one of these threads shoud raise Shutdown + (self._get, (q, go, results)), + (self._get, (q, go, results)), + (self._get, (q, go, results)), + ) threads = [] - for f, params in thrds: - thread = threading.Thread(target=f, args=params) - thread.start() - threads.append(thread) - q.shutdown() + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self._wait() + q.shutdown(immediate) go.set() for t in threads: t.join() + if immediate: + self.assertListEqual(results, [True, True]) + else: + self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1)) - self.assertEqual(results, [True]*len(thrds)) + def test_shutdown_get(self): + return self._shutdown_get(False) - def test_shutdown_put(self): + def test_shutdown_immediate_get(self): + return self._shutdown_get(True) + + def _shutdown_put(self, immediate): q = self.type2test(2) results = [] go = threading.Event() @@ -316,24 +423,15 @@ def test_shutdown_put(self): q.put("D") # queue fulled - def put_once(q, msg, go): - go.wait() - try: - q.put(msg) - results.append(False) - except self.queue.ShutDown: - results.append(True) - return msg - thrds = ( - (put_once, (q, 100, go)), - (put_once, (q, 200, go)), + (self._put_shutdown, (q, "E", go, results)), + (self._put_shutdown, (q, "W", go, results)), ) threads = [] - for f, params in thrds: - thread = threading.Thread(target=f, args=params) - thread.start() - threads.append(thread) + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self._wait() q.shutdown() go.set() for t in threads: @@ -341,27 +439,42 @@ def put_once(q, msg, go): self.assertEqual(results, [True]*len(thrds)) + def test_shutdown_put(self): + return self._shutdown_put(False) + + def test_shutdown_immediate_put(self): + return self._shutdown_put(True) + def _shutdown_join(self, immediate): q = self.type2test() results = [] + q.put("Y") go = threading.Event() + nb = q.qsize() - def join(q, go): - go.wait() - q.join() - results.append(True) - - thrds = ( - (join, (q, go)), - (join, (q, go)), + if immediate: + thrds = ( + (self._join_shutdown, (q, results)), + (self._join_shutdown, (q, results)), + ) + else: + thrds = ( + (self._join, (q, results)), + (self._join, (q, results)), ) threads = [] - for f, params in thrds: - thread = threading.Thread(target=f, args=params) - thread.start() - threads.append(thread) - go.set() + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self._wait() + if not immediate: + res = [] + for i in range(nb): + threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) + threads[-1].start() + self._wait() q.shutdown(immediate) + go.set() for t in threads: t.join() @@ -373,101 +486,76 @@ def test_shutdown_immediate_join(self): def test_shutdown_join(self): return self._shutdown_join(False) - def _shutdown_put_and_join(self, immediate): + def _shutdown_put_join(self, immediate): q = self.type2test(2) results = [] go = threading.Event() q.put("Y") - q.put("D") + nb = q.qsize() # queue fulled - def put_once(q, msg, go): - go.wait() - try: - q.put(msg) - results.append(False) - except self.queue.ShutDown: - results.append(True) - return msg - - def join(q, go): - go.wait() - q.join() - results.append(True) - - thrds = ( - (put_once, (q, 100, go)), - (put_once, (q, 200, go)), - (join, (q, go)), - (join, (q, go)), - ) + if immediate: + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._join_shutdown, (q, results)), + ) + else: + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._join, (q, results)), + ) threads = [] - for f, params in thrds: - thread = threading.Thread(target=f, args=params) - thread.start() - threads.append(thread) + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self._wait() + if not immediate: + self.assertEqual(q.unfinished_tasks, nb) + for i in range(nb): + t = threading.Thread(target=q.task_done) + t.start() + threads.append(t) + self._wait() go.set() q.shutdown(immediate) - if not immediate: - self.assertTrue(q.unfinished_tasks, 2) - for i in range(2): - thread = threading.Thread(target=q.task_done) - thread.start() - threads.append(thread) - for t in threads: t.join() self.assertEqual(results, [True]*len(thrds)) + self.assertEqual(q.unfinished_tasks, 0) - def test_shutdown_immediate_put_and_join(self): - return self._shutdown_put_and_join(True) + def test_shutdown_immediate_put_join(self): + return self._shutdown_put_join(True) - def test_shutdown_put_and_join(self): - return self._shutdown_put_and_join(False) + def test_shutdown_put_join(self): + return self._shutdown_put_join(False) - def _shutdown_get_and_join(self, immediate): - q = self.type2test() + def test_shutdown_get_task_done_join(self): + q = self.type2test(2) results = [] go = threading.Event() - - def get_once(q, go): - go.wait() - try: - msg = q.get() - results.append(False) - except self.queue.ShutDown: - results.append(True) - return True - - def join(q, go): - go.wait() - q.join() - results.append(True) + q.put("Y") + q.put("D") + self.assertEqual(q.unfinished_tasks, q.qsize()) thrds = ( - (get_once, (q, go)), - (get_once, (q, go)), - (join, (q, go)), - (join, (q, go)), + (self._get_task_done, (q, go, results)), + (self._get_task_done, (q, go, results)), + (self._join, (q, results)), + (self._join, (q, results)), ) threads = [] - for f, params in thrds: - thread = threading.Thread(target=f, args=params) - thread.start() - threads.append(thread) + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self._wait() go.set() - q.shutdown(immediate) + q.shutdown(False) for t in threads: t.join() self.assertEqual(results, [True]*len(thrds)) - def test_shutdown_immediate_get_and_join(self): - return self._shutdown_get_and_join(True) - - def test__shutdown_get_and_join(self): - return self._shutdown_get_and_join(False) class QueueTest(BaseQueueTestMixin): From 18bb995fad54c0085679876028084ff714df29df Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Mar 2023 19:21:11 +0100 Subject: [PATCH 23/62] Fixes bugs --- Lib/multiprocessing/queues.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 9d76d8d389c651..9ecf52e648eed5 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -114,7 +114,8 @@ def get(self, block=True, timeout=None): raise ShutDown if block and timeout is None: with self._rlock: - if self._shutdown_state.value == _queue_shutdown_immediate: + if self._shutdown_state.value == _queue_shutdown_immediate or\ + (self._shutdown_state.value == _queue_shutdown and self.empty()): raise ShutDown res = self._recv_bytes() self._sem.release() @@ -122,17 +123,19 @@ def get(self, block=True, timeout=None): if block: deadline = time.monotonic() + timeout if not self._rlock.acquire(block, timeout): + if self._shutdown_state.value == _queue_shutdown: + raise ShutDown raise Empty try: if block: timeout = deadline - time.monotonic() if not self._poll(timeout): - if self._shutdown_state.value == _queue_shutdown_immediate: + if self._shutdown_state.value == _queue_shutdown: raise ShutDown raise Empty - if self._shutdown_state.value == _queue_shutdown_immediate: - raise ShutDown elif not self._poll(): + if self._shutdown_state.value == _queue_shutdown: + raise ShutDown raise Empty res = self._recv_bytes() self._sem.release() @@ -159,7 +162,7 @@ def get_nowait(self): def put_nowait(self, obj): return self.put(obj, False) - def shutdown(self, immediate=True): + def shutdown(self, immediate=False): with self._shutdown_state.get_lock(): if self._shutdown_state.value == _queue_shutdown_immediate: return @@ -364,14 +367,14 @@ def task_done(self): def join(self): with self._cond: - if self._shutdown_state.value != _queue_alive: + if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown if not self._unfinished_tasks._semlock._is_zero(): self._cond.wait() if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown - def shutdown(self, immediate=True): + def shutdown(self, immediate=False): initial_shutdown = self._shutdown_state.value super().shutdown(immediate) if initial_shutdown == _queue_alive: From 05700d54b43a5083bd1bdb4193aec42ba37bc489 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Mar 2023 19:21:33 +0100 Subject: [PATCH 24/62] Add first tests --- Lib/test/_test_multiprocessing.py | 138 ++++++++++++++++++++++++------ 1 file changed, 110 insertions(+), 28 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 1f9c9aec36b312..72520aff53f65c 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1279,39 +1279,121 @@ def test_closed_queue_put_get_exceptions(self): q.get() def test_shutdown_empty(self): - q = multiprocessing.Queue() - q.shutdown() - try: - q.put("data") - self.fail("Didn't appear to shut-down queue") - except pyqueue.ShutDown: - pass - try: - q.get() - self.fail("Didn't appear to shut-down queue") - except pyqueue.ShutDown: - pass + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.shutdown() + with self.assertRaises( + pyqueue.ShutDown, msg="Didn't appear to shut-down queue" + ): + q.put("data") + with self.assertRaises( + pyqueue.ShutDown, msg="Didn't appear to shut-down queue" + ): + q.get() def test_shutdown_nonempty(self): - q = multiprocessing.Queue() - q.put("data") - q.shutdown() - q.get() - try: + for q in multiprocessing.Queue(1), multiprocessing.JoinableQueue(1): + q.put("data") + q.shutdown() q.get() - self.fail("Didn't appear to shut-down queue") - except pyqueue.ShutDown: - pass + with self.assertRaises( + pyqueue.ShutDown, msg="Didn't appear to shut-down queue" + ): + q.get() def test_shutdown_immediate(self): - q = multiprocessing.Queue() - q.put("data") - q.shutdown(immediate=True) - try: - q.get() - self.fail("Didn't appear to shut-down queue") - except pyqueue.ShutDown: - pass + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.put("data") + q.shutdown(immediate=True) + with self.assertRaises( + pyqueue.ShutDown, msg="Didn't appear to shut-down queue" + ): + q.get() + + def test_shutdown_allowed_transitions(self): + # allowed transitions would be from `alive`` via `shutdown` to `shutdown_immediate`` + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + self.assertEqual(0, q._shutdown_state.value) + + # default -> immediate=False + q.shutdown() + self.assertEqual(1, q._shutdown_state.value) + + q.shutdown(immediate=True) + self.assertEqual(2, q._shutdown_state.value) + + q.shutdown(immediate=False) + self.assertNotEqual(1, q._shutdown_state.value) + + def _shutdown_all_methods(self, immediate): + # part 1: Queue + q = multiprocessing.Queue(2) + q.put("L") + _wait() # Give time to simulate delay of starting internal thread + q.put_nowait("O") + q.shutdown(immediate) + _wait() # simulate time of synchro primitive + + with self.assertRaises(pyqueue.ShutDown): + q.put("E") + with self.assertRaises(pyqueue.ShutDown): + q.put_nowait("W") + if immediate: + with self.assertRaises(pyqueue.ShutDown): + q.get() + with self.assertRaises(pyqueue.ShutDown): + q.get_nowait() + else: + # Neither `task_done`, neither `join`methods` to test + self.assertEqual(q.get(), "L") + self.assertEqual(q.get_nowait(), "O") + _wait() + + # on shutdown(immediate=False) + # when queue is empty, should raise ShutDown Exception + with self.assertRaises(pyqueue.ShutDown): + q.get() # p.get(True) + with self.assertRaises(pyqueue.ShutDown): + q.get_nowait() # q.get(False) + with self.assertRaises(pyqueue.ShutDown): + q.get(True, 1.0) + + # part 2: JoinableQueue + q = multiprocessing.JoinableQueue(2) + q.put("L") + _wait() + q.put_nowait("O") + q.shutdown(immediate) + _wait() + + with self.assertRaises(pyqueue.ShutDown): + q.put("E") + with self.assertRaises(pyqueue.ShutDown): + q.put_nowait("W") + if immediate: + with self.assertRaises(pyqueue.ShutDown): + q.get() + with self.assertRaises(pyqueue.ShutDown): + q.get_nowait() + with self.assertRaises(pyqueue.ShutDown): + q.task_done() + with self.assertRaises(pyqueue.ShutDown): + q.join() + else: + self.assertEqual(q.get(), "L") + q.task_done() + _wait() + self.assertEqual(q.get(), "O") + q.task_done() + _wait() + q.join() + # when `shutdown` queue is empty, should raise ShutDown Exception + with self.assertRaises(pyqueue.ShutDown): + q.get() # p.get(True) + with self.assertRaises(pyqueue.ShutDown): + q.get_nowait() # p.get(False) + with self.assertRaises(pyqueue.ShutDown): + q.get(True, 1.0) + # # # From 7d01747b5390d67181d0005df30283363aefb7d8 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Mar 2023 19:21:49 +0100 Subject: [PATCH 25/62] Update tests --- Lib/test/test_asyncio/test_queues.py | 169 +++++++++++++++------------ 1 file changed, 93 insertions(+), 76 deletions(-) diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 16b30a5126feda..1403b9bd2cd8b2 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -525,35 +525,29 @@ class PriorityQueueJoinTests(_QueueJoinTestMixin, unittest.IsolatedAsyncioTestCa class _QueueShutdownTestMixin: q_class = None - async def _get(self, q, go, results): - await go.wait() - try: - msg = await q.get() - results.append(True) - return msg - except asyncio.QueueShutDown: - results.append(False) - return False + async def asyncSetUp(self): + await super().asyncSetUp() + self.delay = 0.001 - async def _get_shutdown(self, q, go, results): + async def _get(self, q, go, results, shutdown=False): await go.wait() try: msg = await q.get() - results.append(False) + results.append(not shutdown) return msg except asyncio.QueueShutDown: - results.append(True) - return False + results.append(shutdown) + return shutdown - async def _get_nowait(self, q, go, results): + async def _get_nowait(self, q, go, results, shutdown=False): await go.wait() try: msg = q.get_nowait() - results.append(True) + results.append(not shutdown) return msg except asyncio.QueueShutDown: - results.append(False) - return False + results.append(shutdown) + return shutdown async def _get_task_done(self, q, go, results): await go.wait() @@ -566,64 +560,45 @@ async def _get_task_done(self, q, go, results): results.append(False) return False - async def _get_nowait_shutdown(self, q, go, results): - await go.wait() - try: - msg = q.get_nowait() - results.append(False) - except asyncio.QueueShutDown: - results.append(True) - return True - - async def _put_shutdown(self, q, go, msg, results): + async def _put(self, q, go, msg, results, shutdown=False): await go.wait() try: await q.put(msg) - results.append(False) + results.append(not shutdown) + return not shutdown except asyncio.QueueShutDown: - results.append(True) - return msg + results.append(shutdown) + return shutdown - async def _put_nowait_shutdown(self, q, go, msg, results): + async def _put_nowait(self, q, go, msg, results, shutdown=False): await go.wait() try: q.put_nowait(msg) results.append(False) + return not shutdown except asyncio.QueueShutDown: results.append(True) - return msg + return shutdown async def _shutdown(self, q, go, immediate): - await asyncio.sleep(0.001) q.shutdown(immediate) - await asyncio.sleep(0.001) + await asyncio.sleep(self.delay) go.set() - await asyncio.sleep(0.001) + await asyncio.sleep(self.delay) - async def _join(self, q, go, results): - await go.wait() + async def _join(self, q, results, shutdown=False): try: await q.join() - results.append(True) + results.append(not shutdown) return True except asyncio.QueueShutDown: - results.append(False) - return False - - async def _join_shutdown(self, q, go, results): - await go.wait() - try: - await q.join() - results.append(False) + results.append(shutdown) return False - except asyncio.QueueShutDown: - results.append(True) - return True except asyncio.CancelledError: - results.append(True) + results.append(shutdown) raise - async def test_empty(self): + async def test_shutdown_empty(self): q = self.q_class() q.shutdown() with self.assertRaises( @@ -635,7 +610,7 @@ async def test_empty(self): ): await q.get() - async def test_nonempty(self): + async def test_shutdown_nonempty(self): q = self.q_class() q.put_nowait("data") q.shutdown() @@ -645,7 +620,7 @@ async def test_nonempty(self): ): await q.get() - async def test_immediate(self): + async def test_shutdown_immediate(self): q = self.q_class() q.put_nowait("data") q.shutdown(immediate=True) @@ -679,8 +654,46 @@ async def test_shutdown_allowed_transitions(self): q.shutdown(immediate=False) self.assertNotEqual("shutdown", q._shutdown_state.value) + async def _shutdown_all_methods(self, immediate): + q = asyncio.Queue() + await q.put("L") + q.put_nowait("O") + q.shutdown(immediate) + with self.assertRaises(asyncio.QueueShutDown): + await q.put("E") + with self.assertRaises(asyncio.QueueShutDown): + q.put_nowait("W") + + if immediate: + with self.assertRaises(asyncio.QueueShutDown): + await q.get() + with self.assertRaises(asyncio.QueueShutDown): + q.get_nowait() + with self.assertRaises(asyncio.QueueShutDown): + q.task_done() + with self.assertRaises(asyncio.QueueShutDown): + await q.join() + else: + self.assertIn(await q.get(), "LO") + q.task_done() + self.assertIn(q.get_nowait(), "LO") + q.task_done() + await q.join() + # on shutdown(immediate=False) + # when queue is empty, should raise ShutDown Exception + with self.assertRaises(asyncio.QueueShutDown): + await q.get() + with self.assertRaises(asyncio.QueueShutDown): + q.get_nowait() + + async def test_shutdown_all_methods(self): + return await self._shutdown_all_methods(False) + + async def test_shutdown_immediate_get(self): + return await self._shutdown_all_methods(True) + async def _shutdown_putters(self, immediate): - delay = 0.001 + delay = self.delay q = self.q_class(2) results = [] await q.put("E") @@ -702,11 +715,13 @@ async def test_shutdown_immediate_putters_deque(self): return await self._shutdown_putters(True) async def _shutdown_getters(self, immediate): - delay = 0.001 + delay = self.delay q = self.q_class(1) results = [] await q.put("Y") + nb = q.qsize() # queue full + asyncio.create_task(q.get()) await asyncio.sleep(delay) t = asyncio.create_task(q.get()) @@ -723,6 +738,8 @@ async def _shutdown_getters(self, immediate): q.shutdown(immediate) await asyncio.sleep(delay) self.assertTrue(q._getters) + self.assertEqual(q._unfinished_tasks, nb) + async def test_shutdown_getters_deque(self): return await self._shutdown_getters(False) @@ -730,7 +747,7 @@ async def test_shutdown_getters_deque(self): async def test_shutdown_immediate_getters_deque(self): return await self._shutdown_getters(True) - async def _shutdown_get_nowait(self, immediate): + async def _shutdown_get(self, immediate): q = self.q_class(2) results = [] go = asyncio.Event() @@ -741,11 +758,13 @@ async def _shutdown_get_nowait(self, immediate): if immediate: coros = ( - (self._get_nowait_shutdown(q, go, results)), - (self._get_nowait_shutdown(q, go, results)), + (self._get(q, go, results, shutdown=True)), + (self._get_nowait(q, go, results, shutdown=True)), ) else: coros = ( + # one of these tasks shoud raise Shutdown + (self._get(q, go, results)), (self._get_nowait(q, go, results)), (self._get_nowait(q, go, results)), ) @@ -754,20 +773,18 @@ async def _shutdown_get_nowait(self, immediate): t.append(asyncio.create_task(coro)) t.append(asyncio.create_task(self._shutdown(q, go, immediate))) res = await asyncio.gather(*t) - - self.assertEqual(results, [True]*len(coros)) - self.assertEqual(len(q._putters), 0) if immediate: - self.assertEqual(len(q._getters), 0) - self.assertEqual(q._unfinished_tasks, nb) + self.assertEqual(results, [True]*len(coros)) + else: + self.assertListEqual(sorted(results), [False] + [True]*(len(coros)-1)) - async def test_shutdown_get_nowait(self): - return await self._shutdown_get_nowait(False) + async def test_shutdown_get(self): + return await self._shutdown_get(False) - async def test_shutdown_immediate_get_nowait(self): - return await self._shutdown_get_nowait(True) + async def test_shutdown_immediate_get(self): + return await self._shutdown_get(True) - async def test_shutdown_get_task_done_join(self, immediate=False): + async def test_shutdown_get_task_done_join(self): q = self.q_class(2) results = [] go = asyncio.Event() @@ -780,8 +797,8 @@ async def test_shutdown_get_task_done_join(self, immediate=False): coros = ( (self._get_task_done(q, go, results)), (self._get_task_done(q, go, results)), - (self._join(q, go, results)), - (self._join(q, go, results)), + (self._join(q, results)), + (self._join(q, results)), ) t = [] for coro in coros: @@ -802,8 +819,8 @@ async def _shutdown_put(self, immediate): # queue not empty coros = ( - (self._put_shutdown(q, go, "Y", results)), - (self._put_nowait_shutdown(q, go, "D", results)), + (self._put(q, go, "Y", results, shutdown=True)), + (self._put_nowait(q, go, "D", results, shutdown=True)), ) t = [] for coro in coros: @@ -835,9 +852,9 @@ async def _cancel_join_task(q, delay, t): q._finished.set() coros = ( - (self._put_shutdown(q, go, "E", results)), - (self._put_nowait_shutdown(q, go, "W", results)), - (self._join_shutdown(q, go, results)), + (self._put(q, go, "E", results, shutdown=True)), + (self._put_nowait(q, go, "W", results, shutdown=True)), + (self._join(q, results, shutdown=True)), ) t = [] for coro in coros: @@ -855,7 +872,7 @@ async def _cancel_join_task(q, delay, t): self.assertEqual(results, [True]*len(coros)) self.assertTrue(q._finished.is_set()) - async def test_shutdown_put_and_join(self): + async def test_shutdown_put_join(self): return await self._shutdown_put_join(False) async def test_shutdown_immediate_put_and_join(self): From aad0cba43bc82689053246e320bfdb81a5e8b948 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Mar 2023 20:02:09 +0100 Subject: [PATCH 26/62] add _wait() --- Lib/test/_test_multiprocessing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 72520aff53f65c..955af66af65b03 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1281,6 +1281,7 @@ def test_closed_queue_put_get_exceptions(self): def test_shutdown_empty(self): for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): q.shutdown() + _wait() with self.assertRaises( pyqueue.ShutDown, msg="Didn't appear to shut-down queue" ): @@ -1294,6 +1295,7 @@ def test_shutdown_nonempty(self): for q in multiprocessing.Queue(1), multiprocessing.JoinableQueue(1): q.put("data") q.shutdown() + _wait() q.get() with self.assertRaises( pyqueue.ShutDown, msg="Didn't appear to shut-down queue" @@ -1304,6 +1306,7 @@ def test_shutdown_immediate(self): for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): q.put("data") q.shutdown(immediate=True) + _wait() with self.assertRaises( pyqueue.ShutDown, msg="Didn't appear to shut-down queue" ): From d9dbb333c29e772b55faee4e929cecfb0a97cfbc Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Mar 2023 20:02:55 +0100 Subject: [PATCH 27/62] Move some tests about shutdwon state and empty queue --- Lib/multiprocessing/queues.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 9ecf52e648eed5..0185ce0fb4283e 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -113,29 +113,24 @@ def get(self, block=True, timeout=None): if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown if block and timeout is None: + if self._shutdown_state.value == _queue_shutdown and self.empty(): + raise ShutDown with self._rlock: - if self._shutdown_state.value == _queue_shutdown_immediate or\ - (self._shutdown_state.value == _queue_shutdown and self.empty()): - raise ShutDown res = self._recv_bytes() self._sem.release() else: if block: deadline = time.monotonic() + timeout + if self._shutdown_state.value == _queue_shutdown and self.empty(): + raise ShutDown if not self._rlock.acquire(block, timeout): - if self._shutdown_state.value == _queue_shutdown: - raise ShutDown raise Empty try: if block: timeout = deadline - time.monotonic() if not self._poll(timeout): - if self._shutdown_state.value == _queue_shutdown: - raise ShutDown raise Empty elif not self._poll(): - if self._shutdown_state.value == _queue_shutdown: - raise ShutDown raise Empty res = self._recv_bytes() self._sem.release() From db6a2577721df815dd3ec47a6f70fe1d91942aef Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 7 Mar 2023 15:42:29 +0000 Subject: [PATCH 28/62] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst diff --git a/Misc/NEWS.d/next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst b/Misc/NEWS.d/next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst new file mode 100644 index 00000000000000..497462bd64162a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst @@ -0,0 +1 @@ +Add `shutdown` method to multithreading.Queue, asyncio.Queue and multiprocessing.queue.Queue and multiprocessing.queue.JoinableQueue From 01e5880f77a7950c74f1ab6f3e7ae8e1bebb380d Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 7 Mar 2023 16:44:59 +0100 Subject: [PATCH 29/62] Update 2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst --- .../next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst b/Misc/NEWS.d/next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst index 497462bd64162a..61af8d77ae94ee 100644 --- a/Misc/NEWS.d/next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst +++ b/Misc/NEWS.d/next/Library/2023-03-07-15-42-27.gh-issue-96471.oWZtwQ.rst @@ -1 +1 @@ -Add `shutdown` method to multithreading.Queue, asyncio.Queue and multiprocessing.queue.Queue and multiprocessing.queue.JoinableQueue +Add "shutdown" method to multithreading.Queue, asyncio.Queue and multiprocessing.queue.Queue and multiprocessing.queue.JoinableQueue From 37da705ab8eb9da19ae3abce2a4a55a59ed9ea7e Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 7 Mar 2023 17:59:37 +0100 Subject: [PATCH 30/62] Update test _shutdown_all_methods --- Lib/test/_test_multiprocessing.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 878250b768b5c8..e8d9ac0bbdfb5f 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1397,6 +1397,12 @@ def _shutdown_all_methods(self, immediate): with self.assertRaises(pyqueue.ShutDown): q.get(True, 1.0) + def test_shutdown_all_methods(self): + return self._shutdown_all_methods(False) + + def test_shutdown_immediate_all_methods(self): + return self._shutdown_all_methods(True) + # # # From 795fb2d01061262d009c86f5f45415fb904b326d Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 8 Mar 2023 15:57:48 +0100 Subject: [PATCH 31/62] Fix some bugs, Refactoring code --- Lib/asyncio/queues.py | 4 ++-- Lib/queue.py | 20 +++++--------------- Lib/test/test_queue.py | 17 +---------------- 3 files changed, 8 insertions(+), 33 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 095f8ad0d2c3a7..da8cdd49feb897 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -276,9 +276,9 @@ def shutdown(self, immediate=False): getter = self._getters.popleft() if not getter.done(): getter.set_result(None) - # Release 'blocked' tasks/coros via `.join()` + # Release all 'blocked' tasks/coros in `join()` self._finished.set() - elif self._shutdown_state is _QueueState.ALIVE: # here + else: self._shutdown_state = _QueueState.SHUTDOWN while self._putters: putter = self._putters.popleft() diff --git a/Lib/queue.py b/Lib/queue.py index ca1dfeb1ded41c..031f994d75ec42 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -193,33 +193,27 @@ def get(self, block=True, timeout=None): # here `self.not_empty` uses `self.mutex` if self.shutdown_state == _queue_shutdown_immediate: raise ShutDown + elif self.shutdown_state == _queue_shutdown and not self._qsize(): + raise ShutDown if not block: if not self._qsize(): - if self.shutdown_state != _queue_alive: - raise ShutDown raise Empty elif timeout is None: while not self._qsize(): - if self.shutdown_state != _queue_alive: - raise ShutDown self.not_empty.wait() - if self.shutdown_state != _queue_alive: + if self.shutdown_state == _queue_shutdown_immediate: raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = time() + timeout while not self._qsize(): - if self.shutdown_state != _queue_alive: - raise ShutDown remaining = endtime - time() if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) - if self.shutdown_state != _queue_alive: + if self.shutdown_state == _queue_shutdown_immediate: raise ShutDown - if self.shutdown_state == _queue_shutdown_immediate: - raise ShutDown item = self._get() self.not_full.notify() return item @@ -252,14 +246,10 @@ def shutdown(self, immediate=False): with self.mutex: if self.shutdown_state is _queue_shutdown_immediate: return - if immediate: self.shutdown_state = _queue_shutdown_immediate self.not_empty.notify_all() - # set self.unfinished_tasks to 0 - # to break the loop in 'self.join()' - # when quits from `wait()` - self.unfinished_tasks = 0 + # release all blocked threads in `join()` self.all_tasks_done.notify_all() else: self.shutdown_state = _queue_shutdown diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 6b65d2a0f1169f..00c18e3bc68664 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -277,17 +277,12 @@ def test_shutdown_allowed_transitions(self): q.shutdown(immediate=False) self.assertNotEqual("shutdown", q.shutdown_state) - @staticmethod - def _wait(): - time.sleep(0.01) def _shutdown_all_methods(self, immediate): q = self.type2test(2) q.put("L") - self._wait() q.put_nowait("O") q.shutdown(immediate) - self._wait() with self.assertRaises(self.queue.ShutDown): q.put("E") @@ -305,10 +300,8 @@ def _shutdown_all_methods(self, immediate): else: self.assertIn(q.get(), "LO") q.task_done() - self._wait() self.assertIn(q.get(), "LO") q.task_done() - self._wait() q.join() # on shutdown(immediate=False) # when queue is empty, should raise ShutDown Exception @@ -399,7 +392,6 @@ def _shutdown_get(self, immediate): for func, params in thrds: threads.append(threading.Thread(target=func, args=params)) threads[-1].start() - self._wait() q.shutdown(immediate) go.set() for t in threads: @@ -431,7 +423,6 @@ def _shutdown_put(self, immediate): for func, params in thrds: threads.append(threading.Thread(target=func, args=params)) threads[-1].start() - self._wait() q.shutdown() go.set() for t in threads: @@ -466,13 +457,11 @@ def _shutdown_join(self, immediate): for func, params in thrds: threads.append(threading.Thread(target=func, args=params)) threads[-1].start() - self._wait() if not immediate: res = [] for i in range(nb): threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) threads[-1].start() - self._wait() q.shutdown(immediate) go.set() for t in threads: @@ -492,7 +481,7 @@ def _shutdown_put_join(self, immediate): go = threading.Event() q.put("Y") nb = q.qsize() - # queue fulled + # queue not fulled if immediate: thrds = ( @@ -508,21 +497,18 @@ def _shutdown_put_join(self, immediate): for func, params in thrds: threads.append(threading.Thread(target=func, args=params)) threads[-1].start() - self._wait() if not immediate: self.assertEqual(q.unfinished_tasks, nb) for i in range(nb): t = threading.Thread(target=q.task_done) t.start() threads.append(t) - self._wait() go.set() q.shutdown(immediate) for t in threads: t.join() self.assertEqual(results, [True]*len(thrds)) - self.assertEqual(q.unfinished_tasks, 0) def test_shutdown_immediate_put_join(self): return self._shutdown_put_join(True) @@ -548,7 +534,6 @@ def test_shutdown_get_task_done_join(self): for func, params in thrds: threads.append(threading.Thread(target=func, args=params)) threads[-1].start() - self._wait() go.set() q.shutdown(False) for t in threads: From 99eadd7996b74d83e25c1dff4ab4e5ca18a8c45f Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 15 Mar 2023 17:56:46 +0100 Subject: [PATCH 32/62] Update test names Add new tests --- Lib/test/_test_multiprocessing.py | 140 ++++++++++++++++++++++++++- Lib/test/test_asyncio/test_queues.py | 10 +- Lib/test/test_queue.py | 111 ++++++++++++++++++++- 3 files changed, 246 insertions(+), 15 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e8d9ac0bbdfb5f..37852bd5fab3aa 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1327,7 +1327,7 @@ def test_shutdown_allowed_transitions(self): q.shutdown(immediate=False) self.assertNotEqual(1, q._shutdown_state.value) - def _shutdown_all_methods(self, immediate): + def _shutdown_all_methods_in_one_process(self, immediate): # part 1: Queue q = multiprocessing.Queue(2) q.put("L") @@ -1397,11 +1397,141 @@ def _shutdown_all_methods(self, immediate): with self.assertRaises(pyqueue.ShutDown): q.get(True, 1.0) - def test_shutdown_all_methods(self): - return self._shutdown_all_methods(False) + def test_shutdown_all_methods_in_one_process(self): + return self._shutdown_all_methods_in_one_process(False) + + def test_shutdown_immediate_all_methods_in_one_process(self): + return self._shutdown_all_methods_in_one_process(True) + + @classmethod + def _write_msg_process(cls, q, n, results, delay, + i_when_exec_shutdown, + event_start, event_end): + event_start.wait() + for i in range(1, n+1): + try: + q.put((i, "YDLO")) + results.append(True) + except pyqueue.ShutDown: + results.append(False) + # triggers shutdown of queue + if i == i_when_exec_shutdown: + event_end.set() + time.sleep(delay) + # end of all puts + if isinstance(q, type(multiprocessing.JoinableQueue())): + try: + q.join() + except pyqueue.ShutDown: + pass + + @classmethod + def _read_msg_process(cls, q, nb, results, delay, event_start): + event_start.wait() + block = True + while nb: + time.sleep(delay) + try: + # Get at least one message + q.get(block) + block = False + if isinstance(q, type(multiprocessing.JoinableQueue())): + q.task_done() + results.append(True) + nb -= 1 + except pyqueue.ShutDown: + results.append(False) + nb -= 1 + except pyqueue.Empty: + pass + # end of all gets + if isinstance(q, type(multiprocessing.JoinableQueue())): + try: + q.join() + except pyqueue.ShutDown: + pass + + @classmethod + def _shutdown_process(cls, q, event_end, immediate): + event_end.wait() + q.shutdown(immediate) + if isinstance(q, type(multiprocessing.JoinableQueue())): + try: + q.join() + except pyqueue.ShutDown: + pass + + @classmethod + def _join_process(cls, q, delay, event_start): + event_start.wait() + time.sleep(delay) + try: + q.join() + except pyqueue.ShutDown: + pass + + #@classmethod + def _shutdown_all_methods_in_many_processes(self, immediate): + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + ps = [] + ev_start = multiprocessing.Event() + ev_exec_shutdown = multiprocessing.Event() + m = multiprocessing.Manager() + res_puts = m.list() + res_gets = m.list() + delay = 1e-4 + read_process = 4 + nb_msgs = read_process * 16 + nb_msgs_r = nb_msgs // read_process + when_exec_shutdown = nb_msgs // 2 + if isinstance(q, type(multiprocessing.Queue())): + lprocs = ( + (self._write_msg_process, 1, (q, nb_msgs, res_puts, delay, + when_exec_shutdown, + ev_start, ev_exec_shutdown)), + (self._read_msg_process, read_process, (q, nb_msgs_r, + res_gets, delay*2, + ev_start)), + (self._shutdown_process, 1, (q, ev_exec_shutdown, immediate)), + ) + else: + # add 2 self._join process processes + lprocs = ( + (self._write_msg_process, 1, (q, nb_msgs, res_puts, delay, + when_exec_shutdown, + ev_start, ev_exec_shutdown)), + (self._read_msg_process, read_process, (q, nb_msgs_r, + res_gets, delay*2, + ev_start)), + (self._join_process, 2, (q, delay*2, ev_start)), + (self._shutdown_process, 1, (q, ev_exec_shutdown, immediate)), + ) + # start all processes + for func, n, args in lprocs: + for i in range(n): + ps.append(multiprocessing.Process(target=func, args=args)) + ps[-1].start() + # set event in order to run q.shutdown() + ev_start.set() + _wait() + # wait + if isinstance(q, type(multiprocessing.Queue())): + for p in ps: + p.join() + + if not immediate: + assert(len(res_gets) == len(res_puts)) + assert(res_gets.count(True) == res_puts.count(True)) + else: + assert(len(res_gets) <= len(res_puts)) + assert(res_gets.count(True) <= res_puts.count(True)) + + def test_shutdown_all_methods_in_many_processes(self): + return self._shutdown_all_methods_in_many_processes(False) + + def test_shutdown_immediate_all_methods_in_many_processes(self): + return self._shutdown_all_methods_in_many_processes(True) - def test_shutdown_immediate_all_methods(self): - return self._shutdown_all_methods(True) # # diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 1403b9bd2cd8b2..8316f6cf752da5 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -654,7 +654,7 @@ async def test_shutdown_allowed_transitions(self): q.shutdown(immediate=False) self.assertNotEqual("shutdown", q._shutdown_state.value) - async def _shutdown_all_methods(self, immediate): + async def _shutdown_all_methods_in_one_task(self, immediate): q = asyncio.Queue() await q.put("L") q.put_nowait("O") @@ -686,11 +686,11 @@ async def _shutdown_all_methods(self, immediate): with self.assertRaises(asyncio.QueueShutDown): q.get_nowait() - async def test_shutdown_all_methods(self): - return await self._shutdown_all_methods(False) + async def test_shutdown_all_methods_in_one_task(self): + return await self._shutdown_all_methods_in_one_task(False) - async def test_shutdown_immediate_get(self): - return await self._shutdown_all_methods(True) + async def test_shutdown_immediate_all_methods_in_one_task(self): + return await self._shutdown_all_methods_in_one_task(True) async def _shutdown_putters(self, immediate): delay = self.delay diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 00c18e3bc68664..d9e840a7c861ed 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -278,7 +278,7 @@ def test_shutdown_allowed_transitions(self): q.shutdown(immediate=False) self.assertNotEqual("shutdown", q.shutdown_state) - def _shutdown_all_methods(self, immediate): + def _shutdown_all_methods_in_one_thread(self, immediate): q = self.type2test(2) q.put("L") q.put_nowait("O") @@ -312,11 +312,112 @@ def _shutdown_all_methods(self, immediate): with self.assertRaises(self.queue.ShutDown): q.get(True, 1.0) - def test_shutdown_all_methods(self): - return self._shutdown_all_methods(False) + def test_shutdown_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(False) - def test_shutdown_immediate_get(self): - return self._shutdown_all_methods(True) + def test_shutdown_immediate_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(True) + + def _write_msg_thread(self, q, n, results, delay, + i_when_exec_shutdown, + event_start, event_end): + event_start.wait() + for i in range(1, n+1): + try: + q.put((i, "YDLO")) + results.append(True) + except self.queue.ShutDown: + results.append(False) + # triggers shutdown of queue + if i == i_when_exec_shutdown: + event_end.set() + time.sleep(delay) + # end of all puts + try: + q.join() + except self.queue.ShutDown: + pass + + def _read_msg_thread(self, q, nb, results, delay, event_start): + event_start.wait() + block = True + while nb: + time.sleep(delay) + try: + # Get at least one message + q.get(block) + block = False + q.task_done() + results.append(True) + nb -= 1 + except self.queue.ShutDown: + results.append(False) + nb -= 1 + except self.queue.Empty: + pass + try: + q.join() + except self.queue.ShutDown: + pass + + def _shutdown_thread(self, q, event_end, immediate): + event_end.wait() + q.shutdown(immediate) + try: + q.join() + except self.queue.ShutDown: + pass + + def _join_thread(self, q, delay, event_start): + event_start.wait() + time.sleep(delay) + try: + q.join() + except self.queue.ShutDown: + pass + + def _shutdown_all_methods_in_many_threads(self, immediate): + q = self.type2test() + ps = [] + ev_start = threading.Event() + ev_exec_shutdown = threading.Event() + res_puts = [] + res_gets = [] + delay = 1e-4 + read_process = 4 + nb_msgs = read_process * 16 + nb_msgs_r = nb_msgs // read_process + when_exec_shutdown = nb_msgs // 2 + lprocs = ( + (self._write_msg_thread, 1, (q, nb_msgs, res_puts, delay, + when_exec_shutdown, + ev_start, ev_exec_shutdown)), + (self._read_msg_thread, read_process, (q, nb_msgs_r, + res_gets, delay*2, + ev_start)), + (self._join_thread, 2, (q, delay*2, ev_start)), + (self._shutdown_thread, 1, (q, ev_exec_shutdown, immediate)), + ) + # start all threds + for func, n, args in lprocs: + for i in range(n): + ps.append(threading.Thread(target=func, args=args)) + ps[-1].start() + # set event in order to run q.shutdown() + ev_start.set() + + if not immediate: + assert(len(res_gets) == len(res_puts)) + assert(res_gets.count(True) == res_puts.count(True)) + else: + assert(len(res_gets) <= len(res_puts)) + assert(res_gets.count(True) <= res_puts.count(True)) + + def test_shutdown_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(False) + + def test_shutdown_immediate_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(True) def _get(self, q, go, results, shutdown=False): go.wait() From 670d864f69aec7755b789d70367e2902fbaf02c6 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 15 Mar 2023 17:57:15 +0100 Subject: [PATCH 33/62] Refactoring and fix minor bugs --- Lib/multiprocessing/queues.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index ebca57aa56a560..175bb52d920a06 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -56,8 +56,9 @@ def __init__(self, maxsize=0, *, ctx): self._ignore_epipe = False self._reset() self._shutdown_state = context._default_context.Value( - ctypes.c_uint8, _queue_alive, lock=True - ) + ctypes.c_uint8, _queue_alive, + lock=True + ) if sys.platform != 'win32': register_after_fork(self, Queue._after_fork) @@ -112,17 +113,15 @@ def get(self, block=True, timeout=None): raise ValueError(f"Queue {self!r} is closed") if self._shutdown_state.value == _queue_shutdown_immediate: raise ShutDown + elif self._shutdown_state.value == _queue_shutdown and self.empty(): + raise ShutDown if block and timeout is None: - if self._shutdown_state.value == _queue_shutdown and self.empty(): - raise ShutDown with self._rlock: res = self._recv_bytes() self._sem.release() else: if block: deadline = time.monotonic() + timeout - if self._shutdown_state.value == _queue_shutdown and self.empty(): - raise ShutDown if not self._rlock.acquire(block, timeout): raise Empty try: @@ -136,8 +135,8 @@ def get(self, block=True, timeout=None): self._sem.release() finally: self._rlock.release() - if self._shutdown_state.value == _queue_shutdown_immediate: - raise ShutDown + if self._shutdown_state.value == _queue_shutdown_immediate: + raise ShutDown # unserialize the data after having released the lock return _ForkingPickler.loads(res) @@ -158,6 +157,8 @@ def put_nowait(self, obj): return self.put(obj, False) def shutdown(self, immediate=False): + if self._closed: + raise ValueError(f"Queue {self!r} is closed") with self._shutdown_state.get_lock(): if self._shutdown_state.value == _queue_shutdown_immediate: return @@ -372,10 +373,10 @@ def join(self): raise ShutDown def shutdown(self, immediate=False): - initial_shutdown = self._shutdown_state.value - super().shutdown(immediate) - if initial_shutdown == _queue_alive: - with self._cond: + with self._cond: + initial_shutdown = self._shutdown_state.value + super().shutdown(immediate) + if initial_shutdown == _queue_alive: self._cond.notify_all() # here to check YD # From 6af0e8e5df025e24327f219d5ed8dab3f0c2a869 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 15 Mar 2023 18:16:38 +0100 Subject: [PATCH 34/62] suppress import enum --- Lib/queue.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/queue.py b/Lib/queue.py index 031f994d75ec42..7c5fb480ef2964 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -1,6 +1,5 @@ '''A multi-producer, multi-consumer queue.''' -import enum import threading import types from collections import deque From a3e03c52faeae6d32a7d04a616bc21da10916b75 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 17 Mar 2023 16:13:16 +0100 Subject: [PATCH 35/62] update docstrings --- Lib/asyncio/queues.py | 31 +++++++++++++++++++++++++++---- Lib/queue.py | 19 ++++++++++++++----- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index da8cdd49feb897..227e8e6b3c15e0 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -134,6 +134,8 @@ async def put(self, item): Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. + + Raise a QueueShutDown if _shutdown_state is not set to `ALIVE`. """ if self._shutdown_state is not _QueueState.ALIVE: raise QueueShutDown @@ -164,6 +166,9 @@ def put_nowait(self, item): """Put an item into the queue without blocking. If no free slot is immediately available, raise QueueFull. + + Raise a QueueShutDown if _shutdown_state is not set to + `ALIVE`. """ if self._shutdown_state is not _QueueState.ALIVE: raise QueueShutDown @@ -178,11 +183,17 @@ async def get(self): """Remove and return an item from the queue. If queue is empty, wait until an item is available. + + Raise a QueueShutDown if _shutdown_state is set to + `SHUTDOWN_IMMEDIATE`. + + Raise a QueueShutDown if _shutdown_state is set to + `SHUTDOWN` and queue is empty. """ if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: raise QueueShutDown while self.empty(): - if self._shutdown_state is not _QueueState.ALIVE: + if self._shutdown_state is _QueueState.SHUTDOWN: raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) @@ -210,13 +221,19 @@ def get_nowait(self): """Remove and return an item from the queue. Return an item if one is immediately available, else raise QueueEmpty. + + Raise a QueueShutDown if _shutdown_state is set to + `SHUTDOWN_IMMEDIATE`. + + Raise a QueueShutDown if _shutdown_state is set to + `SHUTDOWN` and queue is empty. """ + if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + raise QueueShutDown if self.empty(): - if self._shutdown_state is not _QueueState.ALIVE: + if self._shutdown_state is _QueueState.SHUTDOWN: raise QueueShutDown raise QueueEmpty - elif self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: - raise QueueShutDown item = self._get() self._wakeup_next(self._putters) return item @@ -234,6 +251,9 @@ def task_done(self): Raises ValueError if called more times than there were items placed in the queue. + + Raise a QueueShutDown if _shutdown_state is set to + `SHUTDOWN_IMMEDIATE`. """ if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: raise QueueShutDown @@ -250,6 +270,9 @@ async def join(self): queue. The count goes down whenever a consumer calls task_done() to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. + + Raise a QueueShutDown if _shutdown_state is set to + `SHUTDOWN_IMMEDIATE`. """ if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: raise QueueShutDown diff --git a/Lib/queue.py b/Lib/queue.py index 7c5fb480ef2964..c61dff12eec511 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -62,7 +62,7 @@ def __init__(self, maxsize=0): self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 - # Queue shut-down state + # Queue shutdown state self.shutdown_state = _queue_alive def task_done(self): @@ -78,9 +78,10 @@ def task_done(self): Raises a ValueError if called more times than there were items placed in the queue. + Raises a ShutDown if shutdown_state attribute is set to + `_queue_shutdown_immediate`. ''' with self.all_tasks_done: - # here `self.all_task_done` uses `self.mutex` if self.shutdown_state == _queue_shutdown_immediate: raise ShutDown unfinished = self.unfinished_tasks - 1 @@ -98,9 +99,11 @@ def join(self): to indicate the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. + + Raises a ShutDown if shutdown_state attribute is set to + `_queue_shutdown_immediate`. ''' with self.all_tasks_done: - # here `self.all_task_done` uses `self.mutex` if self.shutdown_state == _queue_shutdown_immediate: raise ShutDown while self.unfinished_tasks: @@ -148,9 +151,11 @@ def put(self, item, block=True, timeout=None): Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). + + Raises a ShutDown if shutdown_state attribute is not set to + `_queue_alive`. ''' with self.not_full: - # here `self.not_full` uses `self.mutex`` if self.shutdown_state != _queue_alive: raise ShutDown if self.maxsize > 0: @@ -187,9 +192,13 @@ def get(self, block=True, timeout=None): Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). + + Raises a ShutDown if shutdown_state attribute is set to + `_queue_shutdown` and queue is empty. + Raises a ShutDown if shutdown_state attribute is set to + `_queue_shutdown_immediate` ''' with self.not_empty: - # here `self.not_empty` uses `self.mutex` if self.shutdown_state == _queue_shutdown_immediate: raise ShutDown elif self.shutdown_state == _queue_shutdown and not self._qsize(): From bc30db7250bb8b0a8dc53afe3d8f1a84ddaf77cd Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 17 Mar 2023 16:18:48 +0100 Subject: [PATCH 36/62] Suppress `import ctypes`, causes no necessary uses (and moreover crashes tests in ubuntu OS) --- Lib/multiprocessing/queues.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 175bb52d920a06..68ae963157d291 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -17,7 +17,6 @@ import types import weakref import errno -import ctypes from queue import Empty, Full, ShutDown @@ -56,7 +55,7 @@ def __init__(self, maxsize=0, *, ctx): self._ignore_epipe = False self._reset() self._shutdown_state = context._default_context.Value( - ctypes.c_uint8, _queue_alive, + 'i', _queue_alive, lock=True ) From 43824091b90fff5a2ad18f63be8a07df072a1829 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 22 Mar 2023 19:04:49 +0100 Subject: [PATCH 37/62] fix segmentation fault: use ctx.Value, in replacement of context._default_context.Value --- Lib/multiprocessing/queues.py | 51 +++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 68ae963157d291..9b181b1939bcb2 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -37,7 +37,7 @@ # class Queue(object): - + _VALUE=True def __init__(self, maxsize=0, *, ctx): if maxsize <= 0: # Can raise ImportError (see issues #3770 and #23400) @@ -54,10 +54,7 @@ def __init__(self, maxsize=0, *, ctx): # For use by concurrent.futures self._ignore_epipe = False self._reset() - self._shutdown_state = context._default_context.Value( - 'i', _queue_alive, - lock=True - ) + self._shutdown_state = ctx.Value('i', _queue_alive) if sys.platform != 'win32': register_after_fork(self, Queue._after_fork) @@ -93,10 +90,25 @@ def _reset(self, after_fork=False): self._recv_bytes = self._reader.recv_bytes self._poll = self._reader.poll + def _is_alive(self): + return self._shutdown_state.value == _queue_alive + + def _is_shutdown(self): + return self._shutdown_state.value == _queue_shutdown + + def _is_shutdown_immediate(self): + return self._shutdown_state.value == _queue_shutdown_immediate + + def _set_shutdown(self): + self._shutdown_state.value = _queue_shutdown + + def _set_shutdown_immediate(self): + self._shutdown_state.value = _queue_shutdown_immediate + def put(self, obj, block=True, timeout=None): if self._closed: raise ValueError(f"Queue {self!r} is closed") - if self._shutdown_state.value != _queue_alive: + if not self._is_alive(): raise ShutDown if not self._sem.acquire(block, timeout): raise Full @@ -110,9 +122,8 @@ def put(self, obj, block=True, timeout=None): def get(self, block=True, timeout=None): if self._closed: raise ValueError(f"Queue {self!r} is closed") - if self._shutdown_state.value == _queue_shutdown_immediate: - raise ShutDown - elif self._shutdown_state.value == _queue_shutdown and self.empty(): + if self._is_shutdown_immediate() or\ + (self._is_shutdown() and self.empty()): raise ShutDown if block and timeout is None: with self._rlock: @@ -134,7 +145,7 @@ def get(self, block=True, timeout=None): self._sem.release() finally: self._rlock.release() - if self._shutdown_state.value == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): raise ShutDown # unserialize the data after having released the lock return _ForkingPickler.loads(res) @@ -159,14 +170,14 @@ def shutdown(self, immediate=False): if self._closed: raise ValueError(f"Queue {self!r} is closed") with self._shutdown_state.get_lock(): - if self._shutdown_state.value == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): return if immediate: - self._shutdown_state.value = _queue_shutdown_immediate + self._set_shutdown_immediate() with self._notempty: - self._notempty.notify_all() # cf from @EpicWink + self._notempty.notify_all() else: - self._shutdown_state.value = _queue_shutdown + self._set_shutdown() def close(self): self._closed = True @@ -341,7 +352,7 @@ def __setstate__(self, state): def put(self, obj, block=True, timeout=None): if self._closed: raise ValueError(f"Queue {self!r} is closed") - if self._shutdown_state.value != _queue_alive: + if not self._is_alive(): raise ShutDown if not self._sem.acquire(block, timeout): raise Full @@ -355,7 +366,7 @@ def put(self, obj, block=True, timeout=None): def task_done(self): with self._cond: - if self._shutdown_state.value == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): raise ShutDown if not self._unfinished_tasks.acquire(False): raise ValueError('task_done() called too many times') @@ -364,18 +375,18 @@ def task_done(self): def join(self): with self._cond: - if self._shutdown_state.value == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): raise ShutDown if not self._unfinished_tasks._semlock._is_zero(): self._cond.wait() - if self._shutdown_state.value == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): raise ShutDown def shutdown(self, immediate=False): with self._cond: - initial_shutdown = self._shutdown_state.value + is_alive = self._is_alive() super().shutdown(immediate) - if initial_shutdown == _queue_alive: + if is_alive: self._cond.notify_all() # here to check YD # From aa70dc2679ad2449a6576d928730992d643776e9 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 22 Mar 2023 19:05:34 +0100 Subject: [PATCH 38/62] fix segmentation fault: use ctx.Value, in replacement of context._default_context.Value --- Lib/multiprocessing/queues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 9b181b1939bcb2..98d332cb8e90bb 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -37,7 +37,7 @@ # class Queue(object): - _VALUE=True + def __init__(self, maxsize=0, *, ctx): if maxsize <= 0: # Can raise ImportError (see issues #3770 and #23400) From cbfd7718c1c747a004d79ea59b7a2609841548f3 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 22 Mar 2023 21:17:49 +0100 Subject: [PATCH 39/62] Add private method about shutdown_state Fix bugs --- Lib/queue.py | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index c61dff12eec511..2c47ff022d5aab 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -78,11 +78,12 @@ def task_done(self): Raises a ValueError if called more times than there were items placed in the queue. + Raises a ShutDown if shutdown_state attribute is set to `_queue_shutdown_immediate`. ''' with self.all_tasks_done: - if self.shutdown_state == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): raise ShutDown unfinished = self.unfinished_tasks - 1 if unfinished <= 0: @@ -104,11 +105,11 @@ def join(self): `_queue_shutdown_immediate`. ''' with self.all_tasks_done: - if self.shutdown_state == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): raise ShutDown while self.unfinished_tasks: self.all_tasks_done.wait() - if self.shutdown_state == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): raise ShutDown def qsize(self): @@ -156,7 +157,7 @@ def put(self, item, block=True, timeout=None): `_queue_alive`. ''' with self.not_full: - if self.shutdown_state != _queue_alive: + if not self._is_alive(): raise ShutDown if self.maxsize > 0: if not block: @@ -165,7 +166,7 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() - if self.shutdown_state != _queue_alive: + if not self._is_alive(): raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") @@ -176,7 +177,7 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) - if self.shutdown_state != _queue_alive: + if not self._is_alive(): raise ShutDown self._put(item) self.unfinished_tasks += 1 @@ -195,13 +196,13 @@ def get(self, block=True, timeout=None): Raises a ShutDown if shutdown_state attribute is set to `_queue_shutdown` and queue is empty. + Raises a ShutDown if shutdown_state attribute is set to `_queue_shutdown_immediate` ''' with self.not_empty: - if self.shutdown_state == _queue_shutdown_immediate: - raise ShutDown - elif self.shutdown_state == _queue_shutdown and not self._qsize(): + if self._is_shutdown_immediate() or\ + (self._is_shutdown() and not self._qsize()): raise ShutDown if not block: if not self._qsize(): @@ -209,7 +210,7 @@ def get(self, block=True, timeout=None): elif timeout is None: while not self._qsize(): self.not_empty.wait() - if self.shutdown_state == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") @@ -220,7 +221,7 @@ def get(self, block=True, timeout=None): if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) - if self.shutdown_state == _queue_shutdown_immediate: + if self._is_shutdown_immediate(): raise ShutDown item = self._get() self.not_full.notify() @@ -252,17 +253,33 @@ def shutdown(self, immediate=False): and join() if 'immediate'. The ShutDown exception is raised. ''' with self.mutex: - if self.shutdown_state is _queue_shutdown_immediate: + if self._is_shutdown_immediate(): return if immediate: - self.shutdown_state = _queue_shutdown_immediate + self._set_shutdown_immediate() self.not_empty.notify_all() # release all blocked threads in `join()` self.all_tasks_done.notify_all() else: - self.shutdown_state = _queue_shutdown + self._set_shutdown() self.not_full.notify_all() + def _is_alive(self): + return self.shutdown_state == _queue_alive + + def _is_shutdown(self): + return self.shutdown_state == _queue_shutdown + + def _is_shutdown_immediate(self): + return self.shutdown_state == _queue_shutdown_immediate + + def _set_shutdown(self): + self.shutdown_state = _queue_shutdown + + def _set_shutdown_immediate(self): + self.shutdown_state = _queue_shutdown_immediate + + # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held From 0d095bc577df2aadd33d60e462e429b100403520 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 24 Mar 2023 11:31:26 +0100 Subject: [PATCH 40/62] Add private methods to check _shutdown_state attr Update tests --- Lib/asyncio/queues.py | 53 ++++++++++++++++++---------- Lib/test/test_asyncio/test_queues.py | 8 +++-- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 227e8e6b3c15e0..346cf72cd42451 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -101,8 +101,8 @@ def _format(self): result += f' _putters[{len(self._putters)}]' if self._unfinished_tasks: result += f' tasks={self._unfinished_tasks}' - if self._shutdown_state is not _QueueState.ALIVE: - result += f' shutdown={self._shutdown_state.value}' + if not self._is_alive(): + result += f' state={self._shutdown_state.value}' return result def qsize(self): @@ -137,7 +137,7 @@ async def put(self, item): Raise a QueueShutDown if _shutdown_state is not set to `ALIVE`. """ - if self._shutdown_state is not _QueueState.ALIVE: + if not self._is_alive(): raise QueueShutDown while self.full(): putter = self._get_loop().create_future() @@ -151,14 +151,15 @@ async def put(self, item): self._putters.remove(putter) except ValueError: # The putter could be removed from self._putters by a - # previous get_nowait call. + # previous get_nowait call, + # or a shutdown call. pass if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take # the call. Wake up the next in line. self._wakeup_next(self._putters) raise - if self._shutdown_state is not _QueueState.ALIVE: + if not self._is_alive(): raise QueueShutDown return self.put_nowait(item) @@ -170,7 +171,7 @@ def put_nowait(self, item): Raise a QueueShutDown if _shutdown_state is not set to `ALIVE`. """ - if self._shutdown_state is not _QueueState.ALIVE: + if not self._is_alive(): raise QueueShutDown if self.full(): raise QueueFull @@ -190,10 +191,10 @@ async def get(self): Raise a QueueShutDown if _shutdown_state is set to `SHUTDOWN` and queue is empty. """ - if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + if self._is_shutdown_immediate(): raise QueueShutDown while self.empty(): - if self._shutdown_state is _QueueState.SHUTDOWN: + if self._is_shutdown(): raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) @@ -206,14 +207,15 @@ async def get(self): self._getters.remove(getter) except ValueError: # The getter could be removed from self._getters by a - # previous put_nowait call. + # previous put_nowait call, + # or a shutdown call. pass if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take # the call. Wake up the next in line. self._wakeup_next(self._getters) raise - if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + if self._is_shutdown_immediate(): raise QueueShutDown return self.get_nowait() @@ -228,10 +230,10 @@ def get_nowait(self): Raise a QueueShutDown if _shutdown_state is set to `SHUTDOWN` and queue is empty. """ - if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + if self._is_shutdown_immediate(): raise QueueShutDown if self.empty(): - if self._shutdown_state is _QueueState.SHUTDOWN: + if self._is_shutdown(): raise QueueShutDown raise QueueEmpty item = self._get() @@ -255,7 +257,7 @@ def task_done(self): Raise a QueueShutDown if _shutdown_state is set to `SHUTDOWN_IMMEDIATE`. """ - if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + if self._is_shutdown_immediate(): raise QueueShutDown if self._unfinished_tasks <= 0: raise ValueError('task_done() called too many times') @@ -274,11 +276,11 @@ async def join(self): Raise a QueueShutDown if _shutdown_state is set to `SHUTDOWN_IMMEDIATE`. """ - if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + if self._is_shutdown_immediate(): raise QueueShutDown if self._unfinished_tasks > 0: await self._finished.wait() - if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + if self._is_shutdown_immediate(): raise QueueShutDown def shutdown(self, immediate=False): @@ -290,11 +292,11 @@ def shutdown(self, immediate=False): All blocked callers of put() will be unblocked, and also get() and join() if 'immediate'. The QueueShutDown exception is raised. """ - if self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE: + if self._is_shutdown_immediate(): return # here _shutdown_state is ALIVE or SHUTDOWN if immediate: - self._shutdown_state = _QueueState.SHUTDOWN_IMMEDIATE + self._set_shutdown_immediate() while self._getters: getter = self._getters.popleft() if not getter.done(): @@ -302,12 +304,27 @@ def shutdown(self, immediate=False): # Release all 'blocked' tasks/coros in `join()` self._finished.set() else: - self._shutdown_state = _QueueState.SHUTDOWN + self._set_shutdown() while self._putters: putter = self._putters.popleft() if not putter.done(): putter.set_result(None) + def _is_alive(self): + return self._shutdown_state is _QueueState.ALIVE + + def _is_shutdown(self): + return self._shutdown_state is _QueueState.SHUTDOWN + + def _is_shutdown_immediate(self): + return self._shutdown_state is _QueueState.SHUTDOWN_IMMEDIATE + + def _set_shutdown(self): + self._shutdown_state = _QueueState.SHUTDOWN + + def _set_shutdown_immediate(self): + self._shutdown_state = _QueueState.SHUTDOWN_IMMEDIATE + class PriorityQueue(Queue): """A subclass of Queue; retrieves entries in priority order (lowest first). diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 8316f6cf752da5..bf4a5a78a8f0cc 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -630,13 +630,15 @@ async def test_shutdown_immediate(self): await q.get() async def test_shutdown_repr(self): - q = self.q_class() + q = self.q_class(4) + # when alive, not in repr self.assertNotIn("alive", repr(q)) - q.shutdown() + q = self.q_class(6) + q.shutdown(immediate=False) self.assertIn("shutdown", repr(q)) - q = self.q_class() + q = self.q_class(8) q.shutdown(immediate=True) self.assertIn("shutdown-immediate", repr(q)) From 891cffe5cf1ffa1ec01ae0856aa94d1170a64bb7 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Tue, 28 Mar 2023 19:54:16 +1000 Subject: [PATCH 41/62] Update docs for queue shutdown --- Doc/library/asyncio-queue.rst | 32 +++++++++++++++++++++++++++++ Doc/library/multiprocessing.rst | 24 ++++++++++++++++++++-- Doc/library/queue.rst | 36 +++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d86fbc21351e2d..008a7f908f4fbc 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -62,6 +62,9 @@ Queue Remove and return an item from the queue. If queue is empty, wait until an item is available. + Raises :exc:`QueueShutDown` if the queue has been shut down and + is empty, or if the queue has been shut down immediately. + .. method:: get_nowait() Return an item if one is immediately available, else raise @@ -77,11 +80,16 @@ Queue work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. + Raises :exc:`QueueShutDown` if the queue has been shut down + immediately. + .. coroutinemethod:: put(item) Put an item into the queue. If the queue is full, wait until a free slot is available before adding the item. + Raises :exc:`QueueShutDown` if the queue has been shut down. + .. method:: put_nowait(item) Put an item into the queue without blocking. @@ -92,6 +100,19 @@ Queue Return the number of items in the queue. + .. method:: shutdown(immediate=False) + + Shut-down the queue, making queue gets and puts raise + :exc:`QueueShutDown`. + + By default, gets will only raise once the queue is empty. Set + *immediate* to true to make gets raise immediately instead. + + All blocked callers of put() will be unblocked, and also get() + and join() if *immediate* is true. + + .. versionadded:: 3.12 + .. method:: task_done() Indicate that a formerly enqueued task is complete. @@ -108,6 +129,9 @@ Queue Raises :exc:`ValueError` if called more times than there were items placed in the queue. + Raises :exc:`QueueShutDown` if the queue has been shut down + immediately. + Priority Queue ============== @@ -145,6 +169,14 @@ Exceptions on a queue that has reached its *maxsize*. +.. exception:: QueueShutDown + + Exception raised when getting an item from or putting an item onto a + queue which has been shut down. + + .. versionadded:: 3.12 + + Examples ======== diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 0ec47bb956a99e..b44db0ce6f857f 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -845,7 +845,8 @@ For an example of the usage of queues for interprocess communication see free slot was available within that time. Otherwise (*block* is ``False``), put an item on the queue if a free slot is immediately available, else raise the :exc:`queue.Full` exception (*timeout* is - ignored in that case). + ignored in that case). Raises :exc:`ShutDown` if the queue has been shut + down. .. versionchanged:: 3.8 If the queue is closed, :exc:`ValueError` is raised instead of @@ -863,7 +864,9 @@ For an example of the usage of queues for interprocess communication see it blocks at most *timeout* seconds and raises the :exc:`queue.Empty` exception if no item was available within that time. Otherwise (block is ``False``), return an item if one is immediately available, else raise the - :exc:`queue.Empty` exception (*timeout* is ignored in that case). + :exc:`queue.Empty` exception (*timeout* is ignored in that case). Raises + :exc:`queue.ShutDown` if the queue has been shut down and is empty, or if + the queue has been shut down immediately. .. versionchanged:: 3.8 If the queue is closed, :exc:`ValueError` is raised instead of @@ -873,6 +876,19 @@ For an example of the usage of queues for interprocess communication see Equivalent to ``get(False)``. + .. method:: shutdown(immediate=False) + + Shut-down the queue, making queue gets and puts raise + :exc:`queue.ShutDown`. + + By default, gets will only raise once the queue is empty. Set + *immediate* to true to make gets raise immediately instead. + + All blocked callers of put() will be unblocked, and also get() + and join() if *immediate* is true. + + .. versionadded:: 3.12 + :class:`multiprocessing.Queue` has a few additional methods not found in :class:`queue.Queue`. These methods are usually unnecessary for most code: @@ -962,6 +978,8 @@ For an example of the usage of queues for interprocess communication see Raises a :exc:`ValueError` if called more times than there were items placed in the queue. + Raises :exc:`queue.ShutDown` if the queue has been shut down immediately. + .. method:: join() @@ -973,6 +991,8 @@ For an example of the usage of queues for interprocess communication see it is complete. When the count of unfinished tasks drops to zero, :meth:`~queue.Queue.join` unblocks. + Raises :exc:`queue.ShutDown` if the queue has been shut down immediately. + Miscellaneous ~~~~~~~~~~~~~ diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index b2b787c5a8260c..d9952f1882ae05 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -93,6 +93,14 @@ The :mod:`queue` module defines the following classes and exceptions: on a :class:`Queue` object which is full. +.. exception:: ShutDown + + Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is called on + a :class:`Queue` object which has been shut down. + + .. versionadded:: 3.12 + + .. _queueobjects: Queue Objects @@ -135,6 +143,8 @@ provide the public methods described below. immediately available, else raise the :exc:`Full` exception (*timeout* is ignored in that case). + Raises :exc:`ShutDown` if the queue has been shut down. + .. method:: Queue.put_nowait(item) @@ -155,6 +165,9 @@ provide the public methods described below. an uninterruptible wait on an underlying lock. This means that no exceptions can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. + Raises :exc:`ShutDown` if the queue has been shut down and is empty, or if + the queue has been shut down immediately. + .. method:: Queue.get_nowait() @@ -177,6 +190,8 @@ fully processed by daemon consumer threads. Raises a :exc:`ValueError` if called more times than there were items placed in the queue. + Raises :exc:`ShutDown` if the queue has been shut down immediately. + .. method:: Queue.join() @@ -187,6 +202,8 @@ fully processed by daemon consumer threads. indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. + Raises :exc:`ShutDown` if the queue has been shut down immediately. + Example of how to wait for enqueued tasks to be completed:: @@ -214,6 +231,25 @@ Example of how to wait for enqueued tasks to be completed:: print('All work completed') +Terminating queues +^^^^^^^^^^^^^^^^^^ + +:class:`Queue` objects can be made to prevent further interaction by shutting +them down. + +.. method:: Queue.shutdown(immediate=False) + + Shut-down the queue, making queue gets and puts raise :exc:`ShutDown`. + + By default, gets will only raise once the queue is empty. Set + *immediate* to true to make gets raise immediately instead. + + All blocked callers of put() will be unblocked, and also get() + and join() if *immediate* is true. + + .. versionadded:: 3.12 + + SimpleQueue Objects ------------------- From e30933fa32a1d612f9a04eda180bb56e54cc5b04 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:34:50 +0200 Subject: [PATCH 42/62] Update Lib/queue.py Co-authored-by: Laurie O --- Lib/queue.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 2c47ff022d5aab..804b79f310ba8d 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -153,8 +153,7 @@ def put(self, item, block=True, timeout=None): is immediately available, else raise the Full exception ('timeout' is ignored in that case). - Raises a ShutDown if shutdown_state attribute is not set to - `_queue_alive`. + Raises ShutDown if the queue has been shut down. ''' with self.not_full: if not self._is_alive(): From de5714d1818bf87712c8c21f8113464f5f145a26 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:35:52 +0200 Subject: [PATCH 43/62] Update Lib/asyncio/queues.py Co-authored-by: Laurie O --- Lib/asyncio/queues.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 346cf72cd42451..469b51ccb56e4d 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -151,8 +151,7 @@ async def put(self, item): self._putters.remove(putter) except ValueError: # The putter could be removed from self._putters by a - # previous get_nowait call, - # or a shutdown call. + # previous get_nowait call or a shutdown call. pass if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take From da2e3c7b75de972e3eb306de386ce7bfd1c606f9 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:36:21 +0200 Subject: [PATCH 44/62] Update Lib/asyncio/queues.py Co-authored-by: Laurie O --- Lib/asyncio/queues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 469b51ccb56e4d..147ea9cbaff8e2 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -135,7 +135,7 @@ async def put(self, item): Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. - Raise a QueueShutDown if _shutdown_state is not set to `ALIVE`. + Raises QueueShutDown if the queue has been shut down. """ if not self._is_alive(): raise QueueShutDown From 50857c0a344fdae520d8e266314748c44feac73d Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:36:46 +0200 Subject: [PATCH 45/62] Update Lib/asyncio/queues.py Co-authored-by: Laurie O --- Lib/asyncio/queues.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 147ea9cbaff8e2..e68efd744973a0 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -223,11 +223,8 @@ def get_nowait(self): Return an item if one is immediately available, else raise QueueEmpty. - Raise a QueueShutDown if _shutdown_state is set to - `SHUTDOWN_IMMEDIATE`. - - Raise a QueueShutDown if _shutdown_state is set to - `SHUTDOWN` and queue is empty. + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ if self._is_shutdown_immediate(): raise QueueShutDown From 56272b95d1181e29ab996186f1f787e8836e22be Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:37:03 +0200 Subject: [PATCH 46/62] Update Lib/queue.py Co-authored-by: Laurie O --- Lib/queue.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 804b79f310ba8d..05fb788579f055 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -193,11 +193,8 @@ def get(self, block=True, timeout=None): available, else raise the Empty exception ('timeout' is ignored in that case). - Raises a ShutDown if shutdown_state attribute is set to - `_queue_shutdown` and queue is empty. - - Raises a ShutDown if shutdown_state attribute is set to - `_queue_shutdown_immediate` + Raises ShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. ''' with self.not_empty: if self._is_shutdown_immediate() or\ From db7eaff68320768422f50878b1e352c276eff994 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:37:18 +0200 Subject: [PATCH 47/62] Update Lib/queue.py Co-authored-by: Laurie O --- Lib/queue.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 05fb788579f055..3cd54d865440b5 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -101,8 +101,7 @@ def join(self): When the count of unfinished tasks drops to zero, join() unblocks. - Raises a ShutDown if shutdown_state attribute is set to - `_queue_shutdown_immediate`. + Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: if self._is_shutdown_immediate(): From 4099fb88d4ce6a4df9b80a48d602b9e26bfeef28 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:37:36 +0200 Subject: [PATCH 48/62] Update Lib/multiprocessing/queues.py Co-authored-by: Laurie O --- Lib/multiprocessing/queues.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 98d332cb8e90bb..274e6352bac40b 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -122,8 +122,10 @@ def put(self, obj, block=True, timeout=None): def get(self, block=True, timeout=None): if self._closed: raise ValueError(f"Queue {self!r} is closed") - if self._is_shutdown_immediate() or\ - (self._is_shutdown() and self.empty()): + if ( + self._is_shutdown_immediate() + or (self._is_shutdown() and self.empty()) + ): raise ShutDown if block and timeout is None: with self._rlock: From 58007dc1a2ee39cb2aaff3c5a481bc1592e34d03 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:37:48 +0200 Subject: [PATCH 49/62] Update Lib/asyncio/queues.py Co-authored-by: Laurie O --- Lib/asyncio/queues.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index e68efd744973a0..5347ad6c38bdcc 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -167,8 +167,7 @@ def put_nowait(self, item): If no free slot is immediately available, raise QueueFull. - Raise a QueueShutDown if _shutdown_state is not set to - `ALIVE`. + Raises QueueShutDown if the queue has been shut down. """ if not self._is_alive(): raise QueueShutDown From 225387f48880c9f6b56d2e42a1a6df6d80000b5b Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:38:07 +0200 Subject: [PATCH 50/62] Update Lib/asyncio/queues.py Co-authored-by: Laurie O --- Lib/asyncio/queues.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 5347ad6c38bdcc..14f0e3fd519d41 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -183,11 +183,8 @@ async def get(self): If queue is empty, wait until an item is available. - Raise a QueueShutDown if _shutdown_state is set to - `SHUTDOWN_IMMEDIATE`. - - Raise a QueueShutDown if _shutdown_state is set to - `SHUTDOWN` and queue is empty. + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ if self._is_shutdown_immediate(): raise QueueShutDown From 7c6e1c74515bc1e6716c059d7aef109623d4d7f4 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 13:38:24 +0200 Subject: [PATCH 51/62] Update Lib/queue.py Co-authored-by: Laurie O --- Lib/queue.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 3cd54d865440b5..f8a7ba072247f0 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -79,8 +79,7 @@ def task_done(self): Raises a ValueError if called more times than there were items placed in the queue. - Raises a ShutDown if shutdown_state attribute is set to - `_queue_shutdown_immediate`. + Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: if self._is_shutdown_immediate(): From c025766ccd568239d8ef9cb693ade62de6615b80 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 15:50:25 +0200 Subject: [PATCH 52/62] Update Lib/asyncio/queues.py Co-authored-by: Laurie O --- Lib/asyncio/queues.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 14f0e3fd519d41..dd33db14bd970c 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -246,8 +246,7 @@ def task_done(self): Raises ValueError if called more times than there were items placed in the queue. - Raise a QueueShutDown if _shutdown_state is set to - `SHUTDOWN_IMMEDIATE`. + Raises QueueShutDown if the queue has been shut down immediately. """ if self._is_shutdown_immediate(): raise QueueShutDown From c9ae3bec99020e0837e927eb5ba8f24db6d0c2f6 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 29 Mar 2023 15:50:39 +0200 Subject: [PATCH 53/62] Update Lib/asyncio/queues.py Co-authored-by: Laurie O --- Lib/asyncio/queues.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index dd33db14bd970c..ae2d55478342e6 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -264,8 +264,7 @@ async def join(self): indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. - Raise a QueueShutDown if _shutdown_state is set to - `SHUTDOWN_IMMEDIATE`. + Raises QueueShutDown if the queue has been shut down immediately. """ if self._is_shutdown_immediate(): raise QueueShutDown From e9b66b2562ee2697e3649b988b4226408bd4df4a Mon Sep 17 00:00:00 2001 From: Duprat Date: Thu, 6 Apr 2023 17:34:42 +0200 Subject: [PATCH 54/62] Update some tests --- Lib/test/_test_multiprocessing.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 37852bd5fab3aa..02b8ca04e5f14b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1314,24 +1314,25 @@ def test_shutdown_immediate(self): def test_shutdown_allowed_transitions(self): # allowed transitions would be from `alive`` via `shutdown` to `shutdown_immediate`` + mod_q = multiprocessing.queues for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): - self.assertEqual(0, q._shutdown_state.value) + self.assertEqual(mod_q._queue_alive, q._shutdown_state.value) # default -> immediate=False q.shutdown() - self.assertEqual(1, q._shutdown_state.value) + self.assertEqual(mod_q._queue_shutdown, q._shutdown_state.value) q.shutdown(immediate=True) - self.assertEqual(2, q._shutdown_state.value) + self.assertEqual(mod_q._queue_shutdown_immediate, q._shutdown_state.value) q.shutdown(immediate=False) - self.assertNotEqual(1, q._shutdown_state.value) + self.assertNotEqual(mod_q._queue_shutdown, q._shutdown_state.value) def _shutdown_all_methods_in_one_process(self, immediate): # part 1: Queue q = multiprocessing.Queue(2) q.put("L") - _wait() # Give time to simulate delay of starting internal thread + _wait() # Give time to simulate many processes q.put_nowait("O") q.shutdown(immediate) _wait() # simulate time of synchro primitive @@ -1520,11 +1521,11 @@ def _shutdown_all_methods_in_many_processes(self, immediate): p.join() if not immediate: - assert(len(res_gets) == len(res_puts)) - assert(res_gets.count(True) == res_puts.count(True)) + self.assertTrue(q.empty()) + self.assertEqual(res_gets.count(True), res_puts.count(True)) else: - assert(len(res_gets) <= len(res_puts)) - assert(res_gets.count(True) <= res_puts.count(True)) + self.assertFalse(q.empty()) + self.assertTrue(res_gets.count(True) <= res_puts.count(True)) def test_shutdown_all_methods_in_many_processes(self): return self._shutdown_all_methods_in_many_processes(False) From 570158e421bb6b1c6fdc6637ef8951578ebd1b37 Mon Sep 17 00:00:00 2001 From: Duprat Date: Thu, 6 Apr 2023 18:18:31 +0200 Subject: [PATCH 55/62] Suppress a borderline assert --- Lib/test/_test_multiprocessing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 02b8ca04e5f14b..ae76c697c71ac1 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1524,7 +1524,6 @@ def _shutdown_all_methods_in_many_processes(self, immediate): self.assertTrue(q.empty()) self.assertEqual(res_gets.count(True), res_puts.count(True)) else: - self.assertFalse(q.empty()) self.assertTrue(res_gets.count(True) <= res_puts.count(True)) def test_shutdown_all_methods_in_many_processes(self): From eeb47b0853ab6be4b43ff40ec3e97314c1195443 Mon Sep 17 00:00:00 2001 From: Duprat Date: Thu, 6 Apr 2023 18:20:43 +0200 Subject: [PATCH 56/62] Fix bugs --- Lib/multiprocessing/queues.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 274e6352bac40b..1bb56e7347a361 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -111,6 +111,8 @@ def put(self, obj, block=True, timeout=None): if not self._is_alive(): raise ShutDown if not self._sem.acquire(block, timeout): + if not self._is_alive(): + raise ShutDown raise Full with self._notempty: @@ -122,33 +124,43 @@ def put(self, obj, block=True, timeout=None): def get(self, block=True, timeout=None): if self._closed: raise ValueError(f"Queue {self!r} is closed") - if ( - self._is_shutdown_immediate() - or (self._is_shutdown() and self.empty()) - ): - raise ShutDown if block and timeout is None: with self._rlock: + # checks shutdown state + if (self._is_shutdown_immediate() + or (self._is_shutdown() and self.empty())): + raise ShutDown res = self._recv_bytes() self._sem.release() else: if block: deadline = time.monotonic() + timeout if not self._rlock.acquire(block, timeout): + if (self._is_shutdown_immediate() + or (self._is_shutdown() and self.empty())): + raise ShutDown raise Empty try: if block: timeout = deadline - time.monotonic() if not self._poll(timeout): + if not self._is_alive(): + raise ShutDown raise Empty elif not self._poll(): + if not self._is_alive(): + raise ShutDown raise Empty + + # here queue is not empty + if self._is_shutdown_immediate(): + raise ShutDown + # here shutdown state queue is alive or shutdown res = self._recv_bytes() self._sem.release() finally: self._rlock.release() - if self._is_shutdown_immediate(): - raise ShutDown + # unserialize the data after having released the lock return _ForkingPickler.loads(res) @@ -357,6 +369,8 @@ def put(self, obj, block=True, timeout=None): if not self._is_alive(): raise ShutDown if not self._sem.acquire(block, timeout): + if not self._is_alive(): + raise ShutDown raise Full with self._notempty, self._cond: @@ -389,7 +403,7 @@ def shutdown(self, immediate=False): is_alive = self._is_alive() super().shutdown(immediate) if is_alive: - self._cond.notify_all() # here to check YD + self._cond.notify_all() # # Simplified Queue type -- really just a locked pipe From 4e2a19ed414424a088c5ad7c38d49f16e4892303 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 7 Apr 2023 10:26:42 +0200 Subject: [PATCH 57/62] ran patchcheck --- Lib/multiprocessing/queues.py | 2 +- Tools/c-analyzer/cpython/_parser.py | 288 ++++++++++++++-------------- 2 files changed, 145 insertions(+), 145 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 1bb56e7347a361..c9d5d4b567b4cd 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -138,7 +138,7 @@ def get(self, block=True, timeout=None): if not self._rlock.acquire(block, timeout): if (self._is_shutdown_immediate() or (self._is_shutdown() and self.empty())): - raise ShutDown + raise ShutDown raise Empty try: if block: diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index acf30e2c4020b3..e3f4f370dc9593 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -101,25 +101,25 @@ def clean_lines(text): INCL_DIRS = clean_lines(''' # @begin=tsv@ -glob dirname -* . -* ./Include -* ./Include/internal - -Modules/_decimal/**/*.c Modules/_decimal/libmpdec -Modules/_elementtree.c Modules/expat -Modules/_hacl/*.c Modules/_hacl/include -Modules/_hacl/*.h Modules/_hacl/include -Modules/md5module.c Modules/_hacl/include -Modules/sha1module.c Modules/_hacl/include -Modules/sha2module.c Modules/_hacl/include -Objects/stringlib/*.h Objects +glob dirname +* . +* ./Include +* ./Include/internal + +Modules/_decimal/**/*.c Modules/_decimal/libmpdec +Modules/_elementtree.c Modules/expat +Modules/_hacl/*.c Modules/_hacl/include +Modules/_hacl/*.h Modules/_hacl/include +Modules/md5module.c Modules/_hacl/include +Modules/sha1module.c Modules/_hacl/include +Modules/sha2module.c Modules/_hacl/include +Objects/stringlib/*.h Objects # possible system-installed headers, just in case -Modules/_tkinter.c /usr/include/tcl8.6 -Modules/_uuidmodule.c /usr/include/uuid -Modules/nismodule.c /usr/include/tirpc -Modules/tkappinit.c /usr/include/tcl +Modules/_tkinter.c /usr/include/tcl8.6 +Modules/_uuidmodule.c /usr/include/uuid +Modules/nismodule.c /usr/include/tirpc +Modules/tkappinit.c /usr/include/tcl # @end=tsv@ ''')[1:] @@ -127,36 +127,36 @@ def clean_lines(text): INCLUDES = clean_lines(''' # @begin=tsv@ -glob include +glob include -**/*.h Python.h -Include/**/*.h object.h +**/*.h Python.h +Include/**/*.h object.h # for Py_HAVE_CONDVAR -Include/internal/pycore_gil.h pycore_condvar.h -Python/thread_pthread.h pycore_condvar.h +Include/internal/pycore_gil.h pycore_condvar.h +Python/thread_pthread.h pycore_condvar.h # other -Objects/stringlib/join.h stringlib/stringdefs.h -Objects/stringlib/ctype.h stringlib/stringdefs.h -Objects/stringlib/transmogrify.h stringlib/stringdefs.h -#Objects/stringlib/fastsearch.h stringlib/stringdefs.h -#Objects/stringlib/count.h stringlib/stringdefs.h -#Objects/stringlib/find.h stringlib/stringdefs.h -#Objects/stringlib/partition.h stringlib/stringdefs.h -#Objects/stringlib/split.h stringlib/stringdefs.h -Objects/stringlib/fastsearch.h stringlib/ucs1lib.h -Objects/stringlib/count.h stringlib/ucs1lib.h -Objects/stringlib/find.h stringlib/ucs1lib.h -Objects/stringlib/partition.h stringlib/ucs1lib.h -Objects/stringlib/split.h stringlib/ucs1lib.h -Objects/stringlib/find_max_char.h Objects/stringlib/ucs1lib.h -Objects/stringlib/count.h Objects/stringlib/fastsearch.h -Objects/stringlib/find.h Objects/stringlib/fastsearch.h -Objects/stringlib/partition.h Objects/stringlib/fastsearch.h -Objects/stringlib/replace.h Objects/stringlib/fastsearch.h -Objects/stringlib/split.h Objects/stringlib/fastsearch.h +Objects/stringlib/join.h stringlib/stringdefs.h +Objects/stringlib/ctype.h stringlib/stringdefs.h +Objects/stringlib/transmogrify.h stringlib/stringdefs.h +#Objects/stringlib/fastsearch.h stringlib/stringdefs.h +#Objects/stringlib/count.h stringlib/stringdefs.h +#Objects/stringlib/find.h stringlib/stringdefs.h +#Objects/stringlib/partition.h stringlib/stringdefs.h +#Objects/stringlib/split.h stringlib/stringdefs.h +Objects/stringlib/fastsearch.h stringlib/ucs1lib.h +Objects/stringlib/count.h stringlib/ucs1lib.h +Objects/stringlib/find.h stringlib/ucs1lib.h +Objects/stringlib/partition.h stringlib/ucs1lib.h +Objects/stringlib/split.h stringlib/ucs1lib.h +Objects/stringlib/find_max_char.h Objects/stringlib/ucs1lib.h +Objects/stringlib/count.h Objects/stringlib/fastsearch.h +Objects/stringlib/find.h Objects/stringlib/fastsearch.h +Objects/stringlib/partition.h Objects/stringlib/fastsearch.h +Objects/stringlib/replace.h Objects/stringlib/fastsearch.h +Objects/stringlib/split.h Objects/stringlib/fastsearch.h # @end=tsv@ ''')[1:] @@ -164,123 +164,123 @@ def clean_lines(text): MACROS = clean_lines(''' # @begin=tsv@ -glob name value - -Include/internal/*.h Py_BUILD_CORE 1 -Python/**/*.c Py_BUILD_CORE 1 -Python/**/*.h Py_BUILD_CORE 1 -Parser/**/*.c Py_BUILD_CORE 1 -Parser/**/*.h Py_BUILD_CORE 1 -Objects/**/*.c Py_BUILD_CORE 1 -Objects/**/*.h Py_BUILD_CORE 1 - -Modules/_asynciomodule.c Py_BUILD_CORE 1 -Modules/_codecsmodule.c Py_BUILD_CORE 1 -Modules/_collectionsmodule.c Py_BUILD_CORE 1 -Modules/_ctypes/_ctypes.c Py_BUILD_CORE 1 -Modules/_ctypes/cfield.c Py_BUILD_CORE 1 -Modules/_cursesmodule.c Py_BUILD_CORE 1 -Modules/_datetimemodule.c Py_BUILD_CORE 1 -Modules/_functoolsmodule.c Py_BUILD_CORE 1 -Modules/_heapqmodule.c Py_BUILD_CORE 1 -Modules/_io/*.c Py_BUILD_CORE 1 -Modules/_io/*.h Py_BUILD_CORE 1 -Modules/_localemodule.c Py_BUILD_CORE 1 -Modules/_operator.c Py_BUILD_CORE 1 -Modules/_posixsubprocess.c Py_BUILD_CORE 1 -Modules/_sre/sre.c Py_BUILD_CORE 1 -Modules/_threadmodule.c Py_BUILD_CORE 1 -Modules/_tracemalloc.c Py_BUILD_CORE 1 -Modules/_weakref.c Py_BUILD_CORE 1 -Modules/_zoneinfo.c Py_BUILD_CORE 1 -Modules/atexitmodule.c Py_BUILD_CORE 1 -Modules/cmathmodule.c Py_BUILD_CORE 1 -Modules/faulthandler.c Py_BUILD_CORE 1 -Modules/gcmodule.c Py_BUILD_CORE 1 -Modules/getpath.c Py_BUILD_CORE 1 -Modules/getpath_noop.c Py_BUILD_CORE 1 -Modules/itertoolsmodule.c Py_BUILD_CORE 1 -Modules/main.c Py_BUILD_CORE 1 -Modules/mathmodule.c Py_BUILD_CORE 1 -Modules/posixmodule.c Py_BUILD_CORE 1 -Modules/sha256module.c Py_BUILD_CORE 1 -Modules/sha512module.c Py_BUILD_CORE 1 -Modules/signalmodule.c Py_BUILD_CORE 1 -Modules/symtablemodule.c Py_BUILD_CORE 1 -Modules/timemodule.c Py_BUILD_CORE 1 -Modules/unicodedata.c Py_BUILD_CORE 1 - -Modules/_json.c Py_BUILD_CORE_BUILTIN 1 -Modules/_pickle.c Py_BUILD_CORE_BUILTIN 1 -Modules/_testinternalcapi.c Py_BUILD_CORE_BUILTIN 1 - -Include/cpython/abstract.h Py_CPYTHON_ABSTRACTOBJECT_H 1 -Include/cpython/bytearrayobject.h Py_CPYTHON_BYTEARRAYOBJECT_H 1 -Include/cpython/bytesobject.h Py_CPYTHON_BYTESOBJECT_H 1 -Include/cpython/ceval.h Py_CPYTHON_CEVAL_H 1 -Include/cpython/code.h Py_CPYTHON_CODE_H 1 -Include/cpython/dictobject.h Py_CPYTHON_DICTOBJECT_H 1 -Include/cpython/fileobject.h Py_CPYTHON_FILEOBJECT_H 1 -Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 -Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 -Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 -Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 -Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 -Include/cpython/object.h Py_CPYTHON_OBJECT_H 1 -Include/cpython/objimpl.h Py_CPYTHON_OBJIMPL_H 1 -Include/cpython/pyerrors.h Py_CPYTHON_ERRORS_H 1 -Include/cpython/pylifecycle.h Py_CPYTHON_PYLIFECYCLE_H 1 -Include/cpython/pymem.h Py_CPYTHON_PYMEM_H 1 -Include/cpython/pystate.h Py_CPYTHON_PYSTATE_H 1 -Include/cpython/sysmodule.h Py_CPYTHON_SYSMODULE_H 1 -Include/cpython/traceback.h Py_CPYTHON_TRACEBACK_H 1 -Include/cpython/tupleobject.h Py_CPYTHON_TUPLEOBJECT_H 1 -Include/cpython/unicodeobject.h Py_CPYTHON_UNICODEOBJECT_H 1 +glob name value + +Include/internal/*.h Py_BUILD_CORE 1 +Python/**/*.c Py_BUILD_CORE 1 +Python/**/*.h Py_BUILD_CORE 1 +Parser/**/*.c Py_BUILD_CORE 1 +Parser/**/*.h Py_BUILD_CORE 1 +Objects/**/*.c Py_BUILD_CORE 1 +Objects/**/*.h Py_BUILD_CORE 1 + +Modules/_asynciomodule.c Py_BUILD_CORE 1 +Modules/_codecsmodule.c Py_BUILD_CORE 1 +Modules/_collectionsmodule.c Py_BUILD_CORE 1 +Modules/_ctypes/_ctypes.c Py_BUILD_CORE 1 +Modules/_ctypes/cfield.c Py_BUILD_CORE 1 +Modules/_cursesmodule.c Py_BUILD_CORE 1 +Modules/_datetimemodule.c Py_BUILD_CORE 1 +Modules/_functoolsmodule.c Py_BUILD_CORE 1 +Modules/_heapqmodule.c Py_BUILD_CORE 1 +Modules/_io/*.c Py_BUILD_CORE 1 +Modules/_io/*.h Py_BUILD_CORE 1 +Modules/_localemodule.c Py_BUILD_CORE 1 +Modules/_operator.c Py_BUILD_CORE 1 +Modules/_posixsubprocess.c Py_BUILD_CORE 1 +Modules/_sre/sre.c Py_BUILD_CORE 1 +Modules/_threadmodule.c Py_BUILD_CORE 1 +Modules/_tracemalloc.c Py_BUILD_CORE 1 +Modules/_weakref.c Py_BUILD_CORE 1 +Modules/_zoneinfo.c Py_BUILD_CORE 1 +Modules/atexitmodule.c Py_BUILD_CORE 1 +Modules/cmathmodule.c Py_BUILD_CORE 1 +Modules/faulthandler.c Py_BUILD_CORE 1 +Modules/gcmodule.c Py_BUILD_CORE 1 +Modules/getpath.c Py_BUILD_CORE 1 +Modules/getpath_noop.c Py_BUILD_CORE 1 +Modules/itertoolsmodule.c Py_BUILD_CORE 1 +Modules/main.c Py_BUILD_CORE 1 +Modules/mathmodule.c Py_BUILD_CORE 1 +Modules/posixmodule.c Py_BUILD_CORE 1 +Modules/sha256module.c Py_BUILD_CORE 1 +Modules/sha512module.c Py_BUILD_CORE 1 +Modules/signalmodule.c Py_BUILD_CORE 1 +Modules/symtablemodule.c Py_BUILD_CORE 1 +Modules/timemodule.c Py_BUILD_CORE 1 +Modules/unicodedata.c Py_BUILD_CORE 1 + +Modules/_json.c Py_BUILD_CORE_BUILTIN 1 +Modules/_pickle.c Py_BUILD_CORE_BUILTIN 1 +Modules/_testinternalcapi.c Py_BUILD_CORE_BUILTIN 1 + +Include/cpython/abstract.h Py_CPYTHON_ABSTRACTOBJECT_H 1 +Include/cpython/bytearrayobject.h Py_CPYTHON_BYTEARRAYOBJECT_H 1 +Include/cpython/bytesobject.h Py_CPYTHON_BYTESOBJECT_H 1 +Include/cpython/ceval.h Py_CPYTHON_CEVAL_H 1 +Include/cpython/code.h Py_CPYTHON_CODE_H 1 +Include/cpython/dictobject.h Py_CPYTHON_DICTOBJECT_H 1 +Include/cpython/fileobject.h Py_CPYTHON_FILEOBJECT_H 1 +Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 +Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 +Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 +Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 +Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 +Include/cpython/object.h Py_CPYTHON_OBJECT_H 1 +Include/cpython/objimpl.h Py_CPYTHON_OBJIMPL_H 1 +Include/cpython/pyerrors.h Py_CPYTHON_ERRORS_H 1 +Include/cpython/pylifecycle.h Py_CPYTHON_PYLIFECYCLE_H 1 +Include/cpython/pymem.h Py_CPYTHON_PYMEM_H 1 +Include/cpython/pystate.h Py_CPYTHON_PYSTATE_H 1 +Include/cpython/sysmodule.h Py_CPYTHON_SYSMODULE_H 1 +Include/cpython/traceback.h Py_CPYTHON_TRACEBACK_H 1 +Include/cpython/tupleobject.h Py_CPYTHON_TUPLEOBJECT_H 1 +Include/cpython/unicodeobject.h Py_CPYTHON_UNICODEOBJECT_H 1 # implied include of -Include/**/*.h _POSIX_THREADS 1 -Include/**/*.h HAVE_PTHREAD_H 1 +Include/**/*.h _POSIX_THREADS 1 +Include/**/*.h HAVE_PTHREAD_H 1 # from pyconfig.h -Include/cpython/pthread_stubs.h HAVE_PTHREAD_STUBS 1 -Python/thread_pthread_stubs.h HAVE_PTHREAD_STUBS 1 +Include/cpython/pthread_stubs.h HAVE_PTHREAD_STUBS 1 +Python/thread_pthread_stubs.h HAVE_PTHREAD_STUBS 1 # from Objects/bytesobject.c -Objects/stringlib/partition.h STRINGLIB_GET_EMPTY() bytes_get_empty() -Objects/stringlib/join.h STRINGLIB_MUTABLE 0 -Objects/stringlib/partition.h STRINGLIB_MUTABLE 0 -Objects/stringlib/split.h STRINGLIB_MUTABLE 0 -Objects/stringlib/transmogrify.h STRINGLIB_MUTABLE 0 +Objects/stringlib/partition.h STRINGLIB_GET_EMPTY() bytes_get_empty() +Objects/stringlib/join.h STRINGLIB_MUTABLE 0 +Objects/stringlib/partition.h STRINGLIB_MUTABLE 0 +Objects/stringlib/split.h STRINGLIB_MUTABLE 0 +Objects/stringlib/transmogrify.h STRINGLIB_MUTABLE 0 # from Makefile -Modules/getpath.c PYTHONPATH 1 -Modules/getpath.c PREFIX ... -Modules/getpath.c EXEC_PREFIX ... -Modules/getpath.c VERSION ... -Modules/getpath.c VPATH ... -Modules/getpath.c PLATLIBDIR ... -#Modules/_dbmmodule.c USE_GDBM_COMPAT 1 -Modules/_dbmmodule.c USE_NDBM 1 -#Modules/_dbmmodule.c USE_BERKDB 1 +Modules/getpath.c PYTHONPATH 1 +Modules/getpath.c PREFIX ... +Modules/getpath.c EXEC_PREFIX ... +Modules/getpath.c VERSION ... +Modules/getpath.c VPATH ... +Modules/getpath.c PLATLIBDIR ... +#Modules/_dbmmodule.c USE_GDBM_COMPAT 1 +Modules/_dbmmodule.c USE_NDBM 1 +#Modules/_dbmmodule.c USE_BERKDB 1 # See: setup.py -Modules/_decimal/**/*.c CONFIG_64 1 -Modules/_decimal/**/*.c ASM 1 -Modules/expat/xmlparse.c HAVE_EXPAT_CONFIG_H 1 -Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 -Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 +Modules/_decimal/**/*.c CONFIG_64 1 +Modules/_decimal/**/*.c ASM 1 +Modules/expat/xmlparse.c HAVE_EXPAT_CONFIG_H 1 +Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 +Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 # from Modules/_sha3/sha3module.c -Modules/_sha3/kcp/KeccakP-1600-inplace32BI.c PLATFORM_BYTE_ORDER 4321 # force big-endian -Modules/_sha3/kcp/*.c KeccakOpt 64 -Modules/_sha3/kcp/*.c KeccakP200_excluded 1 -Modules/_sha3/kcp/*.c KeccakP400_excluded 1 -Modules/_sha3/kcp/*.c KeccakP800_excluded 1 +Modules/_sha3/kcp/KeccakP-1600-inplace32BI.c PLATFORM_BYTE_ORDER 4321 # force big-endian +Modules/_sha3/kcp/*.c KeccakOpt 64 +Modules/_sha3/kcp/*.c KeccakP200_excluded 1 +Modules/_sha3/kcp/*.c KeccakP400_excluded 1 +Modules/_sha3/kcp/*.c KeccakP800_excluded 1 # others -Modules/_sre/sre_lib.h LOCAL(type) static inline type -Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F -Objects/stringlib/codecs.h STRINGLIB_IS_UNICODE 1 +Modules/_sre/sre_lib.h LOCAL(type) static inline type +Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F +Objects/stringlib/codecs.h STRINGLIB_IS_UNICODE 1 # @end=tsv@ ''')[1:] From 6926e1192f0e7fa42712cb3eae6fe8c0f5130677 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 7 Apr 2023 12:39:36 +0200 Subject: [PATCH 58/62] remove ./Tools/c-analyzer/cpython/_parser.py --- Tools/c-analyzer/cpython/_parser.py | 288 ++++++++++++++-------------- 1 file changed, 144 insertions(+), 144 deletions(-) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index e3f4f370dc9593..acf30e2c4020b3 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -101,25 +101,25 @@ def clean_lines(text): INCL_DIRS = clean_lines(''' # @begin=tsv@ -glob dirname -* . -* ./Include -* ./Include/internal - -Modules/_decimal/**/*.c Modules/_decimal/libmpdec -Modules/_elementtree.c Modules/expat -Modules/_hacl/*.c Modules/_hacl/include -Modules/_hacl/*.h Modules/_hacl/include -Modules/md5module.c Modules/_hacl/include -Modules/sha1module.c Modules/_hacl/include -Modules/sha2module.c Modules/_hacl/include -Objects/stringlib/*.h Objects +glob dirname +* . +* ./Include +* ./Include/internal + +Modules/_decimal/**/*.c Modules/_decimal/libmpdec +Modules/_elementtree.c Modules/expat +Modules/_hacl/*.c Modules/_hacl/include +Modules/_hacl/*.h Modules/_hacl/include +Modules/md5module.c Modules/_hacl/include +Modules/sha1module.c Modules/_hacl/include +Modules/sha2module.c Modules/_hacl/include +Objects/stringlib/*.h Objects # possible system-installed headers, just in case -Modules/_tkinter.c /usr/include/tcl8.6 -Modules/_uuidmodule.c /usr/include/uuid -Modules/nismodule.c /usr/include/tirpc -Modules/tkappinit.c /usr/include/tcl +Modules/_tkinter.c /usr/include/tcl8.6 +Modules/_uuidmodule.c /usr/include/uuid +Modules/nismodule.c /usr/include/tirpc +Modules/tkappinit.c /usr/include/tcl # @end=tsv@ ''')[1:] @@ -127,36 +127,36 @@ def clean_lines(text): INCLUDES = clean_lines(''' # @begin=tsv@ -glob include +glob include -**/*.h Python.h -Include/**/*.h object.h +**/*.h Python.h +Include/**/*.h object.h # for Py_HAVE_CONDVAR -Include/internal/pycore_gil.h pycore_condvar.h -Python/thread_pthread.h pycore_condvar.h +Include/internal/pycore_gil.h pycore_condvar.h +Python/thread_pthread.h pycore_condvar.h # other -Objects/stringlib/join.h stringlib/stringdefs.h -Objects/stringlib/ctype.h stringlib/stringdefs.h -Objects/stringlib/transmogrify.h stringlib/stringdefs.h -#Objects/stringlib/fastsearch.h stringlib/stringdefs.h -#Objects/stringlib/count.h stringlib/stringdefs.h -#Objects/stringlib/find.h stringlib/stringdefs.h -#Objects/stringlib/partition.h stringlib/stringdefs.h -#Objects/stringlib/split.h stringlib/stringdefs.h -Objects/stringlib/fastsearch.h stringlib/ucs1lib.h -Objects/stringlib/count.h stringlib/ucs1lib.h -Objects/stringlib/find.h stringlib/ucs1lib.h -Objects/stringlib/partition.h stringlib/ucs1lib.h -Objects/stringlib/split.h stringlib/ucs1lib.h -Objects/stringlib/find_max_char.h Objects/stringlib/ucs1lib.h -Objects/stringlib/count.h Objects/stringlib/fastsearch.h -Objects/stringlib/find.h Objects/stringlib/fastsearch.h -Objects/stringlib/partition.h Objects/stringlib/fastsearch.h -Objects/stringlib/replace.h Objects/stringlib/fastsearch.h -Objects/stringlib/split.h Objects/stringlib/fastsearch.h +Objects/stringlib/join.h stringlib/stringdefs.h +Objects/stringlib/ctype.h stringlib/stringdefs.h +Objects/stringlib/transmogrify.h stringlib/stringdefs.h +#Objects/stringlib/fastsearch.h stringlib/stringdefs.h +#Objects/stringlib/count.h stringlib/stringdefs.h +#Objects/stringlib/find.h stringlib/stringdefs.h +#Objects/stringlib/partition.h stringlib/stringdefs.h +#Objects/stringlib/split.h stringlib/stringdefs.h +Objects/stringlib/fastsearch.h stringlib/ucs1lib.h +Objects/stringlib/count.h stringlib/ucs1lib.h +Objects/stringlib/find.h stringlib/ucs1lib.h +Objects/stringlib/partition.h stringlib/ucs1lib.h +Objects/stringlib/split.h stringlib/ucs1lib.h +Objects/stringlib/find_max_char.h Objects/stringlib/ucs1lib.h +Objects/stringlib/count.h Objects/stringlib/fastsearch.h +Objects/stringlib/find.h Objects/stringlib/fastsearch.h +Objects/stringlib/partition.h Objects/stringlib/fastsearch.h +Objects/stringlib/replace.h Objects/stringlib/fastsearch.h +Objects/stringlib/split.h Objects/stringlib/fastsearch.h # @end=tsv@ ''')[1:] @@ -164,123 +164,123 @@ def clean_lines(text): MACROS = clean_lines(''' # @begin=tsv@ -glob name value - -Include/internal/*.h Py_BUILD_CORE 1 -Python/**/*.c Py_BUILD_CORE 1 -Python/**/*.h Py_BUILD_CORE 1 -Parser/**/*.c Py_BUILD_CORE 1 -Parser/**/*.h Py_BUILD_CORE 1 -Objects/**/*.c Py_BUILD_CORE 1 -Objects/**/*.h Py_BUILD_CORE 1 - -Modules/_asynciomodule.c Py_BUILD_CORE 1 -Modules/_codecsmodule.c Py_BUILD_CORE 1 -Modules/_collectionsmodule.c Py_BUILD_CORE 1 -Modules/_ctypes/_ctypes.c Py_BUILD_CORE 1 -Modules/_ctypes/cfield.c Py_BUILD_CORE 1 -Modules/_cursesmodule.c Py_BUILD_CORE 1 -Modules/_datetimemodule.c Py_BUILD_CORE 1 -Modules/_functoolsmodule.c Py_BUILD_CORE 1 -Modules/_heapqmodule.c Py_BUILD_CORE 1 -Modules/_io/*.c Py_BUILD_CORE 1 -Modules/_io/*.h Py_BUILD_CORE 1 -Modules/_localemodule.c Py_BUILD_CORE 1 -Modules/_operator.c Py_BUILD_CORE 1 -Modules/_posixsubprocess.c Py_BUILD_CORE 1 -Modules/_sre/sre.c Py_BUILD_CORE 1 -Modules/_threadmodule.c Py_BUILD_CORE 1 -Modules/_tracemalloc.c Py_BUILD_CORE 1 -Modules/_weakref.c Py_BUILD_CORE 1 -Modules/_zoneinfo.c Py_BUILD_CORE 1 -Modules/atexitmodule.c Py_BUILD_CORE 1 -Modules/cmathmodule.c Py_BUILD_CORE 1 -Modules/faulthandler.c Py_BUILD_CORE 1 -Modules/gcmodule.c Py_BUILD_CORE 1 -Modules/getpath.c Py_BUILD_CORE 1 -Modules/getpath_noop.c Py_BUILD_CORE 1 -Modules/itertoolsmodule.c Py_BUILD_CORE 1 -Modules/main.c Py_BUILD_CORE 1 -Modules/mathmodule.c Py_BUILD_CORE 1 -Modules/posixmodule.c Py_BUILD_CORE 1 -Modules/sha256module.c Py_BUILD_CORE 1 -Modules/sha512module.c Py_BUILD_CORE 1 -Modules/signalmodule.c Py_BUILD_CORE 1 -Modules/symtablemodule.c Py_BUILD_CORE 1 -Modules/timemodule.c Py_BUILD_CORE 1 -Modules/unicodedata.c Py_BUILD_CORE 1 - -Modules/_json.c Py_BUILD_CORE_BUILTIN 1 -Modules/_pickle.c Py_BUILD_CORE_BUILTIN 1 -Modules/_testinternalcapi.c Py_BUILD_CORE_BUILTIN 1 - -Include/cpython/abstract.h Py_CPYTHON_ABSTRACTOBJECT_H 1 -Include/cpython/bytearrayobject.h Py_CPYTHON_BYTEARRAYOBJECT_H 1 -Include/cpython/bytesobject.h Py_CPYTHON_BYTESOBJECT_H 1 -Include/cpython/ceval.h Py_CPYTHON_CEVAL_H 1 -Include/cpython/code.h Py_CPYTHON_CODE_H 1 -Include/cpython/dictobject.h Py_CPYTHON_DICTOBJECT_H 1 -Include/cpython/fileobject.h Py_CPYTHON_FILEOBJECT_H 1 -Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 -Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 -Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 -Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 -Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 -Include/cpython/object.h Py_CPYTHON_OBJECT_H 1 -Include/cpython/objimpl.h Py_CPYTHON_OBJIMPL_H 1 -Include/cpython/pyerrors.h Py_CPYTHON_ERRORS_H 1 -Include/cpython/pylifecycle.h Py_CPYTHON_PYLIFECYCLE_H 1 -Include/cpython/pymem.h Py_CPYTHON_PYMEM_H 1 -Include/cpython/pystate.h Py_CPYTHON_PYSTATE_H 1 -Include/cpython/sysmodule.h Py_CPYTHON_SYSMODULE_H 1 -Include/cpython/traceback.h Py_CPYTHON_TRACEBACK_H 1 -Include/cpython/tupleobject.h Py_CPYTHON_TUPLEOBJECT_H 1 -Include/cpython/unicodeobject.h Py_CPYTHON_UNICODEOBJECT_H 1 +glob name value + +Include/internal/*.h Py_BUILD_CORE 1 +Python/**/*.c Py_BUILD_CORE 1 +Python/**/*.h Py_BUILD_CORE 1 +Parser/**/*.c Py_BUILD_CORE 1 +Parser/**/*.h Py_BUILD_CORE 1 +Objects/**/*.c Py_BUILD_CORE 1 +Objects/**/*.h Py_BUILD_CORE 1 + +Modules/_asynciomodule.c Py_BUILD_CORE 1 +Modules/_codecsmodule.c Py_BUILD_CORE 1 +Modules/_collectionsmodule.c Py_BUILD_CORE 1 +Modules/_ctypes/_ctypes.c Py_BUILD_CORE 1 +Modules/_ctypes/cfield.c Py_BUILD_CORE 1 +Modules/_cursesmodule.c Py_BUILD_CORE 1 +Modules/_datetimemodule.c Py_BUILD_CORE 1 +Modules/_functoolsmodule.c Py_BUILD_CORE 1 +Modules/_heapqmodule.c Py_BUILD_CORE 1 +Modules/_io/*.c Py_BUILD_CORE 1 +Modules/_io/*.h Py_BUILD_CORE 1 +Modules/_localemodule.c Py_BUILD_CORE 1 +Modules/_operator.c Py_BUILD_CORE 1 +Modules/_posixsubprocess.c Py_BUILD_CORE 1 +Modules/_sre/sre.c Py_BUILD_CORE 1 +Modules/_threadmodule.c Py_BUILD_CORE 1 +Modules/_tracemalloc.c Py_BUILD_CORE 1 +Modules/_weakref.c Py_BUILD_CORE 1 +Modules/_zoneinfo.c Py_BUILD_CORE 1 +Modules/atexitmodule.c Py_BUILD_CORE 1 +Modules/cmathmodule.c Py_BUILD_CORE 1 +Modules/faulthandler.c Py_BUILD_CORE 1 +Modules/gcmodule.c Py_BUILD_CORE 1 +Modules/getpath.c Py_BUILD_CORE 1 +Modules/getpath_noop.c Py_BUILD_CORE 1 +Modules/itertoolsmodule.c Py_BUILD_CORE 1 +Modules/main.c Py_BUILD_CORE 1 +Modules/mathmodule.c Py_BUILD_CORE 1 +Modules/posixmodule.c Py_BUILD_CORE 1 +Modules/sha256module.c Py_BUILD_CORE 1 +Modules/sha512module.c Py_BUILD_CORE 1 +Modules/signalmodule.c Py_BUILD_CORE 1 +Modules/symtablemodule.c Py_BUILD_CORE 1 +Modules/timemodule.c Py_BUILD_CORE 1 +Modules/unicodedata.c Py_BUILD_CORE 1 + +Modules/_json.c Py_BUILD_CORE_BUILTIN 1 +Modules/_pickle.c Py_BUILD_CORE_BUILTIN 1 +Modules/_testinternalcapi.c Py_BUILD_CORE_BUILTIN 1 + +Include/cpython/abstract.h Py_CPYTHON_ABSTRACTOBJECT_H 1 +Include/cpython/bytearrayobject.h Py_CPYTHON_BYTEARRAYOBJECT_H 1 +Include/cpython/bytesobject.h Py_CPYTHON_BYTESOBJECT_H 1 +Include/cpython/ceval.h Py_CPYTHON_CEVAL_H 1 +Include/cpython/code.h Py_CPYTHON_CODE_H 1 +Include/cpython/dictobject.h Py_CPYTHON_DICTOBJECT_H 1 +Include/cpython/fileobject.h Py_CPYTHON_FILEOBJECT_H 1 +Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 +Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 +Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 +Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 +Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 +Include/cpython/object.h Py_CPYTHON_OBJECT_H 1 +Include/cpython/objimpl.h Py_CPYTHON_OBJIMPL_H 1 +Include/cpython/pyerrors.h Py_CPYTHON_ERRORS_H 1 +Include/cpython/pylifecycle.h Py_CPYTHON_PYLIFECYCLE_H 1 +Include/cpython/pymem.h Py_CPYTHON_PYMEM_H 1 +Include/cpython/pystate.h Py_CPYTHON_PYSTATE_H 1 +Include/cpython/sysmodule.h Py_CPYTHON_SYSMODULE_H 1 +Include/cpython/traceback.h Py_CPYTHON_TRACEBACK_H 1 +Include/cpython/tupleobject.h Py_CPYTHON_TUPLEOBJECT_H 1 +Include/cpython/unicodeobject.h Py_CPYTHON_UNICODEOBJECT_H 1 # implied include of -Include/**/*.h _POSIX_THREADS 1 -Include/**/*.h HAVE_PTHREAD_H 1 +Include/**/*.h _POSIX_THREADS 1 +Include/**/*.h HAVE_PTHREAD_H 1 # from pyconfig.h -Include/cpython/pthread_stubs.h HAVE_PTHREAD_STUBS 1 -Python/thread_pthread_stubs.h HAVE_PTHREAD_STUBS 1 +Include/cpython/pthread_stubs.h HAVE_PTHREAD_STUBS 1 +Python/thread_pthread_stubs.h HAVE_PTHREAD_STUBS 1 # from Objects/bytesobject.c -Objects/stringlib/partition.h STRINGLIB_GET_EMPTY() bytes_get_empty() -Objects/stringlib/join.h STRINGLIB_MUTABLE 0 -Objects/stringlib/partition.h STRINGLIB_MUTABLE 0 -Objects/stringlib/split.h STRINGLIB_MUTABLE 0 -Objects/stringlib/transmogrify.h STRINGLIB_MUTABLE 0 +Objects/stringlib/partition.h STRINGLIB_GET_EMPTY() bytes_get_empty() +Objects/stringlib/join.h STRINGLIB_MUTABLE 0 +Objects/stringlib/partition.h STRINGLIB_MUTABLE 0 +Objects/stringlib/split.h STRINGLIB_MUTABLE 0 +Objects/stringlib/transmogrify.h STRINGLIB_MUTABLE 0 # from Makefile -Modules/getpath.c PYTHONPATH 1 -Modules/getpath.c PREFIX ... -Modules/getpath.c EXEC_PREFIX ... -Modules/getpath.c VERSION ... -Modules/getpath.c VPATH ... -Modules/getpath.c PLATLIBDIR ... -#Modules/_dbmmodule.c USE_GDBM_COMPAT 1 -Modules/_dbmmodule.c USE_NDBM 1 -#Modules/_dbmmodule.c USE_BERKDB 1 +Modules/getpath.c PYTHONPATH 1 +Modules/getpath.c PREFIX ... +Modules/getpath.c EXEC_PREFIX ... +Modules/getpath.c VERSION ... +Modules/getpath.c VPATH ... +Modules/getpath.c PLATLIBDIR ... +#Modules/_dbmmodule.c USE_GDBM_COMPAT 1 +Modules/_dbmmodule.c USE_NDBM 1 +#Modules/_dbmmodule.c USE_BERKDB 1 # See: setup.py -Modules/_decimal/**/*.c CONFIG_64 1 -Modules/_decimal/**/*.c ASM 1 -Modules/expat/xmlparse.c HAVE_EXPAT_CONFIG_H 1 -Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 -Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 +Modules/_decimal/**/*.c CONFIG_64 1 +Modules/_decimal/**/*.c ASM 1 +Modules/expat/xmlparse.c HAVE_EXPAT_CONFIG_H 1 +Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 +Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 # from Modules/_sha3/sha3module.c -Modules/_sha3/kcp/KeccakP-1600-inplace32BI.c PLATFORM_BYTE_ORDER 4321 # force big-endian -Modules/_sha3/kcp/*.c KeccakOpt 64 -Modules/_sha3/kcp/*.c KeccakP200_excluded 1 -Modules/_sha3/kcp/*.c KeccakP400_excluded 1 -Modules/_sha3/kcp/*.c KeccakP800_excluded 1 +Modules/_sha3/kcp/KeccakP-1600-inplace32BI.c PLATFORM_BYTE_ORDER 4321 # force big-endian +Modules/_sha3/kcp/*.c KeccakOpt 64 +Modules/_sha3/kcp/*.c KeccakP200_excluded 1 +Modules/_sha3/kcp/*.c KeccakP400_excluded 1 +Modules/_sha3/kcp/*.c KeccakP800_excluded 1 # others -Modules/_sre/sre_lib.h LOCAL(type) static inline type -Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F -Objects/stringlib/codecs.h STRINGLIB_IS_UNICODE 1 +Modules/_sre/sre_lib.h LOCAL(type) static inline type +Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F +Objects/stringlib/codecs.h STRINGLIB_IS_UNICODE 1 # @end=tsv@ ''')[1:] From 31ea16b3be0541555a8a807765e49badc4775c78 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 5 May 2023 08:51:04 +0200 Subject: [PATCH 59/62] Add `shutdown` method documentation --- Doc/library/asyncio-queue.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index 008a7f908f4fbc..bebfaf4b61834d 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -111,7 +111,7 @@ Queue All blocked callers of put() will be unblocked, and also get() and join() if *immediate* is true. - .. versionadded:: 3.12 + .. versionadded:: 3.13 .. method:: task_done() From bed3a4b5d79f3de29c2f5ccf3b85af1c9c995812 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 5 May 2023 08:58:07 +0200 Subject: [PATCH 60/62] Add `shutdown` method to documentation --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index b44db0ce6f857f..b53e474990b182 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -887,7 +887,7 @@ For an example of the usage of queues for interprocess communication see All blocked callers of put() will be unblocked, and also get() and join() if *immediate* is true. - .. versionadded:: 3.12 + .. versionadded:: 3.13 :class:`multiprocessing.Queue` has a few additional methods not found in :class:`queue.Queue`. These methods are usually unnecessary for most From 8e8dcfa0204db11724163ac49b905360474880a6 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 5 May 2023 09:02:25 +0200 Subject: [PATCH 61/62] Add `shutdown` method to documentation --- Doc/library/queue.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index d9952f1882ae05..33d6f2f4c85e1c 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -98,7 +98,7 @@ The :mod:`queue` module defines the following classes and exceptions: Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is called on a :class:`Queue` object which has been shut down. - .. versionadded:: 3.12 + .. versionadded:: 3.13 .. _queueobjects: @@ -247,7 +247,7 @@ them down. All blocked callers of put() will be unblocked, and also get() and join() if *immediate* is true. - .. versionadded:: 3.12 + .. versionadded:: 3.13 SimpleQueue Objects From 9eed14e8f94a2354351fb9676bd50da88993e175 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 5 May 2023 09:03:22 +0200 Subject: [PATCH 62/62] Add `shutdown` method to documentation --- Doc/library/asyncio-queue.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index bebfaf4b61834d..b66bc9341a2081 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -174,7 +174,7 @@ Exceptions Exception raised when getting an item from or putting an item onto a queue which has been shut down. - .. versionadded:: 3.12 + .. versionadded:: 3.13 Examples