diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 8bd2bc99d74b83..ff34bb5d71c22b 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1074,7 +1074,7 @@ Performance My program is too slow. How do I speed it up? --------------------------------------------- -That's a tough one, in general. First, here is list of things to +That's a tough one, in general. First, here is a list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 93850b57af2c65..5260c2ca4add47 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -105,8 +105,8 @@ The complete :class:`!Weekday` enum now looks like this:: Now we can find out what today is! Observe:: - >>> from datetime import date - >>> Weekday.from_date(date.today()) # doctest: +SKIP + >>> import datetime as dt + >>> Weekday.from_date(dt.date.today()) # doctest: +SKIP Of course, if you're reading this on some other day, you'll see that day instead. @@ -1480,8 +1480,8 @@ TimePeriod An example to show the :attr:`~Enum._ignore_` attribute in use:: - >>> from datetime import timedelta - >>> class Period(timedelta, Enum): + >>> import datetime as dt + >>> class Period(dt.timedelta, Enum): ... "different lengths of time" ... _ignore_ = 'Period i' ... Period = vars() diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index b87ac93296b915..21df6ba858d8c2 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1549,10 +1549,10 @@ to this (remembering to first import :mod:`concurrent.futures`):: for i in range(10): executor.submit(worker_process, queue, worker_configurer) -Deploying Web applications using Gunicorn and uWSGI +Deploying web applications using Gunicorn and uWSGI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -When deploying Web applications using `Gunicorn `_ or `uWSGI +When deploying web applications using `Gunicorn `_ or `uWSGI `_ (or similar), multiple worker processes are created to handle client requests. In such environments, avoid creating file-based handlers directly in your web application. Instead, use a @@ -3616,7 +3616,6 @@ detailed information. .. code-block:: python3 - import datetime import logging import random import sys @@ -3851,7 +3850,7 @@ Logging to syslog with RFC5424 support Although :rfc:`5424` dates from 2009, most syslog servers are configured by default to use the older :rfc:`3164`, which hails from 2001. When ``logging`` was added to Python in 2003, it supported the earlier (and only existing) protocol at the time. Since -RFC5424 came out, as there has not been widespread deployment of it in syslog +RFC 5424 came out, as there has not been widespread deployment of it in syslog servers, the :class:`~logging.handlers.SysLogHandler` functionality has not been updated. @@ -3859,7 +3858,7 @@ RFC 5424 contains some useful features such as support for structured data, and need to be able to log to a syslog server with support for it, you can do so with a subclassed handler which looks something like this:: - import datetime + import datetime as dt import logging.handlers import re import socket @@ -3877,7 +3876,7 @@ subclassed handler which looks something like this:: def format(self, record): version = 1 - asctime = datetime.datetime.fromtimestamp(record.created).isoformat() + asctime = dt.datetime.fromtimestamp(record.created).isoformat() m = self.tz_offset.match(time.strftime('%z')) has_offset = False if m and time.timezone: diff --git a/Doc/includes/diff.py b/Doc/includes/diff.py index 001619f5f83fc0..bc4bd58ff3e3f1 100644 --- a/Doc/includes/diff.py +++ b/Doc/includes/diff.py @@ -1,4 +1,4 @@ -""" Command line interface to difflib.py providing diffs in four formats: +""" Command-line interface to difflib.py providing diffs in four formats: * ndiff: lists every line and highlights interline changes. * context: highlights clusters of changes in a before/after format. @@ -8,11 +8,11 @@ """ import sys, os, difflib, argparse -from datetime import datetime, timezone +import datetime as dt def file_mtime(path): - t = datetime.fromtimestamp(os.stat(path).st_mtime, - timezone.utc) + t = dt.datetime.fromtimestamp(os.stat(path).st_mtime, + dt.timezone.utc) return t.astimezone().isoformat() def main(): diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 7831b613bd4a60..f3409bcd2df648 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -248,3 +248,225 @@ Output in debug mode:: File "../t.py", line 4, in bug raise Exception("not consumed") Exception: not consumed + + +Asynchronous generators best practices +====================================== + +Writing correct and efficient asyncio code requires awareness of certain pitfalls. +This section outlines essential best practices that can save you hours of debugging. + + +Close asynchronous generators explicitly +---------------------------------------- + +It is recommended to manually close the +:term:`asynchronous generator `. If a generator +exits early - for example, due to an exception raised in the body of +an ``async for`` loop - its asynchronous cleanup code may run in an +unexpected context. This can occur after the tasks it depends on have completed, +or during the event loop shutdown when the async-generator's garbage collection +hook is called. + +To avoid this, explicitly close the generator by calling its +:meth:`~agen.aclose` method, or use the :func:`contextlib.aclosing` +context manager:: + + import asyncio + import contextlib + + async def gen(): + yield 1 + yield 2 + + async def func(): + async with contextlib.aclosing(gen()) as g: + async for x in g: + break # Don't iterate until the end + + asyncio.run(func()) + +As noted above, the cleanup code for these asynchronous generators is deferred. +The following example demonstrates that the finalization of an asynchronous +generator can occur in an unexpected order:: + + import asyncio + work_done = False + + async def cursor(): + try: + yield 1 + finally: + assert work_done + + async def rows(): + global work_done + try: + yield 2 + finally: + await asyncio.sleep(0.1) # immitate some async work + work_done = True + + + async def main(): + async for c in cursor(): + async for r in rows(): + break + break + + asyncio.run(main()) + +For this example, we get the following output:: + + unhandled exception during asyncio.run() shutdown + task: ()> exception=AssertionError()> + Traceback (most recent call last): + File "example.py", line 6, in cursor + yield 1 + asyncio.exceptions.CancelledError + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "example.py", line 8, in cursor + assert work_done + ^^^^^^^^^ + AssertionError + +The ``cursor()`` asynchronous generator was finalized before the ``rows`` +generator - an unexpected behavior. + +The example can be fixed by explicitly closing the +``cursor`` and ``rows`` async-generators:: + + async def main(): + async with contextlib.aclosing(cursor()) as cursor_gen: + async for c in cursor_gen: + async with contextlib.aclosing(rows()) as rows_gen: + async for r in rows_gen: + break + break + + +Create asynchronous generators only when the event loop is running +------------------------------------------------------------------ + +It is recommended to create +:term:`asynchronous generators ` only after +the event loop has been created. + +To ensure that asynchronous generators close reliably, the event loop uses the +:func:`sys.set_asyncgen_hooks` function to register callback functions. These +callbacks update the list of running asynchronous generators to keep it in a +consistent state. + +When the :meth:`loop.shutdown_asyncgens() ` +function is called, the running generators are stopped gracefully and the +list is cleared. + +The asynchronous generator invokes the corresponding system hook during its +first iteration. At the same time, the generator records that the hook has +been called and does not call it again. + +Therefore, if iteration begins before the event loop is created, +the event loop will not be able to add the generator to its list of active +generators because the hooks are set after the generator attempts to call them. +Consequently, the event loop will not be able to terminate the generator +if necessary. + +Consider the following example:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + + with asyncio.Runner() as runner: + agen = agenfn() + print(runner.run(anext(agen))) + del agen + +Output:: + + 10 + Exception ignored while closing generator : + Traceback (most recent call last): + File "example.py", line 13, in + del agen + ^^^^ + RuntimeError: async generator ignored GeneratorExit + +This example can be fixed as follows:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + async def main(): + agen = agenfn() + print(await anext(agen)) + del agen + + asyncio.run(main()) + + +Avoid concurrent iteration and closure of the same generator +------------------------------------------------------------ + +Async generators may be reentered while another +:meth:`~agen.__anext__` / :meth:`~agen.athrow` / :meth:`~agen.aclose` call is in +progress. This may lead to an inconsistent state of the async generator and can +cause errors. + +Let's consider the following example:: + + import asyncio + + async def consumer(): + for idx in range(100): + await asyncio.sleep(0) + message = yield idx + print('received', message) + + async def amain(): + agenerator = consumer() + await agenerator.asend(None) + + fa = asyncio.create_task(agenerator.asend('A')) + fb = asyncio.create_task(agenerator.asend('B')) + await fa + await fb + + asyncio.run(amain()) + +Output:: + + received A + Traceback (most recent call last): + File "test.py", line 38, in + asyncio.run(amain()) + ~~~~~~~~~~~^^^^^^^^^ + File "Lib/asyncio/runners.py", line 204, in run + return runner.run(main) + ~~~~~~~~~~^^^^^^ + File "Lib/asyncio/runners.py", line 127, in run + return self._loop.run_until_complete(task) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ + File "Lib/asyncio/base_events.py", line 719, in run_until_complete + return future.result() + ~~~~~~~~~~~~~^^ + File "test.py", line 36, in amain + await fb + RuntimeError: anext(): asynchronous generator is already running + + +Therefore, it is recommended to avoid using asynchronous generators in parallel +tasks or across multiple event loops. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index bdb24b3a58c267..d1a5b4e7b4638e 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -4,7 +4,7 @@ .. _asyncio-event-loop: ========== -Event Loop +Event loop ========== **Source code:** :source:`Lib/asyncio/events.py`, @@ -105,7 +105,7 @@ This documentation page contains the following sections: .. _asyncio-event-loop-methods: -Event Loop Methods +Event loop methods ================== Event loops have **low-level** APIs for the following: @@ -361,7 +361,7 @@ clocks to track time. The :func:`asyncio.sleep` function. -Creating Futures and Tasks +Creating futures and tasks ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. method:: loop.create_future() @@ -962,7 +962,7 @@ Transferring files .. versionadded:: 3.7 -TLS Upgrade +TLS upgrade ^^^^^^^^^^^ .. method:: loop.start_tls(transport, protocol, \ @@ -1431,7 +1431,7 @@ Executing code in thread or process pools :class:`~concurrent.futures.ThreadPoolExecutor`. -Error Handling API +Error handling API ^^^^^^^^^^^^^^^^^^ Allows customizing how exceptions are handled in the event loop. @@ -1534,7 +1534,7 @@ Enabling debug mode The :ref:`debug mode of asyncio `. -Running Subprocesses +Running subprocesses ^^^^^^^^^^^^^^^^^^^^ Methods described in this subsections are low-level. In regular @@ -1672,7 +1672,7 @@ async/await code consider using the high-level are going to be used to construct shell commands. -Callback Handles +Callback handles ================ .. class:: Handle @@ -1715,7 +1715,7 @@ Callback Handles .. versionadded:: 3.7 -Server Objects +Server objects ============== Server objects are created by :meth:`loop.create_server`, @@ -1858,7 +1858,7 @@ Do not instantiate the :class:`Server` class directly. .. _asyncio-event-loops: .. _asyncio-event-loop-implementations: -Event Loop Implementations +Event loop implementations ========================== asyncio ships with two different event loop implementations: @@ -1971,10 +1971,10 @@ callback uses the :meth:`loop.call_later` method to reschedule itself after 5 seconds, and then stops the event loop:: import asyncio - import datetime + import datetime as dt def display_date(end_time, loop): - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) < end_time: loop.call_later(1, display_date, end_time, loop) else: diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 5208f14c94a50f..58f77feb311984 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -1037,7 +1037,7 @@ The subprocess is created by the :meth:`loop.subprocess_exec` method:: # low-level APIs. loop = asyncio.get_running_loop() - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' exit_future = asyncio.Future(loop=loop) # Create the subprocess controlled by DateProtocol; diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 9416c758e51d95..a6514649bf9a0a 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -311,8 +311,16 @@ their completion. A ``None`` value indicates that the process has not terminated yet. - A negative value ``-N`` indicates that the child was terminated - by signal ``N`` (POSIX only). + For processes created with :func:`~asyncio.create_subprocess_exec`, a negative + value ``-N`` indicates that the child was terminated by signal ``N`` + (POSIX only). + + For processes created with :func:`~asyncio.create_subprocess_shell`, the + return code reflects the exit status of the shell itself (e.g. ``/bin/sh``), + which may map signals to codes such as ``128+N``. See the + documentation of the shell (for example, the Bash manual's Exit Status) + for details. + .. _asyncio-subprocess-threads: @@ -351,7 +359,7 @@ function:: import sys async def get_date(): - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' # Create the subprocess; redirect the standard output # into a pipe. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index e2a6752be12b67..4e60eee44290af 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -2,7 +2,7 @@ ==================== -Coroutines and Tasks +Coroutines and tasks ==================== This section outlines high-level asyncio APIs to work with coroutines @@ -231,7 +231,7 @@ A good example of a low-level function that returns a Future object is :meth:`loop.run_in_executor`. -Creating Tasks +Creating tasks ============== **Source code:** :source:`Lib/asyncio/tasks.py` @@ -300,7 +300,7 @@ Creating Tasks Added the *eager_start* parameter by passing on all *kwargs*. -Task Cancellation +Task cancellation ================= Tasks can easily and safely be cancelled. @@ -324,7 +324,7 @@ remove the cancellation state. .. _taskgroups: -Task Groups +Task groups =========== Task groups combine a task creation API with a convenient @@ -427,7 +427,7 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. -Terminating a Task Group +Terminating a task group ------------------------ While terminating a task group is not natively supported by the standard @@ -498,13 +498,13 @@ Sleeping for 5 seconds:: import asyncio - import datetime + import datetime as dt async def display_date(): loop = asyncio.get_running_loop() end_time = loop.time() + 5.0 while True: - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) >= end_time: break await asyncio.sleep(1) @@ -519,7 +519,7 @@ Sleeping Raises :exc:`ValueError` if *delay* is :data:`~math.nan`. -Running Tasks Concurrently +Running tasks concurrently ========================== .. awaitablefunction:: gather(*aws, return_exceptions=False) @@ -621,7 +621,7 @@ Running Tasks Concurrently .. _eager-task-factory: -Eager Task Factory +Eager task factory ================== .. function:: eager_task_factory(loop, coro, *, name=None, context=None) @@ -664,7 +664,7 @@ Eager Task Factory .. versionadded:: 3.12 -Shielding From Cancellation +Shielding from cancellation =========================== .. awaitablefunction:: shield(aw) @@ -894,7 +894,7 @@ Timeouts Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`. -Waiting Primitives +Waiting primitives ================== .. function:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) @@ -1014,7 +1014,7 @@ Waiting Primitives or as a plain :term:`iterator` (previously it was only a plain iterator). -Running in Threads +Running in threads ================== .. function:: to_thread(func, /, *args, **kwargs) @@ -1074,7 +1074,7 @@ Running in Threads .. versionadded:: 3.9 -Scheduling From Other Threads +Scheduling from other threads ============================= .. function:: run_coroutine_threadsafe(coro, loop) @@ -1198,7 +1198,7 @@ Introspection .. _asyncio-task-obj: -Task Object +Task object =========== .. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index e56c4f5e7dfbf7..8b812c173b5953 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -362,7 +362,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. _sequence-matcher: -SequenceMatcher Objects +SequenceMatcher objects ----------------------- The :class:`SequenceMatcher` class has this constructor: @@ -590,7 +590,7 @@ are always at least as large as :meth:`~SequenceMatcher.ratio`: .. _sequencematcher-examples: -SequenceMatcher Examples +SequenceMatcher examples ------------------------ This example compares two strings, considering blanks to be "junk": @@ -641,7 +641,7 @@ If you want to know how to change the first sequence into the second, use .. _differ-objects: -Differ Objects +Differ objects -------------- Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** @@ -690,7 +690,7 @@ The :class:`Differ` class has this constructor: .. _differ-examples: -Differ Example +Differ example -------------- This example compares two texts. First we set up the texts, sequences of diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 8a8a2edc9e542d..0a1d2a0bac33ab 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -56,7 +56,7 @@ are not normal Python classes. See --------------- -Module Contents +Module contents --------------- :class:`EnumType` @@ -161,7 +161,7 @@ Module Contents --------------- -Data Types +Data types ---------- @@ -341,7 +341,7 @@ Data Types any public methods defined on *self.__class__*:: >>> from enum import Enum - >>> from datetime import date + >>> import datetime as dt >>> class Weekday(Enum): ... MONDAY = 1 ... TUESDAY = 2 @@ -352,7 +352,7 @@ Data Types ... SUNDAY = 7 ... @classmethod ... def today(cls): - ... print('today is %s' % cls(date.today().isoweekday()).name) + ... print(f'today is {cls(dt.date.today().isoweekday()).name}') ... >>> dir(Weekday.SATURDAY) ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] @@ -970,7 +970,7 @@ Supported ``_sunder_`` names --------------- -Utilities and Decorators +Utilities and decorators ------------------------ .. class:: auto diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index e5fc19f495226e..fa15cd4267eef4 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -180,7 +180,7 @@ Examples Generating a plist:: - import datetime + import datetime as dt import plistlib pl = dict( @@ -196,7 +196,7 @@ Generating a plist:: ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.now() + aDate = dt.datetime.now() ) print(plistlib.dumps(pl).decode()) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 40d103c13f8f38..ff1676c5bfb561 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2285,7 +2285,7 @@ This section shows recipes for common adapters and converters. .. testcode:: - import datetime + import datetime as dt import sqlite3 def adapt_date_iso(val): @@ -2300,21 +2300,21 @@ This section shows recipes for common adapters and converters. """Adapt datetime.datetime to Unix timestamp.""" return int(val.timestamp()) - sqlite3.register_adapter(datetime.date, adapt_date_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_epoch) + sqlite3.register_adapter(dt.date, adapt_date_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_epoch) def convert_date(val): """Convert ISO 8601 date to datetime.date object.""" - return datetime.date.fromisoformat(val.decode()) + return dt.date.fromisoformat(val.decode()) def convert_datetime(val): """Convert ISO 8601 datetime to datetime.datetime object.""" - return datetime.datetime.fromisoformat(val.decode()) + return dt.datetime.fromisoformat(val.decode()) def convert_timestamp(val): """Convert Unix epoch timestamp to datetime.datetime object.""" - return datetime.datetime.fromtimestamp(int(val)) + return dt.datetime.fromtimestamp(int(val)) sqlite3.register_converter("date", convert_date) sqlite3.register_converter("datetime", convert_datetime) @@ -2323,17 +2323,17 @@ This section shows recipes for common adapters and converters. .. testcode:: :hide: - dt = datetime.datetime(2019, 5, 18, 15, 17, 8, 123456) + when = dt.datetime(2019, 5, 18, 15, 17, 8, 123456) - assert adapt_date_iso(dt.date()) == "2019-05-18" - assert convert_date(b"2019-05-18") == dt.date() + assert adapt_date_iso(when.date()) == "2019-05-18" + assert convert_date(b"2019-05-18") == when.date() - assert adapt_datetime_iso(dt) == "2019-05-18T15:17:08.123456" - assert convert_datetime(b"2019-05-18T15:17:08.123456") == dt + assert adapt_datetime_iso(when) == "2019-05-18T15:17:08.123456" + assert convert_datetime(b"2019-05-18T15:17:08.123456") == when # Using current time as fromtimestamp() returns local date/time. # Dropping microseconds as adapt_datetime_epoch truncates fractional second part. - now = datetime.datetime.now().replace(microsecond=0) + now = dt.datetime.now().replace(microsecond=0) current_timestamp = int(now.timestamp()) assert adapt_datetime_epoch(now) == current_timestamp diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index e83c2c9a8bc792..f2c35d1897a77f 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -67,7 +67,7 @@ by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. Use of deprecated constants and functions result in deprecation warnings. -Functions, Constants, and Exceptions +Functions, constants, and exceptions ------------------------------------ @@ -374,7 +374,7 @@ Certificate handling .. function:: cert_time_to_seconds(cert_time) - Return the time in seconds since the Epoch, given the ``cert_time`` + Return the time in seconds since the epoch, given the ``cert_time`` string representing the "notBefore" or "notAfter" date from a certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C locale). @@ -384,12 +384,12 @@ Certificate handling .. doctest:: newcontext >>> import ssl + >>> import datetime as dt >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT") >>> timestamp # doctest: +SKIP 1515144883 - >>> from datetime import datetime - >>> print(datetime.utcfromtimestamp(timestamp)) # doctest: +SKIP - 2018-01-05 09:34:43 + >>> print(dt.datetime.fromtimestamp(timestamp, dt.UTC)) # doctest: +SKIP + 2018-01-05 09:34:43+00:00 "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`). @@ -1072,7 +1072,7 @@ Constants :attr:`TLSVersion.TLSv1_3` are deprecated. -SSL Sockets +SSL sockets ----------- .. class:: SSLSocket(socket.socket) @@ -1462,7 +1462,7 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.6 -SSL Contexts +SSL contexts ------------ .. versionadded:: 3.2 @@ -2653,7 +2653,7 @@ thus several things you need to be aware of: as well. -Memory BIO Support +Memory BIO support ------------------ .. versionadded:: 3.5 diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 8096d90317d93f..08ccdfa3f454f8 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -82,7 +82,7 @@ The constants defined in this module are: .. _string-formatting: -Custom String Formatting +Custom string formatting ------------------------ The built-in string class provides the ability to do complex variable @@ -192,7 +192,7 @@ implementation as the built-in :meth:`~str.format` method. .. _formatstrings: -Format String Syntax +Format string syntax -------------------- The :meth:`str.format` method and the :class:`Formatter` class share the same @@ -304,7 +304,7 @@ See the :ref:`formatexamples` section for some examples. .. _formatspec: -Format Specification Mini-Language +Format specification mini-language ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ "Format specifications" are used within replacement fields contained within a @@ -759,8 +759,8 @@ Expressing a percentage:: Using type-specific formatting:: - >>> import datetime - >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> import datetime as dt + >>> d = dt.datetime(2010, 7, 4, 12, 15, 58) >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) '2010-07-04 12:15:58' diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index def6d58eabbeee..9e261a0ca03902 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -964,6 +964,11 @@ Reassigning them to new values is unsupported: A negative value ``-N`` indicates that the child was terminated by signal ``N`` (POSIX only). + When ``shell=True``, the return code reflects the exit status of the shell + itself (e.g. ``/bin/sh``), which may map signals to codes such as + ``128+N``. See the documentation of the shell (for example, the Bash + manual's Exit Status) for details. + Windows Popen Helpers --------------------- diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 7c81f52165972a..b8aa04b9ab246d 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -25,7 +25,7 @@ Using Mock ---------- -Mock Patching Methods +Mock patching methods ~~~~~~~~~~~~~~~~~~~~~ Common uses for :class:`Mock` objects include: @@ -71,7 +71,7 @@ the ``something`` method: -Mock for Method Calls on an Object +Mock for method calls on an object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In the last example we patched a method directly on an object to check that it @@ -101,7 +101,7 @@ accessing it in the test will create it, but :meth:`~Mock.assert_called_with` will raise a failure exception. -Mocking Classes +Mocking classes ~~~~~~~~~~~~~~~ A common use case is to mock out classes instantiated by your code under test. @@ -139,7 +139,7 @@ name is also propagated to attributes or methods of the mock: -Tracking all Calls +Tracking all calls ~~~~~~~~~~~~~~~~~~ Often you want to track more than a single call to a method. The @@ -176,7 +176,7 @@ possible to track nested calls where the parameters used to create ancestors are True -Setting Return Values and Attributes +Setting return values and attributes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Setting the return values on a mock object is trivially easy: @@ -317,7 +317,7 @@ return an async function. >>> mock_instance.__aexit__.assert_awaited_once() -Creating a Mock from an Existing Object +Creating a mock from an existing object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ One problem with over use of mocking is that it couples your tests to the @@ -384,7 +384,7 @@ contents per file stored in a dictionary:: assert file2.read() == "default" -Patch Decorators +Patch decorators ---------------- .. note:: @@ -518,7 +518,7 @@ decorator individually to every method whose name starts with "test". .. _further-examples: -Further Examples +Further examples ---------------- @@ -614,13 +614,13 @@ attribute on the mock date class is then set to a lambda function that returns a real date. When the mock date class is called a real date will be constructed and returned by ``side_effect``. :: - >>> from datetime import date + >>> import datetime as dt >>> with patch('mymodule.date') as mock_date: - ... mock_date.today.return_value = date(2010, 10, 8) - ... mock_date.side_effect = lambda *args, **kw: date(*args, **kw) + ... mock_date.today.return_value = dt.date(2010, 10, 8) + ... mock_date.side_effect = lambda *args, **kw: dt.date(*args, **kw) ... - ... assert mymodule.date.today() == date(2010, 10, 8) - ... assert mymodule.date(2009, 6, 8) == date(2009, 6, 8) + ... assert mymodule.date.today() == dt.date(2010, 10, 8) + ... assert mymodule.date(2009, 6, 8) == dt.date(2009, 6, 8) Note that we don't patch :class:`datetime.date` globally, we patch ``date`` in the module that *uses* it. See :ref:`where to patch `. @@ -638,7 +638,7 @@ is discussed in `this blog entry `_. -Mocking a Generator Method +Mocking a generator method ~~~~~~~~~~~~~~~~~~~~~~~~~~ A Python generator is a function or method that uses the :keyword:`yield` statement @@ -739,7 +739,7 @@ exception is raised in the setUp then tearDown is not called. >>> MyTest('test_foo').run() -Mocking Unbound Methods +Mocking unbound methods ~~~~~~~~~~~~~~~~~~~~~~~ Sometimes a test needs to patch an *unbound method*, which means patching the @@ -937,7 +937,7 @@ and the ``return_value`` will use your subclass automatically. That means all children of a ``CopyingMock`` will also have the type ``CopyingMock``. -Nesting Patches +Nesting patches ~~~~~~~~~~~~~~~ Using patch as a context manager is nice, but if you do multiple patches you diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 8b3b3dcaa13447..458e67dbe51dcc 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -272,12 +272,12 @@ DateTime Objects A working example follows. The server code:: - import datetime + import datetime as dt from xmlrpc.server import SimpleXMLRPCServer import xmlrpc.client def today(): - today = datetime.datetime.today() + today = dt.datetime.today() return xmlrpc.client.DateTime(today) server = SimpleXMLRPCServer(("localhost", 8000)) @@ -288,14 +288,14 @@ A working example follows. The server code:: The client code for the preceding server:: import xmlrpc.client - import datetime + import datetime as dt proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") today = proxy.today() - # convert the ISO8601 string to a datetime object - converted = datetime.datetime.strptime(today.value, "%Y%m%dT%H:%M:%S") - print("Today: %s" % converted.strftime("%d.%m.%Y, %H:%M")) + # convert the ISO 8601 string to a datetime object + converted = dt.datetime.strptime(today.value, "%Y%m%dT%H:%M:%S") + print(f"Today: {converted.strftime('%d.%m.%Y, %H:%M')}") .. _binary-objects: diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 2c130785be0db0..5702257dfe2eba 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -69,7 +69,7 @@ servers written in Python. Servers can either be free standing, using .. _simple-xmlrpc-servers: -SimpleXMLRPCServer Objects +SimpleXMLRPCServer objects -------------------------- The :class:`SimpleXMLRPCServer` class is based on @@ -140,7 +140,7 @@ alone XML-RPC servers. .. _simplexmlrpcserver-example: -SimpleXMLRPCServer Example +SimpleXMLRPCServer example ^^^^^^^^^^^^^^^^^^^^^^^^^^ Server code:: @@ -231,7 +231,7 @@ a server allowing dotted names and registering a multicall function. :: - import datetime + import datetime as dt class ExampleService: def getData(self): @@ -240,7 +240,7 @@ a server allowing dotted names and registering a multicall function. class currentTime: @staticmethod def getCurrentTime(): - return datetime.datetime.now() + return dt.datetime.now() with SimpleXMLRPCServer(("localhost", 8000)) as server: server.register_function(pow) @@ -387,7 +387,7 @@ to HTTP GET requests. Servers can either be free standing, using .. _doc-xmlrpc-servers: -DocXMLRPCServer Objects +DocXMLRPCServer objects ----------------------- The :class:`DocXMLRPCServer` class is derived from :class:`SimpleXMLRPCServer` diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index cba08d6614fc08..f5d3ade478ffb2 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -37,24 +37,24 @@ the constructor, the :meth:`datetime.replace ` method or :meth:`datetime.astimezone `:: >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta + >>> import datetime as dt - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-10-31 12:00:00-07:00 - >>> dt.tzname() + >>> when.tzname() 'PDT' Datetimes constructed in this way are compatible with datetime arithmetic and handle daylight saving time transitions with no further intervention:: - >>> dt_add = dt + timedelta(days=1) + >>> when_add = when + dt.timedelta(days=1) - >>> print(dt_add) + >>> print(when_add) 2020-11-01 12:00:00-08:00 - >>> dt_add.tzname() + >>> when_add.tzname() 'PST' These time zones also support the :attr:`~datetime.datetime.fold` attribute @@ -63,26 +63,25 @@ times (such as a daylight saving time to standard time transition), the offset from *before* the transition is used when ``fold=0``, and the offset *after* the transition is used when ``fold=1``, for example:: - >>> dt = datetime(2020, 11, 1, 1, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 11, 1, 1, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-11-01 01:00:00-07:00 - >>> print(dt.replace(fold=1)) + >>> print(when.replace(fold=1)) 2020-11-01 01:00:00-08:00 When converting from another time zone, the fold will be set to the correct value:: - >>> from datetime import timezone >>> LOS_ANGELES = ZoneInfo("America/Los_Angeles") - >>> dt_utc = datetime(2020, 11, 1, 8, tzinfo=timezone.utc) + >>> when_utc = dt.datetime(2020, 11, 1, 8, tzinfo=dt.timezone.utc) >>> # Before the PDT -> PST transition - >>> print(dt_utc.astimezone(LOS_ANGELES)) + >>> print(when_utc.astimezone(LOS_ANGELES)) 2020-11-01 01:00:00-07:00 >>> # After the PDT -> PST transition - >>> print((dt_utc + timedelta(hours=1)).astimezone(LOS_ANGELES)) + >>> print((when_utc + dt.timedelta(hours=1)).astimezone(LOS_ANGELES)) 2020-11-01 01:00:00-08:00 Data sources @@ -276,8 +275,8 @@ the note on usage in the attribute documentation):: >>> str(zone) 'Pacific/Kwajalein' - >>> dt = datetime(2020, 4, 1, 3, 15, tzinfo=zone) - >>> f"{dt.isoformat()} [{dt.tzinfo}]" + >>> when = dt.datetime(2020, 4, 1, 3, 15, tzinfo=zone) + >>> f"{when.isoformat()} [{when.tzinfo}]" '2020-04-01T03:15:00+12:00 [Pacific/Kwajalein]' For objects constructed from a file without specifying a ``key`` parameter, diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 342c1a00193959..e7c64104474050 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -1,13 +1,13 @@ .. _tut-brieftour: ********************************** -Brief Tour of the Standard Library +Brief tour of the standard library ********************************** .. _tut-os-interface: -Operating System Interface +Operating system interface ========================== The :mod:`os` module provides dozens of functions for interacting with the @@ -47,7 +47,7 @@ a higher level interface that is easier to use:: .. _tut-file-wildcards: -File Wildcards +File wildcards ============== The :mod:`glob` module provides a function for making file lists from directory @@ -60,7 +60,7 @@ wildcard searches:: .. _tut-command-line-arguments: -Command Line Arguments +Command-line arguments ====================== Common utility scripts often need to process command line arguments. These @@ -97,7 +97,7 @@ to ``['alpha.txt', 'beta.txt']``. .. _tut-stderr: -Error Output Redirection and Program Termination +Error output redirection and program termination ================================================ The :mod:`sys` module also has attributes for *stdin*, *stdout*, and *stderr*. @@ -112,7 +112,7 @@ The most direct way to terminate a script is to use ``sys.exit()``. .. _tut-string-pattern-matching: -String Pattern Matching +String pattern matching ======================= The :mod:`re` module provides regular expression tools for advanced string @@ -175,7 +175,7 @@ computations. .. _tut-internet-access: -Internet Access +Internet access =============== There are a number of modules for accessing the internet and processing internet @@ -206,7 +206,7 @@ from URLs and :mod:`smtplib` for sending mail:: .. _tut-dates-and-times: -Dates and Times +Dates and times =============== The :mod:`datetime` module supplies classes for manipulating dates and times in @@ -216,15 +216,15 @@ formatting and manipulation. The module also supports objects that are timezone aware. :: >>> # dates are easily constructed and formatted - >>> from datetime import date - >>> now = date.today() + >>> import datetime as dt + >>> now = dt.date.today() >>> now datetime.date(2003, 12, 2) >>> now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.") '12-02-03. 02 Dec 2003 is a Tuesday on the 02 day of December.' >>> # dates support calendar arithmetic - >>> birthday = date(1964, 7, 31) + >>> birthday = dt.date(1964, 7, 31) >>> age = now - birthday >>> age.days 14368 @@ -232,7 +232,7 @@ aware. :: .. _tut-data-compression: -Data Compression +Data compression ================ Common data archiving and compression formats are directly supported by modules @@ -254,7 +254,7 @@ including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`lzma`, :mod:`zipfile` and .. _tut-performance-measurement: -Performance Measurement +Performance measurement ======================= Some Python users develop a deep interest in knowing the relative performance of @@ -278,7 +278,7 @@ larger blocks of code. .. _tut-quality-control: -Quality Control +Quality control =============== One approach for developing high quality software is to write tests for each @@ -324,7 +324,7 @@ file:: .. _tut-batteries-included: -Batteries Included +Batteries included ================== Python has a "batteries included" philosophy. This is best seen through the diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index f43692b3dce9e8..43ab19037d2627 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1698,8 +1698,8 @@ current local date. Once created, instances of the date/time classes are all immutable. There are a number of methods for producing formatted strings from objects:: - >>> import datetime - >>> now = datetime.datetime.now() + >>> import datetime as dt + >>> now = dt.datetime.now() >>> now.isoformat() '2002-12-30T21:27:03.994956' >>> now.ctime() # Only available on date, datetime @@ -1710,10 +1710,10 @@ number of methods for producing formatted strings from objects:: The :meth:`~datetime.datetime.replace` method allows modifying one or more fields of a :class:`~datetime.date` or :class:`~datetime.datetime` instance, returning a new instance:: - >>> d = datetime.datetime.now() + >>> d = dt.datetime.now() >>> d datetime.datetime(2002, 12, 30, 22, 15, 38, 827738) - >>> d.replace(year=2001, hour = 12) + >>> d.replace(year=2001, hour=12) datetime.datetime(2001, 12, 30, 12, 15, 38, 827738) >>> diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 9b8f36862c1335..03e612fb651e71 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1313,10 +1313,10 @@ complete list of changes, or look through the SVN logs for all the details. by Josh Spoerri. It uses the same format characters as :func:`time.strptime` and :func:`time.strftime`:: - from datetime import datetime + import datetime as dt - ts = datetime.strptime('10:13:15 2006-03-07', - '%H:%M:%S %Y-%m-%d') + ts = dt.datetime.strptime('10:13:15 2006-03-07', + '%H:%M:%S %Y-%m-%d') * The :meth:`SequenceMatcher.get_matching_blocks` method in the :mod:`difflib` module now guarantees to return a minimal list of blocks describing matching diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index f5e3a47037c65f..1215601a09d681 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -2822,10 +2822,10 @@ Using the module is simple:: import sys import plistlib - import datetime + import datetime as dt # Create data structure - data_struct = dict(lastAccessed=datetime.datetime.now(), + data_struct = dict(lastAccessed=dt.datetime.now(), version=1, categories=('Personal','Shared','Private')) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 3b13d90f7692cd..48c461d891e861 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -992,12 +992,12 @@ datetime and time offset and timezone name. This makes it easier to create timezone-aware datetime objects:: - >>> from datetime import datetime, timezone + >>> import datetime as dt - >>> datetime.now(timezone.utc) + >>> dt.datetime.now(dt.timezone.utc) datetime.datetime(2010, 12, 8, 21, 4, 2, 923754, tzinfo=datetime.timezone.utc) - >>> datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") + >>> dt.datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") datetime.datetime(2000, 1, 1, 12, 0, tzinfo=datetime.timezone.utc) * Also, :class:`~datetime.timedelta` objects can now be multiplied by diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 545a17aecab8ee..91cd23f6f2bbb9 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -50,7 +50,6 @@ For full details, see the :ref:`changelog `. .. testsetup:: - from datetime import date from math import cos, radians from unicodedata import normalize import re @@ -259,15 +258,16 @@ Added an ``=`` specifier to :term:`f-string`\s. An f-string such as ``f'{expr=}'`` will expand to the text of the expression, an equal sign, then the representation of the evaluated expression. For example: + >>> import datetime as dt >>> user = 'eric_idle' - >>> member_since = date(1975, 7, 31) + >>> member_since = dt.date(1975, 7, 31) >>> f'{user=} {member_since=}' "user='eric_idle' member_since=datetime.date(1975, 7, 31)" The usual :ref:`f-string format specifiers ` allow more control over how the result of the expression is displayed:: - >>> delta = date.today() - member_since + >>> delta = dt.date.today() - member_since >>> f'{user=!s} {delta.days=:,d}' 'user=eric_idle delta.days=16,075' diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 40d4a27bff9fee..49a52b7504bc95 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -282,20 +282,20 @@ the standard library. It adds :class:`zoneinfo.ZoneInfo`, a concrete Example:: >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta + >>> import datetime as dt >>> # Daylight saving time - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-10-31 12:00:00-07:00 - >>> dt.tzname() + >>> when.tzname() 'PDT' >>> # Standard time - >>> dt += timedelta(days=7) - >>> print(dt) + >>> when += dt.timedelta(days=7) + >>> print(when) 2020-11-07 12:00:00-08:00 - >>> print(dt.tzname()) + >>> print(when.tzname()) PST diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 77c70aaa7b986e..7a6837546d930f 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -14,6 +14,7 @@ """ import collections +import contextvars import collections.abc import concurrent.futures import errno @@ -290,6 +291,7 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, self._ssl_shutdown_timeout = ssl_shutdown_timeout self._serving = False self._serving_forever_fut = None + self._context = contextvars.copy_context() def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' @@ -319,7 +321,7 @@ def _start_serving(self): self._loop._start_serving( self._protocol_factory, sock, self._ssl_context, self, self._backlog, self._ssl_handshake_timeout, - self._ssl_shutdown_timeout) + self._ssl_shutdown_timeout, context=self._context) def get_loop(self): return self._loop @@ -509,7 +511,8 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - call_connection_made=True): + call_connection_made=True, + context=None): """Create SSL transport.""" raise NotImplementedError @@ -1213,9 +1216,10 @@ async def _create_connection_transport( self, sock, protocol_factory, ssl, server_hostname, server_side=False, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): sock.setblocking(False) + context = context if context is not None else contextvars.copy_context() protocol = protocol_factory() waiter = self.create_future() @@ -1225,9 +1229,10 @@ async def _create_connection_transport( sock, protocol, sslcontext, waiter, server_side=server_side, server_hostname=server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: - transport = self._make_socket_transport(sock, protocol, waiter) + transport = self._make_socket_transport(sock, protocol, waiter, context=context) try: await waiter diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 3fa93b14a6787f..2dc1569d780791 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -642,7 +642,7 @@ def __init__(self, proactor): signal.set_wakeup_fd(self._csock.fileno()) def _make_socket_transport(self, sock, protocol, waiter=None, - extra=None, server=None): + extra=None, server=None, context=None): return _ProactorSocketTransport(self, sock, protocol, waiter, extra, server) @@ -651,7 +651,7 @@ def _make_ssl_transport( *, server_side=False, server_hostname=None, extra=None, server=None, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, @@ -837,7 +837,7 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): def loop(f=None): try: diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index ff7e16df3c6273..9685e7fc05d241 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -67,10 +67,10 @@ def __init__(self, selector=None): self._transports = weakref.WeakValueDictionary() def _make_socket_transport(self, sock, protocol, waiter=None, *, - extra=None, server=None): + extra=None, server=None, context=None): self._ensure_fd_no_transport(sock) return _SelectorSocketTransport(self, sock, protocol, waiter, - extra, server) + extra, server, context=context) def _make_ssl_transport( self, rawsock, protocol, sslcontext, waiter=None, @@ -78,16 +78,17 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, + context=None, ): self._ensure_fd_no_transport(rawsock) ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout + ssl_shutdown_timeout=ssl_shutdown_timeout, ) _SelectorSocketTransport(self, rawsock, ssl_protocol, - extra=extra, server=server) + extra=extra, server=server, context=context) return ssl_protocol._app_transport def _make_datagram_transport(self, sock, protocol, @@ -159,16 +160,16 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): self._add_reader(sock.fileno(), self._accept_connection, protocol_factory, sock, sslcontext, server, backlog, - ssl_handshake_timeout, ssl_shutdown_timeout) + ssl_handshake_timeout, ssl_shutdown_timeout, context) def _accept_connection( self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): # This method is only called once for each event loop tick where the # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. @@ -204,21 +205,22 @@ def _accept_connection( self._start_serving, protocol_factory, sock, sslcontext, server, backlog, ssl_handshake_timeout, - ssl_shutdown_timeout) + ssl_shutdown_timeout, context) else: raise # The event loop will catch, log and ignore it. else: extra = {'peername': addr} + conn_context = context.copy() if context is not None else None accept = self._accept_connection2( protocol_factory, conn, extra, sslcontext, server, - ssl_handshake_timeout, ssl_shutdown_timeout) - self.create_task(accept) + ssl_handshake_timeout, ssl_shutdown_timeout, context=conn_context) + self.create_task(accept, context=conn_context) async def _accept_connection2( self, protocol_factory, conn, extra, sslcontext=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): protocol = None transport = None try: @@ -229,11 +231,12 @@ async def _accept_connection2( conn, protocol, sslcontext, waiter=waiter, server_side=True, extra=extra, server=server, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: transport = self._make_socket_transport( conn, protocol, waiter=waiter, extra=extra, - server=server) + server=server, context=context) try: await waiter @@ -275,9 +278,9 @@ def _ensure_fd_no_transport(self, fd): f'File descriptor {fd!r} is used by transport ' f'{transport!r}') - def _add_reader(self, fd, callback, *args): + def _add_reader(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_READ, @@ -309,9 +312,9 @@ def _remove_reader(self, fd): else: return False - def _add_writer(self, fd, callback, *args): + def _add_writer(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_WRITE, @@ -770,7 +773,7 @@ class _SelectorTransport(transports._FlowControlMixin, # exception) _sock = None - def __init__(self, loop, sock, protocol, extra=None, server=None): + def __init__(self, loop, sock, protocol, extra=None, server=None, context=None): super().__init__(extra, loop) self._extra['socket'] = trsock.TransportSocket(sock) try: @@ -784,7 +787,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._extra['peername'] = None self._sock = sock self._sock_fd = sock.fileno() - + self._context = context self._protocol_connected = False self.set_protocol(protocol) @@ -866,7 +869,7 @@ def close(self): if not self._buffer: self._conn_lost += 1 self._loop._remove_writer(self._sock_fd) - self._loop.call_soon(self._call_connection_lost, None) + self._call_soon(self._call_connection_lost, None) def __del__(self, _warn=warnings.warn): if self._sock is not None: @@ -899,7 +902,7 @@ def _force_close(self, exc): self._closing = True self._loop._remove_reader(self._sock_fd) self._conn_lost += 1 - self._loop.call_soon(self._call_connection_lost, exc) + self._call_soon(self._call_connection_lost, exc) def _call_connection_lost(self, exc): try: @@ -921,8 +924,13 @@ def get_write_buffer_size(self): def _add_reader(self, fd, callback, *args): if not self.is_reading(): return - self._loop._add_reader(fd, callback, *args) + self._loop._add_reader(fd, callback, *args, context=self._context) + def _add_writer(self, fd, callback, *args): + self._loop._add_writer(fd, callback, *args, context=self._context) + + def _call_soon(self, callback, *args): + self._loop.call_soon(callback, *args, context=self._context) class _SelectorSocketTransport(_SelectorTransport): @@ -930,10 +938,9 @@ class _SelectorSocketTransport(_SelectorTransport): _sendfile_compatible = constants._SendfileMode.TRY_NATIVE def __init__(self, loop, sock, protocol, waiter=None, - extra=None, server=None): - + extra=None, server=None, context=None): self._read_ready_cb = None - super().__init__(loop, sock, protocol, extra, server) + super().__init__(loop, sock, protocol, extra, server, context) self._eof = False self._empty_waiter = None if _HAS_SENDMSG: @@ -945,14 +952,12 @@ def __init__(self, loop, sock, protocol, waiter=None, # decreases the latency (in some cases significantly.) base_events._set_nodelay(self._sock) - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def set_protocol(self, protocol): if isinstance(protocol, protocols.BufferedProtocol): @@ -1081,7 +1086,7 @@ def write(self, data): if not data: return # Not all was written; register write handler. - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) # Add it to the buffer. self._buffer.append(data) @@ -1185,7 +1190,7 @@ def writelines(self, list_of_data): self._write_ready() # If the entire buffer couldn't be written, register a write handler if self._buffer: - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) self._maybe_pause_protocol() def can_write_eof(self): @@ -1226,14 +1231,12 @@ def __init__(self, loop, sock, protocol, address=None, super().__init__(loop, sock, protocol, extra) self._address = address self._buffer_size = 0 - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def get_write_buffer_size(self): return self._buffer_size @@ -1280,7 +1283,7 @@ def sendto(self, data, addr=None): self._sock.sendto(data, addr) return except (BlockingIOError, InterruptedError): - self._loop._add_writer(self._sock_fd, self._sendto_ready) + self._add_writer(self._sock_fd, self._sendto_ready) except OSError as exc: self._protocol.error_received(exc) return diff --git a/Lib/curses/has_key.py b/Lib/curses/has_key.py index 4e37b480f13353..3471b28cbbe017 100644 --- a/Lib/curses/has_key.py +++ b/Lib/curses/has_key.py @@ -7,7 +7,7 @@ # Table mapping curses keys to the terminfo capability name -_capability_names = { +_capability_names = frozendict({ _curses.KEY_A1: 'ka1', _curses.KEY_A3: 'ka3', _curses.KEY_B2: 'kb2', @@ -157,7 +157,7 @@ _curses.KEY_SUSPEND: 'kspd', _curses.KEY_UNDO: 'kund', _curses.KEY_UP: 'kcuu1' - } + }) def has_key(ch): if isinstance(ch, str): @@ -170,7 +170,7 @@ def has_key(ch): #Check the current terminal description for that capability; #if present, return true, else return false. - if _curses.tigetstr( capability_name ): + if _curses.tigetstr(capability_name): return True else: return False diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 3c6a6b7bdc44d2..01c7aa96261abe 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -21,7 +21,7 @@ Generate Plist example: - import datetime + import datetime as dt import plistlib pl = dict( @@ -37,7 +37,7 @@ ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.now() + aDate = dt.datetime.now() ) print(plistlib.dumps(pl).decode()) diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 8c02de77c24740..e59bc25668b4cb 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -1696,7 +1696,8 @@ def mock_make_ssl_transport(sock, protocol, sslcontext, waiter, server_side=False, server_hostname='python.org', ssl_handshake_timeout=handshake_timeout, - ssl_shutdown_timeout=shutdown_timeout) + ssl_shutdown_timeout=shutdown_timeout, + context=ANY) # Next try an explicit server_hostname. self.loop._make_ssl_transport.reset_mock() coro = self.loop.create_connection( @@ -1711,7 +1712,8 @@ def mock_make_ssl_transport(sock, protocol, sslcontext, waiter, server_side=False, server_hostname='perl.com', ssl_handshake_timeout=handshake_timeout, - ssl_shutdown_timeout=shutdown_timeout) + ssl_shutdown_timeout=shutdown_timeout, + context=ANY) # Finally try an explicit empty server_hostname. self.loop._make_ssl_transport.reset_mock() coro = self.loop.create_connection( @@ -1726,7 +1728,8 @@ def mock_make_ssl_transport(sock, protocol, sslcontext, waiter, server_side=False, server_hostname='', ssl_handshake_timeout=handshake_timeout, - ssl_shutdown_timeout=shutdown_timeout) + ssl_shutdown_timeout=shutdown_timeout, + context=ANY) def test_create_connection_no_ssl_server_hostname_errors(self): # When not using ssl, server_hostname must be None. @@ -2104,7 +2107,7 @@ def test_accept_connection_exception(self, m_log): constants.ACCEPT_RETRY_DELAY, # self.loop._start_serving mock.ANY, - MyProto, sock, None, None, mock.ANY, mock.ANY, mock.ANY) + MyProto, sock, None, None, mock.ANY, mock.ANY, mock.ANY, mock.ANY) def test_call_coroutine(self): async def simple_coroutine(): diff --git a/Lib/test/test_asyncio/test_server_context.py b/Lib/test/test_asyncio/test_server_context.py new file mode 100644 index 00000000000000..3f15654a1af467 --- /dev/null +++ b/Lib/test/test_asyncio/test_server_context.py @@ -0,0 +1,314 @@ +import asyncio +import contextvars +import unittest +import sys + +from unittest import TestCase + +try: + import ssl +except ImportError: + ssl = None + +from test.test_asyncio import utils as test_utils + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + +class ServerContextvarsTestCase: + loop_factory = None # To be defined in subclasses + server_ssl_context = None # To be defined in subclasses for SSL tests + client_ssl_context = None # To be defined in subclasses for SSL tests + + def run_coro(self, coro): + return asyncio.run(coro, loop_factory=self.loop_factory) + + def test_start_server1(self): + # Test that asyncio.start_server captures the context at the time of server creation + async def test(): + var = contextvars.ContextVar("var", default="default") + + async def handle_client(reader, writer): + value = var.get() + writer.write(value.encode()) + await writer.drain() + writer.close() + + server = await asyncio.start_server(handle_client, '127.0.0.1', 0, + ssl=self.server_ssl_context) + # change the value + var.set("after_server") + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + writer.close() + await writer.wait_closed() + return data.decode() + + async with server: + addr = server.sockets[0].getsockname() + self.assertEqual(await client(addr), "default") + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_start_server2(self): + # Test that mutations to the context in one handler don't affect other handlers or the server's context + async def test(): + var = contextvars.ContextVar("var", default="default") + + async def handle_client(reader, writer): + value = var.get() + writer.write(value.encode()) + var.set("in_handler") + await writer.drain() + writer.close() + + server = await asyncio.start_server(handle_client, '127.0.0.1', 0, + ssl=self.server_ssl_context) + var.set("after_server") + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + writer.close() + await writer.wait_closed() + return data.decode() + + async with server: + addr = server.sockets[0].getsockname() + self.assertEqual(await client(addr), "default") + self.assertEqual(await client(addr), "default") + self.assertEqual(await client(addr), "default") + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_start_server3(self): + # Test that mutations to context in concurrent handlers don't affect each other or the server's context + async def test(): + var = contextvars.ContextVar("var", default="default") + var.set("before_server") + + async def handle_client(reader, writer): + writer.write(var.get().encode()) + await writer.drain() + writer.close() + + server = await asyncio.start_server(handle_client, '127.0.0.1', 0, + ssl=self.server_ssl_context) + var.set("after_server") + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + self.assertEqual(data.decode(), "before_server") + writer.close() + await writer.wait_closed() + + async with server: + addr = server.sockets[0].getsockname() + async with asyncio.TaskGroup() as tg: + for _ in range(100): + tg.create_task(client(addr)) + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_create_server1(self): + # Test that loop.create_server captures the context at the time of server creation + # and that mutations to the context in protocol callbacks don't affect the server's context + async def test(): + var = contextvars.ContextVar("var", default="default") + + class EchoProtocol(asyncio.Protocol): + def connection_made(self, transport): + self.transport = transport + value = var.get() + var.set("in_handler") + self.transport.write(value.encode()) + self.transport.close() + + server = await asyncio.get_running_loop().create_server( + lambda: EchoProtocol(), '127.0.0.1', 0, + ssl=self.server_ssl_context) + var.set("after_server") + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + self.assertEqual(data.decode(), "default") + writer.close() + await writer.wait_closed() + + async with server: + addr = server.sockets[0].getsockname() + await client(addr) + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_create_server2(self): + # Test that mutations to context in one protocol instance don't affect other instances or the server's context + async def test(): + var = contextvars.ContextVar("var", default="default") + + class EchoProtocol(asyncio.Protocol): + def __init__(self): + super().__init__() + assert var.get() == "default", var.get() + def connection_made(self, transport): + self.transport = transport + value = var.get() + var.set("in_handler") + self.transport.write(value.encode()) + self.transport.close() + + server = await asyncio.get_running_loop().create_server( + lambda: EchoProtocol(), '127.0.0.1', 0, + ssl=self.server_ssl_context) + + var.set("after_server") + + async def client(addr, expected): + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + data = await reader.read(100) + self.assertEqual(data.decode(), expected) + writer.close() + await writer.wait_closed() + + async with server: + addr = server.sockets[0].getsockname() + await client(addr, "default") + await client(addr, "default") + + self.assertEqual(var.get(), "after_server") + + self.run_coro(test()) + + def test_gh140947(self): + # See https://github.com/python/cpython/issues/140947 + + cvar1 = contextvars.ContextVar("cvar1") + cvar2 = contextvars.ContextVar("cvar2") + cvar3 = contextvars.ContextVar("cvar3") + results = {} + is_ssl = self.server_ssl_context is not None + + def capture_context(meth): + result = [] + for k,v in contextvars.copy_context().items(): + if k.name.startswith("cvar"): + result.append((k.name, v)) + results[meth] = sorted(result) + + class DemoProtocol(asyncio.Protocol): + def __init__(self, on_conn_lost): + self.transport = None + self.on_conn_lost = on_conn_lost + self.tasks = set() + + def connection_made(self, transport): + capture_context("connection_made") + self.transport = transport + + def data_received(self, data): + capture_context("data_received") + + task = asyncio.create_task(self.asgi()) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + + self.transport.pause_reading() + + def connection_lost(self, exc): + capture_context("connection_lost") + if not self.on_conn_lost.done(): + self.on_conn_lost.set_result(True) + + async def asgi(self): + capture_context("asgi start") + cvar1.set(True) + # make sure that we only resume after the pause + # otherwise the resume does nothing + if is_ssl: + while not self.transport._ssl_protocol._app_reading_paused: + await asyncio.sleep(0.01) + else: + while not self.transport._paused: + await asyncio.sleep(0.01) + cvar2.set(True) + self.transport.resume_reading() + cvar3.set(True) + capture_context("asgi end") + + async def main(): + loop = asyncio.get_running_loop() + on_conn_lost = loop.create_future() + + server = await loop.create_server( + lambda: DemoProtocol(on_conn_lost), '127.0.0.1', 0, + ssl=self.server_ssl_context) + async with server: + addr = server.sockets[0].getsockname() + reader, writer = await asyncio.open_connection(*addr, + ssl=self.client_ssl_context) + writer.write(b"anything") + await writer.drain() + writer.close() + await writer.wait_closed() + await on_conn_lost + + self.run_coro(main()) + self.assertDictEqual(results, { + "connection_made": [], + "data_received": [], + "asgi start": [], + "asgi end": [("cvar1", True), ("cvar2", True), ("cvar3", True)], + "connection_lost": [], + }) + + +class AsyncioEventLoopTests(TestCase, ServerContextvarsTestCase): + loop_factory = staticmethod(asyncio.new_event_loop) + +@unittest.skipUnless(ssl, "SSL not available") +class AsyncioEventLoopSSLTests(AsyncioEventLoopTests): + def setUp(self): + super().setUp() + self.server_ssl_context = test_utils.simple_server_sslcontext() + self.client_ssl_context = test_utils.simple_client_sslcontext() + +if sys.platform == "win32": + class AsyncioProactorEventLoopTests(TestCase, ServerContextvarsTestCase): + loop_factory = asyncio.ProactorEventLoop + + class AsyncioSelectorEventLoopTests(TestCase, ServerContextvarsTestCase): + loop_factory = asyncio.SelectorEventLoop + + @unittest.skipUnless(ssl, "SSL not available") + class AsyncioProactorEventLoopSSLTests(AsyncioProactorEventLoopTests): + def setUp(self): + super().setUp() + self.server_ssl_context = test_utils.simple_server_sslcontext() + self.client_ssl_context = test_utils.simple_client_sslcontext() + + @unittest.skipUnless(ssl, "SSL not available") + class AsyncioSelectorEventLoopSSLTests(AsyncioSelectorEventLoopTests): + def setUp(self): + super().setUp() + self.server_ssl_context = test_utils.simple_server_sslcontext() + self.client_ssl_context = test_utils.simple_client_sslcontext() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index a480e16e81bb91..62cfcf8ceb5f2a 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -388,8 +388,8 @@ def close(self): else: # pragma: no cover raise AssertionError("Time generator is not finished") - def _add_reader(self, fd, callback, *args): - self.readers[fd] = events.Handle(callback, args, self, None) + def _add_reader(self, fd, callback, *args, context=None): + self.readers[fd] = events.Handle(callback, args, self, context) def _remove_reader(self, fd): self.remove_reader_count[fd] += 1 @@ -414,8 +414,8 @@ def assert_no_reader(self, fd): if fd in self.readers: raise AssertionError(f'fd {fd} is registered') - def _add_writer(self, fd, callback, *args): - self.writers[fd] = events.Handle(callback, args, self, None) + def _add_writer(self, fd, callback, *args, context=None): + self.writers[fd] = events.Handle(callback, args, self, context) def _remove_writer(self, fd): self.remove_writer_count[fd] += 1 diff --git a/Misc/NEWS.d/next/Library/2026-03-21-08-23-26.gh-issue-140947.owZ4r_.rst b/Misc/NEWS.d/next/Library/2026-03-21-08-23-26.gh-issue-140947.owZ4r_.rst new file mode 100644 index 00000000000000..88e787e8549a1d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-21-08-23-26.gh-issue-140947.owZ4r_.rst @@ -0,0 +1 @@ +Fix incorrect contextvars handling in server tasks created by :mod:`asyncio`. Patch by Kumar Aditya.