Merge branch 'main' into patch-13

Author: benediktjohannes

Doc/library/pyexpat.rst

@@ -63,12 +63,26 @@ The :mod:`!xml.parsers.expat` module contains two functions:
 
 .. function:: ParserCreate(encoding=None, namespace_separator=None)
 
-   Creates and returns a new :class:`xmlparser` object.   *encoding*, if specified,
-   must be a string naming the encoding  used by the XML data.  Expat doesn't
-   support as many encodings as Python does, and its repertoire of encodings can't
-   be extended; it supports UTF-8, UTF-16, ISO-8859-1 (Latin1), and ASCII.  If
-   *encoding* [1]_ is given it will override the implicit or explicit encoding of the
-   document.
+   Creates and returns a new :class:`xmlparser` object.
+   *encoding* [1]_, if specified, must be a string naming the encoding
+   used by the XML data.
+   If it is given it will override the implicit or explicit encoding
+   of the document.
+
+   .. impl-detail::
+
+      Expat natively understands and processes UTF-8, UTF-16, UTF-16BE,
+      UTF-16LE, ISO-8859-1, and US-ASCII.
+      For other encodings (including aliases like Latin1 and ASCII) it
+      falls back to Python.
+      It supports most of 8-bit encodings and many multi-byte encodings
+      like Shift_JIS, although only BMP characters (``U+0000-U+FFFF``)
+      are supported with non-native encodings (this restriction is also
+      applied to aliases like UTF8).
+      These restrictions only apply if *encoding* is not given.
+
+      .. versionchanged:: next
+         Added support for multi-byte encodings.
 
    .. _xmlparser-non-root:
 
@@ -113,7 +127,6 @@ The :mod:`!xml.parsers.expat` module contains two functions:
    XML document.  Call ``ParserCreate`` for each document to provide unique
    parser instances.
 
-
 .. seealso::
 
    `The Expat XML Parser <http://www.libexpat.org/>`_
@@ -1083,9 +1096,11 @@ The ``errors`` module has the following attributes:
 
 .. rubric:: Footnotes
 
-.. [1] The encoding string included in XML output should conform to the
-   appropriate standards. For example, "UTF-8" is valid, but "UTF8" is
-   not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl
+.. [1] The encoding string included in XML output should conform to
+   the appropriate standards. For example, "UTF-8" is valid, but
+   "UTF8" is not valid in an XML document's declaration, even though
+   Python accepts it as an encoding name.
+   See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl
    and https://www.iana.org/assignments/character-sets/character-sets.xhtml.
 
 

Doc/whatsnew/3.16.rst

@@ -86,7 +86,6 @@ New modules
 Improved modules
 ================
 
-
 gzip
 ----
 
@@ -101,6 +100,21 @@ os
   process via a pidfd.  Available on Linux 5.6+.
   (Contributed by Maurycy Pawłowski-Wieroński in :gh:`149464`.)
 
+xml
+---
+
+* Add support for multiple multi-byte encodings in the :mod:`XML parser
+  <xml.parsers.expat>`: "cp932", "cp949", "cp950", "Big5","EUC-JP",
+  "GB2312", "GBK", "johab", and "Shift_JIS".
+  Add partial support (only BMP characters) for multi-byte encodings
+  "Big5-HKSCS", "EUC_JIS-2004", "EUC_JISX0213", "Shift_JIS-2004",
+  "Shift_JISX0213", "utf-8-sig" and non-standard aliases like "UTF8"
+  (without hyphen).
+  The parser now raises :exc:`ValueError` for known unsupported
+  multi-byte encodings such us "ISO-2022-JP" or "raw-unicode-escape"
+  instead of failing later, when encounter non-ASCII data.
+  (Contributed by Serhiy Storchaka in :gh:`62259`.)
+
 .. Add improved modules above alphabetically, not here at the end.
 
 Optimizations

Include/internal/pycore_ceval.h

@@ -208,16 +208,16 @@ extern void _PyEval_DeactivateOpCache(void);
 
 /* --- _Py_EnterRecursiveCall() ----------------------------------------- */
 
-static inline int _Py_MakeRecCheck(PyThreadState *tstate)  {
+static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate)  {
     uintptr_t here_addr = _Py_get_machine_stack_pointer();
     _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
-    // Overflow if stack pointer is between soft limit and the base of the hardware stack.
-    // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing.
-    // We could have the wrong stack limits because of limited platform support, or user-space threads.
+    // Possible overflow if stack pointer is beyond the soft limit.
+    // _Py_CheckRecursiveCall will check for corner cases and
+    // report an error if there is an overflow.
 #if _Py_STACK_GROWS_DOWN
-    return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES;
+    return here_addr < _tstate->c_stack_soft_limit;
 #else
-    return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES;
+    return here_addr > _tstate->c_stack_soft_limit;
 #endif
 }
 
@@ -232,7 +232,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCallPy(
 
 static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate,
                                                const char *where) {
-    return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where));
+    return (_Py_ReachedRecursionLimit(tstate) && _Py_CheckRecursiveCall(tstate, where));
 }
 
 static inline int _Py_EnterRecursiveCall(const char *where) {
@@ -246,8 +246,6 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) {
 
 PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate);
 
-PyAPI_FUNC(int) _Py_ReachedRecursionLimit(PyThreadState *tstate);
-
 // Export for test_peg_generator
 PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(
     PyThreadState *tstate,

Include/internal/pycore_codecs.h

@@ -45,7 +45,7 @@ extern int _PyCodec_UnregisterError(const char *name);
    in Python 3.5+?
 
  */
-extern PyObject* _PyCodec_LookupTextEncoding(
+PyAPI_FUNC(PyObject*) _PyCodec_LookupTextEncoding(
    const char *encoding,
    const char *alternate_command);
 

Include/internal/pycore_pystate.h

@@ -316,15 +316,20 @@ static uintptr_t return_pointer_as_int(char* p) {
 
 static inline uintptr_t
 _Py_get_machine_stack_pointer(void) {
-#if _Py__has_builtin(__builtin_frame_address) || defined(__GNUC__)
-    return (uintptr_t)__builtin_frame_address(0);
-#elif defined(_MSC_VER)
-    return (uintptr_t)_AddressOfReturnAddress();
+    uintptr_t result;
+#if defined(_M_ARM64)
+    result = __getReg(31);
+#elif defined(_M_X64) || defined(_M_IX86)
+    result = (uintptr_t)_AddressOfReturnAddress();
+#elif defined(__aarch64__)
+    __asm__ ("mov %0, sp" : "=r" (result));
+#elif defined(__x86_64__)
+    __asm__("{movq %%rsp, %0" : "=r" (result));
 #else
     char here;
-    /* Avoid compiler warning about returning stack address */
-    return return_pointer_as_int(&here);
+    result = (uintptr_t)&here;
 #endif
+    return result;
 }
 
 static inline intptr_t

Include/internal/pycore_pythonrun.h

@@ -46,7 +46,8 @@ extern PyObject * _Py_CompileStringObjectWithModule(
  * stack consumption of PyEval_EvalDefault */
 #if (defined(Py_DEBUG) \
      || defined(_Py_ADDRESS_SANITIZER) \
-     || defined(_Py_THREAD_SANITIZER))
+     || defined(_Py_THREAD_SANITIZER)) \
+     || defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER)
 #  define _PyOS_LOG2_STACK_MARGIN 12
 #else
 #  define _PyOS_LOG2_STACK_MARGIN 11

Lib/asyncio/selector_events.py

@@ -1233,7 +1233,6 @@ def close(self):
 
 class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport):
 
-    _buffer_factory = collections.deque
     _header_size = 8
 
     def __init__(self, loop, sock, protocol, address=None,

Lib/codecs.py

@@ -93,7 +93,7 @@ class CodecInfo(tuple):
 
     def __new__(cls, encode, decode, streamreader=None, streamwriter=None,
         incrementalencoder=None, incrementaldecoder=None, name=None,
-        *, _is_text_encoding=None):
+        *, _is_text_encoding=None, _expat_decoding_table=None):
         self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter))
         self.name = name
         self.encode = encode
@@ -104,6 +104,8 @@ def __new__(cls, encode, decode, streamreader=None, streamwriter=None,
         self.streamreader = streamreader
         if _is_text_encoding is not None:
             self._is_text_encoding = _is_text_encoding
+        if _expat_decoding_table is not None:
+            self._expat_decoding_table = _expat_decoding_table
         return self
 
     def __repr__(self):

Lib/email/charset.py

@@ -93,8 +93,6 @@
 
 # Map charsets to their Unicode codec strings.
 CODEC_MAP = {
-    'gb2312':      'eucgb2312_cn',
-    'big5':        'big5_tw',
     # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
     # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
     # Let that stuff pass through without conversion to/from Unicode.

Lib/encodings/big5.py

@@ -36,4 +36,13 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _expat_decoding_table=(*range(128),
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+            -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
+            -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
+            -2, -2, -2, -2, -2, -2, -2, -2, -1, -2, -2, -2, -2, -2, -2, -2,
+            -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
+            -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
+            -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1, -1, -1, -1),
     )