[3.14] gh-87587: Fix os.device_encoding() for any console on Windows (GH-155410) (GH-156596)

It was hard coded to map file descriptors 0, 1 and 2 to the console code
page, but a console can be opened as any file descriptor, and other
character devices, like NUL, are not consoles.

The file handle is now queried.  The UTF-8 code page is also now reported
as "utf-8" instead of "cp65001".

(cherry picked from commit b4aea41)

Author: serhiy-storchaka

Lib/test/test_os.py

@@ -3575,6 +3575,45 @@ def test_device_encoding(self):
         self.assertTrue(codecs.lookup(encoding))
 
 
+@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+class Win32DeviceEncodingTests(unittest.TestCase):
+    # gh-87587: any console file descriptor is supported, not only 0, 1 and 2,
+    # and other character devices are not consoles.
+
+    @staticmethod
+    def expected_encoding(cp):
+        return 'utf-8' if cp == 65001 else 'cp%d' % cp
+
+    def test_console(self):
+        import ctypes
+        kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
+        try:
+            fin = open('CONIN$')
+        except OSError:
+            self.skipTest('no console')
+        with fin:
+            self.assertEqual(os.device_encoding(fin.fileno()),
+                             self.expected_encoding(kernel32.GetConsoleCP()))
+        with open('CONOUT$', 'w') as fout:
+            self.assertEqual(
+                os.device_encoding(fout.fileno()),
+                self.expected_encoding(kernel32.GetConsoleOutputCP()))
+
+    def test_not_a_console(self):
+        with open('NUL', 'w') as f:
+            self.assertTrue(os.isatty(f.fileno()))
+            self.assertIsNone(os.device_encoding(f.fileno()))
+            # Not a console even if it is a standard file descriptor.
+            saved = os.dup(1)
+            try:
+                os.dup2(f.fileno(), 1)
+                encoding = os.device_encoding(1)
+            finally:
+                os.dup2(saved, 1)
+                os.close(saved)
+            self.assertIsNone(encoding)
+
+
 @support.requires_subprocess()
 class PidTests(unittest.TestCase):
     @unittest.skipUnless(hasattr(os, 'getppid'), "test needs os.getppid")

Misc/NEWS.d/next/Windows/2026-08-09-04-00-00.gh-issue-87587.devenc.rst

@@ -0,0 +1,4 @@
+:func:`os.device_encoding` on Windows now returns the code page of any
+console file descriptor, not only 0, 1 and 2, and returns ``None`` for other
+character devices like ``NUL``.  The UTF-8 code page is now reported as
+``"utf-8"`` instead of ``"cp65001"``.

Python/fileutils.c

@@ -80,41 +80,67 @@ get_surrogateescape(_Py_error_handler errors, int *surrogateescape)
 PyObject *
 _Py_device_encoding(int fd)
 {
-    int valid;
-    Py_BEGIN_ALLOW_THREADS
+#if defined(MS_WINDOWS) && defined(HAVE_WINDOWS_CONSOLE_IO)
+    HANDLE handle;
+    DWORD temp;
+    UINT cp = 0;
+
     _Py_BEGIN_SUPPRESS_IPH
-    valid = isatty(fd);
+    handle = (HANDLE)_get_osfhandle(fd);
     _Py_END_SUPPRESS_IPH
-    Py_END_ALLOW_THREADS
-    if (!valid)
+    if (handle == INVALID_HANDLE_VALUE) {
         Py_RETURN_NONE;
+    }
+
+    Py_BEGIN_ALLOW_THREADS
+    if (GetFileType(handle) == FILE_TYPE_CHAR) {
+        /* GetConsoleMode() only succeeds for a console handle. */
+        if (!GetConsoleMode(handle, &temp)) {
+            /* Assume that access denied implies an output handle. */
+            if (GetLastError() == ERROR_ACCESS_DENIED) {
+                cp = GetConsoleOutputCP();
+            }
+        }
+        else if (GetNumberOfConsoleInputEvents(handle, &temp)) {
+            cp = GetConsoleCP();
+        }
+        else {
+            cp = GetConsoleOutputCP();
+        }
+    }
+    Py_END_ALLOW_THREADS
 
-#ifdef MS_WINDOWS
-#ifdef HAVE_WINDOWS_CONSOLE_IO
-    UINT cp;
-    if (fd == 0)
-        cp = GetConsoleCP();
-    else if (fd == 1 || fd == 2)
-        cp = GetConsoleOutputCP();
-    else
-        cp = 0;
     /* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application
        has no console */
+    if (cp == CP_UTF8) {
+        _Py_DECLARE_STR(utf_8, "utf-8");
+        return &_Py_STR(utf_8);
+    }
     if (cp == 0) {
         Py_RETURN_NONE;
     }
-
     return PyUnicode_FromFormat("cp%u", (unsigned int)cp);
 #else
+    int valid;
+    Py_BEGIN_ALLOW_THREADS
+    _Py_BEGIN_SUPPRESS_IPH
+    valid = isatty(fd);
+    _Py_END_SUPPRESS_IPH
+    Py_END_ALLOW_THREADS
+    if (!valid) {
+        Py_RETURN_NONE;
+    }
+
+#ifdef MS_WINDOWS
     Py_RETURN_NONE;
-#endif /* HAVE_WINDOWS_CONSOLE_IO */
 #else
     if (_PyRuntime.preconfig.utf8_mode) {
         _Py_DECLARE_STR(utf_8, "utf-8");
         return &_Py_STR(utf_8);
     }
     return _Py_GetLocaleEncodingObject();
 #endif
+#endif /* MS_WINDOWS && HAVE_WINDOWS_CONSOLE_IO */
 }