From 1724a3c7d88c71ce7da4dcd28e62530c4c8e4163 Mon Sep 17 00:00:00 2001 From: Wulian233 <1055917385@qq.com> Date: Sat, 6 Jul 2024 12:38:38 +0800 Subject: [PATCH 01/11] Speed up `socket.errotTab` and lazy import `selectors` --- Lib/socket.py | 199 +++++++++--------- ...-07-06-12-37-10.gh-issue-121423.vnxrl4.rst | 1 + 2 files changed, 102 insertions(+), 98 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst diff --git a/Lib/socket.py b/Lib/socket.py index 524ce1361b9091..b008a2a45f3848 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -52,7 +52,7 @@ import _socket from _socket import * -import os, sys, io, selectors +import os, sys, io from enum import IntEnum, IntFlag try: @@ -109,103 +109,104 @@ def _intenum_converter(value, enum_klass): # WSA error codes -if sys.platform.lower().startswith("win"): - errorTab = {} - errorTab[6] = "Specified event object handle is invalid." - errorTab[8] = "Insufficient memory available." - errorTab[87] = "One or more parameters are invalid." - errorTab[995] = "Overlapped operation aborted." - errorTab[996] = "Overlapped I/O event object not in signaled state." - errorTab[997] = "Overlapped operation will complete later." - errorTab[10004] = "The operation was interrupted." - errorTab[10009] = "A bad file handle was passed." - errorTab[10013] = "Permission denied." - errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT - errorTab[10022] = "An invalid operation was attempted." - errorTab[10024] = "Too many open files." - errorTab[10035] = "The socket operation would block." - errorTab[10036] = "A blocking operation is already in progress." - errorTab[10037] = "Operation already in progress." - errorTab[10038] = "Socket operation on nonsocket." - errorTab[10039] = "Destination address required." - errorTab[10040] = "Message too long." - errorTab[10041] = "Protocol wrong type for socket." - errorTab[10042] = "Bad protocol option." - errorTab[10043] = "Protocol not supported." - errorTab[10044] = "Socket type not supported." - errorTab[10045] = "Operation not supported." - errorTab[10046] = "Protocol family not supported." - errorTab[10047] = "Address family not supported by protocol family." - errorTab[10048] = "The network address is in use." - errorTab[10049] = "Cannot assign requested address." - errorTab[10050] = "Network is down." - errorTab[10051] = "Network is unreachable." - errorTab[10052] = "Network dropped connection on reset." - errorTab[10053] = "Software caused connection abort." - errorTab[10054] = "The connection has been reset." - errorTab[10055] = "No buffer space available." - errorTab[10056] = "Socket is already connected." - errorTab[10057] = "Socket is not connected." - errorTab[10058] = "The network has been shut down." - errorTab[10059] = "Too many references." - errorTab[10060] = "The operation timed out." - errorTab[10061] = "Connection refused." - errorTab[10062] = "Cannot translate name." - errorTab[10063] = "The name is too long." - errorTab[10064] = "The host is down." - errorTab[10065] = "The host is unreachable." - errorTab[10066] = "Directory not empty." - errorTab[10067] = "Too many processes." - errorTab[10068] = "User quota exceeded." - errorTab[10069] = "Disk quota exceeded." - errorTab[10070] = "Stale file handle reference." - errorTab[10071] = "Item is remote." - errorTab[10091] = "Network subsystem is unavailable." - errorTab[10092] = "Winsock.dll version out of range." - errorTab[10093] = "Successful WSAStartup not yet performed." - errorTab[10101] = "Graceful shutdown in progress." - errorTab[10102] = "No more results from WSALookupServiceNext." - errorTab[10103] = "Call has been canceled." - errorTab[10104] = "Procedure call table is invalid." - errorTab[10105] = "Service provider is invalid." - errorTab[10106] = "Service provider failed to initialize." - errorTab[10107] = "System call failure." - errorTab[10108] = "Service not found." - errorTab[10109] = "Class type not found." - errorTab[10110] = "No more results from WSALookupServiceNext." - errorTab[10111] = "Call was canceled." - errorTab[10112] = "Database query was refused." - errorTab[11001] = "Host not found." - errorTab[11002] = "Nonauthoritative host not found." - errorTab[11003] = "This is a nonrecoverable error." - errorTab[11004] = "Valid name, no data record requested type." - errorTab[11005] = "QoS receivers." - errorTab[11006] = "QoS senders." - errorTab[11007] = "No QoS senders." - errorTab[11008] = "QoS no receivers." - errorTab[11009] = "QoS request confirmed." - errorTab[11010] = "QoS admission error." - errorTab[11011] = "QoS policy failure." - errorTab[11012] = "QoS bad style." - errorTab[11013] = "QoS bad object." - errorTab[11014] = "QoS traffic control error." - errorTab[11015] = "QoS generic error." - errorTab[11016] = "QoS service type error." - errorTab[11017] = "QoS flowspec error." - errorTab[11018] = "Invalid QoS provider buffer." - errorTab[11019] = "Invalid QoS filter style." - errorTab[11020] = "Invalid QoS filter style." - errorTab[11021] = "Incorrect QoS filter count." - errorTab[11022] = "Invalid QoS object length." - errorTab[11023] = "Incorrect QoS flow count." - errorTab[11024] = "Unrecognized QoS object." - errorTab[11025] = "Invalid QoS policy object." - errorTab[11026] = "Invalid QoS flow descriptor." - errorTab[11027] = "Invalid QoS provider-specific flowspec." - errorTab[11028] = "Invalid QoS provider-specific filterspec." - errorTab[11029] = "Invalid QoS shape discard mode object." - errorTab[11030] = "Invalid QoS shaping rate object." - errorTab[11031] = "Reserved policy QoS element type." +if sys.platform == "win32": + errorTab = { + 6: "Specified event object handle is invalid.", + 8: "Insufficient memory available.", + 87: "One or more parameters are invalid.", + 995: "Overlapped operation aborted.", + 996: "Overlapped I/O event object not in signaled state.", + 997: "Overlapped operation will complete later.", + 10004: "The operation was interrupted.", + 10009: "A bad file handle was passed.", + 10013: "Permission denied.", + 10014: "A fault occurred on the network??", + 10022: "An invalid operation was attempted.", + 10024: "Too many open files.", + 10035: "The socket operation would block.", + 10036: "A blocking operation is already in progress.", + 10037: "Operation already in progress.", + 10038: "Socket operation on nonsocket.", + 10039: "Destination address required.", + 10040: "Message too long.", + 10041: "Protocol wrong type for socket.", + 10042: "Bad protocol option.", + 10043: "Protocol not supported.", + 10044: "Socket type not supported.", + 10045: "Operation not supported.", + 10046: "Protocol family not supported.", + 10047: "Address family not supported by protocol family.", + 10048: "The network address is in use.", + 10049: "Cannot assign requested address.", + 10050: "Network is down.", + 10051: "Network is unreachable.", + 10052: "Network dropped connection on reset.", + 10053: "Software caused connection abort.", + 10054: "The connection has been reset.", + 10055: "No buffer space available.", + 10056: "Socket is already connected.", + 10057: "Socket is not connected.", + 10058: "The network has been shut down.", + 10059: "Too many references.", + 10060: "The operation timed out.", + 10061: "Connection refused.", + 10062: "Cannot translate name.", + 10063: "The name is too long.", + 10064: "The host is down.", + 10065: "The host is unreachable.", + 10066: "Directory not empty.", + 10067: "Too many processes.", + 10068: "User quota exceeded.", + 10069: "Disk quota exceeded.", + 10070: "Stale file handle reference.", + 10071: "Item is remote.", + 10091: "Network subsystem is unavailable.", + 10092: "Winsock.dll version out of range.", + 10093: "Successful WSAStartup not yet performed.", + 10101: "Graceful shutdown in progress.", + 10102: "No more results from WSALookupServiceNext.", + 10103: "Call has been canceled.", + 10104: "Procedure call table is invalid.", + 10105: "Service provider is invalid.", + 10106: "Service provider failed to initialize.", + 10107: "System call failure.", + 10108: "Service not found.", + 10109: "Class type not found.", + 10110: "No more results from WSALookupServiceNext.", + 10111: "Call was canceled.", + 10112: "Database query was refused.", + 11001: "Host not found.", + 11002: "Nonauthoritative host not found.", + 11003: "This is a nonrecoverable error.", + 11004: "Valid name, no data record requested type.", + 11005: "QoS receivers.", + 11006: "QoS senders.", + 11007: "No QoS senders.", + 11008: "QoS no receivers.", + 11009: "QoS request confirmed.", + 11010: "QoS admission error.", + 11011: "QoS policy failure.", + 11012: "QoS bad style.", + 11013: "QoS bad object.", + 11014: "QoS traffic control error.", + 11015: "QoS generic error.", + 11016: "QoS service type error.", + 11017: "QoS flowspec error.", + 11018: "Invalid QoS provider buffer.", + 11019: "Invalid QoS filter style.", + 11020: "Invalid QoS filter style.", + 11021: "Incorrect QoS filter count.", + 11022: "Invalid QoS object length.", + 11023: "Incorrect QoS flow count.", + 11024: "Unrecognized QoS object.", + 11025: "Invalid QoS policy object.", + 11026: "Invalid QoS flow descriptor.", + 11027: "Invalid QoS provider-specific flowspec.", + 11028: "Invalid QoS provider-specific filterspec.", + 11029: "Invalid QoS shape discard mode object.", + 11030: "Invalid QoS shaping rate object.", + 11031: "Reserved policy QoS element type." + } __all__.append("errorTab") @@ -348,6 +349,8 @@ def makefile(self, mode="r", buffering=None, *, if hasattr(os, 'sendfile'): def _sendfile_use_sendfile(self, file, offset=0, count=None): + import selectors + self._check_sendfile_params(file, offset, count) sockno = self.fileno() try: diff --git a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst new file mode 100644 index 00000000000000..21a1911f483f9a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst @@ -0,0 +1 @@ +Speed up `socket.errotTab` and lazy import `selectors` From 6559ebb00abc6ac7d5afcbb2c92ae8f1f567b241 Mon Sep 17 00:00:00 2001 From: Wulian233 <1055917385@qq.com> Date: Sat, 6 Jul 2024 12:43:20 +0800 Subject: [PATCH 02/11] lint --- .../next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst index 21a1911f483f9a..a6536070d74532 100644 --- a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst +++ b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst @@ -1 +1 @@ -Speed up `socket.errotTab` and lazy import `selectors` +Speed up socket.errotTab and lazy import selectors From 984a3b217ccc3f0ff7a34d9bc4e20c400c80f9ab Mon Sep 17 00:00:00 2001 From: Wulian233 <1055917385@qq.com> Date: Sat, 6 Jul 2024 14:01:38 +0800 Subject: [PATCH 03/11] typo --- .../next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst index a6536070d74532..f37c4500e6c13e 100644 --- a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst +++ b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst @@ -1 +1 @@ -Speed up socket.errotTab and lazy import selectors +Speed up socket.errorTab and lazy import selectors From 1823c9aa8f26dfce45c4c9329d48f90fbd7f7e93 Mon Sep 17 00:00:00 2001 From: Wulian233 <1055917385@qq.com> Date: Sat, 6 Jul 2024 17:22:03 +0800 Subject: [PATCH 04/11] lower().startswith("win") --- Lib/socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/socket.py b/Lib/socket.py index b008a2a45f3848..48bfc919a73e84 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -109,7 +109,7 @@ def _intenum_converter(value, enum_klass): # WSA error codes -if sys.platform == "win32": +if sys.platform.lower().startswith("win"): errorTab = { 6: "Specified event object handle is invalid.", 8: "Insufficient memory available.", From dce6cd26f334ca6f35764438c2163f38d2bf2ad3 Mon Sep 17 00:00:00 2001 From: Wulian233 <1055917385@qq.com> Date: Sun, 7 Jul 2024 16:00:19 +0800 Subject: [PATCH 05/11] Update Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst Co-authored-by: Pieter Eendebak --- .../next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst index f37c4500e6c13e..b3d573f81bd9a3 100644 --- a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst +++ b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst @@ -1 +1 @@ -Speed up socket.errorTab and lazy import selectors +Improve import time of `mod`:`socket` by writing `socket.errorTab` as a constant and lazy import of `mod`:`selectors`. From 276d0bb5c426313167743902d7bb7720647cc188 Mon Sep 17 00:00:00 2001 From: Wulian233 <1055917385@qq.com> Date: Sun, 7 Jul 2024 16:05:54 +0800 Subject: [PATCH 06/11] lint --- .../Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst index b3d573f81bd9a3..2688071e0c4e39 100644 --- a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst +++ b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst @@ -1 +1,2 @@ -Improve import time of `mod`:`socket` by writing `socket.errorTab` as a constant and lazy import of `mod`:`selectors`. +Improve import time of `mod`:`socket` by writing `socket.errorTab` as a +constant and lazy import of `mod`:`selectors` From feae2fa7dbbe4d88b64396db4f45ddc3d5d7686a Mon Sep 17 00:00:00 2001 From: Wulian233 <1055917385@qq.com> Date: Sun, 7 Jul 2024 17:25:19 +0800 Subject: [PATCH 07/11] Update Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst index 2688071e0c4e39..e645286c3d5dc4 100644 --- a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst +++ b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst @@ -1,2 +1,2 @@ -Improve import time of `mod`:`socket` by writing `socket.errorTab` as a -constant and lazy import of `mod`:`selectors` +Improve import time of :mod:`socket` by lazy importing :mod:`selectors` and +writing :data:`!socket.errorTab` as a constant. From bd771c831e78be812d4dde6ae28270abbd769e09 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 31 Aug 2024 10:54:37 -0700 Subject: [PATCH 08/11] comment describing the intentional lazy import Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/socket.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/socket.py b/Lib/socket.py index 48bfc919a73e84..4949b51a89dc28 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -349,6 +349,7 @@ def makefile(self, mode="r", buffering=None, *, if hasattr(os, 'sendfile'): def _sendfile_use_sendfile(self, file, offset=0, count=None): + # Lazy import to improve module import time import selectors self._check_sendfile_params(file, offset, count) From 1773d04a41d27cf1945f5fc5907f51857a556dd1 Mon Sep 17 00:00:00 2001 From: Wulian <1055917385@qq.com> Date: Tue, 3 Sep 2024 20:10:12 +0800 Subject: [PATCH 09/11] lazy array --- Doc/whatsnew/3.14.rst | 9 +++++++++ Lib/socket.py | 8 ++++---- .../2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2ab4102f32ab0b..3dbfb7ee501305 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -238,6 +238,15 @@ asyncio reduces memory usage. (Contributed by Kumar Aditya in :gh:`107803`.) + +socket +------- + +* Improve import time of :mod:`socket` by lazy importing modules and + writing :data:`!socket.errorTab` as a constant, which results in + a 30% speed up in standard pyperformance benchmarks. + (Contributed by Jiahao Li in :gh:`121424`.) + Deprecated ========== diff --git a/Lib/socket.py b/Lib/socket.py index 3228d85b9b6f72..f250840df0b161 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -553,20 +553,18 @@ def fromfd(fd, family, type, proto=0): return socket(family, type, proto, nfd) if hasattr(_socket.socket, "sendmsg"): - import array - def send_fds(sock, buffers, fds, flags=0, address=None): """ send_fds(sock, buffers, fds[, flags[, address]]) -> integer Send the list of file descriptors fds over an AF_UNIX socket. """ + import array + return sock.sendmsg(buffers, [(_socket.SOL_SOCKET, _socket.SCM_RIGHTS, array.array("i", fds))]) __all__.append("send_fds") if hasattr(_socket.socket, "recvmsg"): - import array - def recv_fds(sock, bufsize, maxfds, flags=0): """ recv_fds(sock, bufsize, maxfds[, flags]) -> (data, list of file descriptors, msg_flags, address) @@ -574,6 +572,8 @@ def recv_fds(sock, bufsize, maxfds, flags=0): Receive up to maxfds file descriptors returning the message data and a list containing the descriptors. """ + import array + # Array of ints fds = array.array("i") msg, ancdata, flags, addr = sock.recvmsg(bufsize, diff --git a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst index e645286c3d5dc4..0fd89a99681292 100644 --- a/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst +++ b/Misc/NEWS.d/next/Library/2024-07-06-12-37-10.gh-issue-121423.vnxrl4.rst @@ -1,2 +1,2 @@ -Improve import time of :mod:`socket` by lazy importing :mod:`selectors` and +Improve import time of :mod:`socket` by lazy importing modules and writing :data:`!socket.errorTab` as a constant. From 412a3d2a6b276e98b1fd9b7cf2069876126acd2f Mon Sep 17 00:00:00 2001 From: Wulian <1055917385@qq.com> Date: Tue, 3 Sep 2024 20:27:43 +0800 Subject: [PATCH 10/11] separate imports --- Lib/socket.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/socket.py b/Lib/socket.py index f250840df0b161..be37c24d6174a2 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -52,7 +52,9 @@ import _socket from _socket import * -import os, sys, io +import io +import os +import sys from enum import IntEnum, IntFlag try: From fa493950862a325007bbf9e056f8da957b01bd60 Mon Sep 17 00:00:00 2001 From: Wulian <1055917385@qq.com> Date: Tue, 3 Sep 2024 21:14:27 +0800 Subject: [PATCH 11/11] undo 3.14.rst --- Doc/whatsnew/3.14.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 3dbfb7ee501305..2ab4102f32ab0b 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -238,15 +238,6 @@ asyncio reduces memory usage. (Contributed by Kumar Aditya in :gh:`107803`.) - -socket -------- - -* Improve import time of :mod:`socket` by lazy importing modules and - writing :data:`!socket.errorTab` as a constant, which results in - a 30% speed up in standard pyperformance benchmarks. - (Contributed by Jiahao Li in :gh:`121424`.) - Deprecated ==========