From 0153d2e2dfd663b661118b246efd9439f8b21b61 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 22 Feb 2022 10:21:13 +0000 Subject: [PATCH 1/3] skip getaddrinfo thread if host is already resolved, using socket.AI_NUMERIC... --- Lib/asyncio/base_events.py | 77 ++--------- Lib/test/test_asyncio/test_base_events.py | 126 +----------------- .../2022-02-22-10-22-44.bpo-46824.IflDPF.rst | 1 + 3 files changed, 14 insertions(+), 190 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-02-22-10-22-44.bpo-46824.IflDPF.rst diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index f9215c5e0182a89..57c2eee1a8dbf3c 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -96,65 +96,6 @@ def _set_reuseport(sock): 'SO_REUSEPORT defined but not implemented.') -def _ipaddr_info(host, port, family, type, proto, flowinfo=0, scopeid=0): - # Try to skip getaddrinfo if "host" is already an IP. Users might have - # handled name resolution in their own code and pass in resolved IPs. - if not hasattr(socket, 'inet_pton'): - return - - if proto not in {0, socket.IPPROTO_TCP, socket.IPPROTO_UDP} or \ - host is None: - return None - - if type == socket.SOCK_STREAM: - proto = socket.IPPROTO_TCP - elif type == socket.SOCK_DGRAM: - proto = socket.IPPROTO_UDP - else: - return None - - if port is None: - port = 0 - elif isinstance(port, bytes) and port == b'': - port = 0 - elif isinstance(port, str) and port == '': - port = 0 - else: - # If port's a service name like "http", don't skip getaddrinfo. - try: - port = int(port) - except (TypeError, ValueError): - return None - - if family == socket.AF_UNSPEC: - afs = [socket.AF_INET] - if _HAS_IPv6: - afs.append(socket.AF_INET6) - else: - afs = [family] - - if isinstance(host, bytes): - host = host.decode('idna') - if '%' in host: - # Linux's inet_pton doesn't accept an IPv6 zone index after host, - # like '::1%lo0'. - return None - - for af in afs: - try: - socket.inet_pton(af, host) - # The host has already been resolved. - if _HAS_IPv6 and af == socket.AF_INET6: - return af, type, proto, '', (host, port, flowinfo, scopeid) - else: - return af, type, proto, '', (host, port) - except OSError: - pass - - # "host" is not an IP address. - return None - - def _interleave_addrinfos(addrinfos, first_address_family_count=1): """Interleave list of addrinfo tuples by family.""" # Group addresses by family @@ -856,6 +797,15 @@ async def getaddrinfo(self, host, port, *, else: getaddr_func = socket.getaddrinfo + try: + return getaddr_func( + host, port, family, type, proto, + flags | socket.AI_NUMERICHOST | socket.AI_NUMERICSERV, + ) + except socket.gaierror as e: + if e.errno != socket.EAI_NONAME: + raise + return await self.run_in_executor( None, getaddr_func, host, port, family, type, proto, flags) @@ -1392,13 +1342,8 @@ async def _ensure_resolved(self, address, *, family=0, type=socket.SOCK_STREAM, proto=0, flags=0, loop): host, port = address[:2] - info = _ipaddr_info(host, port, family, type, proto, *address[2:]) - if info is not None: - # "host" is already a resolved IP. - return [info] - else: - return await loop.getaddrinfo(host, port, family=family, type=type, - proto=proto, flags=flags) + return await loop.getaddrinfo(host, port, family=family, type=type, + proto=proto, flags=flags) async def _create_server_getaddrinfo(self, host, port, family, flags): infos = await self._ensure_resolved((host, port), family=family, diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index c6671bd0ad3d852..4848deae11207ba 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -31,7 +31,8 @@ def mock_socket_module(): m_socket = mock.MagicMock(spec=socket) for name in ( 'AF_INET', 'AF_INET6', 'AF_UNSPEC', 'IPPROTO_TCP', 'IPPROTO_UDP', - 'SOCK_STREAM', 'SOCK_DGRAM', 'SOL_SOCKET', 'SO_REUSEADDR', 'inet_pton' + 'SOCK_STREAM', 'SOCK_DGRAM', 'SOL_SOCKET', 'SO_REUSEADDR', 'inet_pton', + 'gaierror', 'AI_NUMERICHOST', 'AI_NUMERICSERV', 'EAI_NONAME', ): if hasattr(socket, name): setattr(m_socket, name, getattr(socket, name)) @@ -50,103 +51,6 @@ def patch_socket(f): new_callable=mock_socket_module)(f) -class BaseEventTests(test_utils.TestCase): - - def test_ipaddr_info(self): - UNSPEC = socket.AF_UNSPEC - INET = socket.AF_INET - INET6 = socket.AF_INET6 - STREAM = socket.SOCK_STREAM - DGRAM = socket.SOCK_DGRAM - TCP = socket.IPPROTO_TCP - UDP = socket.IPPROTO_UDP - - self.assertEqual( - (INET, STREAM, TCP, '', ('1.2.3.4', 1)), - base_events._ipaddr_info('1.2.3.4', 1, INET, STREAM, TCP)) - - self.assertEqual( - (INET, STREAM, TCP, '', ('1.2.3.4', 1)), - base_events._ipaddr_info(b'1.2.3.4', 1, INET, STREAM, TCP)) - - self.assertEqual( - (INET, STREAM, TCP, '', ('1.2.3.4', 1)), - base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, STREAM, TCP)) - - self.assertEqual( - (INET, DGRAM, UDP, '', ('1.2.3.4', 1)), - base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, DGRAM, UDP)) - - # Socket type STREAM implies TCP protocol. - self.assertEqual( - (INET, STREAM, TCP, '', ('1.2.3.4', 1)), - base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, STREAM, 0)) - - # Socket type DGRAM implies UDP protocol. - self.assertEqual( - (INET, DGRAM, UDP, '', ('1.2.3.4', 1)), - base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, DGRAM, 0)) - - # No socket type. - self.assertIsNone( - base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, 0, 0)) - - if socket_helper.IPV6_ENABLED: - # IPv4 address with family IPv6. - self.assertIsNone( - base_events._ipaddr_info('1.2.3.4', 1, INET6, STREAM, TCP)) - - self.assertEqual( - (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)), - base_events._ipaddr_info('::3', 1, INET6, STREAM, TCP)) - - self.assertEqual( - (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)), - base_events._ipaddr_info('::3', 1, UNSPEC, STREAM, TCP)) - - # IPv6 address with family IPv4. - self.assertIsNone( - base_events._ipaddr_info('::3', 1, INET, STREAM, TCP)) - - # IPv6 address with zone index. - self.assertIsNone( - base_events._ipaddr_info('::3%lo0', 1, INET6, STREAM, TCP)) - - def test_port_parameter_types(self): - # Test obscure kinds of arguments for "port". - INET = socket.AF_INET - STREAM = socket.SOCK_STREAM - TCP = socket.IPPROTO_TCP - - self.assertEqual( - (INET, STREAM, TCP, '', ('1.2.3.4', 0)), - base_events._ipaddr_info('1.2.3.4', None, INET, STREAM, TCP)) - - self.assertEqual( - (INET, STREAM, TCP, '', ('1.2.3.4', 0)), - base_events._ipaddr_info('1.2.3.4', b'', INET, STREAM, TCP)) - - self.assertEqual( - (INET, STREAM, TCP, '', ('1.2.3.4', 0)), - base_events._ipaddr_info('1.2.3.4', '', INET, STREAM, TCP)) - - self.assertEqual( - (INET, STREAM, TCP, '', ('1.2.3.4', 1)), - base_events._ipaddr_info('1.2.3.4', '1', INET, STREAM, TCP)) - - self.assertEqual( - (INET, STREAM, TCP, '', ('1.2.3.4', 1)), - base_events._ipaddr_info('1.2.3.4', b'1', INET, STREAM, TCP)) - - @patch_socket - def test_ipaddr_info_no_inet_pton(self, m_socket): - del m_socket.inet_pton - self.assertIsNone(base_events._ipaddr_info('1.2.3.4', 1, - socket.AF_INET, - socket.SOCK_STREAM, - socket.IPPROTO_TCP)) - - class BaseEventLoopTests(test_utils.TestCase): def setUp(self): @@ -1827,32 +1731,6 @@ def test_create_datagram_endpoint_nosoreuseport(self, m_socket): self.assertRaises(ValueError, self.loop.run_until_complete, coro) - @patch_socket - def test_create_datagram_endpoint_ip_addr(self, m_socket): - def getaddrinfo(*args, **kw): - self.fail('should not have called getaddrinfo') - - m_socket.getaddrinfo = getaddrinfo - m_socket.socket.return_value.bind = bind = mock.Mock() - self.loop._add_reader = mock.Mock() - self.loop._add_reader._is_coroutine = False - - reuseport_supported = hasattr(socket, 'SO_REUSEPORT') - coro = self.loop.create_datagram_endpoint( - lambda: MyDatagramProto(loop=self.loop), - local_addr=('1.2.3.4', 0), - reuse_port=reuseport_supported) - - t, p = self.loop.run_until_complete(coro) - try: - bind.assert_called_with(('1.2.3.4', 0)) - m_socket.socket.assert_called_with(family=m_socket.AF_INET, - proto=m_socket.IPPROTO_UDP, - type=m_socket.SOCK_DGRAM) - finally: - t.close() - test_utils.run_briefly(self.loop) # allow transport to close - def test_accept_connection_retry(self): sock = mock.Mock() sock.accept.side_effect = BlockingIOError() diff --git a/Misc/NEWS.d/next/Library/2022-02-22-10-22-44.bpo-46824.IflDPF.rst b/Misc/NEWS.d/next/Library/2022-02-22-10-22-44.bpo-46824.IflDPF.rst new file mode 100644 index 000000000000000..05ccd84333d8475 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-02-22-10-22-44.bpo-46824.IflDPF.rst @@ -0,0 +1 @@ +skip getaddrinfo thread if host is already resolved, using socket.AI_NUMERIC From 9120c95fd5eb084c990579a4d851d877af38735a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 23 Feb 2022 09:46:41 +0000 Subject: [PATCH 2/3] allow getaddrinfo flags in test_sock_connect_resolve_using_socket_params --- Lib/test/test_asyncio/test_selector_events.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index d9d30fc25b513bb..219114cd29177c4 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -164,7 +164,9 @@ def test_sock_connect_resolve_using_socket_params(self, m_gai): con = self.loop.create_task(self.loop.sock_connect(sock, addr)) self.loop.run_until_complete(con) m_gai.assert_called_with( - addr[0], addr[1], sock.family, sock.type, sock.proto, 0) + addr[0], addr[1], sock.family, sock.type, sock.proto, + mock.ANY, + ) self.loop.run_until_complete(con) sock.connect.assert_called_with(('127.0.0.1', 0)) From d305816a640d92f6056ac8ce6a9ddf90e3390796 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 23 Feb 2022 09:49:35 +0000 Subject: [PATCH 3/3] respect explicit flowinfo and scope in sock_connect --- Lib/asyncio/base_events.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 57c2eee1a8dbf3c..e78b771067f6e90 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -1341,9 +1341,17 @@ async def create_datagram_endpoint(self, protocol_factory, async def _ensure_resolved(self, address, *, family=0, type=socket.SOCK_STREAM, proto=0, flags=0, loop): - host, port = address[:2] - return await loop.getaddrinfo(host, port, family=family, type=type, - proto=proto, flags=flags) + host, port, *rest = address + if not rest: + return await loop.getaddrinfo( + host, port, family=family, type=type, proto=proto, flags=flags, + ) + return [ + (*first, (host, port, *rest)) for *first, (host, port, *_) in + await loop.getaddrinfo( + host, port, family=family, type=type, proto=proto, flags=flags, + ) + ] async def _create_server_getaddrinfo(self, host, port, family, flags): infos = await self._ensure_resolved((host, port), family=family,