mirror of
https://github.com/EttusResearch/uhd.git
synced 2026-10-25 17:36:10 +00:00
mpm: net.py: optimize netlink ops to fix RPC starvation
usrp_hwd runs all gRPC greenlets on a single gevent hub, so any non-yielding C call blocks every other RPC. Under a uhd_find_devices discovery flood, the GetDeviceInfo interface enumeration run long enough And may starve most claimed RPC past its default 2s client deadline, causing "Deadline Exceeded" failures. Three places that this commit optimizes, all in net.py: - get_local_ip_addrs() used pyroute2 IPDB, which spawns helper threads and mirrors the full link/addr/route/neighbour tables on every call (~100s of ms, non-yielding) --> Replace it with a single stateless IPRoute get_addr() dump. Keep IFA_LOCAL (falling back to IFA_ADDRESS) so the returned set matches IPDB's behaviour, including point-to-point links. - get_link_speed() used to create a new pyudev.Context() per interface, which forks on creation, tripping gRPC's fork handlers. gRPC then freezes all workder threads and restarts after fork is done. Fix is to reuse one PID-guarded pyudev context per process. - get_valid_interfaces() did an O(N) netlink round-trip per interface. This stalls gRPC server. --> Dump all links and IPv4 addresses once and filter in Python. IPRoute stays per-call: it opens its own netlink socket and does not fork, so there is no shared state to lock.
This commit is contained in:
parent
cf14224aa6
commit
6e11078807
@ -7,13 +7,35 @@
|
|||||||
Network utilities for MPM
|
Network utilities for MPM
|
||||||
"""
|
"""
|
||||||
import itertools
|
import itertools
|
||||||
|
import os
|
||||||
import socket
|
import socket
|
||||||
|
|
||||||
import pyudev
|
import pyudev
|
||||||
from pyroute2 import IPDB, IPRoute
|
from pyroute2 import IPRoute
|
||||||
from six import iteritems
|
from six import iteritems
|
||||||
from usrp_mpm.mpmlog import get_logger
|
from usrp_mpm.mpmlog import get_logger
|
||||||
|
|
||||||
|
# pyudev.Context() forks (find_library -> ldconfig) on creation; doing that on
|
||||||
|
# every GetDeviceInfo interface enumeration trips gRPC's fork handlers and can
|
||||||
|
# stall unrelated RPCs. Keep one context per process instead. IPRoute does not
|
||||||
|
# fork, so it stays per-call (its own netlink socket, no shared state to lock).
|
||||||
|
_udev_ctx = None
|
||||||
|
_udev_pid = None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_udev_context():
|
||||||
|
"""Return the shared pyudev.Context, recreating it if the PID changed.
|
||||||
|
|
||||||
|
The PID guard ensures a child process (e.g. spawned via multiprocessing)
|
||||||
|
never reuses a context inherited from its parent.
|
||||||
|
"""
|
||||||
|
global _udev_ctx, _udev_pid
|
||||||
|
pid = os.getpid()
|
||||||
|
if _udev_ctx is None or _udev_pid != pid:
|
||||||
|
_udev_ctx = pyudev.Context()
|
||||||
|
_udev_pid = pid
|
||||||
|
return _udev_ctx
|
||||||
|
|
||||||
|
|
||||||
def get_hostname():
|
def get_hostname():
|
||||||
"""Return the current device's hostname"""
|
"""Return the current device's hostname"""
|
||||||
@ -27,22 +49,24 @@ def get_valid_interfaces(iface_list):
|
|||||||
Interfaces are checked for if they actually exist, and if so, if they're up.
|
Interfaces are checked for if they actually exist, and if so, if they're up.
|
||||||
"""
|
"""
|
||||||
valid_ifaces = []
|
valid_ifaces = []
|
||||||
|
from usrp_mpm import __simulated__
|
||||||
|
|
||||||
with IPRoute() as ipr:
|
with IPRoute() as ipr:
|
||||||
|
# Dump all links and addresses once and filter in Python, so the netlink
|
||||||
|
# round-trips stay constant instead of scaling with len(iface_list).
|
||||||
|
links_by_name = {link.get_attr("IFLA_IFNAME"): link for link in ipr.get_links()}
|
||||||
|
addrs_by_index = {}
|
||||||
|
for addr in ipr.get_addr(family=socket.AF_INET):
|
||||||
|
for ip_addr in addr.get_attrs("IFA_ADDRESS"):
|
||||||
|
addrs_by_index.setdefault(addr.get("index"), []).append(ip_addr)
|
||||||
for iface in iface_list:
|
for iface in iface_list:
|
||||||
valid_iface_idx = ipr.link_lookup(ifname=iface)
|
link_info = links_by_name.get(iface)
|
||||||
if not valid_iface_idx:
|
if link_info is None:
|
||||||
continue
|
continue
|
||||||
valid_iface_idx = valid_iface_idx[0]
|
|
||||||
link_list = ipr.get_links(valid_iface_idx)
|
|
||||||
if not link_list:
|
|
||||||
continue
|
|
||||||
link_info = link_list[0]
|
|
||||||
# IFLA_OPERSTATE attribute isn't implemented on WSL
|
# IFLA_OPERSTATE attribute isn't implemented on WSL
|
||||||
# Workaround is ignore it in the simulator
|
# Workaround is ignore it in the simulator
|
||||||
from usrp_mpm import __simulated__
|
if (link_info.get_attr("IFLA_OPERSTATE") == "UP" or __simulated__) and addrs_by_index.get(
|
||||||
|
link_info.get("index")
|
||||||
if (link_info.get_attr("IFLA_OPERSTATE") == "UP" or __simulated__) and len(
|
|
||||||
get_iface_addrs(link_info.get_attr("IFLA_ADDRESS"))
|
|
||||||
):
|
):
|
||||||
assert link_info.get_attr("IFLA_IFNAME") == iface
|
assert link_info.get_attr("IFLA_IFNAME") == iface
|
||||||
valid_ifaces.append(iface)
|
valid_ifaces.append(iface)
|
||||||
@ -116,7 +140,7 @@ def get_link_speed(ifname):
|
|||||||
return 1000
|
return 1000
|
||||||
net_sysfs = [
|
net_sysfs = [
|
||||||
device
|
device
|
||||||
for device in pyudev.Context().list_devices(subsystem="net")
|
for device in _get_udev_context().list_devices(subsystem="net")
|
||||||
if device.sys_name == ifname
|
if device.sys_name == ifname
|
||||||
][0]
|
][0]
|
||||||
|
|
||||||
@ -224,10 +248,17 @@ def get_local_ip_addrs(ipv4_only=False):
|
|||||||
"""
|
"""
|
||||||
Return a set of IP addresses which are bound to local interfaces.
|
Return a set of IP addresses which are bound to local interfaces.
|
||||||
"""
|
"""
|
||||||
with IPDB() as ipdb:
|
# Use a single IPRoute netlink dump instead of IPDB, which spawns helper
|
||||||
return {
|
# threads and builds a full interface/route database on every call
|
||||||
ip_subnet[0]
|
# (~100s of ms, non-yielding) - that starves other RPCs when GetDeviceInfo
|
||||||
for ip_subnet_list in [x["ipaddr"] for x in ipdb.interfaces.values()]
|
# is flooded during discovery.
|
||||||
for ip_subnet in ip_subnet_list
|
with IPRoute() as ipr:
|
||||||
if not ipv4_only or ip_subnet[0].find(":") == -1
|
addrs = ipr.get_addr()
|
||||||
}
|
# Prefer IFA_LOCAL, fall back to IFA_ADDRESS, matching how IPDB keyed
|
||||||
|
# addresses.
|
||||||
|
return {
|
||||||
|
ip_addr
|
||||||
|
for addr in addrs
|
||||||
|
for ip_addr in (addr.get_attrs("IFA_LOCAL") or addr.get_attrs("IFA_ADDRESS"))
|
||||||
|
if not ipv4_only or ip_addr.find(":") == -1
|
||||||
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user