From 05443d301dcb624b6249cc1ed6a0e22a652100be Mon Sep 17 00:00:00 2001 From: Abhishek Verma Date: Wed, 19 Aug 2026 11:44:33 +0200 Subject: [PATCH] host/mpm: make gRPC robust to idle teardowns The main RPC channel can sit idle for a long time (e.g. an interactive Python session) while only the claim-loop channel sees traffic. Without keepalive, gRPC self-closes the idle transport (~30 min) and the next RPC fails re-establishing within its per-call deadline. With this change the client now pings every 30 s to keep the channel warm, and the server is configured to accept that cadence instead of replying GOAWAY "too_many_pings". --- host/lib/usrp/common/mpm_client.cpp.mako | 31 ++++++++++++++++++++++++ mpm/python/usrp_mpm/rpc_server.py | 13 ++++++++++ 2 files changed, 44 insertions(+) diff --git a/host/lib/usrp/common/mpm_client.cpp.mako b/host/lib/usrp/common/mpm_client.cpp.mako index 785534d61..7a31dbcc9 100644 --- a/host/lib/usrp/common/mpm_client.cpp.mako +++ b/host/lib/usrp/common/mpm_client.cpp.mako @@ -29,6 +29,18 @@ #ifndef GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL #define GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL "grpc.use_local_subchannel_pool" #endif +// Keepalive channel-arg keys. These string keys are stable +// across gRPC versions; provide fallback definitions for older releases that +// do not export the macros. +#ifndef GRPC_ARG_KEEPALIVE_TIME_MS +#define GRPC_ARG_KEEPALIVE_TIME_MS "grpc.keepalive_time_ms" +#endif +#ifndef GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS +#define GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS "grpc.keepalive_permit_without_calls" +#endif +#ifndef GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA +#define GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA "grpc.http2.max_pings_without_data" +#endif using uhd::rpc_exception; @@ -222,6 +234,25 @@ public: // session cannot drop the active session's connection. args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); + // Keep idle channels alive. The main RPC channel can sit idle for a + // long time (e.g. an interactive Python session left open) while only + // the separate claimer channel sees traffic. Without keepalive, gRPC + // sends nothing on an idle channel, so it self-closes the transport + // (GRPC_ARG_CLIENT_IDLE_TIMEOUT_MS). The next RPC then has to re- + // establish the connection within its short per-call deadline and may + // reports "Deadline Exceeded". Sending HTTP/2 pings every 60 s keeps + // the flow warm. The server must permit this ping cadence otherwise + // it responds with GOAWAY "too_many_pings". + args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, 60000); + // gRPC pings keepalive even if the channel is idle (no active RPC calls) + args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1); + // By default gRPC stops sending keepalive pings after 2 pings without + // any data frame (GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA default = 2), + // which would silently defeat keepalive on an idle channel. + // Set to 0 (unlimited) so pings continue for the lifetime of an idle + // connection. + args.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0); + auto channel = grpc::CreateCustomChannel(server_address, grpc::InsecureChannelCredentials(), args); _channel = channel; stub_ = ${package_name}::${service_name}::NewStub(channel); diff --git a/mpm/python/usrp_mpm/rpc_server.py b/mpm/python/usrp_mpm/rpc_server.py index 69e9f287b..299349625 100644 --- a/mpm/python/usrp_mpm/rpc_server.py +++ b/mpm/python/usrp_mpm/rpc_server.py @@ -1116,6 +1116,19 @@ def _rpc_server_process(shared_state, port, default_args): options=[ ("grpc.max_send_message_length", MAX_GRPC_MESSAGE_SIZE), ("grpc.max_receive_message_length", MAX_GRPC_MESSAGE_SIZE), + # Permit the client's keepalive pings (see mpm_client.cpp.mako). + # The host pings every 30 s to keep an otherwise-idle channel alive + # to avoid silently closing the channel which would result into + # "Deadline Exceeded" on the next RPC after a long idle. + # Without these options the server treats frequent pings on an + # idle connection as abusive and sends GOAWAY "too_many_pings", + # tearing down the connection. + ("grpc.keepalive_permit_without_calls", 1), + # Accept a ping as often as every 30 s (must be <= the client's 60 s + # keepalive interval to avoid too_many_pings). + ("grpc.http2.min_ping_interval_without_data_ms", 30000), + # Do not cap the number of pings the client may send without data. + ("grpc.http2.max_pings_without_data", 0), # Disable the server's BDP (bandwidth-estimation) ping. It fires # repeatedly and its ACK must return within grpcio's internal BDP # timeout. The ACK may be missed if the server is stalled by ClkMgr