From 2df58957d6a0d19c62c0bddc889a9f36d5d58f7e Mon Sep 17 00:00:00 2001 From: huniu20 Date: Thu, 2 Jul 2026 15:59:13 +0800 Subject: [PATCH 1/5] [feat] Support P2PHANDSHAKE metadata mode and SSD offload for MooncakeStore - Add P2PHANDSHAKE metadata mode support in mooncake_bootstrap.py and mooncake_client.py. This is Mooncake's official peer-to-peer metadata discovery mode, which avoids IP detection issues in multi-NIC environments (e.g., clusters with both RDMA and business network interfaces). - Add SSD offload support via config.yaml's offload sub-section under MooncakeStore. When enabled, the bootstrap automatically starts a standalone mooncake_client process that offloads data from DRAM to NVMe SSD when memory usage exceeds the configured high watermark. This is essential for environments where total CPU DRAM < GPU HBM (increasingly common with H200/B200/B300 generation GPUs). - All offload parameters are configured through TQ's standard config system (OmegaConf/YAML), not environment variables. Default behavior is unchanged (offload.enabled: false). Backward compatible: existing users are not affected. Signed-off-by: huniu20 --- transfer_queue/config.yaml | 27 +++ transfer_queue/interface.py | 7 + .../storage/bootstrap/mooncake_bootstrap.py | 182 ++++++++++++++++-- .../storage/clients/mooncake_client.py | 20 +- 4 files changed, 212 insertions(+), 24 deletions(-) diff --git a/transfer_queue/config.yaml b/transfer_queue/config.yaml index 8a6df3df..48e57a22 100644 --- a/transfer_queue/config.yaml +++ b/transfer_queue/config.yaml @@ -39,6 +39,7 @@ backend: # WARNING: When set to `true`, TQ will attempt to terminate any existing mooncake_master process. auto_init: true # Address of the metadata coordination server. + # Set to "P2PHANDSHAKE" to use peer-to-peer handshake mode (recommended for multi-NIC environments). metadata_server: localhost:50050 # Address of the Mooncake master server. master_server_address: localhost:50051 @@ -54,6 +55,32 @@ backend: # Network device name. # Set to "" to let Mooncake auto-select available devices. device_name: "" + # Whether to hard-pin objects in memory. Hard-pinned objects are never evicted. + # When offload is enabled, this is automatically set to false unless explicitly overridden. + # Set to true for maximum read performance when offload is not needed. + hard_pin: true + + # SSD Offload configuration. + # When enabled, TQ will start a standalone mooncake_client process that offloads + # data from DRAM to NVMe SSD when memory usage exceeds the high watermark. + # This is essential for environments where total CPU DRAM < GPU HBM. + offload: + # Master switch for SSD offload. + enabled: false + # Path to the SSD directory for offloaded data. + file_storage_path: /tmp/mooncake_offload + # Local buffer size in bytes for the offload client (default: 2GB). + local_buffer_size_bytes: 2147483648 + # Whether to use io_uring for async I/O (requires kernel >= 5.1 and liburing). + use_uring: false + # Heartbeat interval in seconds for the offload client. + heartbeat_interval_seconds: 2 + # Port for the standalone mooncake_client process. + client_port: 42052 + # Eviction high watermark ratio (0.0-1.0). Eviction starts when memory usage exceeds this ratio. + eviction_high_watermark_ratio: 0.9 + # Fraction of memory to free during each eviction cycle. + eviction_ratio: 0.1 # For Yuanrong: Yuanrong: diff --git a/transfer_queue/interface.py b/transfer_queue/interface.py index 82ceacaf..ab312954 100644 --- a/transfer_queue/interface.py +++ b/transfer_queue/interface.py @@ -246,6 +246,13 @@ def close(): f"Consider manually killing the mooncake_master." ) + # Terminate offload client process if it was started + if isinstance(value, dict): + offload_proc = value.get("offload_client_process") + if offload_proc is not None and offload_proc.poll() is None: + offload_proc.terminate() + logger.info(f"Terminated mooncake_client offload process (PID: {offload_proc.pid}).") + if _TQ_CLIENT: try: ret = _TQ_CLIENT.storage_manager.storage_client._store.remove_all() diff --git a/transfer_queue/storage/bootstrap/mooncake_bootstrap.py b/transfer_queue/storage/bootstrap/mooncake_bootstrap.py index 536599ca..43c7973e 100644 --- a/transfer_queue/storage/bootstrap/mooncake_bootstrap.py +++ b/transfer_queue/storage/bootstrap/mooncake_bootstrap.py @@ -27,15 +27,27 @@ @StorageBootstrapProvider.register_provider("MooncakeStore") -def initialize_mooncake_storage(conf: DictConfig) -> subprocess.Popen | None: +def initialize_mooncake_storage(conf: DictConfig) -> subprocess.Popen | dict | None: """ Initialize Mooncake store backend. + + Supports two metadata modes: + - HTTP metadata server (default): metadata_server = "host:port" + - P2P handshake (recommended for multi-NIC environments): metadata_server = "P2PHANDSHAKE" + + When ``offload.enabled`` is set to ``true`` in the config, this function also starts + a standalone ``mooncake_client`` process that offloads data from DRAM to NVMe SSD. + Args: conf (DictConfig): Configuration dictionary for the Mooncake store backend. Returns: - subprocess.Popen | None: Process object for the Mooncake store backend process. + subprocess.Popen | dict | None: + - None if auto_init is disabled. + - subprocess.Popen: master process (when offload is disabled). + - dict: {"master_process": Popen, "offload_client_process": Popen} (when offload is enabled). Raises: ValueError: If the Mooncake store is not initialized successfully. + RuntimeError: If mooncake_master or mooncake_client fails to start. """ if not conf.backend.MooncakeStore.auto_init: return None @@ -52,21 +64,28 @@ def initialize_mooncake_storage(conf: DictConfig) -> subprocess.Popen | None: else: raise RuntimeError(f"Failed to kill existing mooncake_master processes (exit code: {result}).") - # process metadata_server - metadata_server_raw_address = conf.backend.MooncakeStore.metadata_server - if "://" not in metadata_server_raw_address: - metadata_server_raw_address = "//" + metadata_server_raw_address + # process metadata_server (normalize and validate) + metadata_server_raw_address = str(conf.backend.MooncakeStore.metadata_server).strip() + use_p2p_handshake = metadata_server_raw_address.upper() == "P2PHANDSHAKE" + + if use_p2p_handshake: + metadata_server_host = None + metadata_server_port = None + logger.info("mooncake_master: Using P2PHANDSHAKE mode (no HTTP metadata server)") + else: + if "://" not in metadata_server_raw_address: + metadata_server_raw_address = "//" + metadata_server_raw_address - metadata_server_parsed = urlparse(metadata_server_raw_address) + metadata_server_parsed = urlparse(metadata_server_raw_address) - if not metadata_server_parsed.hostname or metadata_server_parsed.port is None: - raise ValueError( - f"Invalid metadata_server '{conf.backend.MooncakeStore.metadata_server}'. " - f"Host and port are required (e.g., host:port)." - ) + if not metadata_server_parsed.hostname or metadata_server_parsed.port is None: + raise ValueError( + f"Invalid metadata_server '{conf.backend.MooncakeStore.metadata_server}'. " + f"Host and port are required (e.g., host:port)." + ) - metadata_server_host = metadata_server_parsed.hostname - metadata_server_port = str(metadata_server_parsed.port) + metadata_server_host = metadata_server_parsed.hostname + metadata_server_port = str(metadata_server_parsed.port) # process master_server master_server_raw_address = conf.backend.MooncakeStore.master_server_address @@ -83,20 +102,62 @@ def initialize_mooncake_storage(conf: DictConfig) -> subprocess.Popen | None: master_server_port = str(master_server_parsed.port) + # Read offload configuration from config + offload_conf = conf.backend.MooncakeStore.get("offload", {}) + enable_offload = offload_conf.get("enabled", False) + cmd = [ "mooncake_master", "-client_ttl=30", "-default_kv_lease_ttl=999999", "-default_kv_soft_pin_ttl=999999", - "--eviction_high_watermark_ratio=1.0", - "--eviction_ratio=0.0", - "--enable_http_metadata_server=true", "--allow_evict_soft_pinned_objects=false", - f"--http_metadata_server_host={metadata_server_host}", - f"--http_metadata_server_port={metadata_server_port}", f"--rpc_port={master_server_port}", ] + if not use_p2p_handshake: + # Enable HTTP metadata server for non-P2P mode + cmd.extend( + [ + "--enable_http_metadata_server=true", + f"--http_metadata_server_host={metadata_server_host}", + f"--http_metadata_server_port={metadata_server_port}", + ] + ) + + if enable_offload: + eviction_high_watermark = offload_conf.get("eviction_high_watermark_ratio", 0.9) + eviction_ratio = offload_conf.get("eviction_ratio", 0.1) + + # Validate offload parameters + if not (0.0 < eviction_high_watermark <= 1.0): + raise ValueError( + f"offload.eviction_high_watermark_ratio must be in (0.0, 1.0], got {eviction_high_watermark}" + ) + if not (0.0 < eviction_ratio <= 1.0): + raise ValueError(f"offload.eviction_ratio must be in (0.0, 1.0], got {eviction_ratio}") + + # Enable SSD offload: lower watermark to trigger eviction, offload on evict + cmd.extend( + [ + "--enable_offload=true", + f"--eviction_high_watermark_ratio={eviction_high_watermark}", + f"--eviction_ratio={eviction_ratio}", + "--offload_on_evict=true", + "--offload_force_evict=false", + "--offloading_queue_limit=10000", + ] + ) + logger.info("mooncake_master: SSD offload enabled (offload_on_evict=true)") + else: + # Default: no eviction, no offload + cmd.extend( + [ + "--eviction_high_watermark_ratio=1.0", + "--eviction_ratio=0.0", + ] + ) + log_file_path = "/tmp/mooncake_master.log" with open(log_file_path, "w") as log_file: process = subprocess.Popen( @@ -124,4 +185,87 @@ def initialize_mooncake_storage(conf: DictConfig) -> subprocess.Popen | None: f"mooncake_master exited with error. Check {log_file_path} for detailed logs. Output:\n{error_msg}" ) + # Start standalone mooncake_client for SSD offload if enabled + if enable_offload: + ssd_path = offload_conf.get("file_storage_path", "/tmp/mooncake_offload") + client_port = str(offload_conf.get("client_port", 42052)) + local_buffer_size = str(offload_conf.get("local_buffer_size_bytes", 2147483648)) + use_uring = "1" if offload_conf.get("use_uring", False) else "0" + heartbeat_interval = str(offload_conf.get("heartbeat_interval_seconds", 2)) + global_segment_size = str(conf.backend.MooncakeStore.get("global_segment_size", 4294967296)) + + # Get local hostname + local_hostname = conf.backend.MooncakeStore.get("local_hostname", "") + if not local_hostname: + try: + from transfer_queue.utils.zmq_utils import get_node_ip_address + + local_hostname = get_node_ip_address() + except Exception: + import socket + + local_hostname = socket.gethostbyname(socket.gethostname()) + + os.makedirs(ssd_path, exist_ok=True) + + client_env = os.environ.copy() + client_env["MOONCAKE_OFFLOAD_FILE_STORAGE_PATH"] = ssd_path + client_env["MOONCAKE_OFFLOAD_LOCAL_BUFFER_SIZE_BYTES"] = local_buffer_size + client_env["MOONCAKE_OFFLOAD_USE_URING"] = use_uring + client_env["MOONCAKE_OFFLOAD_HEARTBEAT_INTERVAL_SECONDS"] = heartbeat_interval + + if use_p2p_handshake: + metadata_server_url = "P2PHANDSHAKE" + else: + metadata_server_url = f"http://{metadata_server_host}:{metadata_server_port}/metadata" + master_address = f"{master_server_parsed.hostname}:{master_server_port}" + + client_cmd = [ + "mooncake_client", + f"-host={local_hostname}", + f"-global_segment_size={global_segment_size}", + f"-master_server_address={master_address}", + f"-metadata_server={metadata_server_url}", + f"-protocol={conf.backend.MooncakeStore.get('protocol', 'tcp')}", + "-enable_offload=true", + f"-port={client_port}", + ] + + client_log_path = "/tmp/mooncake_client.log" + with open(client_log_path, "w") as client_log: + client_process = subprocess.Popen( + client_cmd, + stdout=client_log, + stderr=subprocess.STDOUT, + env=client_env, + start_new_session=True, + ) + time.sleep(5) + + if client_process.poll() is None: + logger.info( + f"mooncake_client started for SSD offload, PID: {client_process.pid}. " + f"SSD path: {ssd_path}. Logs: {client_log_path}" + ) + else: + # Offload client is a required component when offload is enabled. + # Hard fail to prevent silent degradation. + error_msg = "" + try: + with open(client_log_path) as f: + error_msg = f.read() + except Exception as e: + error_msg = f"Failed to read log file: {e}" + + # Terminate the master process since offload cannot work without the client + process.terminate() + raise RuntimeError( + f"mooncake_client exited unexpectedly (exit code: {client_process.returncode}). " + f"SSD offload is enabled but the offload client failed to start. " + f"Check {client_log_path} for details. Output:\n{error_msg}" + ) + + # Return structured resources for lifecycle management + if enable_offload: + return {"master_process": process, "offload_client_process": client_process} return process diff --git a/transfer_queue/storage/clients/mooncake_client.py b/transfer_queue/storage/clients/mooncake_client.py index 297b7cf3..d3bba37c 100644 --- a/transfer_queue/storage/clients/mooncake_client.py +++ b/transfer_queue/storage/clients/mooncake_client.py @@ -78,13 +78,23 @@ def __init__(self, config: dict[str, Any]): if self.master_server_address is None or not isinstance(self.master_server_address, str): raise ValueError("Missing or invalid 'master_server_address' in config") - if not self.metadata_server.startswith("http://") and not self.metadata_server.startswith("etcd://"): - self.metadata_server = f"http://{self.metadata_server}" - if not self.metadata_server.startswith("etcd://") and not self.metadata_server.endswith("/metadata"): - self.metadata_server = self.metadata_server + "/metadata" + # Support P2PHANDSHAKE mode: if metadata_server is "P2PHANDSHAKE" (case-insensitive), + # normalize to the exact string "P2PHANDSHAKE" and pass it directly without adding + # http:// prefix. This avoids IP detection issues in multi-NIC environments. + if str(self.metadata_server).strip().upper() == "P2PHANDSHAKE": + self.metadata_server = "P2PHANDSHAKE" + else: + if not self.metadata_server.startswith("http://") and not self.metadata_server.startswith("etcd://"): + self.metadata_server = f"http://{self.metadata_server}" + if not self.metadata_server.startswith("etcd://") and not self.metadata_server.endswith("/metadata"): + self.metadata_server = self.metadata_server + "/metadata" self.replica_config = ReplicateConfig() - self.replica_config.with_hard_pin = True + # When offload is enabled, hard_pin must be disabled so that objects can be evicted + # and offloaded to SSD. Hard-pinned objects are never evicted by Mooncake. + offload_conf = config.get("offload", {}) + offload_enabled = offload_conf.get("enabled", False) if isinstance(offload_conf, dict) else False + self.replica_config.with_hard_pin = bool(config.get("hard_pin", not offload_enabled)) self._store = MooncakeDistributedStore() ret = self._store.setup( From 375710b30b79dd6f8a9774f65af7370a6bd32d61 Mon Sep 17 00:00:00 2001 From: huniu20 Date: Fri, 3 Jul 2026 14:48:27 +0800 Subject: [PATCH 2/5] fix(review): sync hard_pin and offload config to perftest_config.yaml Signed-off-by: huniu20 --- scripts/performance_test/perftest_config.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scripts/performance_test/perftest_config.yaml b/scripts/performance_test/perftest_config.yaml index 310653b4..9eec70f4 100644 --- a/scripts/performance_test/perftest_config.yaml +++ b/scripts/performance_test/perftest_config.yaml @@ -38,6 +38,7 @@ backend: # WARNING: When set to `true`, TQ will attempt to terminate any existing mooncake_master process. auto_init: true # Address of the metadata coordination server. + # Set to "P2PHANDSHAKE" to use peer-to-peer handshake mode (recommended for multi-NIC environments). metadata_server: localhost:50050 # Address of the Mooncake master server. master_server_address: localhost:50051 @@ -53,6 +54,32 @@ backend: # Network device name. # Set to "" to let Mooncake auto-select available devices. device_name: "" + # Whether to hard-pin objects in memory. Hard-pinned objects are never evicted. + # When offload is enabled, this is automatically set to false unless explicitly overridden. + # Set to true for maximum read performance when offload is not needed. + hard_pin: true + + # SSD Offload configuration. + # When enabled, TQ will start a standalone mooncake_client process that offloads + # data from DRAM to NVMe SSD when memory usage exceeds the high watermark. + # This is essential for environments where total CPU DRAM < GPU HBM. + offload: + # Master switch for SSD offload. + enabled: false + # Path to the SSD directory for offloaded data. + file_storage_path: /tmp/mooncake_offload + # Local buffer size in bytes for the offload client (default: 2GB). + local_buffer_size_bytes: 2147483648 + # Whether to use io_uring for async I/O (requires kernel >= 5.1 and liburing). + use_uring: false + # Heartbeat interval in seconds for the offload client. + heartbeat_interval_seconds: 2 + # Port for the standalone mooncake_client process. + client_port: 42052 + # Eviction high watermark ratio (0.0-1.0). Eviction starts when memory usage exceeds this ratio. + eviction_high_watermark_ratio: 0.9 + # Fraction of memory to free during each eviction cycle. + eviction_ratio: 0.1 # For Yuanrong: Yuanrong: From 415e98c498da36cb713f71f4ca184c0dea0ea2bb Mon Sep 17 00:00:00 2001 From: huniu20 Date: Fri, 3 Jul 2026 15:06:47 +0800 Subject: [PATCH 3/5] fix(review): address Copilot suggestions for process lifecycle and hard_pin auto-management - mooncake_bootstrap.py: Add wait(timeout=5) + fallback kill() after process.terminate() when offload client fails to start, preventing stray mooncake_master processes if SIGTERM is ignored. - interface.py: Add wait(timeout=5) + fallback kill() after offload_proc.terminate() during tq.close() cleanup, preventing zombie processes. - mooncake_client.py: Treat hard_pin=None as 'auto' mode. When not explicitly set by user, hard_pin is automatically disabled when offload is enabled. This fixes a bug where OmegaConf merge always provides hard_pin=true from defaults, preventing auto-disable. - config.yaml + perftest_config.yaml: Change hard_pin default from true to null to enable auto-management. Signed-off-by: huniu20 --- scripts/performance_test/perftest_config.yaml | 6 +++--- transfer_queue/config.yaml | 6 +++--- transfer_queue/interface.py | 4 ++++ transfer_queue/storage/bootstrap/mooncake_bootstrap.py | 4 ++++ transfer_queue/storage/clients/mooncake_client.py | 6 +++++- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/scripts/performance_test/perftest_config.yaml b/scripts/performance_test/perftest_config.yaml index 9eec70f4..d0814538 100644 --- a/scripts/performance_test/perftest_config.yaml +++ b/scripts/performance_test/perftest_config.yaml @@ -55,9 +55,9 @@ backend: # Set to "" to let Mooncake auto-select available devices. device_name: "" # Whether to hard-pin objects in memory. Hard-pinned objects are never evicted. - # When offload is enabled, this is automatically set to false unless explicitly overridden. - # Set to true for maximum read performance when offload is not needed. - hard_pin: true + # Set to null (default) for auto-management: hard_pin is disabled when offload is enabled, + # and enabled otherwise. Set to true/false to explicitly override. + hard_pin: null # SSD Offload configuration. # When enabled, TQ will start a standalone mooncake_client process that offloads diff --git a/transfer_queue/config.yaml b/transfer_queue/config.yaml index 48e57a22..54b8bba2 100644 --- a/transfer_queue/config.yaml +++ b/transfer_queue/config.yaml @@ -56,9 +56,9 @@ backend: # Set to "" to let Mooncake auto-select available devices. device_name: "" # Whether to hard-pin objects in memory. Hard-pinned objects are never evicted. - # When offload is enabled, this is automatically set to false unless explicitly overridden. - # Set to true for maximum read performance when offload is not needed. - hard_pin: true + # Set to null (default) for auto-management: hard_pin is disabled when offload is enabled, + # and enabled otherwise. Set to true/false to explicitly override. + hard_pin: null # SSD Offload configuration. # When enabled, TQ will start a standalone mooncake_client process that offloads diff --git a/transfer_queue/interface.py b/transfer_queue/interface.py index ab312954..c3dd8018 100644 --- a/transfer_queue/interface.py +++ b/transfer_queue/interface.py @@ -251,6 +251,10 @@ def close(): offload_proc = value.get("offload_client_process") if offload_proc is not None and offload_proc.poll() is None: offload_proc.terminate() + try: + offload_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + offload_proc.kill() logger.info(f"Terminated mooncake_client offload process (PID: {offload_proc.pid}).") if _TQ_CLIENT: diff --git a/transfer_queue/storage/bootstrap/mooncake_bootstrap.py b/transfer_queue/storage/bootstrap/mooncake_bootstrap.py index 43c7973e..ddc5f6d5 100644 --- a/transfer_queue/storage/bootstrap/mooncake_bootstrap.py +++ b/transfer_queue/storage/bootstrap/mooncake_bootstrap.py @@ -259,6 +259,10 @@ def initialize_mooncake_storage(conf: DictConfig) -> subprocess.Popen | dict | N # Terminate the master process since offload cannot work without the client process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() raise RuntimeError( f"mooncake_client exited unexpectedly (exit code: {client_process.returncode}). " f"SSD offload is enabled but the offload client failed to start. " diff --git a/transfer_queue/storage/clients/mooncake_client.py b/transfer_queue/storage/clients/mooncake_client.py index d3bba37c..e6878304 100644 --- a/transfer_queue/storage/clients/mooncake_client.py +++ b/transfer_queue/storage/clients/mooncake_client.py @@ -94,7 +94,11 @@ def __init__(self, config: dict[str, Any]): # and offloaded to SSD. Hard-pinned objects are never evicted by Mooncake. offload_conf = config.get("offload", {}) offload_enabled = offload_conf.get("enabled", False) if isinstance(offload_conf, dict) else False - self.replica_config.with_hard_pin = bool(config.get("hard_pin", not offload_enabled)) + hard_pin = config.get("hard_pin", None) + if hard_pin is None: + # Auto-manage: disable hard_pin when offload is enabled + hard_pin = not offload_enabled + self.replica_config.with_hard_pin = bool(hard_pin) self._store = MooncakeDistributedStore() ret = self._store.setup( From 695c05d07504144b59238c0c6e95307a705d675f Mon Sep 17 00:00:00 2001 From: huniu20 Date: Fri, 3 Jul 2026 15:56:20 +0800 Subject: [PATCH 4/5] docs: clarify single-node offload architecture and heartbeat/client_ttl relationship Signed-off-by: nexhu Signed-off-by: huniu20 --- scripts/performance_test/perftest_config.yaml | 15 +++++++++++++++ transfer_queue/config.yaml | 15 +++++++++++++++ .../storage/bootstrap/mooncake_bootstrap.py | 13 ++++++++++++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/scripts/performance_test/perftest_config.yaml b/scripts/performance_test/perftest_config.yaml index d0814538..94445910 100644 --- a/scripts/performance_test/perftest_config.yaml +++ b/scripts/performance_test/perftest_config.yaml @@ -63,6 +63,17 @@ backend: # When enabled, TQ will start a standalone mooncake_client process that offloads # data from DRAM to NVMe SSD when memory usage exceeds the high watermark. # This is essential for environments where total CPU DRAM < GPU HBM. + # + # Architecture note: + # The offload client is a **single-node centralized SSD storage pool** that runs + # on the node where tq.init() is called. It registers itself with mooncake_master + # and serves the entire cluster — when eviction is triggered, the master moves data + # from ANY node's memory to this offload client via TCP/RDMA. + # Ensure that `local_hostname` (above) is set to a cluster-reachable address so that + # all nodes can transfer data to the offload client. + # + # TODO: Support multi-node distributed offload (one client per node) for better + # scalability in large clusters. offload: # Master switch for SSD offload. enabled: false @@ -73,6 +84,10 @@ backend: # Whether to use io_uring for async I/O (requires kernel >= 5.1 and liburing). use_uring: false # Heartbeat interval in seconds for the offload client. + # Note: This is paired with mooncake_master's -client_ttl (default: 30s). + # The client sends heartbeats at this interval; the master considers the client + # dead if no heartbeat is received within client_ttl. Ensure: + # heartbeat_interval_seconds << client_ttl (currently 2s << 30s = tolerates ~15 missed beats) heartbeat_interval_seconds: 2 # Port for the standalone mooncake_client process. client_port: 42052 diff --git a/transfer_queue/config.yaml b/transfer_queue/config.yaml index 54b8bba2..0556533c 100644 --- a/transfer_queue/config.yaml +++ b/transfer_queue/config.yaml @@ -64,6 +64,17 @@ backend: # When enabled, TQ will start a standalone mooncake_client process that offloads # data from DRAM to NVMe SSD when memory usage exceeds the high watermark. # This is essential for environments where total CPU DRAM < GPU HBM. + # + # Architecture note: + # The offload client is a **single-node centralized SSD storage pool** that runs + # on the node where tq.init() is called. It registers itself with mooncake_master + # and serves the entire cluster — when eviction is triggered, the master moves data + # from ANY node's memory to this offload client via TCP/RDMA. + # Ensure that `local_hostname` (above) is set to a cluster-reachable address so that + # all nodes can transfer data to the offload client. + # + # TODO: Support multi-node distributed offload (one client per node) for better + # scalability in large clusters. offload: # Master switch for SSD offload. enabled: false @@ -74,6 +85,10 @@ backend: # Whether to use io_uring for async I/O (requires kernel >= 5.1 and liburing). use_uring: false # Heartbeat interval in seconds for the offload client. + # Note: This is paired with mooncake_master's -client_ttl (default: 30s). + # The client sends heartbeats at this interval; the master considers the client + # dead if no heartbeat is received within client_ttl. Ensure: + # heartbeat_interval_seconds << client_ttl (currently 2s << 30s = tolerates ~15 missed beats) heartbeat_interval_seconds: 2 # Port for the standalone mooncake_client process. client_port: 42052 diff --git a/transfer_queue/storage/bootstrap/mooncake_bootstrap.py b/transfer_queue/storage/bootstrap/mooncake_bootstrap.py index ddc5f6d5..2e30f1da 100644 --- a/transfer_queue/storage/bootstrap/mooncake_bootstrap.py +++ b/transfer_queue/storage/bootstrap/mooncake_bootstrap.py @@ -185,7 +185,18 @@ def initialize_mooncake_storage(conf: DictConfig) -> subprocess.Popen | dict | N f"mooncake_master exited with error. Check {log_file_path} for detailed logs. Output:\n{error_msg}" ) - # Start standalone mooncake_client for SSD offload if enabled + # Start standalone mooncake_client for SSD offload if enabled. + # + # Architecture: The offload client is a single-node centralized SSD storage pool. + # It runs only on the node that calls tq.init(), but serves the entire cluster: + # when mooncake_master triggers eviction, it moves data from ANY node's DRAM segment + # to this offload client's SSD via TCP/RDMA. The client registers itself with the + # master using `local_hostname`, which must be reachable from all cluster nodes. + # + # Heartbeat relationship: + # - mooncake_master uses -client_ttl=30 (seconds) to detect dead clients. + # - mooncake_client sends heartbeats every `heartbeat_interval_seconds` (default: 2s). + # - Ratio: 30s / 2s = tolerates up to ~15 consecutive missed heartbeats. if enable_offload: ssd_path = offload_conf.get("file_storage_path", "/tmp/mooncake_offload") client_port = str(offload_conf.get("client_port", 42052)) From 22e20f8bc3037429755da2c9cf111486e7735239 Mon Sep 17 00:00:00 2001 From: huniu20 Date: Fri, 3 Jul 2026 16:51:24 +0800 Subject: [PATCH 5/5] docs: clarify 'the first node that calls tq.init()' per review Signed-off-by: huniu20 --- scripts/performance_test/perftest_config.yaml | 2 +- transfer_queue/config.yaml | 2 +- transfer_queue/storage/bootstrap/mooncake_bootstrap.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/performance_test/perftest_config.yaml b/scripts/performance_test/perftest_config.yaml index 94445910..a1026ca0 100644 --- a/scripts/performance_test/perftest_config.yaml +++ b/scripts/performance_test/perftest_config.yaml @@ -66,7 +66,7 @@ backend: # # Architecture note: # The offload client is a **single-node centralized SSD storage pool** that runs - # on the node where tq.init() is called. It registers itself with mooncake_master + # on the first node where tq.init() is called. It registers itself with mooncake_master # and serves the entire cluster — when eviction is triggered, the master moves data # from ANY node's memory to this offload client via TCP/RDMA. # Ensure that `local_hostname` (above) is set to a cluster-reachable address so that diff --git a/transfer_queue/config.yaml b/transfer_queue/config.yaml index 0556533c..331cd80b 100644 --- a/transfer_queue/config.yaml +++ b/transfer_queue/config.yaml @@ -67,7 +67,7 @@ backend: # # Architecture note: # The offload client is a **single-node centralized SSD storage pool** that runs - # on the node where tq.init() is called. It registers itself with mooncake_master + # on the first node where tq.init() is called. It registers itself with mooncake_master # and serves the entire cluster — when eviction is triggered, the master moves data # from ANY node's memory to this offload client via TCP/RDMA. # Ensure that `local_hostname` (above) is set to a cluster-reachable address so that diff --git a/transfer_queue/storage/bootstrap/mooncake_bootstrap.py b/transfer_queue/storage/bootstrap/mooncake_bootstrap.py index 2e30f1da..c5d7ba1e 100644 --- a/transfer_queue/storage/bootstrap/mooncake_bootstrap.py +++ b/transfer_queue/storage/bootstrap/mooncake_bootstrap.py @@ -188,7 +188,7 @@ def initialize_mooncake_storage(conf: DictConfig) -> subprocess.Popen | dict | N # Start standalone mooncake_client for SSD offload if enabled. # # Architecture: The offload client is a single-node centralized SSD storage pool. - # It runs only on the node that calls tq.init(), but serves the entire cluster: + # It runs only on the first node that calls tq.init(), but serves the entire cluster: # when mooncake_master triggers eviction, it moves data from ANY node's DRAM segment # to this offload client's SSD via TCP/RDMA. The client registers itself with the # master using `local_hostname`, which must be reachable from all cluster nodes.