From b736620564add7a8dcb86b6347fffa79bb44b76d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Tue, 14 Jul 2026 14:42:13 +0200 Subject: [PATCH 1/2] Mesh hub: never persist its transient node; never NACK a message addressed to it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The root mesh hub's address (Address.Type == "mesh") is transient per-process infrastructure (a fresh guid every process via CreateMeshAddress), NOT an addressable node. Two leaks, each a known catastrophe class: B — never persist (StoragePostCommitFlush.Flush): when the flushing hub is the mesh hub, log an error and skip the write (the ack still resolves true, so read-after-write is intact). Persisting mesh/{id} just orphaned a DB row on every restart ("puke standing") — the stray mesh/{id} rows another agent found in the DB. Same as portals: routable, never persisted. Receive-guard (MessageHub.FinishDelivery): a message addressed to the mesh hub that reaches no handler is LOGGED as an error (to expose the abusing sender) and returns delivery.Processed() — NOT a DeliveryFailure/NACK back to the sender. Echoing an error back wedges the sender (it treats it as a fault, retries → storm → the 60s-timeout mesh-wide-outage class). NOTE the non-obvious part: delivery.Ignored() is NOT enough — MessageService turns an on-target Ignored into a DeliveryFailure{Ignored} sent back to the sender (see UnhandledMessageReportsFailureTest). Only Processed() ("handled, no failure") fully suppresses the echo; a requesting sender simply times out. Dedicated test MeshHubNotAnEndpointTest: an unhandled IRequest to the mesh hub must yield a TimeoutException (Ignored/no-NACK), never a DeliveryFailureException — the exact inverse of UnhandledMessageReportsFailureTest. Deferred (separate): registering the mesh hub as a routing participant (so routing resolves mesh/{id} instead of CreateHub-ing it) — the naive WithInitialization → RegisterStream deadlocks the mesh at startup because IRoutingService is mesh-scoped (circular: mesh hub construction → resolve IRoutingService → needs the mesh hub). Needs a post-construction hook or routing-side self-recognition of the mesh address. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Persistence/StoragePostCommitFlush.cs | 16 +++++++ src/MeshWeaver.Messaging.Hub/MessageHub.cs | 22 ++++++++++ .../MeshHubNotAnEndpointTest.cs | 42 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 test/MeshWeaver.Messaging.Hub.Test/MeshHubNotAnEndpointTest.cs diff --git a/src/MeshWeaver.Hosting/Persistence/StoragePostCommitFlush.cs b/src/MeshWeaver.Hosting/Persistence/StoragePostCommitFlush.cs index 61b53d1c9..76a70fea2 100644 --- a/src/MeshWeaver.Hosting/Persistence/StoragePostCommitFlush.cs +++ b/src/MeshWeaver.Hosting/Persistence/StoragePostCommitFlush.cs @@ -5,6 +5,7 @@ using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace MeshWeaver.Hosting.Persistence; @@ -25,6 +26,21 @@ public IObservable Flush(object committed) if (committed is not MeshNode node) return Observable.Return(true); + // 🚨 The root mesh hub's own node is transient infrastructure, NOT an addressable mesh node + // — its `mesh/{id}` id is a fresh guid every process, so persisting it just orphans a row on + // each restart ("puke standing"). The mesh hub is routable (registered with the routing + // service like portal/), but must never hit storage — same as portals, which route yet + // persist nothing. Log an error (so a future writer that commits it is visible in the logs) + // and skip: the ack still resolves true, so the read-after-write contract is unaffected. + if (hub.Address.Type == AddressExtensions.MeshType) + { + hub.ServiceProvider.GetService>()?.LogError( + "Refused to persist the root mesh hub's own node {Path}: a mesh/{{id}} address is transient " + + "infrastructure, not an addressable node, and must never be committed. Investigate the writer.", + node.Path); + return Observable.Return(true); + } + var storage = hub.ServiceProvider.GetService(); if (storage is null) return Observable.Return(true); diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 7902d3227..d88d1402b 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -801,6 +801,28 @@ private IMessageDelivery FinishDelivery(IMessageDelivery delivery) if (delivery.State == MessageDeliveryState.Submitted) { + // 🚨 The root mesh hub is routing INFRASTRUCTURE, not a message endpoint — nothing + // should ever address it directly (its `mesh/{id}` is a transient per-process id, not + // an addressable node). If something did and no handler matched, LOG loudly so the + // abusing sender is visible, but return Ignored: answering a DeliveryFailure/NACK back + // to the sender is a WEDGE (the sender treats it as a fault → retries → storm — the + // exact 60s-timeout mesh-wide outage class). Legit lifecycle/routing is already handled + // upstream (never reaches Submitted); never react to a DeliveryFailure (ping-pong). + if (Address.Type == AddressExtensions.MeshType && delivery.Message is not DeliveryFailure) + { + logger.LogError( + "Message {MessageType} (ID: {MessageId}) was addressed to the root mesh hub {Address} " + + "(sender={Sender}) but the mesh hub is routing infrastructure, not an endpoint. Dropping — " + + "no failure is returned (that would wedge the sender). This must not happen; investigate the sender.", + delivery.Message.GetType().Name, delivery.Id, Address, delivery.Sender); + // Processed, NOT Ignored: an Ignored on-target delivery is exactly what MessageService + // turns into a DeliveryFailure{Ignored} back to the sender (the wedge). Processed means + // "handled, no failure" — nothing is echoed back; a requesting sender simply times out + // (correct: it should never have addressed the mesh hub). The [Error] log above is how + // the abusing sender is found. + return delivery.Processed(); + } + // Fallback-hub contract (UnhandledMessageNack policy): this hub stands in // for a node whose NodeType couldn't produce a real configuration. Anything // its (default/overlay) config didn't handle — including RawJson deliveries diff --git a/test/MeshWeaver.Messaging.Hub.Test/MeshHubNotAnEndpointTest.cs b/test/MeshWeaver.Messaging.Hub.Test/MeshHubNotAnEndpointTest.cs new file mode 100644 index 000000000..8e84c2142 --- /dev/null +++ b/test/MeshWeaver.Messaging.Hub.Test/MeshHubNotAnEndpointTest.cs @@ -0,0 +1,42 @@ +using System; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Threading.Tasks; +using MeshWeaver.Fixture; +using Xunit; + +namespace MeshWeaver.Messaging.Hub.Test; + +/// +/// Pins the wedge-safety invariant for the root mesh hub: it is routing INFRASTRUCTURE, not a +/// message endpoint. A message addressed directly to it (its mesh/{id} address is a transient +/// per-process id, never an addressable node) reaches no handler — and the mesh hub must +/// Ignore it (logging an error to expose the abusing sender), never answer a +/// back. Echoing a failure to the sender is the classic wedge: the +/// sender treats it as a fault and retries → storm → the 60s-timeout mesh-wide outage class. +/// +/// This is the exact inverse of (a normal host +/// hub DOES NACK unhandled requests) — here the sender must observe a +/// (no response) rather than . +/// +public class MeshHubNotAnEndpointTest(ITestOutputHelper output) : HubTestBase(output) +{ + record ProbeRequest : IRequest; + record ProbeResponse; + + [Fact] + public async Task RequestAddressedToMeshHub_IsIgnored_NeverNacksBackToSender() + { + // Mesh is a mesh-typed hub (HubTestBase creates it at CreateMeshAddress()); it has no + // handler for ProbeRequest, so the delivery reaches FinishDelivery unhandled. The + // mesh-hub guard must Ignore it — so the client's observe TIMES OUT (no response), + // it must NOT throw DeliveryFailureException (which a normal hub would). + var client = GetClient(); + + await Assert.ThrowsAsync(() => + client.Observe(new ProbeRequest(), o => o.WithTarget(Mesh.Address)) + .Timeout(TimeSpan.FromSeconds(3)) + .FirstAsync() + .ToTask()); + } +} From 06478b27fe0f5c33a65d876cf9334d8a368b7309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Tue, 14 Jul 2026 21:31:29 +0200 Subject: [PATCH 2/2] Revert the mesh-hub receive-guard (it broke message delivery); keep only never-persist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI on #475 failed 5/6 shards — 16 message-flow / data-change / activity tests (Acme, AI, Content, GitSync, Orleans, Persistence, Threading) timed out at 60s. Root cause: the FinishDelivery receive-guard was built on a wrong premise. The root mesh hub is NOT a pure non-endpoint — legitimate framework flows POST requests to it and AWAIT the response (even an unhandled-NACK unblocks the caller). Returning Processed() gave those callers NO response, so they hung until timeout. Revert the FinishDelivery change entirely (MessageHub back to baseline) and drop MeshHubNotAnEndpointTest. Keep the safe, independent, valuable part: StoragePostCommitFlush still refuses to persist the mesh hub's own transient mesh/{id} node (the actual DB-puke bug). A correct wedge-safe receive-guard needs a much narrower predicate and its own analysis — deferred. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 22 ---------- .../MeshHubNotAnEndpointTest.cs | 42 ------------------- 2 files changed, 64 deletions(-) delete mode 100644 test/MeshWeaver.Messaging.Hub.Test/MeshHubNotAnEndpointTest.cs diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index d88d1402b..7902d3227 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -801,28 +801,6 @@ private IMessageDelivery FinishDelivery(IMessageDelivery delivery) if (delivery.State == MessageDeliveryState.Submitted) { - // 🚨 The root mesh hub is routing INFRASTRUCTURE, not a message endpoint — nothing - // should ever address it directly (its `mesh/{id}` is a transient per-process id, not - // an addressable node). If something did and no handler matched, LOG loudly so the - // abusing sender is visible, but return Ignored: answering a DeliveryFailure/NACK back - // to the sender is a WEDGE (the sender treats it as a fault → retries → storm — the - // exact 60s-timeout mesh-wide outage class). Legit lifecycle/routing is already handled - // upstream (never reaches Submitted); never react to a DeliveryFailure (ping-pong). - if (Address.Type == AddressExtensions.MeshType && delivery.Message is not DeliveryFailure) - { - logger.LogError( - "Message {MessageType} (ID: {MessageId}) was addressed to the root mesh hub {Address} " + - "(sender={Sender}) but the mesh hub is routing infrastructure, not an endpoint. Dropping — " + - "no failure is returned (that would wedge the sender). This must not happen; investigate the sender.", - delivery.Message.GetType().Name, delivery.Id, Address, delivery.Sender); - // Processed, NOT Ignored: an Ignored on-target delivery is exactly what MessageService - // turns into a DeliveryFailure{Ignored} back to the sender (the wedge). Processed means - // "handled, no failure" — nothing is echoed back; a requesting sender simply times out - // (correct: it should never have addressed the mesh hub). The [Error] log above is how - // the abusing sender is found. - return delivery.Processed(); - } - // Fallback-hub contract (UnhandledMessageNack policy): this hub stands in // for a node whose NodeType couldn't produce a real configuration. Anything // its (default/overlay) config didn't handle — including RawJson deliveries diff --git a/test/MeshWeaver.Messaging.Hub.Test/MeshHubNotAnEndpointTest.cs b/test/MeshWeaver.Messaging.Hub.Test/MeshHubNotAnEndpointTest.cs deleted file mode 100644 index 8e84c2142..000000000 --- a/test/MeshWeaver.Messaging.Hub.Test/MeshHubNotAnEndpointTest.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Reactive.Linq; -using System.Reactive.Threading.Tasks; -using System.Threading.Tasks; -using MeshWeaver.Fixture; -using Xunit; - -namespace MeshWeaver.Messaging.Hub.Test; - -/// -/// Pins the wedge-safety invariant for the root mesh hub: it is routing INFRASTRUCTURE, not a -/// message endpoint. A message addressed directly to it (its mesh/{id} address is a transient -/// per-process id, never an addressable node) reaches no handler — and the mesh hub must -/// Ignore it (logging an error to expose the abusing sender), never answer a -/// back. Echoing a failure to the sender is the classic wedge: the -/// sender treats it as a fault and retries → storm → the 60s-timeout mesh-wide outage class. -/// -/// This is the exact inverse of (a normal host -/// hub DOES NACK unhandled requests) — here the sender must observe a -/// (no response) rather than . -/// -public class MeshHubNotAnEndpointTest(ITestOutputHelper output) : HubTestBase(output) -{ - record ProbeRequest : IRequest; - record ProbeResponse; - - [Fact] - public async Task RequestAddressedToMeshHub_IsIgnored_NeverNacksBackToSender() - { - // Mesh is a mesh-typed hub (HubTestBase creates it at CreateMeshAddress()); it has no - // handler for ProbeRequest, so the delivery reaches FinishDelivery unhandled. The - // mesh-hub guard must Ignore it — so the client's observe TIMES OUT (no response), - // it must NOT throw DeliveryFailureException (which a normal hub would). - var client = GetClient(); - - await Assert.ThrowsAsync(() => - client.Observe(new ProbeRequest(), o => o.WithTarget(Mesh.Address)) - .Timeout(TimeSpan.FromSeconds(3)) - .FirstAsync() - .ToTask()); - } -}