Fix idle-page crash: don't poison the read cache with transient grain-reactivation rejects

src/MeshWeaver.Hosting/MeshNodeStreamCache.cs

@@ -616,9 +616,22 @@ private Entry CreateEntry(string p)
             // 5 min cap (the atioz `_Activity` / missing-node NotFound resubscribe storm
             // that saturated the action block and tripped the liveness probe). A real
             // resolution is a non-null node; nothing else clears the breaker.
+            //
+            // 🚨 Record ONLY a genuine missing-node failure — never a TRANSIENT owner miss.
+            // A per-node grain that Orleans idle-collected answers the next read's
+            // SubscribeRequest with a transient reactivation reject ("Forwarding failed …
+            // to invalid activation. Rejecting now.") or a 60 s request timeout. Poisoning
+            // the negative cache with THAT made the storm-breaker replay the raw Orleans
+            // reject to every reader for the backoff window AND refuse to re-probe the grain
+            // that already reactivated — so navigating to a just-idle page crashed until a
+            // manual reload outlasted the window. The node exists; it is momentarily
+            // unreachable. Forward the error to THIS subscriber (its caller / the area's
+            // transient classifier retries) but leave the cache clean so the very next read
+            // re-probes the fresh activation. Symmetric with the write path (see the
+            // [UpdateQueue] FAILED branch), which already guards RecordNegative the same way.
             var bookkeeping = inner.AsObservable().Subscribe(
                 node => { if (node is not null) _negative.TryRemove(p, out _); },
-                ex => RecordNegative(p, ex));
+                ex => { if (IsMissingNodeFailure(ex)) RecordNegative(p, ex); });
             var disposal = new System.Reactive.Disposables.CompositeDisposable(hydrationSub, bookkeeping);
             // Store the disposal on the Entry so the mesh hub's pre-Quiescing
             // disposal hook (registered in the ctor) can cancel it — and so the
@@ -805,13 +818,70 @@ private void RecordNegative(string path, Exception error)
 
     /// <summary>
     /// True when an owner failure means the node/hub does not exist (NotFound / activation
-    /// failed) — the only failure class the storm-breaker suppresses on the WRITE path. RLS
-    /// denials and transient routing errors are excluded so an existing-but-busy node is never
-    /// falsely blocked from writes.
+    /// failed) — the only failure class the storm-breaker suppresses. RLS denials and
+    /// <see cref="IsTransientOwnerFailure">transient routing errors</see> are excluded so an
+    /// existing-but-busy (or a just-idle-collected, mid-reactivation) node is never falsely
+    /// blocked from reads OR writes.
+    ///
+    /// <para>🚨 Transient takes PRECEDENCE. A grain that idle-collected and is mid-
+    /// <c>DeactivateOnIdle</c> answers the next delivery with an Orleans forwarding-reject
+    /// ("Forwarding failed … to invalid activation. Rejecting now.") or, if reactivation is
+    /// slow, the 60&#160;s hub-request timeout ("… target hub was not found"). Without the
+    /// transient guard first, that transient miss would be POISONED into the negative cache:
+    /// the storm-breaker would then replay the raw Orleans reject to every reader for the whole
+    /// backoff window and refuse to re-probe the grain that already reactivated — the exact
+    /// "navigating to an idle page intermittently crashes; a manual reload fixes it" bug (the
+    /// reload just outlasts the 2&#160;s window and the re-probe lands on the fresh activation).
+    /// The <c>&amp;&amp; !IsTransientOwnerFailure</c> guard also protects the "activation failed"
+    /// substring below — a transient activation miss must never be read as a permanent absence.
+    /// The node exists; it is transiently unreachable — never suppress it.</para>
+    /// </summary>
+    internal static bool IsMissingNodeFailure(Exception error) =>
+        !IsTransientOwnerFailure(error)
+        && (error.Message.Contains("No node found", StringComparison.OrdinalIgnoreCase)
+            || error.Message.Contains("activation failed", StringComparison.OrdinalIgnoreCase));
+
+    /// <summary>
+    /// True when an owner failure is TRANSIENT — the node exists but is momentarily
+    /// unreachable, so a later read/write is likely to succeed and the storm-breaker must
+    /// NEVER record it as a negative (missing-node) entry. Chiefly the Orleans grain-
+    /// reactivation window: a per-node grain that Orleans idle-collected answers the next
+    /// delivery with an <c>OrleansMessageRejectionException</c> ("… to invalid activation.
+    /// Rejecting now.", surfaced through routing as a <c>DeliveryFailure{ErrorType.Failed}</c>
+    /// whose message carries "Forwarding failed" / "invalid activation" / "Rejecting now"),
+    /// or — when reactivation outruns the request budget — a <see cref="TimeoutException"/>
+    /// ("No response received in hub …" / "target hub was not found" / "undeliverable").
+    /// Each self-heals: the grain reactivates and the very next probe lands on the fresh
+    /// instance, so the error is forwarded to the current subscriber (whose caller retries)
+    /// but the cache stays clean. Mirrors <c>RoutingGrain.IsTransientFailure</c> and
+    /// <c>AreaErrorClassifier.IsTransientHubFailure</c> so all three layers agree on which
+    /// failures are worth a retry rather than a suppress.
     /// </summary>
-    private static bool IsMissingNodeFailure(Exception error) =>
-        error.Message.Contains("No node found", StringComparison.OrdinalIgnoreCase)
-        || error.Message.Contains("activation failed", StringComparison.OrdinalIgnoreCase);
+    internal static bool IsTransientOwnerFailure(Exception? error)
+    {
+        for (var e = error; e != null; e = e.InnerException)
+        {
+            if (e is TimeoutException) return true;
+            // OrleansMessageRejectionException lives in Orleans.Core, which this project does
+            // not (and must not) reference — match by type name so the classifier stays free
+            // of the Orleans dependency while still catching the raw grain-reject that reaches
+            // the cache from the silo-side routing grain.
+            if (e.GetType().Name == "OrleansMessageRejectionException") return true;
+            var msg = e.Message ?? string.Empty;
+            // The routing grain surfaces the exhausted-forward reject as a DeliveryFailure whose
+            // message is "Delivery to '{path}' failed: Forwarding failed: tried to forward … to
+            // invalid activation. Rejecting now." — a TRANSIENT reactivation miss, not a real
+            // missing node. The 60 s hub-request timeout banner adds the "… hub was not found" /
+            // "undeliverable" / "No response received in hub" forms.
+            if (msg.Contains("invalid activation", StringComparison.OrdinalIgnoreCase)) return true;
+            if (msg.Contains("Rejecting now", StringComparison.OrdinalIgnoreCase)) return true;
+            if (msg.Contains("Forwarding failed", StringComparison.OrdinalIgnoreCase)) return true;
+            if (msg.Contains("target hub was not found", StringComparison.OrdinalIgnoreCase)) return true;
+            if (msg.Contains("No response received in hub", StringComparison.OrdinalIgnoreCase)) return true;
+            if (msg.Contains("undeliverable", StringComparison.OrdinalIgnoreCase)) return true;
+        }
+        return false;
+    }
 
     /// <summary>
     /// Returns a per-user access-gated view of the cached shared stream. The

test/MeshWeaver.Hosting.Monolith.Test/MeshNodeStreamCacheNegativeClassifierTest.cs

@@ -0,0 +1,133 @@
+using System;
+using MeshWeaver.Messaging;
+using Xunit;
+
+namespace MeshWeaver.Hosting.Monolith.Test;
+
+/// <summary>
+/// Deterministic classifier tests for the <see cref="MeshNodeStreamCache"/> storm-breaker's
+/// TRANSIENT-vs-MISSING decision — the fix for the recurring "navigating to a just-idle page
+/// intermittently crashes; a manual reload fixes it" production bug.
+///
+/// <para><b>The bug.</b> A per-node grain that Orleans idle-collects is deactivated. The next
+/// read's <c>SubscribeRequest</c> hits the mid-<c>DeactivateOnIdle</c> activation and Orleans,
+/// after forwarding <c>MaxForwardCount</c>=2 times, rejects it transiently — surfacing through
+/// the silo routing grain as a <c>DeliveryFailure</c> whose message is
+/// <c>"Delivery to '{path}' failed: Forwarding failed: tried to forward message … for 2 times
+/// after \"DeactivateOnIdle was called.\" to invalid activation. Rejecting now."</c> (or, when
+/// reactivation is slow, the 60&#160;s hub-request timeout "… target hub was not found"). The
+/// cache's READ path recorded THAT into the negative cache exactly like a genuine missing node,
+/// so the storm-breaker replayed the raw Orleans reject to every reader for the whole backoff
+/// window AND refused to re-probe the grain that had already reactivated. A manual reload just
+/// outlasted the 2&#160;s window so the re-probe landed on the fresh activation.</para>
+///
+/// <para><b>The invariant.</b> Only a GENUINE missing-node failure (NotFound / "No node found")
+/// may be recorded as a negative (storm-breaker) entry. A transient reactivation reject / request
+/// timeout is NEVER recorded — the node exists and self-heals on the next probe. Mirrors
+/// <c>RoutingGrain.IsTransientFailure</c> and <c>AreaErrorClassifier.IsTransientHubFailure</c> so
+/// all three layers agree.</para>
+/// </summary>
+public class MeshNodeStreamCacheNegativeClassifierTest
+{
+    // The exact string the silo routing grain (RoutingGrain.DeliverToGrainWithRetry) posts to the
+    // sender once the transient Orleans forwarding-reject exhausts its retries — with the verbatim
+    // Orleans reject text ("Forwarding failed … invalid activation. Rejecting now.") inlined. This
+    // is the message that reached the browser as a crashed page.
+    private const string ForwardingRejectMessage =
+        "Delivery to 'AgenticPension/Statement' failed: Forwarding failed: tried to forward message " +
+        "Request [S1]->[messagehub/AgenticPension/Statement] DeliverMessage(...) #42[ForwardCount=2] " +
+        "for 2 times after \"DeactivateOnIdle was called.\" to invalid activation. Rejecting now.";
+
+    // The 60 s hub-request timeout banner (MessageHub.BuildTimeoutMessage) — the "activation timeout"
+    // symptom. It contains "target hub was not found" / "undeliverable", which the transient
+    // classifier catches so a slow reactivation is never mistaken for a permanently-absent node.
+    private const string RequestTimeoutMessage =
+        "No response received in hub cache/mesh-node-cache within 00:01:00 for request SubscribeRequest " +
+        "(id=abc) → target AgenticPension/Statement. The request may have been undeliverable or the " +
+        "target hub was not found.";
+
+    // A locally-defined stand-in with the SAME type name as the real Orleans exception, so the
+    // classifier's by-name match can be exercised WITHOUT this project referencing Orleans.Core.
+    private sealed class OrleansMessageRejectionException(string message) : Exception(message);
+
+    private static DeliveryFailureException Nack(string message, ErrorType errorType)
+    {
+        var delivery = new MessageDelivery<object>(
+            new Address("client", "1"), new Address("host", "1"), new object(), new System.Text.Json.JsonSerializerOptions());
+        return new DeliveryFailureException(new DeliveryFailure(delivery, message) { ErrorType = errorType });
+    }
+
+    // ---- TRANSIENT: must be classified transient and therefore NOT recorded as a negative entry ----
+
+    [Fact]
+    public void ForwardingReject_IsTransient_NotMissingNode()
+    {
+        var nack = Nack(ForwardingRejectMessage, ErrorType.Failed);
+        MeshNodeStreamCache.IsTransientOwnerFailure(nack).Should().BeTrue(
+            "the exhausted-forward Orleans reject ('… invalid activation. Rejecting now.') is a mid-reactivation miss");
+        MeshNodeStreamCache.IsMissingNodeFailure(nack).Should().BeFalse(
+            "a transient reactivation reject must NEVER be recorded as a missing-node negative entry (the bug)");
+    }
+
+    [Fact]
+    public void RawOrleansRejectionException_IsTransient()
+    {
+        var ex = new OrleansMessageRejectionException("invalid activation. Rejecting now.");
+        MeshNodeStreamCache.IsTransientOwnerFailure(ex).Should().BeTrue(
+            "the raw OrleansMessageRejectionException (matched by type name) is transient");
+        MeshNodeStreamCache.IsMissingNodeFailure(ex).Should().BeFalse();
+    }
+
+    [Fact]
+    public void RequestTimeout_TargetHubNotFound_IsTransient()
+    {
+        var nack = Nack(RequestTimeoutMessage, ErrorType.Exception);
+        MeshNodeStreamCache.IsTransientOwnerFailure(nack).Should().BeTrue(
+            "the 60 s 'target hub was not found' timeout is a slow-reactivation miss, not a permanent absence");
+        MeshNodeStreamCache.IsMissingNodeFailure(nack).Should().BeFalse();
+    }
+
+    [Fact]
+    public void RawTimeoutException_IsTransient()
+    {
+        MeshNodeStreamCache.IsTransientOwnerFailure(new TimeoutException("no response")).Should().BeTrue();
+    }
+
+    [Fact]
+    public void TransientCause_WrappedAsInnerException_IsStillTransient()
+    {
+        var wrapped = new InvalidOperationException("outer", new TimeoutException("target hub was not found"));
+        MeshNodeStreamCache.IsTransientOwnerFailure(wrapped).Should().BeTrue(
+            "the transient classifier walks the inner-exception chain, like the sibling classifiers");
+    }
+
+    // ---- MISSING NODE: a genuine absence IS recorded (the storm-breaker's legitimate job) ----
+
+    [Fact]
+    public void NoNodeFound_IsMissingNode_NotTransient()
+    {
+        var nack = Nack(
+            "No node found at 'rbuergi/_Activity/markdown-abc'. Closest ancestor is 'rbuergi' (remainder='…').",
+            ErrorType.NotFound);
+        MeshNodeStreamCache.IsTransientOwnerFailure(nack).Should().BeFalse(
+            "a genuine 'No node found' is a permanent absence, not a transient reactivation miss");
+        MeshNodeStreamCache.IsMissingNodeFailure(nack).Should().BeTrue(
+            "a genuinely-missing node IS recorded — that is the storm-breaker's legitimate purpose");
+    }
+
+    [Fact]
+    public void UnrelatedError_IsNeitherTransientNorMissing()
+    {
+        // An RLS denial or a random processing fault: not transient (don't retry-forever) and not
+        // missing (don't suppress the node) — recorded by NEITHER predicate, matching prior behaviour.
+        var nack = Nack("Access denied: user 'x' lacks Read permission on 'y'.", ErrorType.Unauthorized);
+        MeshNodeStreamCache.IsTransientOwnerFailure(nack).Should().BeFalse();
+        MeshNodeStreamCache.IsMissingNodeFailure(nack).Should().BeFalse();
+    }
+
+    [Fact]
+    public void Null_IsNeither()
+    {
+        MeshNodeStreamCache.IsTransientOwnerFailure(null).Should().BeFalse();
+    }
+}