Fix concurrent hub-construction SIGSEGV: evict Autofac reflection cache on collectible ALC unload

.github/workflows/alc-unload-probe.yml

@@ -0,0 +1,152 @@
+# Manually-triggered ROOT-CAUSE probe for the teardown SIGSEGV (exit=139) in the
+# ALC-heavy native test hosts (Hosting.Monolith.Test / FutuRe.Test). The crash is a
+# native use-after-unload: a process-wide reflection/serialization cache retains an
+# accessor/handle into a collectible node AssemblyLoadContext; when a node-hub disposes
+# it unloads that ALC (MeshDataSource.RegisterForDisposal → UnloadNodeContexts), and a
+# LATER background GC dereferences the freed metadata → SIGSEGV. The post-crash mini-dump
+# is a delayed-detection snapshot (the ALC is already gone → nothing to gcroot), so it
+# can't name the offending cache.
+#
+# This workflow flips MESHWEAVER_ALC_UNLOAD_GC_PROBE=1, which makes NodeAssemblyLoadContext
+# drive a SYNCHRONOUS full GC immediately after each Unload. A dangling native pointer then
+# faults AT that unload — on the Dispose thread, right after a
+# "ALC_UNLOAD_PROBE forcing GC after unloading <node>" log line — so the culprit node (and a
+# corruption-time dump with a managed Dispose→GC.Collect stack) is captured instead of the
+# opaque delayed crash. GCStress would be the classic tool but needs a checked runtime the
+# hosted runner doesn't have; forcing the collection at the unload point is the retail-runtime
+# equivalent for this specific use-after-unload.
+#
+# Run from the Actions tab (workflow_dispatch). Re-dispatch with project=MeshWeaver.FutuRe.Test
+# to probe the other crasher.
+name: ALC unload probe (manual)
+
+on:
+  workflow_dispatch:
+    inputs:
+      project:
+        description: "Test project NAME whose native host to loop under the probe"
+        type: string
+        default: "MeshWeaver.Hosting.Monolith.Test"
+      iterations:
+        description: "Max loop iterations (stops on the first native crash)"
+        type: string
+        default: "10"
+
+permissions:
+  contents: read
+
+env:
+  NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages
+
+jobs:
+  probe:
+    name: "Loop native host under ALC-unload GC probe"
+    # ubuntu-latest == the 2-vCPU hosted runner the crash reproduces on.
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    env:
+      PROBE_PROJECT: ${{ inputs.project || 'MeshWeaver.Hosting.Monolith.Test' }}
+      PROBE_ITERS: ${{ inputs.iterations || '10' }}
+      DOTNET_ENVIRONMENT: Development
+      Logging__LogLevel__Default: Warning
+      # THE PROBE: force a synchronous GC after each collectible node-ALC Unload so a
+      # use-after-unload dangling native pointer faults at the unload (naming the node).
+      MESHWEAVER_ALC_UNLOAD_GC_PROBE: "1"
+      # Native-crash mini dump, same as dotnet-test.yml — the corruption-time forensic payload.
+      DOTNET_DbgEnableMiniDump: 1
+      DOTNET_DbgMiniDumpType: 2
+      DOTNET_DbgMiniDumpName: /tmp/coredumps/%e-%p.dmp
+    steps:
+      - uses: actions/checkout@v6
+      - name: Free disk space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          dotnet: false
+          android: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v5
+        with:
+          dotnet-version: 10.0.x
+      - name: Restore workloads
+        run: dotnet workload restore
+      - name: Restore dependencies
+        run: dotnet restore
+      # Build Release exactly as the real CI build job, so the looped native host is the
+      # identical binary that crashes in CI.
+      - name: Build
+        run: dotnet build --no-restore -c Release -p:CIRun=true -warnaserror
+      # The mesh-local #r feed dynamic-compilation tests resolve at runtime (version-less
+      # `#r "nuget:MeshWeaver.X"`) — packed exactly as dotnet-test.yml / flake-repro.yml do.
+      - name: Pack mesh-local #r packages
+        run: |
+          set -euo pipefail
+          dotnet pack src/MeshWeaver.BusinessRules/MeshWeaver.BusinessRules.csproj \
+            -c Release --no-build --no-restore -o dist/packages --nologo
+          dotnet pack src/MeshWeaver.BusinessRules.Generator/MeshWeaver.BusinessRules.Generator.csproj \
+            -c Release --no-build --no-restore -o dist/packages --nologo
+      - name: Loop the native host under the probe until it crashes
+        run: |
+          set -uo pipefail
+          mkdir -p /tmp/coredumps probe-logs
+          name="$PROBE_PROJECT"
+          dir="test/$name/bin/Release/net10.0"
+          if [ ! -f "$dir/$name.dll" ]; then
+            echo "::error::no native xUnit v3 host at $dir/$name.dll"; exit 1
+          fi
+          echo "Looping $name native host up to $PROBE_ITERS× under MESHWEAVER_ALC_UNLOAD_GC_PROBE=1 (nproc=$(nproc))."
+          crashed=0
+          for i in $(seq 1 "$PROBE_ITERS"); do
+            echo "::group::iteration $i / $PROBE_ITERS"
+            mkdir -p "$dir/TestResults"
+            # CWD = bin dir so appsettings/xunit.runner.json/TestData resolve as in CI.
+            # tee the host's stdout+stderr so the "ALC_UNLOAD_PROBE forcing GC after unloading
+            # <node>" line right before a crash is captured even if the ILogger sink isn't console.
+            ( cd "$dir" && timeout --signal=TERM --kill-after=30s 15m \
+                dotnet "$name.dll" -trx "TestResults/$name-$i.trx" ) \
+                2>&1 | tee "probe-logs/run-$i.log"
+            rc=${PIPESTATUS[0]}
+            echo "iteration $i exit=$rc"
+            echo "::endgroup::"
+            # Signal-induced death (SIGSEGV 139 / SIGABRT 134 / SIGILL 132 / SIGFPE 136) is the
+            # crash we hunt; rc==1 is ordinary test failures, 0 is clean — keep looping those.
+            if [ "$rc" = "139" ] || [ "$rc" = "134" ] || [ "$rc" = "132" ] || [ "$rc" = "136" ]; then
+              echo "::error::NATIVE CRASH on iteration $i (exit $rc). Culprit ALC unload (last probe lines):"
+              grep 'ALC_UNLOAD_PROBE' "probe-logs/run-$i.log" | tail -5 || echo "(no probe line in stdout — see collected test-logs + dump)"
+              crashed=1
+              break
+            fi
+          done
+          if [ "$crashed" = "0" ]; then
+            echo "No native crash in $PROBE_ITERS iterations (the flake did not surface this run)."
+          fi
+          # Red the run on repro so it's obvious in the Actions list; diagnostics upload via always().
+          exit "$crashed"
+      - name: Collect diagnostics
+        if: always()
+        run: |
+          mkdir -p probe-diagnostics
+          cp -r probe-logs probe-diagnostics/ 2>/dev/null || true
+          # Corruption-time mini dump(s).
+          cp /tmp/coredumps/*.dmp probe-diagnostics/ 2>/dev/null || true
+          # MonolithMeshTestBase phase/dispose/memory traces (which test class was tearing down).
+          for f in meshweaver-test-trace meshweaver-dispose-trace meshweaver-memory-delta; do
+            [ -f "/tmp/$f.log" ] && cp "/tmp/$f.log" "probe-diagnostics/_$f.log" || true
+          done
+          # ILogger sink (the ALC_UNLOAD_PROBE warnings land here if routed to the file helper).
+          find . -path '*/bin/*/test-logs/*.log' -exec cp {} probe-diagnostics/ \; 2>/dev/null || true
+          echo "=== last ALC_UNLOAD_PROBE lines (culprit node whose unload tripped the crash) ==="
+          grep -h 'ALC_UNLOAD_PROBE' probe-logs/*.log probe-diagnostics/*.log 2>/dev/null | tail -12 || true
+          ls -lh probe-diagnostics/ || true
+      - name: Upload probe diagnostics
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: alc-unload-probe-diagnostics
+          path: probe-diagnostics/
+          retention-days: 15
+          compression-level: 9

src/MeshWeaver.Documentation/Data/WhatsNew/2026-07-14-node-recompile-stability.md

@@ -0,0 +1,14 @@
+---
+Name: More reliable node recompilation
+Category: What's New
+Description: Recompiling code/scope nodes no longer leaks unloaded assemblies or risks a rare crash under load.
+Icon: Sparkle
+---
+
+# More reliable node recompilation
+
+Compiling and recompiling code and scope nodes now cleanly releases the previous
+version's compiled assembly. Previously the runtime's dependency-injection cache kept a
+hidden reference to each unloaded assembly, which slowly grew memory over many recompiles
+and, under concurrent activity, could trigger a rare hard crash. Portals that frequently
+edit and recompile nodes will be more stable and use less memory over time.

src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs

@@ -2,6 +2,7 @@
 using System.Collections.Immutable;
 using System.Reflection;
 using System.Runtime.Loader;
+using MeshWeaver.ServiceProvider;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -216,6 +217,14 @@ internal sealed class NodeAssemblyLoadContext : AssemblyLoadContext, IDisposable
     private volatile bool _disposed;
     private ImmutableArray<string> _probingDirs = ImmutableArray<string>.Empty;
 
+    // Opt-in diagnostic (env MESHWEAVER_ALC_UNLOAD_GC_PROBE=1) — OFF by default, so zero cost in
+    // prod and normal CI. When on, Dispose drives a synchronous full GC right after Unload so a
+    // use-after-unload dangling NATIVE pointer into this now-collectible assembly faults HERE
+    // (pinning the culprit node in the log + a corruption-time dump) instead of as a delayed
+    // background-GC SIGSEGV. Immutable config constant read once at type init — never written.
+    private static readonly bool UnloadGcProbe =
+        Environment.GetEnvironmentVariable("MESHWEAVER_ALC_UNLOAD_GC_PROBE") == "1";
+
     public void SetProbingDirectories(System.Collections.Generic.IReadOnlyList<string> dirs)
     {
         _probingDirs = dirs.ToImmutableArray();
@@ -242,6 +251,15 @@ public NodeAssemblyLoadContext(string nodeName, string? dllPath, ILogger? logger
         _nodeName = nodeName;
         _dllPath = dllPath;
         _logger = logger;
+
+        // Purge Autofac's process-static reflection cache of this context's assemblies the instant
+        // it starts unloading — while the metadata the predicate walks is still valid. A cached
+        // ConstructorInfo/Assembly key otherwise (1) roots this collectible context so it can never
+        // be collected, and (2) makes a later, unrelated concurrent GetOrAdd bucket-probe compare
+        // against the freed key → AccessViolationException/SIGSEGV under concurrent hub construction.
+        // Autofac does this automatically for BeginLoadContextLifetimeScope; we manage the context
+        // by hand, so we mirror it. Static handler ⇒ no self-reference that would defeat collection.
+        Unloading += ReflectionCacheEviction.EvictFor;
     }
 
     /// <summary>
@@ -401,6 +419,19 @@ public void Dispose()
 
         // Initiate unload outside the lock - the context will be collected when all references are released
         Unload();
+
+        // Diagnostic probe (opt-in, off by default): drive the collection synchronously so a
+        // use-after-unload dangling native pointer trips at THIS unload — naming the culprit node
+        // and yielding a corruption-time dump — instead of a delayed background-GC SIGSEGV. Used
+        // by the alc-unload-probe workflow to pin the reflection/serialization cache that retains
+        // an accessor into a collectible node assembly (the exit=139 teardown crash).
+        if (UnloadGcProbe)
+        {
+            _logger?.LogWarning("ALC_UNLOAD_PROBE forcing GC after unloading {ContextName}", Name);
+            GC.Collect();
+            GC.WaitForPendingFinalizers();
+            GC.Collect();
+        }
     }
 }
 

src/MeshWeaver.ServiceProvider/ReflectionCacheEviction.cs

@@ -0,0 +1,52 @@
+#nullable enable
+using System.Runtime.Loader;
+using Autofac.Core;
+
+namespace MeshWeaver.ServiceProvider;
+
+/// <summary>
+/// Evicts entries from Autofac's <b>process-static</b> shared reflection cache
+/// (<see cref="ReflectionCacheSet.Shared"/>) that reference assemblies loaded into a
+/// collectible <see cref="AssemblyLoadContext"/> which is about to be unloaded.
+/// </summary>
+/// <remarks>
+/// Autofac keys its reflection caches (constructor-binder factories, parameter maps,
+/// assembly scans) on <see cref="System.Reflection.MemberInfo"/>/<see cref="System.Reflection.Assembly"/>.
+/// Those keys strongly root the declaring assembly and therefore its
+/// <see cref="AssemblyLoadContext"/>. MeshWeaver compiles nodes into collectible load
+/// contexts (<c>NodeAssemblyLoadContext</c>) and unloads them on recompile / release /
+/// teardown. If the shared cache still holds an entry for the unloaded assembly, two
+/// things go wrong:
+/// <list type="number">
+/// <item>the context can never be collected (the cache roots it) — an unbounded leak; and</item>
+/// <item>a later, unrelated concurrent <c>GetOrAdd</c> whose key hashes into the same bucket
+/// compares against the stale key and dereferences <b>freed metadata</b> →
+/// <see cref="AccessViolationException"/> / SIGSEGV.</item>
+/// </list>
+/// Autofac performs exactly this eviction automatically for scopes created via
+/// <c>BeginLoadContextLifetimeScope</c>; because MeshWeaver manages its collectible
+/// contexts by hand, it must run the eviction itself — <b>before</b> unload, while the
+/// assembly metadata the predicate walks is still valid.
+/// </remarks>
+public static class ReflectionCacheEviction
+{
+    /// <summary>
+    /// Removes every shared-reflection-cache entry that references an assembly loaded into
+    /// <paramref name="loadContext"/>. Safe to call concurrently with cache reads (the
+    /// underlying stores are <see cref="System.Collections.Concurrent.ConcurrentDictionary{TKey,TValue}"/>);
+    /// call it <b>before</b> unloading so the walked metadata is still live.
+    /// </summary>
+    /// <param name="loadContext">The collectible context whose entries should be purged.</param>
+    public static void EvictFor(AssemblyLoadContext loadContext)
+    {
+        // Access Shared fresh each call — it is a WeakReference-backed singleton and must
+        // never be stored (Autofac's own guidance on the property).
+        ReflectionCacheSet.Shared.Clear((_, referencedAssemblies) =>
+        {
+            foreach (var assembly in referencedAssemblies)
+                if (AssemblyLoadContext.GetLoadContext(assembly) == loadContext)
+                    return true;
+            return false;
+        });
+    }
+}

test/MeshWeaver.Graph.Test/MeshWeaver.Graph.Test.csproj

@@ -4,6 +4,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Autofac" />
     <PackageReference Include="Namotion.Reflection" />
     <PackageReference Include="NSubstitute" />
   </ItemGroup>

test/MeshWeaver.Graph.Test/ReflectionCacheEvictionTest.cs

@@ -0,0 +1,98 @@
+using System.Runtime.CompilerServices;
+using Autofac;
+using Autofac.Core;
+using MeshWeaver.Graph.Configuration;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Xunit;
+
+namespace MeshWeaver.Graph.Test;
+
+/// <summary>
+/// Pins the fix for the concurrent-hub-construction SIGSEGV: a collectible
+/// <see cref="NodeAssemblyLoadContext"/> whose types were activated through Autofac must be
+/// GC-collectable once unloaded. Autofac's process-static reflection cache keys on
+/// <c>ConstructorInfo</c>, which strongly roots the assembly (and its load context); if the
+/// cache isn't evicted on unload, the context leaks AND a later concurrent cache probe
+/// dereferences the freed metadata → <see cref="System.AccessViolationException"/>.
+/// This asserts the (deterministic) root cause — a retained reference — not the flaky crash.
+/// </summary>
+public class ReflectionCacheEvictionTest
+{
+    private const string WidgetSource =
+        "namespace Dyn { public sealed class Widget { public Widget() { } } }";
+
+    [Fact]
+    public void UnloadedNodeContext_IsCollectible_AfterAutofacActivation()
+    {
+        // Pin the shared reflection cache alive for the whole test. It is held only by a
+        // WeakReference, so without this a plain GC would collect the entire cache and release the
+        // widget ctor on its own — masking the leak. A running app keeps the cache alive through
+        // constant container/scope activity; that is exactly when a retained stale key both roots
+        // the context (leak) and, on a concurrent probe, dereferences freed metadata (SIGSEGV).
+        var pinnedCache = ReflectionCacheSet.Shared;
+
+        var weak = ActivateThroughAutofacThenUnload();
+
+        for (var i = 0; i < 15 && weak.IsAlive; i++)
+        {
+            GC.Collect();
+            GC.WaitForPendingFinalizers();
+        }
+
+        weak.IsAlive.Should().BeFalse(
+            "a collectible node AssemblyLoadContext must be collectable once unloaded; if Autofac's " +
+            "process-static reflection cache still holds the widget's ConstructorInfo it roots the " +
+            "context forever (leak) AND a later concurrent GetOrAdd bucket-probe would dereference the " +
+            "freed metadata → AccessViolationException. NodeAssemblyLoadContext must evict the shared " +
+            "reflection cache before Unload.");
+
+        GC.KeepAlive(pinnedCache);
+    }
+
+    // Kept out-of-line + non-inlined so no local (assembly / type / instance) stays rooted on the
+    // caller's frame — otherwise the context could not be collected regardless of the fix.
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static WeakReference ActivateThroughAutofacThenUnload()
+    {
+        var bytes = CompileWidget();
+        var ctx = new NodeAssemblyLoadContext("EvictionTestWidget", dllPath: null, logger: null);
+        var assembly = ctx.LoadFromBytes(bytes, null);
+        var widgetType = assembly.GetType("Dyn.Widget")!;
+
+        // Activate through Autofac exactly as a hosted-hub container would: this creates a
+        // ConstructorBinder for the widget's ctor and caches it in ReflectionCacheSet.Shared.
+        var builder = new ContainerBuilder();
+        builder.RegisterType(widgetType);
+        using (var container = builder.Build())
+        {
+            container.Resolve(widgetType).Should().NotBeNull();
+        }
+
+        var weak = new WeakReference(ctx);
+        ctx.Dispose(); // → Unloading → ReflectionCacheEviction.EvictFor → Unload
+        return weak;
+    }
+
+    private static byte[] CompileWidget()
+    {
+        var tpa = (string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!;
+        var references = tpa
+            .Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries)
+            .Select(p => (MetadataReference)MetadataReference.CreateFromFile(p))
+            .ToArray();
+
+        var compilation = CSharpCompilation.Create(
+            "EvictionTestWidget",
+            new[] { CSharpSyntaxTree.ParseText(WidgetSource) },
+            references,
+            new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary));
+
+        using var ms = new MemoryStream();
+        var result = compilation.Emit(ms);
+        result.Success.Should().BeTrue(
+            "the widget must compile: " +
+            string.Join("; ", result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error)));
+        return ms.ToArray();
+    }
+}