From 1b8a921219ce8af03e0e86f2efd1c6175a6c8f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Tue, 14 Jul 2026 06:38:48 +0200 Subject: [PATCH 1/2] Fix concurrent hub-construction SIGSEGV: evict Autofac reflection cache on collectible ALC unload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: NodeAssemblyLoadContext (collectible, isCollectible:true) unloaded compiled-node assemblies without evicting Autofac's process-static shared reflection cache (ReflectionCacheSet.Shared). Autofac keys that cache on ConstructorInfo, so a retained key (1) rooted the collectible context — it could never be collected (a memory leak on every recompile) — and (2) let a later, unrelated CONCURRENT GetOrAdd during framework hub construction (AddMeshDataSource) compare against the freed metadata → AccessViolationException in ConcurrentDictionary.TryGetValueInternal. That is the native-host exit=139 the #447 exit-marker gate unmasked (FutuRe.Test / Hosting.Monolith.Test). Fix: NodeAssemblyLoadContext subscribes its Unloading event to ReflectionCacheEviction.EvictFor, which clears every shared-cache entry referencing the context's assemblies BEFORE unload (while the metadata the predicate walks is still valid). This mirrors exactly what Autofac performs automatically for BeginLoadContextLifetimeScope; MeshWeaver manages its collectible node contexts by hand, so it must run the eviction itself. No lock, no serialization, no convoy — the earlier global SetupModules lock was the wrong fix and is not used. Test: ReflectionCacheEvictionTest is a deterministic root-cause repro (not the flaky AVE) — a collectible NodeAssemblyLoadContext activated through Autofac must be GC-collectable after unload. It pins ReflectionCacheSet.Shared alive (it is WeakReference-backed, else a plain GC collects the whole cache and false-passes). RED pre-fix (context rooted), GREEN post-fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../2026-07-14-node-recompile-stability.md | 14 +++ .../Configuration/CompilationCacheService.cs | 10 ++ .../ReflectionCacheEviction.cs | 52 ++++++++++ .../MeshWeaver.Graph.Test.csproj | 1 + .../ReflectionCacheEvictionTest.cs | 98 +++++++++++++++++++ 5 files changed, 175 insertions(+) create mode 100644 src/MeshWeaver.Documentation/Data/WhatsNew/2026-07-14-node-recompile-stability.md create mode 100644 src/MeshWeaver.ServiceProvider/ReflectionCacheEviction.cs create mode 100644 test/MeshWeaver.Graph.Test/ReflectionCacheEvictionTest.cs diff --git a/src/MeshWeaver.Documentation/Data/WhatsNew/2026-07-14-node-recompile-stability.md b/src/MeshWeaver.Documentation/Data/WhatsNew/2026-07-14-node-recompile-stability.md new file mode 100644 index 000000000..0a81459c4 --- /dev/null +++ b/src/MeshWeaver.Documentation/Data/WhatsNew/2026-07-14-node-recompile-stability.md @@ -0,0 +1,14 @@ +--- +Name: More reliable node recompilation +Category: What's New +Description: Recompiling code/scope nodes no longer leaks unloaded assemblies or risks a rare crash under load. +Icon: Sparkle +--- + +# More reliable node recompilation + +Compiling and recompiling code and scope nodes now cleanly releases the previous +version's compiled assembly. Previously the runtime's dependency-injection cache kept a +hidden reference to each unloaded assembly, which slowly grew memory over many recompiles +and, under concurrent activity, could trigger a rare hard crash. Portals that frequently +edit and recompile nodes will be more stable and use less memory over time. diff --git a/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs b/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs index 918410e3e..769a23bcc 100644 --- a/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs +++ b/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs @@ -2,6 +2,7 @@ using System.Collections.Immutable; using System.Reflection; using System.Runtime.Loader; +using MeshWeaver.ServiceProvider; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -242,6 +243,15 @@ public NodeAssemblyLoadContext(string nodeName, string? dllPath, ILogger? logger _nodeName = nodeName; _dllPath = dllPath; _logger = logger; + + // Purge Autofac's process-static reflection cache of this context's assemblies the instant + // it starts unloading — while the metadata the predicate walks is still valid. A cached + // ConstructorInfo/Assembly key otherwise (1) roots this collectible context so it can never + // be collected, and (2) makes a later, unrelated concurrent GetOrAdd bucket-probe compare + // against the freed key → AccessViolationException/SIGSEGV under concurrent hub construction. + // Autofac does this automatically for BeginLoadContextLifetimeScope; we manage the context + // by hand, so we mirror it. Static handler ⇒ no self-reference that would defeat collection. + Unloading += ReflectionCacheEviction.EvictFor; } /// diff --git a/src/MeshWeaver.ServiceProvider/ReflectionCacheEviction.cs b/src/MeshWeaver.ServiceProvider/ReflectionCacheEviction.cs new file mode 100644 index 000000000..015b5022a --- /dev/null +++ b/src/MeshWeaver.ServiceProvider/ReflectionCacheEviction.cs @@ -0,0 +1,52 @@ +#nullable enable +using System.Runtime.Loader; +using Autofac.Core; + +namespace MeshWeaver.ServiceProvider; + +/// +/// Evicts entries from Autofac's process-static shared reflection cache +/// () that reference assemblies loaded into a +/// collectible which is about to be unloaded. +/// +/// +/// Autofac keys its reflection caches (constructor-binder factories, parameter maps, +/// assembly scans) on /. +/// Those keys strongly root the declaring assembly and therefore its +/// . MeshWeaver compiles nodes into collectible load +/// contexts (NodeAssemblyLoadContext) and unloads them on recompile / release / +/// teardown. If the shared cache still holds an entry for the unloaded assembly, two +/// things go wrong: +/// +/// the context can never be collected (the cache roots it) — an unbounded leak; and +/// a later, unrelated concurrent GetOrAdd whose key hashes into the same bucket +/// compares against the stale key and dereferences freed metadata → +/// / SIGSEGV. +/// +/// Autofac performs exactly this eviction automatically for scopes created via +/// BeginLoadContextLifetimeScope; because MeshWeaver manages its collectible +/// contexts by hand, it must run the eviction itself — before unload, while the +/// assembly metadata the predicate walks is still valid. +/// +public static class ReflectionCacheEviction +{ + /// + /// Removes every shared-reflection-cache entry that references an assembly loaded into + /// . Safe to call concurrently with cache reads (the + /// underlying stores are ); + /// call it before unloading so the walked metadata is still live. + /// + /// The collectible context whose entries should be purged. + public static void EvictFor(AssemblyLoadContext loadContext) + { + // Access Shared fresh each call — it is a WeakReference-backed singleton and must + // never be stored (Autofac's own guidance on the property). + ReflectionCacheSet.Shared.Clear((_, referencedAssemblies) => + { + foreach (var assembly in referencedAssemblies) + if (AssemblyLoadContext.GetLoadContext(assembly) == loadContext) + return true; + return false; + }); + } +} diff --git a/test/MeshWeaver.Graph.Test/MeshWeaver.Graph.Test.csproj b/test/MeshWeaver.Graph.Test/MeshWeaver.Graph.Test.csproj index 5870ae090..a8bff9496 100644 --- a/test/MeshWeaver.Graph.Test/MeshWeaver.Graph.Test.csproj +++ b/test/MeshWeaver.Graph.Test/MeshWeaver.Graph.Test.csproj @@ -4,6 +4,7 @@ + diff --git a/test/MeshWeaver.Graph.Test/ReflectionCacheEvictionTest.cs b/test/MeshWeaver.Graph.Test/ReflectionCacheEvictionTest.cs new file mode 100644 index 000000000..8147140f5 --- /dev/null +++ b/test/MeshWeaver.Graph.Test/ReflectionCacheEvictionTest.cs @@ -0,0 +1,98 @@ +using System.Runtime.CompilerServices; +using Autofac; +using Autofac.Core; +using MeshWeaver.Graph.Configuration; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Xunit; + +namespace MeshWeaver.Graph.Test; + +/// +/// Pins the fix for the concurrent-hub-construction SIGSEGV: a collectible +/// whose types were activated through Autofac must be +/// GC-collectable once unloaded. Autofac's process-static reflection cache keys on +/// ConstructorInfo, which strongly roots the assembly (and its load context); if the +/// cache isn't evicted on unload, the context leaks AND a later concurrent cache probe +/// dereferences the freed metadata → . +/// This asserts the (deterministic) root cause — a retained reference — not the flaky crash. +/// +public class ReflectionCacheEvictionTest +{ + private const string WidgetSource = + "namespace Dyn { public sealed class Widget { public Widget() { } } }"; + + [Fact] + public void UnloadedNodeContext_IsCollectible_AfterAutofacActivation() + { + // Pin the shared reflection cache alive for the whole test. It is held only by a + // WeakReference, so without this a plain GC would collect the entire cache and release the + // widget ctor on its own — masking the leak. A running app keeps the cache alive through + // constant container/scope activity; that is exactly when a retained stale key both roots + // the context (leak) and, on a concurrent probe, dereferences freed metadata (SIGSEGV). + var pinnedCache = ReflectionCacheSet.Shared; + + var weak = ActivateThroughAutofacThenUnload(); + + for (var i = 0; i < 15 && weak.IsAlive; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + + weak.IsAlive.Should().BeFalse( + "a collectible node AssemblyLoadContext must be collectable once unloaded; if Autofac's " + + "process-static reflection cache still holds the widget's ConstructorInfo it roots the " + + "context forever (leak) AND a later concurrent GetOrAdd bucket-probe would dereference the " + + "freed metadata → AccessViolationException. NodeAssemblyLoadContext must evict the shared " + + "reflection cache before Unload."); + + GC.KeepAlive(pinnedCache); + } + + // Kept out-of-line + non-inlined so no local (assembly / type / instance) stays rooted on the + // caller's frame — otherwise the context could not be collected regardless of the fix. + [MethodImpl(MethodImplOptions.NoInlining)] + private static WeakReference ActivateThroughAutofacThenUnload() + { + var bytes = CompileWidget(); + var ctx = new NodeAssemblyLoadContext("EvictionTestWidget", dllPath: null, logger: null); + var assembly = ctx.LoadFromBytes(bytes, null); + var widgetType = assembly.GetType("Dyn.Widget")!; + + // Activate through Autofac exactly as a hosted-hub container would: this creates a + // ConstructorBinder for the widget's ctor and caches it in ReflectionCacheSet.Shared. + var builder = new ContainerBuilder(); + builder.RegisterType(widgetType); + using (var container = builder.Build()) + { + container.Resolve(widgetType).Should().NotBeNull(); + } + + var weak = new WeakReference(ctx); + ctx.Dispose(); // → Unloading → ReflectionCacheEviction.EvictFor → Unload + return weak; + } + + private static byte[] CompileWidget() + { + var tpa = (string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!; + var references = tpa + .Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries) + .Select(p => (MetadataReference)MetadataReference.CreateFromFile(p)) + .ToArray(); + + var compilation = CSharpCompilation.Create( + "EvictionTestWidget", + new[] { CSharpSyntaxTree.ParseText(WidgetSource) }, + references, + new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)); + + using var ms = new MemoryStream(); + var result = compilation.Emit(ms); + result.Success.Should().BeTrue( + "the widget must compile: " + + string.Join("; ", result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error))); + return ms.ToArray(); + } +} From c47ae8adf792a644cedff17532c7b282377a2240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Tue, 14 Jul 2026 08:41:07 +0200 Subject: [PATCH 2/2] Add ALC-unload-GC probe + workflow to pin the teardown SIGSEGV corruptor The exit=139 crash in the ALC-heavy native test hosts (Hosting.Monolith.Test / FutuRe.Test) is a NATIVE use-after-unload during teardown: a process-wide reflection/serialization cache retains an accessor into a collectible node AssemblyLoadContext; when a node-hub disposes it unloads that ALC, and a later background GC dereferences the freed metadata -> SIGSEGV. The post-crash minidump is a delayed-detection snapshot (the ALC already unloaded, nothing to gcroot), so it can't name the offending cache. This is the known .NET collectible-ALC hazard class (dotnet/runtime #1388, #13283) -- the same SHAPE as the Autofac reflection cache this branch already evicts, but a DIFFERENT cache (that fix is orthogonal to the segfault). Diagnostic (NOT a fix), gated OFF by default so zero prod/CI cost: MESHWEAVER_ALC_UNLOAD_GC_PROBE=1 makes NodeAssemblyLoadContext.Dispose drive a synchronous full GC immediately after Unload, logging the node name first. A dangling native pointer then faults AT that unload -- naming the culprit node and yielding a corruption-time managed-stack dump -- instead of the opaque delayed crash. GCStress is the classic tool but needs a checked runtime the hosted runner lacks; forcing the collection is the retail-runtime equivalent for use-after-unload. Workflow .github/workflows/alc-unload-probe.yml (workflow_dispatch) loops the native host under the probe, breaks on a signal exit (139/134/132/136), greps the last ALC_UNLOAD_PROBE line = culprit node, and uploads the dump + logs. Run it to name the cache, then apply the evict-on-unload pattern to it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/alc-unload-probe.yml | 152 ++++++++++++++++++ .../Configuration/CompilationCacheService.cs | 21 +++ 2 files changed, 173 insertions(+) create mode 100644 .github/workflows/alc-unload-probe.yml diff --git a/.github/workflows/alc-unload-probe.yml b/.github/workflows/alc-unload-probe.yml new file mode 100644 index 000000000..bf340978d --- /dev/null +++ b/.github/workflows/alc-unload-probe.yml @@ -0,0 +1,152 @@ +# Manually-triggered ROOT-CAUSE probe for the teardown SIGSEGV (exit=139) in the +# ALC-heavy native test hosts (Hosting.Monolith.Test / FutuRe.Test). The crash is a +# native use-after-unload: a process-wide reflection/serialization cache retains an +# accessor/handle into a collectible node AssemblyLoadContext; when a node-hub disposes +# it unloads that ALC (MeshDataSource.RegisterForDisposal → UnloadNodeContexts), and a +# LATER background GC dereferences the freed metadata → SIGSEGV. The post-crash mini-dump +# is a delayed-detection snapshot (the ALC is already gone → nothing to gcroot), so it +# can't name the offending cache. +# +# This workflow flips MESHWEAVER_ALC_UNLOAD_GC_PROBE=1, which makes NodeAssemblyLoadContext +# drive a SYNCHRONOUS full GC immediately after each Unload. A dangling native pointer then +# faults AT that unload — on the Dispose thread, right after a +# "ALC_UNLOAD_PROBE forcing GC after unloading " log line — so the culprit node (and a +# corruption-time dump with a managed Dispose→GC.Collect stack) is captured instead of the +# opaque delayed crash. GCStress would be the classic tool but needs a checked runtime the +# hosted runner doesn't have; forcing the collection at the unload point is the retail-runtime +# equivalent for this specific use-after-unload. +# +# Run from the Actions tab (workflow_dispatch). Re-dispatch with project=MeshWeaver.FutuRe.Test +# to probe the other crasher. +name: ALC unload probe (manual) + +on: + workflow_dispatch: + inputs: + project: + description: "Test project NAME whose native host to loop under the probe" + type: string + default: "MeshWeaver.Hosting.Monolith.Test" + iterations: + description: "Max loop iterations (stops on the first native crash)" + type: string + default: "10" + +permissions: + contents: read + +env: + NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages + +jobs: + probe: + name: "Loop native host under ALC-unload GC probe" + # ubuntu-latest == the 2-vCPU hosted runner the crash reproduces on. + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + PROBE_PROJECT: ${{ inputs.project || 'MeshWeaver.Hosting.Monolith.Test' }} + PROBE_ITERS: ${{ inputs.iterations || '10' }} + DOTNET_ENVIRONMENT: Development + Logging__LogLevel__Default: Warning + # THE PROBE: force a synchronous GC after each collectible node-ALC Unload so a + # use-after-unload dangling native pointer faults at the unload (naming the node). + MESHWEAVER_ALC_UNLOAD_GC_PROBE: "1" + # Native-crash mini dump, same as dotnet-test.yml — the corruption-time forensic payload. + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 2 + DOTNET_DbgMiniDumpName: /tmp/coredumps/%e-%p.dmp + steps: + - uses: actions/checkout@v6 + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + dotnet: false + android: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + - name: Restore workloads + run: dotnet workload restore + - name: Restore dependencies + run: dotnet restore + # Build Release exactly as the real CI build job, so the looped native host is the + # identical binary that crashes in CI. + - name: Build + run: dotnet build --no-restore -c Release -p:CIRun=true -warnaserror + # The mesh-local #r feed dynamic-compilation tests resolve at runtime (version-less + # `#r "nuget:MeshWeaver.X"`) — packed exactly as dotnet-test.yml / flake-repro.yml do. + - name: Pack mesh-local #r packages + run: | + set -euo pipefail + dotnet pack src/MeshWeaver.BusinessRules/MeshWeaver.BusinessRules.csproj \ + -c Release --no-build --no-restore -o dist/packages --nologo + dotnet pack src/MeshWeaver.BusinessRules.Generator/MeshWeaver.BusinessRules.Generator.csproj \ + -c Release --no-build --no-restore -o dist/packages --nologo + - name: Loop the native host under the probe until it crashes + run: | + set -uo pipefail + mkdir -p /tmp/coredumps probe-logs + name="$PROBE_PROJECT" + dir="test/$name/bin/Release/net10.0" + if [ ! -f "$dir/$name.dll" ]; then + echo "::error::no native xUnit v3 host at $dir/$name.dll"; exit 1 + fi + echo "Looping $name native host up to $PROBE_ITERS× under MESHWEAVER_ALC_UNLOAD_GC_PROBE=1 (nproc=$(nproc))." + crashed=0 + for i in $(seq 1 "$PROBE_ITERS"); do + echo "::group::iteration $i / $PROBE_ITERS" + mkdir -p "$dir/TestResults" + # CWD = bin dir so appsettings/xunit.runner.json/TestData resolve as in CI. + # tee the host's stdout+stderr so the "ALC_UNLOAD_PROBE forcing GC after unloading + # " line right before a crash is captured even if the ILogger sink isn't console. + ( cd "$dir" && timeout --signal=TERM --kill-after=30s 15m \ + dotnet "$name.dll" -trx "TestResults/$name-$i.trx" ) \ + 2>&1 | tee "probe-logs/run-$i.log" + rc=${PIPESTATUS[0]} + echo "iteration $i exit=$rc" + echo "::endgroup::" + # Signal-induced death (SIGSEGV 139 / SIGABRT 134 / SIGILL 132 / SIGFPE 136) is the + # crash we hunt; rc==1 is ordinary test failures, 0 is clean — keep looping those. + if [ "$rc" = "139" ] || [ "$rc" = "134" ] || [ "$rc" = "132" ] || [ "$rc" = "136" ]; then + echo "::error::NATIVE CRASH on iteration $i (exit $rc). Culprit ALC unload (last probe lines):" + grep 'ALC_UNLOAD_PROBE' "probe-logs/run-$i.log" | tail -5 || echo "(no probe line in stdout — see collected test-logs + dump)" + crashed=1 + break + fi + done + if [ "$crashed" = "0" ]; then + echo "No native crash in $PROBE_ITERS iterations (the flake did not surface this run)." + fi + # Red the run on repro so it's obvious in the Actions list; diagnostics upload via always(). + exit "$crashed" + - name: Collect diagnostics + if: always() + run: | + mkdir -p probe-diagnostics + cp -r probe-logs probe-diagnostics/ 2>/dev/null || true + # Corruption-time mini dump(s). + cp /tmp/coredumps/*.dmp probe-diagnostics/ 2>/dev/null || true + # MonolithMeshTestBase phase/dispose/memory traces (which test class was tearing down). + for f in meshweaver-test-trace meshweaver-dispose-trace meshweaver-memory-delta; do + [ -f "/tmp/$f.log" ] && cp "/tmp/$f.log" "probe-diagnostics/_$f.log" || true + done + # ILogger sink (the ALC_UNLOAD_PROBE warnings land here if routed to the file helper). + find . -path '*/bin/*/test-logs/*.log' -exec cp {} probe-diagnostics/ \; 2>/dev/null || true + echo "=== last ALC_UNLOAD_PROBE lines (culprit node whose unload tripped the crash) ===" + grep -h 'ALC_UNLOAD_PROBE' probe-logs/*.log probe-diagnostics/*.log 2>/dev/null | tail -12 || true + ls -lh probe-diagnostics/ || true + - name: Upload probe diagnostics + if: always() + uses: actions/upload-artifact@v6 + with: + name: alc-unload-probe-diagnostics + path: probe-diagnostics/ + retention-days: 15 + compression-level: 9 diff --git a/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs b/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs index 769a23bcc..2f9b6af42 100644 --- a/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs +++ b/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs @@ -217,6 +217,14 @@ internal sealed class NodeAssemblyLoadContext : AssemblyLoadContext, IDisposable private volatile bool _disposed; private ImmutableArray _probingDirs = ImmutableArray.Empty; + // Opt-in diagnostic (env MESHWEAVER_ALC_UNLOAD_GC_PROBE=1) — OFF by default, so zero cost in + // prod and normal CI. When on, Dispose drives a synchronous full GC right after Unload so a + // use-after-unload dangling NATIVE pointer into this now-collectible assembly faults HERE + // (pinning the culprit node in the log + a corruption-time dump) instead of as a delayed + // background-GC SIGSEGV. Immutable config constant read once at type init — never written. + private static readonly bool UnloadGcProbe = + Environment.GetEnvironmentVariable("MESHWEAVER_ALC_UNLOAD_GC_PROBE") == "1"; + public void SetProbingDirectories(System.Collections.Generic.IReadOnlyList dirs) { _probingDirs = dirs.ToImmutableArray(); @@ -411,6 +419,19 @@ public void Dispose() // Initiate unload outside the lock - the context will be collected when all references are released Unload(); + + // Diagnostic probe (opt-in, off by default): drive the collection synchronously so a + // use-after-unload dangling native pointer trips at THIS unload — naming the culprit node + // and yielding a corruption-time dump — instead of a delayed background-GC SIGSEGV. Used + // by the alc-unload-probe workflow to pin the reflection/serialization cache that retains + // an accessor into a collectible node assembly (the exit=139 teardown crash). + if (UnloadGcProbe) + { + _logger?.LogWarning("ALC_UNLOAD_PROBE forcing GC after unloading {ContextName}", Name); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } } }