diff --git a/.github/workflows/alc-unload-probe.yml b/.github/workflows/alc-unload-probe.yml new file mode 100644 index 000000000..bf340978d --- /dev/null +++ b/.github/workflows/alc-unload-probe.yml @@ -0,0 +1,152 @@ +# Manually-triggered ROOT-CAUSE probe for the teardown SIGSEGV (exit=139) in the +# ALC-heavy native test hosts (Hosting.Monolith.Test / FutuRe.Test). The crash is a +# native use-after-unload: a process-wide reflection/serialization cache retains an +# accessor/handle into a collectible node AssemblyLoadContext; when a node-hub disposes +# it unloads that ALC (MeshDataSource.RegisterForDisposal → UnloadNodeContexts), and a +# LATER background GC dereferences the freed metadata → SIGSEGV. The post-crash mini-dump +# is a delayed-detection snapshot (the ALC is already gone → nothing to gcroot), so it +# can't name the offending cache. +# +# This workflow flips MESHWEAVER_ALC_UNLOAD_GC_PROBE=1, which makes NodeAssemblyLoadContext +# drive a SYNCHRONOUS full GC immediately after each Unload. A dangling native pointer then +# faults AT that unload — on the Dispose thread, right after a +# "ALC_UNLOAD_PROBE forcing GC after unloading " log line — so the culprit node (and a +# corruption-time dump with a managed Dispose→GC.Collect stack) is captured instead of the +# opaque delayed crash. GCStress would be the classic tool but needs a checked runtime the +# hosted runner doesn't have; forcing the collection at the unload point is the retail-runtime +# equivalent for this specific use-after-unload. +# +# Run from the Actions tab (workflow_dispatch). Re-dispatch with project=MeshWeaver.FutuRe.Test +# to probe the other crasher. +name: ALC unload probe (manual) + +on: + workflow_dispatch: + inputs: + project: + description: "Test project NAME whose native host to loop under the probe" + type: string + default: "MeshWeaver.Hosting.Monolith.Test" + iterations: + description: "Max loop iterations (stops on the first native crash)" + type: string + default: "10" + +permissions: + contents: read + +env: + NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages + +jobs: + probe: + name: "Loop native host under ALC-unload GC probe" + # ubuntu-latest == the 2-vCPU hosted runner the crash reproduces on. + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + PROBE_PROJECT: ${{ inputs.project || 'MeshWeaver.Hosting.Monolith.Test' }} + PROBE_ITERS: ${{ inputs.iterations || '10' }} + DOTNET_ENVIRONMENT: Development + Logging__LogLevel__Default: Warning + # THE PROBE: force a synchronous GC after each collectible node-ALC Unload so a + # use-after-unload dangling native pointer faults at the unload (naming the node). + MESHWEAVER_ALC_UNLOAD_GC_PROBE: "1" + # Native-crash mini dump, same as dotnet-test.yml — the corruption-time forensic payload. + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 2 + DOTNET_DbgMiniDumpName: /tmp/coredumps/%e-%p.dmp + steps: + - uses: actions/checkout@v6 + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + dotnet: false + android: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + - name: Restore workloads + run: dotnet workload restore + - name: Restore dependencies + run: dotnet restore + # Build Release exactly as the real CI build job, so the looped native host is the + # identical binary that crashes in CI. + - name: Build + run: dotnet build --no-restore -c Release -p:CIRun=true -warnaserror + # The mesh-local #r feed dynamic-compilation tests resolve at runtime (version-less + # `#r "nuget:MeshWeaver.X"`) — packed exactly as dotnet-test.yml / flake-repro.yml do. + - name: Pack mesh-local #r packages + run: | + set -euo pipefail + dotnet pack src/MeshWeaver.BusinessRules/MeshWeaver.BusinessRules.csproj \ + -c Release --no-build --no-restore -o dist/packages --nologo + dotnet pack src/MeshWeaver.BusinessRules.Generator/MeshWeaver.BusinessRules.Generator.csproj \ + -c Release --no-build --no-restore -o dist/packages --nologo + - name: Loop the native host under the probe until it crashes + run: | + set -uo pipefail + mkdir -p /tmp/coredumps probe-logs + name="$PROBE_PROJECT" + dir="test/$name/bin/Release/net10.0" + if [ ! -f "$dir/$name.dll" ]; then + echo "::error::no native xUnit v3 host at $dir/$name.dll"; exit 1 + fi + echo "Looping $name native host up to $PROBE_ITERS× under MESHWEAVER_ALC_UNLOAD_GC_PROBE=1 (nproc=$(nproc))." + crashed=0 + for i in $(seq 1 "$PROBE_ITERS"); do + echo "::group::iteration $i / $PROBE_ITERS" + mkdir -p "$dir/TestResults" + # CWD = bin dir so appsettings/xunit.runner.json/TestData resolve as in CI. + # tee the host's stdout+stderr so the "ALC_UNLOAD_PROBE forcing GC after unloading + # " line right before a crash is captured even if the ILogger sink isn't console. + ( cd "$dir" && timeout --signal=TERM --kill-after=30s 15m \ + dotnet "$name.dll" -trx "TestResults/$name-$i.trx" ) \ + 2>&1 | tee "probe-logs/run-$i.log" + rc=${PIPESTATUS[0]} + echo "iteration $i exit=$rc" + echo "::endgroup::" + # Signal-induced death (SIGSEGV 139 / SIGABRT 134 / SIGILL 132 / SIGFPE 136) is the + # crash we hunt; rc==1 is ordinary test failures, 0 is clean — keep looping those. + if [ "$rc" = "139" ] || [ "$rc" = "134" ] || [ "$rc" = "132" ] || [ "$rc" = "136" ]; then + echo "::error::NATIVE CRASH on iteration $i (exit $rc). Culprit ALC unload (last probe lines):" + grep 'ALC_UNLOAD_PROBE' "probe-logs/run-$i.log" | tail -5 || echo "(no probe line in stdout — see collected test-logs + dump)" + crashed=1 + break + fi + done + if [ "$crashed" = "0" ]; then + echo "No native crash in $PROBE_ITERS iterations (the flake did not surface this run)." + fi + # Red the run on repro so it's obvious in the Actions list; diagnostics upload via always(). + exit "$crashed" + - name: Collect diagnostics + if: always() + run: | + mkdir -p probe-diagnostics + cp -r probe-logs probe-diagnostics/ 2>/dev/null || true + # Corruption-time mini dump(s). + cp /tmp/coredumps/*.dmp probe-diagnostics/ 2>/dev/null || true + # MonolithMeshTestBase phase/dispose/memory traces (which test class was tearing down). + for f in meshweaver-test-trace meshweaver-dispose-trace meshweaver-memory-delta; do + [ -f "/tmp/$f.log" ] && cp "/tmp/$f.log" "probe-diagnostics/_$f.log" || true + done + # ILogger sink (the ALC_UNLOAD_PROBE warnings land here if routed to the file helper). + find . -path '*/bin/*/test-logs/*.log' -exec cp {} probe-diagnostics/ \; 2>/dev/null || true + echo "=== last ALC_UNLOAD_PROBE lines (culprit node whose unload tripped the crash) ===" + grep -h 'ALC_UNLOAD_PROBE' probe-logs/*.log probe-diagnostics/*.log 2>/dev/null | tail -12 || true + ls -lh probe-diagnostics/ || true + - name: Upload probe diagnostics + if: always() + uses: actions/upload-artifact@v6 + with: + name: alc-unload-probe-diagnostics + path: probe-diagnostics/ + retention-days: 15 + compression-level: 9 diff --git a/src/MeshWeaver.Documentation/Data/WhatsNew/2026-07-14-node-recompile-stability.md b/src/MeshWeaver.Documentation/Data/WhatsNew/2026-07-14-node-recompile-stability.md new file mode 100644 index 000000000..0a81459c4 --- /dev/null +++ b/src/MeshWeaver.Documentation/Data/WhatsNew/2026-07-14-node-recompile-stability.md @@ -0,0 +1,14 @@ +--- +Name: More reliable node recompilation +Category: What's New +Description: Recompiling code/scope nodes no longer leaks unloaded assemblies or risks a rare crash under load. +Icon: Sparkle +--- + +# More reliable node recompilation + +Compiling and recompiling code and scope nodes now cleanly releases the previous +version's compiled assembly. Previously the runtime's dependency-injection cache kept a +hidden reference to each unloaded assembly, which slowly grew memory over many recompiles +and, under concurrent activity, could trigger a rare hard crash. Portals that frequently +edit and recompile nodes will be more stable and use less memory over time. diff --git a/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs b/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs index 918410e3e..2f9b6af42 100644 --- a/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs +++ b/src/MeshWeaver.Graph/Configuration/CompilationCacheService.cs @@ -2,6 +2,7 @@ using System.Collections.Immutable; using System.Reflection; using System.Runtime.Loader; +using MeshWeaver.ServiceProvider; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -216,6 +217,14 @@ internal sealed class NodeAssemblyLoadContext : AssemblyLoadContext, IDisposable private volatile bool _disposed; private ImmutableArray _probingDirs = ImmutableArray.Empty; + // Opt-in diagnostic (env MESHWEAVER_ALC_UNLOAD_GC_PROBE=1) — OFF by default, so zero cost in + // prod and normal CI. When on, Dispose drives a synchronous full GC right after Unload so a + // use-after-unload dangling NATIVE pointer into this now-collectible assembly faults HERE + // (pinning the culprit node in the log + a corruption-time dump) instead of as a delayed + // background-GC SIGSEGV. Immutable config constant read once at type init — never written. + private static readonly bool UnloadGcProbe = + Environment.GetEnvironmentVariable("MESHWEAVER_ALC_UNLOAD_GC_PROBE") == "1"; + public void SetProbingDirectories(System.Collections.Generic.IReadOnlyList dirs) { _probingDirs = dirs.ToImmutableArray(); @@ -242,6 +251,15 @@ public NodeAssemblyLoadContext(string nodeName, string? dllPath, ILogger? logger _nodeName = nodeName; _dllPath = dllPath; _logger = logger; + + // Purge Autofac's process-static reflection cache of this context's assemblies the instant + // it starts unloading — while the metadata the predicate walks is still valid. A cached + // ConstructorInfo/Assembly key otherwise (1) roots this collectible context so it can never + // be collected, and (2) makes a later, unrelated concurrent GetOrAdd bucket-probe compare + // against the freed key → AccessViolationException/SIGSEGV under concurrent hub construction. + // Autofac does this automatically for BeginLoadContextLifetimeScope; we manage the context + // by hand, so we mirror it. Static handler ⇒ no self-reference that would defeat collection. + Unloading += ReflectionCacheEviction.EvictFor; } /// @@ -401,6 +419,19 @@ public void Dispose() // Initiate unload outside the lock - the context will be collected when all references are released Unload(); + + // Diagnostic probe (opt-in, off by default): drive the collection synchronously so a + // use-after-unload dangling native pointer trips at THIS unload — naming the culprit node + // and yielding a corruption-time dump — instead of a delayed background-GC SIGSEGV. Used + // by the alc-unload-probe workflow to pin the reflection/serialization cache that retains + // an accessor into a collectible node assembly (the exit=139 teardown crash). + if (UnloadGcProbe) + { + _logger?.LogWarning("ALC_UNLOAD_PROBE forcing GC after unloading {ContextName}", Name); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } } } diff --git a/src/MeshWeaver.ServiceProvider/ReflectionCacheEviction.cs b/src/MeshWeaver.ServiceProvider/ReflectionCacheEviction.cs new file mode 100644 index 000000000..015b5022a --- /dev/null +++ b/src/MeshWeaver.ServiceProvider/ReflectionCacheEviction.cs @@ -0,0 +1,52 @@ +#nullable enable +using System.Runtime.Loader; +using Autofac.Core; + +namespace MeshWeaver.ServiceProvider; + +/// +/// Evicts entries from Autofac's process-static shared reflection cache +/// () that reference assemblies loaded into a +/// collectible which is about to be unloaded. +/// +/// +/// Autofac keys its reflection caches (constructor-binder factories, parameter maps, +/// assembly scans) on /. +/// Those keys strongly root the declaring assembly and therefore its +/// . MeshWeaver compiles nodes into collectible load +/// contexts (NodeAssemblyLoadContext) and unloads them on recompile / release / +/// teardown. If the shared cache still holds an entry for the unloaded assembly, two +/// things go wrong: +/// +/// the context can never be collected (the cache roots it) — an unbounded leak; and +/// a later, unrelated concurrent GetOrAdd whose key hashes into the same bucket +/// compares against the stale key and dereferences freed metadata → +/// / SIGSEGV. +/// +/// Autofac performs exactly this eviction automatically for scopes created via +/// BeginLoadContextLifetimeScope; because MeshWeaver manages its collectible +/// contexts by hand, it must run the eviction itself — before unload, while the +/// assembly metadata the predicate walks is still valid. +/// +public static class ReflectionCacheEviction +{ + /// + /// Removes every shared-reflection-cache entry that references an assembly loaded into + /// . Safe to call concurrently with cache reads (the + /// underlying stores are ); + /// call it before unloading so the walked metadata is still live. + /// + /// The collectible context whose entries should be purged. + public static void EvictFor(AssemblyLoadContext loadContext) + { + // Access Shared fresh each call — it is a WeakReference-backed singleton and must + // never be stored (Autofac's own guidance on the property). + ReflectionCacheSet.Shared.Clear((_, referencedAssemblies) => + { + foreach (var assembly in referencedAssemblies) + if (AssemblyLoadContext.GetLoadContext(assembly) == loadContext) + return true; + return false; + }); + } +} diff --git a/test/MeshWeaver.Graph.Test/MeshWeaver.Graph.Test.csproj b/test/MeshWeaver.Graph.Test/MeshWeaver.Graph.Test.csproj index 5870ae090..a8bff9496 100644 --- a/test/MeshWeaver.Graph.Test/MeshWeaver.Graph.Test.csproj +++ b/test/MeshWeaver.Graph.Test/MeshWeaver.Graph.Test.csproj @@ -4,6 +4,7 @@ + diff --git a/test/MeshWeaver.Graph.Test/ReflectionCacheEvictionTest.cs b/test/MeshWeaver.Graph.Test/ReflectionCacheEvictionTest.cs new file mode 100644 index 000000000..8147140f5 --- /dev/null +++ b/test/MeshWeaver.Graph.Test/ReflectionCacheEvictionTest.cs @@ -0,0 +1,98 @@ +using System.Runtime.CompilerServices; +using Autofac; +using Autofac.Core; +using MeshWeaver.Graph.Configuration; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Xunit; + +namespace MeshWeaver.Graph.Test; + +/// +/// Pins the fix for the concurrent-hub-construction SIGSEGV: a collectible +/// whose types were activated through Autofac must be +/// GC-collectable once unloaded. Autofac's process-static reflection cache keys on +/// ConstructorInfo, which strongly roots the assembly (and its load context); if the +/// cache isn't evicted on unload, the context leaks AND a later concurrent cache probe +/// dereferences the freed metadata → . +/// This asserts the (deterministic) root cause — a retained reference — not the flaky crash. +/// +public class ReflectionCacheEvictionTest +{ + private const string WidgetSource = + "namespace Dyn { public sealed class Widget { public Widget() { } } }"; + + [Fact] + public void UnloadedNodeContext_IsCollectible_AfterAutofacActivation() + { + // Pin the shared reflection cache alive for the whole test. It is held only by a + // WeakReference, so without this a plain GC would collect the entire cache and release the + // widget ctor on its own — masking the leak. A running app keeps the cache alive through + // constant container/scope activity; that is exactly when a retained stale key both roots + // the context (leak) and, on a concurrent probe, dereferences freed metadata (SIGSEGV). + var pinnedCache = ReflectionCacheSet.Shared; + + var weak = ActivateThroughAutofacThenUnload(); + + for (var i = 0; i < 15 && weak.IsAlive; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + + weak.IsAlive.Should().BeFalse( + "a collectible node AssemblyLoadContext must be collectable once unloaded; if Autofac's " + + "process-static reflection cache still holds the widget's ConstructorInfo it roots the " + + "context forever (leak) AND a later concurrent GetOrAdd bucket-probe would dereference the " + + "freed metadata → AccessViolationException. NodeAssemblyLoadContext must evict the shared " + + "reflection cache before Unload."); + + GC.KeepAlive(pinnedCache); + } + + // Kept out-of-line + non-inlined so no local (assembly / type / instance) stays rooted on the + // caller's frame — otherwise the context could not be collected regardless of the fix. + [MethodImpl(MethodImplOptions.NoInlining)] + private static WeakReference ActivateThroughAutofacThenUnload() + { + var bytes = CompileWidget(); + var ctx = new NodeAssemblyLoadContext("EvictionTestWidget", dllPath: null, logger: null); + var assembly = ctx.LoadFromBytes(bytes, null); + var widgetType = assembly.GetType("Dyn.Widget")!; + + // Activate through Autofac exactly as a hosted-hub container would: this creates a + // ConstructorBinder for the widget's ctor and caches it in ReflectionCacheSet.Shared. + var builder = new ContainerBuilder(); + builder.RegisterType(widgetType); + using (var container = builder.Build()) + { + container.Resolve(widgetType).Should().NotBeNull(); + } + + var weak = new WeakReference(ctx); + ctx.Dispose(); // → Unloading → ReflectionCacheEviction.EvictFor → Unload + return weak; + } + + private static byte[] CompileWidget() + { + var tpa = (string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!; + var references = tpa + .Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries) + .Select(p => (MetadataReference)MetadataReference.CreateFromFile(p)) + .ToArray(); + + var compilation = CSharpCompilation.Create( + "EvictionTestWidget", + new[] { CSharpSyntaxTree.ParseText(WidgetSource) }, + references, + new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)); + + using var ms = new MemoryStream(); + var result = compilation.Emit(ms); + result.Success.Should().BeTrue( + "the widget must compile: " + + string.Join("; ", result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error))); + return ms.ToArray(); + } +}