diff --git a/common/config.go b/common/config.go index f392e488d..95c16bd3e 100644 --- a/common/config.go +++ b/common/config.go @@ -2259,6 +2259,37 @@ type EvmNetworkConfig struct { // to work safely with transaction broadcasting. // Set to false to disable this behavior and return raw upstream errors. IdempotentTransactionBroadcast *bool `yaml:"idempotentTransactionBroadcast,omitempty" json:"idempotentTransactionBroadcast,omitempty"` + + // FinalityStallWindow and FinalityStallMargin together classify a primary + // upstream as "finality-stalled": a node whose `finalized` has frozen (a + // node-software finality bug) while its `latest` keeps advancing. Such a + // primary is circuit-CLOSED and serves requests normally, so it is otherwise + // "up", yet it is an untrustworthy SOURCE OF FINALIZED — serving its frozen + // value keeps the network finalized far behind the true chain finalized and + // trips strict downstream finality guards. eRPC excludes a finality-stalled + // primary from finalized resolution and fails over to a healthy fallback. + // + // A primary is treated as finality-stalled only when BOTH conditions hold: + // - its finalized has not advanced for longer than FinalityStallWindow, AND + // - its latest is more than FinalityStallMargin blocks ahead of its finalized. + // + // Requiring BOTH is what prevents false-positives on chains with legitimately + // deep-but-healthy finality: a chain whose latest sits close to finalized + // never crosses the margin, and a chain whose finalized keeps advancing never + // crosses the window. Set FinalityStallMargin comfortably ABOVE the chain's + // normal latest-minus-finalized gap; for very fast block times or unusually + // deep finality, raise it. Default: 90s window, 8192-block margin. + // Set either knob to 0 to disable finality-stall detection for the network. + FinalityStallWindow *Duration `yaml:"finalityStallWindow,omitempty" json:"finalityStallWindow,omitempty" tstype:"Duration"` + FinalityStallMargin *int64 `yaml:"finalityStallMargin,omitempty" json:"finalityStallMargin,omitempty"` + + // FinalizedCorroborationReorgWindow caps the finalized value eRPC will adopt + // from a fallback during a primary outage at `fallbackLatest - reorgWindow`. + // It is a rogue-high guard: a single fallback cannot push the network + // finalized up to (or past) the chain tip. Default 0 means "cap at the + // corroborating fallback's own latest" (reject only a finalized that exceeds + // it); raise it per-network for a stricter reorg-safety margin. + FinalizedCorroborationReorgWindow int64 `yaml:"finalizedCorroborationReorgWindow,omitempty" json:"finalizedCorroborationReorgWindow,omitempty"` } // EvmIntegrityConfig is deprecated. Use DirectiveDefaultsConfig for validation settings. @@ -2629,8 +2660,8 @@ const tsLoaderWalker = ` // This means closures, imports, and module-level helpers in the user's // TS file flow naturally into the evalFunc: // -// const weights = { hot: { errorRate: 8 }, cold: { errorRate: 4 } }; -// selectionPolicy: { evalFunc: (u, ctx) => u.sortByScore((u) => weights[u.id] || PREFER_FASTEST) } +// const weights = { hot: { errorRate: 8 }, cold: { errorRate: 4 } }; +// selectionPolicy: { evalFunc: (u, ctx) => u.sortByScore((u) => weights[u.id] || PREFER_FASTEST) } // // works as written, because `weights` exists in the same module scope // as the function in every pool runtime. diff --git a/common/defaults.go b/common/defaults.go index 19fcecc18..510e986dd 100644 --- a/common/defaults.go +++ b/common/defaults.go @@ -2007,6 +2007,8 @@ func (n *NetworkConfig) SetDefaults(upstreams []*UpstreamConfig, defaults *Netwo const DefaultEvmFinalityDepth = 1024 const DefaultEvmStatePollerDebounce = Duration(5 * time.Second) +const DefaultEvmFinalityStallWindow = Duration(90 * time.Second) +const DefaultEvmFinalityStallMargin = int64(8192) const DefaultDynamicBlockTimeDebounceMultiplier = 0.7 const DefaultBlockUnavailableDelayMultiplier = 0.8 @@ -2091,6 +2093,14 @@ func (e *EvmNetworkConfig) SetDefaults() error { if e.FallbackStatePollerDebounce == 0 { e.FallbackStatePollerDebounce = DefaultEvmStatePollerDebounce } + if e.FinalityStallWindow == nil { + d := DefaultEvmFinalityStallWindow + e.FinalityStallWindow = &d + } + if e.FinalityStallMargin == nil { + d := DefaultEvmFinalityStallMargin + e.FinalityStallMargin = &d + } if e.DynamicBlockTimeDebounceMultiplier == nil { d := DefaultDynamicBlockTimeDebounceMultiplier e.DynamicBlockTimeDebounceMultiplier = &d diff --git a/erpc.dist.yaml b/erpc.dist.yaml index 9d9cf026c..0172ebda8 100644 --- a/erpc.dist.yaml +++ b/erpc.dist.yaml @@ -76,6 +76,22 @@ projects: - architecture: evm evm: chainId: 1 + # Finality-stall detection: a primary whose `finalized` freezes (a + # node-software finality bug) while its `latest` keeps advancing is + # demoted as a SOURCE OF FINALIZED and the network fails over to a + # healthy fallback. A primary trips ONLY when BOTH hold: finalized has + # not advanced for `finalityStallWindow`, AND latest is more than + # `finalityStallMargin` blocks ahead of finalized. Requiring both + # avoids demoting a chain with legitimately deep-but-advancing finality + # — set the margin comfortably above the chain's normal + # latest-minus-finalized gap (raise it for very fast block times). + # Set either to 0 to disable. Defaults: 90s window, 8192-block margin. + finalityStallWindow: 90s + finalityStallMargin: 8192 + # Rogue-high guard when adopting a fallback's finalized during an + # outage: reject any fallback finalized above `fallbackLatest - N`. + # 0 (default) means "cap at the corroborating fallback's own latest". + finalizedCorroborationReorgWindow: 0 failsafe: timeout: duration: 30s diff --git a/erpc/networks.go b/erpc/networks.go index 3101ef681..cfbd4b332 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -64,6 +64,22 @@ type Network struct { // shared counter, or a race between them. lastReturnedLatestBlock atomic.Int64 lastReturnedFinalizedBlock atomic.Int64 + + // finalizedProgress tracks, per upstream id, the last finalized block we've + // observed for a primary and when it last advanced. It is the state behind + // finality-stall detection (detectFinalityStall) and reuses the poller's + // existing finalized/latest values — no new poller is added. + finalizedProgress sync.Map // map[string]*finalizedProgressEntry +} + +// finalizedProgressEntry records a primary's last-seen finalized block and the +// wall-clock time it last advanced, so we can tell "frozen for N seconds" from +// "advancing normally". stalled is edge-tracked so we log/emit a metric only on +// the transition into and out of the stalled state, not on every resolution. +type finalizedProgressEntry struct { + value atomic.Int64 + sinceMs atomic.Int64 + stalled atomic.Bool } // Bootstrap registers this network with the policy engine. The engine kicks @@ -303,6 +319,116 @@ func (n *Network) EvmHighestFinalizedBlockNumber(ctx context.Context) int64 { return result } +// finalizedProgressFor returns (creating if needed) the finalized-advance record +// for an upstream id. +func (n *Network) finalizedProgressFor(id string) *finalizedProgressEntry { + if v, ok := n.finalizedProgress.Load(id); ok { + return v.(*finalizedProgressEntry) + } + actual, _ := n.finalizedProgress.LoadOrStore(id, &finalizedProgressEntry{}) + return actual.(*finalizedProgressEntry) +} + +// detectFinalityStall updates a primary's finalized-advance record and reports +// whether it is finality-stalled, then emits edge-triggered observability +// (a metric/log fires only on the transition into/out of the stalled state). +// The classification itself lives in the pure evaluateFinalityStall so it can be +// unit-tested exhaustively without upstreams, gock, or wall-clock timing. +func (n *Network) detectFinalityStall(u *upstream.Upstream, finalized, latest, nowMs int64) bool { + entry := n.finalizedProgressFor(u.Id()) + stalled, becameStalled, resumed := evaluateFinalityStall( + entry, finalized, latest, nowMs, n.finalityStallWindowMs(), n.finalityStallMargin()) + + if resumed { + n.logger.Info().Str("upstreamId", u.Id()).Int64("finalized", finalized). + Msg("primary finalized resumed advancing; restored as finalized source") + } + if becameStalled { + n.logger.Warn(). + Str("upstreamId", u.Id()). + Int64("finalized", finalized). + Int64("latest", latest). + Int64("lag", latest-finalized). + Msg("primary finalized has stalled (frozen while latest advances); demoting it as a finalized source") + telemetry.MetricUpstreamFinalityStalled.WithLabelValues(n.projectId, u.VendorName(), n.Label(), u.Id()).Inc() + } + return stalled +} + +// evaluateFinalityStall is the pure classification behind detectFinalityStall. +// It mutates the per-upstream record and reports whether the upstream is +// finality-stalled: its finalized has not advanced for longer than windowMs +// WHILE its latest is more than margin blocks ahead of its finalized. Both +// conditions are required so a chain with legitimately deep-but-advancing +// finality (or one whose latest sits close to finalized) is never misclassified. +// Detection is disabled (always returns false) when either threshold is <= 0 or +// inputs are incomplete. becameStalled/resumed flag the edge transitions so the +// caller can emit observability exactly once per transition. +func evaluateFinalityStall(entry *finalizedProgressEntry, finalized, latest, nowMs, windowMs, margin int64) (stalled, becameStalled, resumed bool) { + // Advancing finalized always clears the stall and resets the freeze timer. + if finalized > entry.value.Load() { + entry.value.Store(finalized) + entry.sinceMs.Store(nowMs) + resumed = entry.stalled.Swap(false) + return false, false, resumed + } + + if windowMs <= 0 || margin <= 0 || finalized <= 0 || latest <= 0 { + return false, false, false + } + + since := entry.sinceMs.Load() + if since <= 0 { + // First observation at this (non-advancing) value — start the timer. + entry.sinceMs.Store(nowMs) + return false, false, false + } + if nowMs-since < windowMs || latest-finalized < margin { + return false, false, false + } + + becameStalled = !entry.stalled.Swap(true) + return true, becameStalled, false +} + +// corroboratedFallbackFinalized validates ONE fallback's finalized against that +// SAME fallback's latest and returns it (or 0 if it can't be corroborated). It +// guards against a single rogue-high fallback: the value is adopted only when the +// fallback's own latest is ahead of it (so finalized is plausibly behind the +// tip), and it is rejected if it exceeds latest - reorgWindow so a bad fallback +// can't push finalized up to (or past) the chain tip. Per-source corroboration is +// what makes the guard meaningful — comparing against the max latest across all +// fallbacks would let a rogue fallback ride another's high tip. A stricter +// >=2-source corroboration remains a separate follow-up. +func (n *Network) corroboratedFallbackFinalized(fallbackFinalized, fallbackLatest int64) int64 { + if fallbackFinalized <= 0 || fallbackLatest <= 0 { + return 0 + } + reorg := int64(0) + if n.cfg != nil && n.cfg.Evm != nil && n.cfg.Evm.FinalizedCorroborationReorgWindow > 0 { + reorg = n.cfg.Evm.FinalizedCorroborationReorgWindow + } + bound := fallbackLatest - reorg + if bound <= 0 || fallbackFinalized > bound { + return 0 + } + return fallbackFinalized +} + +func (n *Network) finalityStallWindowMs() int64 { + if n.cfg == nil || n.cfg.Evm == nil || n.cfg.Evm.FinalityStallWindow == nil { + return 0 + } + return n.cfg.Evm.FinalityStallWindow.Duration().Milliseconds() +} + +func (n *Network) finalityStallMargin() int64 { + if n.cfg == nil || n.cfg.Evm == nil || n.cfg.Evm.FinalityStallMargin == nil { + return 0 + } + return *n.cfg.Evm.FinalityStallMargin +} + // evmHighestBlockNumber aggregates a per-upstream block number across a // network and reconciles it with the cross-pod shared counter. // @@ -328,8 +454,11 @@ func (n *Network) evmHighestBlockNumber( ) int64 { var primaryMax, fallbackMax int64 anyPrimaryUp := false + nowMs := time.Now().UnixMilli() + isFinalized := tag == "finalized" for _, u := range n.upstreamsRegistry.GetNetworkUpstreams(ctx, n.networkId) { - if u.EvmStatePoller() == nil { + sp := u.EvmStatePoller() + if sp == nil { continue } if u.EvmSyncingState() == common.EvmSyncingStateSyncing { @@ -338,12 +467,41 @@ func (n *Network) evmHighestBlockNumber( } upBlock := blockOf(u) if u.Config().HasTag(common.TagTierFallback) { - if upBlock > fallbackMax { + // A circuit-broken fallback's last-known finalized can't be trusted + // as a source once its breaker is open — skip it, same as we treat a + // down primary below. + if u.IsDown() { + continue + } + if isFinalized { + // Per-source corroboration: trust this fallback's finalized only + // when its OWN latest supports it (finalized can't legitimately + // exceed latest, less a reorg margin). Guarding per source — not + // against the max latest across all fallbacks — stops a single + // rogue-high fallback from being adopted just because some other + // fallback happens to report a high latest. + if cf := n.corroboratedFallbackFinalized(upBlock, sp.LatestBlock()); cf > fallbackMax { + fallbackMax = cf + } + } else if upBlock > fallbackMax { fallbackMax = upBlock } continue } if !u.IsDown() { + // Distrust a finality-stalled primary as a finalized source: its + // frozen finalized must not anchor primaryMax and it must not count + // as an "up" primary, so the no-trustworthy-primary failover below + // engages — mirroring how a circuit-broken fallback is skipped above. + // The primary is otherwise healthy (circuit closed) and still serves + // normal traffic; we only demote it as a SOURCE OF FINALIZED. Only + // evaluate stall on a primary that is actually up: a circuit-broken + // primary is excluded from the trusted set anyway, and running + // detection on it would emit misleading stall signals off stale + // poller values. + if isFinalized && n.detectFinalityStall(u, upBlock, sp.LatestBlock(), nowMs) { + continue + } anyPrimaryUp = true } if upBlock > primaryMax { @@ -351,9 +509,19 @@ func (n *Network) evmHighestBlockNumber( } } + // When no primary can be trusted for finalized (all circuit-broken OR all + // finality-stalled), fail over to the fallback tier. Their finalized/latest + // are already tracked by the state poller (probe:off only opts a fallback out + // of the selection policy's shadow-mirror traffic, not out of state polling), + // so the value is available here without any extra on-demand polling. For the + // finalized tag, fallbackMax already holds only per-source-corroborated + // values, so a rogue-high fallback was excluded above. localMax := primaryMax if fallbackMax > 0 && !anyPrimaryUp { localMax = fallbackMax + if isFinalized { + telemetry.MetricNetworkFinalizedServedFromFallback.WithLabelValues(n.projectId, n.Label()).Inc() + } } var result int64 diff --git a/erpc/networks_finality_stall_test.go b/erpc/networks_finality_stall_test.go new file mode 100644 index 000000000..ce56c2c70 --- /dev/null +++ b/erpc/networks_finality_stall_test.go @@ -0,0 +1,133 @@ +package erpc + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/require" +) + +// TestCorroboratedFallbackFinalized covers the per-source rogue-high guard: a +// fallback's finalized is adopted only when its OWN latest supports it. The +// corroboration is per source on purpose — comparing against the max latest +// across all fallbacks would let a single rogue-high fallback ride another +// fallback's high tip, which is exactly the gap this guards. +func TestCorroboratedFallbackFinalized(t *testing.T) { + // reorg window 0 (default): cap at the fallback's own latest. + n := &Network{} + + t.Run("finalized below its own latest is adopted", func(t *testing.T) { + require.Equal(t, int64(1000), n.corroboratedFallbackFinalized(1000, 1050)) + }) + t.Run("finalized equal to its own latest is adopted (reorg 0)", func(t *testing.T) { + require.Equal(t, int64(1050), n.corroboratedFallbackFinalized(1050, 1050)) + }) + t.Run("rogue-high finalized above its own latest is rejected", func(t *testing.T) { + require.Equal(t, int64(0), n.corroboratedFallbackFinalized(1100, 1050), + "a fallback claiming finalized past its own tip must not be adopted") + }) + t.Run("zero finalized or latest is rejected", func(t *testing.T) { + require.Equal(t, int64(0), n.corroboratedFallbackFinalized(0, 1050)) + require.Equal(t, int64(0), n.corroboratedFallbackFinalized(1000, 0)) + }) + + t.Run("reorg window tightens the bound to latest - window", func(t *testing.T) { + nr := &Network{cfg: &common.NetworkConfig{ + Evm: &common.EvmNetworkConfig{FinalizedCorroborationReorgWindow: 64}, + }} + // bound = 1050 - 64 = 986. + require.Equal(t, int64(980), nr.corroboratedFallbackFinalized(980, 1050), + "finalized within latest-window is adopted") + require.Equal(t, int64(0), nr.corroboratedFallbackFinalized(1000, 1050), + "finalized inside the reorg margin (986..1050) is rejected") + }) +} + +// TestEvaluateFinalityStall exhaustively covers the pure finality-stall +// classifier — the timer, the margin, advance-resets-the-timer, the disable +// switches, and the edge transitions — deterministically, with no upstreams, +// gock, or wall-clock sleeps. nowMs is injected so "time" is just an argument. +func TestEvaluateFinalityStall(t *testing.T) { + const window = int64(1000) // ms + const margin = int64(100) // blocks + + t.Run("advancing finalized is never stalled, even with a huge gap", func(t *testing.T) { + e := &finalizedProgressEntry{} + // Finalized advances on every observation while latest sits far ahead. + // Each advance resets the freeze timer, so the window can never elapse. + for i := int64(0); i < 100; i++ { + stalled, became, _ := evaluateFinalityStall(e, 1000+i, 1_000_000, i*10_000, window, margin) + require.False(t, stalled, "advancing finalized must never be flagged (iter %d)", i) + require.False(t, became, "no stall transition while advancing (iter %d)", i) + } + }) + + t.Run("frozen within the window is not stalled", func(t *testing.T) { + e := &finalizedProgressEntry{} + // t=0 seeds the value+timer; t=500ms (< 1000ms window) must not trip. + _, _, _ = evaluateFinalityStall(e, 1000, 1_000_000, 10_000, window, margin) + stalled, became, _ := evaluateFinalityStall(e, 1000, 1_000_000, 10_500, window, margin) + require.False(t, stalled, "frozen but within the window must not be stalled") + require.False(t, became) + }) + + t.Run("frozen past the window with a large gap is stalled (edge-triggered)", func(t *testing.T) { + e := &finalizedProgressEntry{} + _, _, _ = evaluateFinalityStall(e, 1000, 1_000_000, 10_000, window, margin) + stalled, became, _ := evaluateFinalityStall(e, 1000, 1_000_000, 11_500, window, margin) + require.True(t, stalled, "frozen past window with gap > margin must be stalled") + require.True(t, became, "first stalled observation must report the transition") + // A second stalled observation must NOT re-report the transition. + stalled2, became2, _ := evaluateFinalityStall(e, 1000, 1_000_000, 11_600, window, margin) + require.True(t, stalled2) + require.False(t, became2, "stall transition must fire only once") + }) + + t.Run("frozen past the window but small gap is NOT stalled (margin guard)", func(t *testing.T) { + e := &finalizedProgressEntry{} + // latest only 50 ahead of finalized (< margin 100): a tight-finality chain. + _, _, _ = evaluateFinalityStall(e, 1000, 1050, 10_000, window, margin) + stalled, became, _ := evaluateFinalityStall(e, 1000, 1050, 15_000, window, margin) + require.False(t, stalled, "small latest-finalized gap must never be classified as stalled") + require.False(t, became) + }) + + t.Run("resuming after a stall clears it and reports the transition", func(t *testing.T) { + e := &finalizedProgressEntry{} + _, _, _ = evaluateFinalityStall(e, 1000, 1_000_000, 10_000, window, margin) + stalled, became, _ := evaluateFinalityStall(e, 1000, 1_000_000, 12_000, window, margin) + require.True(t, stalled) + require.True(t, became) + // Finalized advances again → no longer stalled, resume transition reported. + stalled2, _, resumed := evaluateFinalityStall(e, 1001, 1_000_000, 12_100, window, margin) + require.False(t, stalled2, "an advance must clear the stall") + require.True(t, resumed, "resume must be reported once") + // And the timer is reset, so it isn't immediately stalled again. + stalled3, _, _ := evaluateFinalityStall(e, 1001, 1_000_000, 12_200, window, margin) + require.False(t, stalled3, "freeze timer must restart after an advance") + }) + + t.Run("disabled when window <= 0", func(t *testing.T) { + e := &finalizedProgressEntry{} + _, _, _ = evaluateFinalityStall(e, 1000, 1_000_000, 10_000, 0, margin) + stalled, _, _ := evaluateFinalityStall(e, 1000, 1_000_000, 1_000_000, 0, margin) + require.False(t, stalled, "window <= 0 disables detection") + }) + + t.Run("disabled when margin <= 0", func(t *testing.T) { + e := &finalizedProgressEntry{} + _, _, _ = evaluateFinalityStall(e, 1000, 1_000_000, 10_000, window, 0) + stalled, _, _ := evaluateFinalityStall(e, 1000, 1_000_000, 1_000_000, window, 0) + require.False(t, stalled, "margin <= 0 disables detection") + }) + + t.Run("incomplete inputs (zero finalized/latest) are not stalled", func(t *testing.T) { + e := &finalizedProgressEntry{} + stalled, _, _ := evaluateFinalityStall(e, 0, 1_000_000, 5000, window, margin) + require.False(t, stalled, "finalized=0 must not be classified") + e2 := &finalizedProgressEntry{} + _, _, _ = evaluateFinalityStall(e2, 1000, 0, 10_000, window, margin) + stalled2, _, _ := evaluateFinalityStall(e2, 1000, 0, 15_000, window, margin) + require.False(t, stalled2, "latest=0 must not be classified") + }) +} diff --git a/erpc/networks_finalized_fallback_test.go b/erpc/networks_finalized_fallback_test.go new file mode 100644 index 000000000..a6dc0a144 --- /dev/null +++ b/erpc/networks_finalized_fallback_test.go @@ -0,0 +1,260 @@ +package erpc + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/erpc/erpc/architecture/evm" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/stretchr/testify/require" +) + +// finalityStallConfigs builds two circuit-CLOSED primaries and two fallback-tier +// upstreams, all with the state poller on. probe:off fallbacks in production are +// still state-polled (probe:off only opts out of selection-policy shadow-mirror +// traffic), so the fallbacks here poll their latest/finalized exactly as prod +// does. The primaries are healthy and serve requests normally; the test mocks +// freeze their finalized while latest stays at tip to simulate a node finality bug. +func finalityStallConfigs() []*common.UpstreamConfig { + return []*common.UpstreamConfig{ + { + Type: common.UpstreamTypeEvm, Id: "primary-1", + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 999, + StatePollerInterval: common.Duration(100 * time.Millisecond), + StatePollerDebounce: common.Duration(20 * time.Millisecond), + }, + }, + { + Type: common.UpstreamTypeEvm, Id: "primary-2", + Endpoint: "http://rpc2.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 999, + StatePollerInterval: common.Duration(100 * time.Millisecond), + StatePollerDebounce: common.Duration(20 * time.Millisecond), + }, + }, + { + Type: common.UpstreamTypeEvm, Id: "fallback-1", + Endpoint: "http://rpc3.localhost", + Tags: []string{common.TagTierFallback}, + Evm: &common.EvmUpstreamConfig{ + ChainId: 999, + StatePollerInterval: common.Duration(100 * time.Millisecond), + StatePollerDebounce: common.Duration(20 * time.Millisecond), + }, + }, + { + Type: common.UpstreamTypeEvm, Id: "fallback-2", + Endpoint: "http://rpc4.localhost", + Tags: []string{common.TagTierFallback}, + Evm: &common.EvmUpstreamConfig{ + ChainId: 999, + StatePollerInterval: common.Duration(100 * time.Millisecond), + StatePollerDebounce: common.Duration(20 * time.Millisecond), + }, + }, + } +} + +// setFinalityStallThresholds tightens the per-network finality-stall knobs so the +// behaviour is testable in milliseconds instead of the 90s/8192 production +// defaults. +func setFinalityStallThresholds(n *Network, window time.Duration, margin int64) { + w := common.Duration(window) + m := margin + n.cfg.Evm.FinalityStallWindow = &w + n.cfg.Evm.FinalityStallMargin = &m +} + +// TestFinalizedResolution_DemotesCircuitClosedFinalityStalledPrimary reproduces +// the Base mainnet incident (2026-06-24): the primary is UP and circuit-CLOSED, +// `latest` tracks tip, but its `finalized` is FROZEN far behind tip (a node +// finality bug). PR #2's all-primaries-down failover is a no-op here because the +// primary is up. eRPC must detect the stalled primary, demote it as a finalized +// source, and fail over to the healthy fallback's higher finalized — forward-only. +func TestFinalizedResolution_DemotesCircuitClosedFinalityStalledPrimary(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const chainIdHex = "0x3e7" + const primaryFinalized = int64(0xF0000) // frozen, far behind tip + const fallbackFinalized = int64(0xFFF00) // true, higher finalized + const fallbackLatest = "0x100010" + + // Primaries: latest at tip (0x100000) but finalized FROZEN at 0xF0000 + // (~65k blocks behind — clearly a finality bug, not normal lag). Circuit + // stays closed (no failures). Fallbacks: poller off, holding the correct + // higher finalized. + mockJsonRpcUpstream("rpc1.localhost", chainIdHex, "0x100000", "0xF0000") + mockJsonRpcUpstream("rpc2.localhost", chainIdHex, "0x100000", "0xF0000") + mockJsonRpcUpstream("rpc3.localhost", chainIdHex, fallbackLatest, "0xFFF00") + mockJsonRpcUpstream("rpc4.localhost", chainIdHex, fallbackLatest, "0xFFF00") + + network, upr, _ := buildFailoverNetwork(t, ctx, finalityStallConfigs(), true) + require.Len(t, upr.GetNetworkUpstreams(ctx, util.EvmNetworkId(999)), 4) + setFinalityStallThresholds(network, 200*time.Millisecond, 100) + + // Seed the finalized-advance record while the primary is still trusted. + require.Eventually(t, func() bool { + return network.EvmHighestFinalizedBlockNumber(ctx) == primaryFinalized + }, 3*time.Second, 50*time.Millisecond, + "baseline: finalized comes from the primary (%d) before the stall window elapses", primaryFinalized) + + // Let the freeze window elapse; the primary's finalized never advances. + time.Sleep(300 * time.Millisecond) + + // The stalled-but-circuit-closed primary must be demoted and finalized must + // fail over to the fallback's higher value. + require.Eventually(t, func() bool { + return network.EvmHighestFinalizedBlockNumber(ctx) == fallbackFinalized + }, 5*time.Second, 100*time.Millisecond, + "finalized must fail over to the fallback's higher value (%d) once the primary is finality-stalled", + fallbackFinalized) + + // Forward-only: never regress below the value already served. + for i := 0; i < 30; i++ { + require.GreaterOrEqual(t, network.EvmHighestFinalizedBlockNumber(ctx), fallbackFinalized, + "finalized must never regress below the served fallback value (iter %d)", i) + } +} + +// TestFinalizedResolution_DoesNotDemoteHealthyPrimaryWithSmallFinalityGap is the +// false-positive guard: a healthy primary whose finalized sits only slightly +// behind latest (gap < margin) must NEVER be demoted, even after the staleness +// window elapses — so a chain with legitimately tight finality is not mistaken +// for stalled. The happy path stays unchanged: the primary serves finalized and +// the higher fallback value is never adopted. +func TestFinalizedResolution_DoesNotDemoteHealthyPrimaryWithSmallFinalityGap(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const chainIdHex = "0x3e7" + const primaryFinalized = int64(0xFFFC0) // only 64 blocks behind latest + const fallbackFinalized = int64(0x1000C0) + + // Primary: latest 0x100000, finalized 0xFFFC0 → gap 64, below the 100-block + // margin, so even with finalized frozen it is NOT finality-stalled. Fallback + // holds a higher finalized that must NOT be adopted. + mockJsonRpcUpstream("rpc1.localhost", chainIdHex, "0x100000", "0xFFFC0") + mockJsonRpcUpstream("rpc2.localhost", chainIdHex, "0x100000", "0xFFFC0") + mockJsonRpcUpstream("rpc3.localhost", chainIdHex, "0x100100", "0x1000C0") + mockJsonRpcUpstream("rpc4.localhost", chainIdHex, "0x100100", "0x1000C0") + + network, upr, _ := buildFailoverNetwork(t, ctx, finalityStallConfigs(), true) + require.Len(t, upr.GetNetworkUpstreams(ctx, util.EvmNetworkId(999)), 4) + setFinalityStallThresholds(network, 200*time.Millisecond, 100) + + require.Eventually(t, func() bool { + return network.EvmHighestFinalizedBlockNumber(ctx) == primaryFinalized + }, 3*time.Second, 50*time.Millisecond, + "baseline: healthy primary serves its finalized (%d)", primaryFinalized) + + // Wait well past the staleness window; the small gap must keep the primary + // trusted (the margin guard, not the window, is what protects it). + time.Sleep(400 * time.Millisecond) + + for i := 0; i < 20; i++ { + got := network.EvmHighestFinalizedBlockNumber(ctx) + require.Equal(t, primaryFinalized, got, + "healthy small-gap primary must not be demoted; must keep serving %d, not the fallback's %d (iter %d)", + primaryFinalized, fallbackFinalized, i) + time.Sleep(20 * time.Millisecond) + } +} + +// mockGetBlockByNumberConcrete answers eth_getBlockByNumber for a specific hex +// block number (the value eRPC re-fetches / interpolates to once it knows the +// true finalized height). Hex must be lowercase to match eRPC's NormalizeHex. +func mockGetBlockByNumberConcrete(host, hexNum string) { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + b := util.SafeReadBody(r) + return strings.Contains(b, "eth_getBlockByNumber") && strings.Contains(b, hexNum) + }). + Reply(200). + JSON([]byte(fmt.Sprintf( + `{"jsonrpc":"2.0","id":1,"result":{"number":"%s","hash":"0xabc","timestamp":"0x6702a8e0"}}`, hexNum))) +} + +// TestEthGetBlockByNumber_Finalized_ReturnsHealthyNodeBlockWhenPrimaryStalled is +// the end-to-end, RPC-method-level proof of the Base incident fix: a client that +// calls eth_getBlockByNumber("finalized") — exactly what Chainlink MultiNode polls +// for finality — must receive the HEALTHY node's higher finalized block when the +// primary is finality-stalled, not the primary's frozen value. +// +// This drives the real network.Forward path, so it exercises the +// enforce-highest-finalized / tag-interpolation machinery on top of the stall +// detection — the whole chain a downstream node actually hits. +func TestEthGetBlockByNumber_Finalized_ReturnsHealthyNodeBlockWhenPrimaryStalled(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const chainIdHex = "0x3e7" + const stalledFinalized = "0xf0000" // frozen, far behind tip (the bug) + const healthyFinalizedHex = "0xfff00" // the true, higher finalized + const healthyFinalized = int64(0xfff00) // == 1048320 + + // Primaries: circuit-closed, latest at tip, finalized FROZEN far behind. + // Fallbacks: healthy, poller on, holding the true higher finalized. + mockJsonRpcUpstream("rpc1.localhost", chainIdHex, "0x100000", stalledFinalized) + mockJsonRpcUpstream("rpc2.localhost", chainIdHex, "0x100000", stalledFinalized) + mockJsonRpcUpstream("rpc3.localhost", chainIdHex, "0x100010", healthyFinalizedHex) + mockJsonRpcUpstream("rpc4.localhost", chainIdHex, "0x100010", healthyFinalizedHex) + // Any upstream may be asked for the concrete higher block (interpolation or + // the enforce-highest-finalized re-fetch, which excludes the stale server). + for _, h := range []string{"rpc1.localhost", "rpc2.localhost", "rpc3.localhost", "rpc4.localhost"} { + mockGetBlockByNumberConcrete(h, healthyFinalizedHex) + } + + // All four pollers on so the fallbacks' finalized is known without waiting on + // the on-demand refresh (that path is covered by the other tests). + network, upr, _ := buildFailoverNetwork(t, ctx, failoverUpstreamConfigs(), true) + require.Len(t, upr.GetNetworkUpstreams(ctx, util.EvmNetworkId(999)), 4) + setFinalityStallThresholds(network, 100*time.Millisecond, 100) + + // Resolution-level guard: once the primaries are demoted as finality-stalled, + // the network finalized resolves to the healthy fallback's higher value. + require.Eventually(t, func() bool { + return network.EvmHighestFinalizedBlockNumber(ctx) == healthyFinalized + }, 5*time.Second, 100*time.Millisecond, + "network finalized must fail over to the healthy node's value (%d) once the primary is stalled", healthyFinalized) + + // Method-level proof: a client's eth_getBlockByNumber("finalized") returns the + // healthy node's block, not the stalled primary's frozen one. + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["finalized",false]}`)) + req.SetNetwork(network) + // eth_getBlockByNumber finality enforcement is gated on this directive + // (defaults to true in production; set explicitly here since the test builds + // requests programmatically rather than via the HTTP server). + req.SetDirectives(&common.RequestDirectives{EnforceHighestBlock: true}) + // Replicate the project's request path: network.Forward wrapped by the EVM + // network post-forward hook (which is where enforce-highest-finalized lives). + resp, err := network.Forward(ctx, req) + resp, err = evm.HandleNetworkPostForward(ctx, network, req, resp, err) + require.NoError(t, err, "eth_getBlockByNumber(finalized) must succeed") + require.NotNil(t, resp) + defer resp.Release() + + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + numHex, err := jrr.PeekStringByPath(ctx, "number") + require.NoError(t, err) + got, err := common.HexToInt64(numHex) + require.NoError(t, err) + require.Equal(t, healthyFinalized, got, + "eth_getBlockByNumber(finalized) must return the healthy node's block %d, not the stalled primary's frozen finalized", healthyFinalized) +} diff --git a/telemetry/metrics.go b/telemetry/metrics.go index e7e05f56e..349fc746e 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -242,6 +242,18 @@ var ( Help: "Total number of times an upstream returned a stale (vs others) finalized block number.", }, []string{"project", "vendor", "network", "upstream"}) + MetricUpstreamFinalityStalled = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "upstream_finality_stalled_total", + Help: "Total number of times a circuit-closed primary upstream was demoted as a finalized source because its finalized stopped advancing while its latest kept advancing.", + }, []string{"project", "vendor", "network", "upstream"}) + + MetricNetworkFinalizedServedFromFallback = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "network_finalized_served_from_fallback_total", + Help: "Total number of times the network finalized was resolved from a fallback upstream because no primary could be trusted (all down or finality-stalled).", + }, []string{"project", "network"}) + MetricUpstreamStaleUpperBound = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "erpc", Name: "upstream_stale_upper_bound_total",