From 3bb56f6f521344397b69abc3374920d4940fe7d6 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Mon, 22 Jun 2026 22:12:37 +0200 Subject: [PATCH 01/14] Simplex orchestration layer preliminary implementation This commit contains an implementation of an orchestration layer that orchestrates epoch transition for validators and non-validators as they move through epochs. The orchestration layer introduces an Instance that drives the epoch lifecycle, switching between validator consensus and non-validator block tracking as the node's role changes across epochs. Key pieces: - Instance: manages the per-epoch lifecycle, ticks the active epoch or non-validator, and handles the handoff at epoch boundaries. - epochChangeSupression: drops outgoing messages and cancels VM operations while an epoch change is in progress, preventing side effects mid-transition. - EpochAwareStorage: ignores blocks from previous epochs and signals when a new epoch is detected. - Config: wiring for the P-chain, storage, crypto, and networking dependencies. Signed-off-by: Yacov Manevich --- adapters.go | 318 ++++++++++++++++ adapters_test.go | 61 +++ common/api.go | 4 +- common/msg.go | 27 +- config.go | 107 ++++++ external.canoto.go | 217 +++++++++++ external.go | 92 +++++ instance.go | 631 +++++++++++++++++++++++++++++++ instance_test.go | 691 ++++++++++++++++++++++++++++++++++ msm/build_decision.go | 13 +- msm/build_decision_test.go | 14 +- msm/encoding.go | 3 +- msm/fake_node_test.go | 45 ++- msm/fuzz_test.go | 28 +- msm/misc.go | 3 + msm/msm.go | 55 ++- msm/msm_test.go | 100 +++-- msm/util_test.go | 74 ++-- nonvalidator/non_validator.go | 6 + testutil/block.go | 27 ++ testutil/util.go | 8 +- 21 files changed, 2366 insertions(+), 158 deletions(-) create mode 100644 adapters.go create mode 100644 adapters_test.go create mode 100644 config.go create mode 100644 external.canoto.go create mode 100644 external.go create mode 100644 instance.go create mode 100644 instance_test.go diff --git a/adapters.go b/adapters.go new file mode 100644 index 00000000..88c6f689 --- /dev/null +++ b/adapters.go @@ -0,0 +1,318 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" +) + +// epochChangeSupression is used to suppress sending messages during an epoch change or using the VM. +// The motivation is that an epoch change occurrs during indexing a block, and we don't want any further +// external side effects to take place before the epoch change finishes. +// We therefore drop all messages and cancel VM operations such as block building, +// and delay VM transaction listening until the epoch change is complete. +type epochChangeSupression struct { + lock sync.RWMutex + sealingBlockSeq uint64 + active bool +} + +func (ecs *epochChangeSupression) isSupressionActive() bool { + ecs.lock.RLock() + defer ecs.lock.RUnlock() + return ecs.active +} + +func (ecs *epochChangeSupression) sendProhibited(seq uint64) bool { + ecs.lock.RLock() + defer ecs.lock.RUnlock() + if !ecs.active { + return false + } + return seq > ecs.sealingBlockSeq +} + +func (ecs *epochChangeSupression) setSupression(sealingBlockSeq uint64) { + ecs.lock.Lock() + defer ecs.lock.Unlock() + ecs.sealingBlockSeq = sealingBlockSeq + ecs.active = true +} + +func (ecs *epochChangeSupression) clearSupression() { + ecs.lock.Lock() + defer ecs.lock.Unlock() + ecs.active = false +} + +type Communication struct { + epochChangeSupression *epochChangeSupression + nodes atomic.Value // common.Nodes + Sender + Broadcaster +} + +func (c *Communication) SetValidators(nodes common.Nodes) { + c.nodes.Store(nodes) +} + +func (c *Communication) Validators() common.Nodes { + nodes, ok := c.nodes.Load().(common.Nodes) + if !ok { + return nil + } + return nodes +} + +func (c *Communication) Broadcast(msg *common.Message) { + if c.epochChangeSupression.sendProhibited(msg.Seq()) { + return + } + + c.Broadcaster.Broadcast(msg) +} + +func (c *Communication) Send(msg *common.Message, destination common.NodeID) { + if c.epochChangeSupression.sendProhibited(msg.Seq()) { + return + } + + c.Sender.Send(msg, destination) +} + +// EpochAwareStorage is a wrapper around Storage that is aware of epoch changes. +// Upon an epoch change, it will ignore blocks from previous epochs +// and will call the OnEpochChange callback when a new epoch is detected. +type EpochAwareStorage struct { + msm *metadata.StateMachine + OnEpochChange func(seq uint64, validators common.Nodes) error + Storage + Epoch uint64 +} + +func (e *EpochAwareStorage) Retrieve(seq uint64) (common.VerifiedBlock, common.Finalization, error) { + block, finalization, err := e.Storage.GetBlock(seq) + if err != nil { + return nil, common.Finalization{}, err + } + parsedBlock := &ParsedBlock{ + msm: e.msm, + StateMachineBlock: block, + } + return parsedBlock, *finalization, nil +} + +func (e *EpochAwareStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { + if block.BlockHeader().Epoch < e.Epoch { + // This is a Telock from a previous h, so we ignore it and do not index it. + return nil + } + if err := e.Storage.Index(ctx, block, certificate); err != nil { + return err + } + if block.SealingBlockInfo() != nil { + if err := e.OnEpochChange(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet); err != nil { + return err + } + // We are now in a new h, so we update the h number to prevent indexing Telocks from the previous h. + e.Epoch = block.BlockHeader().Seq + } + return nil +} + +// cachedBlock is a wrapper around ParsedBlock that caches the block in the CachedStorage upon verification. +// It is needed for the MSM because the MSM needs to be able to retrieve blocks that aren't finalized during its execution. +// These blocks are cached in the CachedStorage upon verification, and removed from the cache upon finalization (indexing). +type cachedBlock struct { + cache *CachedStorage + *ParsedBlock +} + +func (cb *cachedBlock) Verify(ctx context.Context) (common.VerifiedBlock, error) { + vb, err := cb.ParsedBlock.Verify(ctx) + if err == nil { + cb.cache.insertBlock(cb.ParsedBlock) + } + return vb, err +} + +type CachedStorage struct { + msm *metadata.StateMachine + lock sync.RWMutex + Storage + cache map[[32]byte]cachedBlock +} + +func NewCachedStorage(storage Storage) *CachedStorage { + return &CachedStorage{ + Storage: storage, + cache: make(map[[32]byte]cachedBlock), + } +} + +func (cs *CachedStorage) RetrieveBlock(seq uint64, digest [32]byte) (metadata.StateMachineBlock, *common.Finalization, error) { + block, finalization, err := cs.Retrieve(seq, digest) + if err != nil { + return metadata.StateMachineBlock{}, nil, err + } + + return block.(*ParsedBlock).StateMachineBlock, finalization, nil +} + +func (cs *CachedStorage) Retrieve(seq uint64, digest [32]byte) (common.VerifiedBlock, *common.Finalization, error) { + cs.lock.RLock() + item, exists := cs.cache[digest] + if exists { + cs.lock.RUnlock() + // If the block is cached, it means it's not finalized yet, because upon finalizing the block (indexing) + // we also remove it from the cache. Therefore, we return nil for the finalization. + return item.ParsedBlock, nil, nil + } + cs.lock.RUnlock() + + // We don't populate the cache here because we populate it externally. + + block, finalization, err := cs.Storage.GetBlock(seq) + if err != nil { + return nil, nil, err + } + + return &ParsedBlock{ + StateMachineBlock: block, + msm: cs.msm, + }, finalization, nil +} + +func (cs *CachedStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { + err := cs.Storage.Index(ctx, block, certificate) + + if err == nil { + // We delete the block from the cache after it has been indexed because now that it is persisted, + // we can just lookup by sequence number instead of digest. + cs.lock.Lock() + defer cs.lock.Unlock() + delete(cs.cache, block.BlockHeader().Digest) + + // We also delete all blocks that are older than the indexed block, because they are now finalized and persisted. + for digest, cachedBlock := range cs.cache { + if cachedBlock.BlockHeader().Seq < block.BlockHeader().Seq { + delete(cs.cache, digest) + } + } + } + + return err +} + +func (cs *CachedStorage) insertBlock(block *ParsedBlock) { + cs.lock.Lock() + defer cs.lock.Unlock() + + cs.cache[block.Digest()] = cachedBlock{ + ParsedBlock: block, + } +} + +type NoopAuxiliaryInfoApp struct{} + +func (n *NoopAuxiliaryInfoApp) IsLegalAppend(versionID metadata.VersionID, nodes metadata.NodeBLSMappings, history [][]byte, x []byte) error { + if len(x) > 0 { + return fmt.Errorf("input should be empty") + } + return nil +} + +func (n *NoopAuxiliaryInfoApp) IsSufficient(versionID metadata.VersionID, nodes metadata.NodeBLSMappings, history [][]byte) (bool, error) { + return true, nil +} + +func (n *NoopAuxiliaryInfoApp) Generate(metadata.VersionID, metadata.NodeBLSMappings, [][]byte) ([]byte, error) { + return nil, nil +} + +func (n *NoopAuxiliaryInfoApp) DefaultVersionID() metadata.VersionID { + return 0 +} + +type BlockBuilderWaiter struct { + epochChangeSupression *epochChangeSupression + lock sync.Mutex + cancel context.CancelFunc + msm *metadata.StateMachine + vm VM +} + +func (bw *BlockBuilderWaiter) stop() { + bw.lock.Lock() + defer bw.lock.Unlock() + if bw.cancel != nil { + bw.cancel() + bw.cancel = nil + } +} + +func (bw *BlockBuilderWaiter) WaitForPendingBlock(ctx context.Context) { + if bw.epochChangeSupression.isSupressionActive() { + <-ctx.Done() // We wait for the context to be cancelled once the epoch is tore down. + return + } + + bw.lock.Lock() + if bw.cancel != nil { + bw.cancel() + } + ctx, cancel := context.WithCancel(ctx) + bw.cancel = cancel + bw.lock.Unlock() + defer cancel() + bw.vm.WaitForPendingBlock(ctx) +} + +func (bw *BlockBuilderWaiter) BuildBlock(ctx context.Context, metadata common.ProtocolMetadata, blacklist common.Blacklist) (common.VerifiedBlock, bool) { + if bw.epochChangeSupression.isSupressionActive() { + return nil, false + } + + block, err := bw.msm.BuildBlock(ctx, metadata, &blacklist) + if err != nil { + return nil, false + } + + pb := ParsedBlock{ + StateMachineBlock: *block, + msm: bw.msm, + } + + return &pb, true +} + +type blockDeserializer struct { + vm VM + msm *metadata.StateMachine +} + +func (bp *blockDeserializer) DeserializeBlock(ctx context.Context, bytes []byte) (common.Block, error) { + var rawBlock RawBlock + if err := rawBlock.UnmarshalCanoto(bytes); err != nil { + return nil, err + } + + block, err := bp.vm.ParseBlock(ctx, rawBlock.InnerBlockBytes) + if err != nil { + return nil, err + } + return &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + InnerBlock: block, + Metadata: rawBlock.Metadata, + }, + msm: bp.msm, + }, nil +} diff --git a/adapters_test.go b/adapters_test.go new file mode 100644 index 00000000..117adf9c --- /dev/null +++ b/adapters_test.go @@ -0,0 +1,61 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEpochChangeSupression(t *testing.T) { + t.Run("Inactive by default", func(t *testing.T) { + var ecs epochChangeSupression + require.False(t, ecs.isSupressionActive()) + // When inactive, nothing is prohibited regardless of sequence. + require.False(t, ecs.sendProhibited(0)) + require.False(t, ecs.sendProhibited(100)) + }) + + t.Run("setSupression activates and prohibits sequences after the sealing block", func(t *testing.T) { + var ecs epochChangeSupression + const sealingBlockSeq = uint64(10) + ecs.setSupression(sealingBlockSeq) + + require.True(t, ecs.isSupressionActive()) + + // Sequences up to and including the sealing block are allowed. + require.False(t, ecs.sendProhibited(sealingBlockSeq-1)) + require.False(t, ecs.sendProhibited(sealingBlockSeq)) + + // Sequences after the sealing block are prohibited. + require.True(t, ecs.sendProhibited(sealingBlockSeq+1)) + require.True(t, ecs.sendProhibited(sealingBlockSeq+100)) + }) + + t.Run("clearSupression deactivates and allows all sequences", func(t *testing.T) { + var ecs epochChangeSupression + const sealingBlockSeq = uint64(10) + ecs.setSupression(sealingBlockSeq) + require.True(t, ecs.sendProhibited(sealingBlockSeq+1)) + + ecs.clearSupression() + + require.False(t, ecs.isSupressionActive()) + // Once cleared, previously prohibited sequences are allowed again. + require.False(t, ecs.sendProhibited(sealingBlockSeq+1)) + }) + + t.Run("setSupression overwrites the previous sealing block sequence", func(t *testing.T) { + var ecs epochChangeSupression + ecs.setSupression(5) + require.True(t, ecs.sendProhibited(8)) + + ecs.setSupression(10) + require.True(t, ecs.isSupressionActive()) + // The new, higher sealing block sequence now permits what was previously prohibited. + require.False(t, ecs.sendProhibited(8)) + require.True(t, ecs.sendProhibited(11)) + }) +} diff --git a/common/api.go b/common/api.go index cee833e4..c12dcd2b 100644 --- a/common/api.go +++ b/common/api.go @@ -75,7 +75,7 @@ type Signer interface { } type SignatureVerifier interface { - Verify(message []byte, signature []byte, publicKey []byte) error + VerifySignature(message []byte, signature []byte, publicKey []byte) error } type WriteAheadLog interface { @@ -126,7 +126,7 @@ type VerifiedBlock interface { // BlockDeserializer deserializes blocks according to formatting // enforced by the application. type BlockDeserializer interface { - // DeserializeBlock parses the given bytes and initializes a VerifiedBlock. + // DeserializeBlock deserializes the given bytes and initializes a VerifiedBlock. // Returns an error upon failure. DeserializeBlock(ctx context.Context, bytes []byte) (Block, error) } diff --git a/common/msg.go b/common/msg.go index 6679451f..539a7f90 100644 --- a/common/msg.go +++ b/common/msg.go @@ -40,6 +40,31 @@ func (m *Message) IsReplicationMessage() bool { } } +func (m *Message) Seq() uint64 { + switch { + case m.BlockMessage != nil: + return m.BlockMessage.Block.BlockHeader().Seq + case m.VerifiedBlockMessage != nil: + return m.VerifiedBlockMessage.VerifiedBlock.BlockHeader().Seq + case m.EmptyNotarization != nil: + // Empty notarizations have no sequence, only a round. + return m.EmptyNotarization.Vote.Round + case m.VoteMessage != nil: + return m.VoteMessage.Vote.Seq + case m.EmptyVoteMessage != nil: + // Empty votes have no sequence, only a round. + return m.EmptyVoteMessage.Vote.Round + case m.Notarization != nil: + return m.Notarization.Vote.Seq + case m.FinalizeVote != nil: + return m.FinalizeVote.Finalization.Seq + case m.Finalization != nil: + return m.Finalization.Finalization.Seq + default: + return 0 + } +} + type EmptyVoteMetadata struct { Round uint64 Epoch uint64 @@ -136,7 +161,7 @@ func verifyContext(signature []byte, verifier SignatureVerifier, msg []byte, con if err != nil { return err } - return verifier.Verify(toBeSigned, signature, pk) + return verifier.VerifySignature(toBeSigned, signature, pk) } func verifyContextQC(qc QuorumCertificate, msg []byte, context string, nodes Nodes) error { diff --git a/config.go b/config.go new file mode 100644 index 00000000..b5f9eb99 --- /dev/null +++ b/config.go @@ -0,0 +1,107 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "context" + "time" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/wal" +) + +type ParameterConfig struct { + // WalMaxEntryCount is the maximum number of entries in the write-ahead log before it is closed. + WALMaxEntryCount int + // MaxnetworkDelay is the assumed upper bound on the network delay for messages to be delivered. + MaxNetworkDelay time.Duration + // MaxRoundWindow is the maximum number of rounds that can be stored in memory. + MaxRoundWindow uint64 +} + +// PlatformChain is an interface that abstracts the interaction with the P-chain. +type PlatformChain interface { + GetValidatorSet(uint64) (metadata.NodeBLSMappings, error) + // GenesisValidatorSet returns the first ever validator set for this network. + GenesisValidatorSet() metadata.NodeBLSMappings + // GetMinimumHeight returns the minimum height of the block still in the proposal window. + GetMinimumHeight(context.Context) (uint64, error) + // GetCurrentHeight returns the current height of the P-chain. + GetCurrentHeight(context.Context) (uint64, error) + // WaitForProgress should block until either the context is cancelled, or the P-chain height has increased from the provided pChainHeight. + WaitForProgress(ctx context.Context, pChainHeight uint64) error + // LastNonSimplexBlockPChainHeight returns the P-chain height of the last non-simplex block in the chain. + LastNonSimplexBlockPChainHeight() uint64 +} + +type Broadcaster interface { + Broadcast(msg *common.Message) +} + +// Sender is an interface that defines the ability to send messages to other nodes in the network. +type Sender interface { + // Send sends a message to the given destination node + Send(msg *common.Message, destination common.NodeID) +} + +type VM interface { + // BuildBlock builds a block given the current context and the P-chain height. + BuildBlock(ctx context.Context, pChainHeight uint64) (metadata.VMBlock, error) + + // WaitForPendingBlock returns when either the given context is cancelled, + // or when the VM signals that a block should be built. + WaitForPendingBlock(ctx context.Context) + + // ParseBlock parses the given block bytes into a VMBlock. + ParseBlock(context.Context, []byte) (metadata.VMBlock, error) + + // ComputeICMEpoch computes the ICM epoch transition given the input parameters. + ComputeICMEpoch(input metadata.ICMEpochInput) metadata.ICMEpochInfo +} + +type Storage interface { + // GetBlock retrieves the block and finalization at [seq] with the given digest. + // If the digest is nil, the block with the given sequence number is returned. + // If [seq] the block cannot be found, returns ErrBlockNotFound. + GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) + + // NumBlocks returns the number of blocks stored in the storage. + NumBlocks() uint64 + + // Index indexes the given block and finalization in the storage. + Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error + + // CreateWAL creates a new Write-Ahead Log (WAL). + CreateWAL() (wal.DeletableWAL, error) +} + +type CryptoOps interface { + Sign(message []byte) ([]byte, error) + AggregateKeys(keys ...[]byte) ([]byte, error) + VerifySignature(message []byte, signature []byte, publicKey []byte) error + CreateSignatureAggregator([]common.Node) common.SignatureAggregator + DeserializeQuorumCertificate(bytes []byte) (common.QuorumCertificate, error) +} + +type MSMConfig struct { + LastNonSimplexInnerBlock metadata.VMBlock + ComputeICMEpoch metadata.ICMEpochTransition +} + +type EpochParams struct { + ID common.NodeID + Sender Sender + Storage Storage + QCDeserializer common.QCDeserializer + BlockDeserializer common.BlockDeserializer + Verifier common.SignatureVerifier + MaxProposalWait time.Duration + MaxRoundWindow uint64 + MaxRebroadcastWait time.Duration + FinalizeRebroadcastTimeout time.Duration + MaxWALSize int + WALCreator wal.Creator + WALs []wal.DeletableWAL +} diff --git a/external.canoto.go b/external.canoto.go new file mode 100644 index 00000000..2c3df92f --- /dev/null +++ b/external.canoto.go @@ -0,0 +1,217 @@ +// Code generated by canoto. DO NOT EDIT. +// versions: +// canoto v0.19.0 +// source: external.go + +package simplex + +import ( + "io" + "reflect" + "sync/atomic" + + "github.com/StephenButtolph/canoto" +) + +// Ensure that the generated code is compatible with the library version. +const ( + _ uint = canoto.VersionCompatibility - 1 + _ uint = 1 - canoto.VersionCompatibility +) + +// Ensure that unused imports do not error +var _ = io.ErrUnexpectedEOF + +const ( + canotoNumber_RawBlock__Metadata = 1 + canotoNumber_RawBlock__InnerBlockBytes = 2 + + canotoTag_RawBlock__Metadata = "\x0a" // canoto.Tag(canotoNumber_RawBlock__Metadata, canoto.Len) + canotoTag_RawBlock__InnerBlockBytes = "\x12" // canoto.Tag(canotoNumber_RawBlock__InnerBlockBytes, canoto.Len) +) + +type canotoData_RawBlock struct { + size uint64 +} + +// CanotoSpec returns the specification of this canoto message. +func (*RawBlock) CanotoSpec(types ...reflect.Type) *canoto.Spec { + types = append(types, reflect.TypeFor[RawBlock]()) + var zero RawBlock + s := &canoto.Spec{ + Name: "RawBlock", + Fields: []canoto.FieldType{ + canoto.FieldTypeFromField( + /*type inference:*/ (&zero.Metadata), + /*FieldNumber: */ canotoNumber_RawBlock__Metadata, + /*Name: */ "Metadata", + /*FixedLength: */ 0, + /*Repeated: */ false, + /*OneOf: */ "", + /*Pointer: */ false, + /*types: */ types, + ), + { + FieldNumber: canotoNumber_RawBlock__InnerBlockBytes, + Name: "InnerBlockBytes", + OneOf: "", + TypeBytes: true, + }, + }, + } + s.CalculateCanotoCache() + return s +} + +// UnmarshalCanoto unmarshals a Canoto-rawBlock byte slice into the struct. +// +// During parsing, the canoto cache is saved. +func (c *RawBlock) UnmarshalCanoto(bytes []byte) error { + r := canoto.Reader{ + B: bytes, + } + return c.UnmarshalCanotoFrom(r) +} + +// UnmarshalCanotoFrom populates the struct from a [canoto.Reader]. Most users +// should just use UnmarshalCanoto. +// +// During parsing, the canoto cache is saved. +// +// This function enables configuration of reader options. +func (c *RawBlock) UnmarshalCanotoFrom(r canoto.Reader) error { + // Zero the struct before unmarshaling. + *c = RawBlock{} + atomic.StoreUint64(&c.canotoData.size, uint64(len(r.B))) + + var minField uint32 + for canoto.HasNext(&r) { + field, wireType, err := canoto.ReadTag(&r) + if err != nil { + return err + } + if field < minField { + return canoto.ErrInvalidFieldOrder + } + + switch field { + case canotoNumber_RawBlock__Metadata: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + // Read the bytes for the field. + originalUnsafe := r.Unsafe + r.Unsafe = true + var msgBytes []byte + if err := canoto.ReadBytes(&r, &msgBytes); err != nil { + return err + } + if len(msgBytes) == 0 { + return canoto.ErrZeroValue + } + r.Unsafe = originalUnsafe + + // Unmarshal the field from the bytes. + remainingBytes := r.B + r.B = msgBytes + if err := (&c.Metadata).UnmarshalCanotoFrom(r); err != nil { + return err + } + r.B = remainingBytes + case canotoNumber_RawBlock__InnerBlockBytes: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadBytes(&r, &c.InnerBlockBytes); err != nil { + return err + } + if len(c.InnerBlockBytes) == 0 { + return canoto.ErrZeroValue + } + default: + return canoto.ErrUnknownField + } + + minField = field + 1 + } + return nil +} + +// ValidCanoto validates that the struct can be correctly marshaled into the +// Canoto format. +// +// Specifically, ValidCanoto ensures: +// 1. All OneOfs are specified at most once. +// 2. All strings are valid utf-8. +// 3. All custom fields are ValidCanoto. +func (c *RawBlock) ValidCanoto() bool { + if !(&c.Metadata).ValidCanoto() { + return false + } + return true +} + +// CalculateCanotoCache populates size and OneOf caches based on the current +// values in the struct. +// +// It is not safe to copy this struct concurrently. +func (c *RawBlock) CalculateCanotoCache() { + var size uint64 + (&c.Metadata).CalculateCanotoCache() + if fieldSize := (&c.Metadata).CachedCanotoSize(); fieldSize != 0 { + size += uint64(len(canotoTag_RawBlock__Metadata)) + canoto.SizeUint(fieldSize) + fieldSize + } + if len(c.InnerBlockBytes) != 0 { + size += uint64(len(canotoTag_RawBlock__InnerBlockBytes)) + canoto.SizeBytes(c.InnerBlockBytes) + } + atomic.StoreUint64(&c.canotoData.size, size) +} + +// CachedCanotoSize returns the previously calculated size of the Canoto +// representation from CalculateCanotoCache. +// +// If CalculateCanotoCache has not yet been called, it will return 0. +// +// If the struct has been modified since the last call to CalculateCanotoCache, +// the returned size may be incorrect. +func (c *RawBlock) CachedCanotoSize() uint64 { + return atomic.LoadUint64(&c.canotoData.size) +} + +// MarshalCanoto returns the Canoto representation of this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *RawBlock) MarshalCanoto() []byte { + c.CalculateCanotoCache() + w := canoto.Writer{ + B: make([]byte, 0, c.CachedCanotoSize()), + } + w = c.MarshalCanotoInto(w) + return w.B +} + +// MarshalCanotoInto writes the struct into a [canoto.Writer] and returns the +// resulting [canoto.Writer]. Most users should just use MarshalCanoto. +// +// It is assumed that CalculateCanotoCache has been called since the last +// modification to this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *RawBlock) MarshalCanotoInto(w canoto.Writer) canoto.Writer { + if fieldSize := (&c.Metadata).CachedCanotoSize(); fieldSize != 0 { + canoto.Append(&w, canotoTag_RawBlock__Metadata) + canoto.AppendUint(&w, fieldSize) + w = (&c.Metadata).MarshalCanotoInto(w) + } + if len(c.InnerBlockBytes) != 0 { + canoto.Append(&w, canotoTag_RawBlock__InnerBlockBytes) + canoto.AppendBytes(&w, c.InnerBlockBytes) + } + return w +} diff --git a/external.go b/external.go new file mode 100644 index 00000000..3ceb0f18 --- /dev/null +++ b/external.go @@ -0,0 +1,92 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "context" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" +) + +type RawBlock struct { + Metadata metadata.StateMachineMetadata `canoto:"value,1"` + InnerBlockBytes []byte `canoto:"bytes,2"` + + canotoData canotoData_RawBlock +} + +type ParsedBlock struct { + metadata.StateMachineBlock + msm *metadata.StateMachine +} + +func (p *ParsedBlock) Bytes() ([]byte, error) { + var innerBlockBytes []byte + if p.InnerBlock != nil { + rawInnerBlock, err := p.InnerBlock.Bytes() + if err != nil { + return nil, err + } + innerBlockBytes = rawInnerBlock + } + rawBlock := &RawBlock{ + Metadata: p.Metadata, + InnerBlockBytes: innerBlockBytes, + } + return rawBlock.MarshalCanoto(), nil +} + +func (p *ParsedBlock) BlockHeader() common.BlockHeader { + var md *common.ProtocolMetadata + var err error + if len(p.Metadata.SimplexProtocolMetadata) > 0 { + md, err = common.ProtocolMetadataFromBytes(p.Metadata.SimplexProtocolMetadata) + if err != nil { + panic(err) // TODO: handle error + } + } else { + md = &common.ProtocolMetadata{} + } + + digest := p.StateMachineBlock.Digest() + return common.BlockHeader{ + ProtocolMetadata: *md, + Digest: digest, + } +} + +func (p *ParsedBlock) Blacklist() common.Blacklist { + var blacklist common.Blacklist + _ = blacklist.FromBytes(p.Metadata.SimplexBlacklist) // TODO: encode blacklist with Canoto + return blacklist +} + +func (p *ParsedBlock) Verify(ctx context.Context) (common.VerifiedBlock, error) { + if err := p.msm.VerifyBlock(ctx, &p.StateMachineBlock); err != nil { + return nil, err + } + return p, nil +} + +func (p *ParsedBlock) SealingBlockInfo() *common.SealingBlockInfo { + if p.Metadata.SimplexEpochInfo.BlockValidationDescriptor == nil { + return nil + } + + bdc := p.Metadata.SimplexEpochInfo.BlockValidationDescriptor + var nodes common.Nodes + + for _, vdr := range bdc.AggregatedMembership.Members { + nodes = append(nodes, common.Node{ + Id: vdr.NodeID[:], + Weight: vdr.Weight, + PK: vdr.BLSKey, + }) + } + + return &common.SealingBlockInfo{ + ValidatorSet: nodes, + } +} diff --git a/instance.go b/instance.go new file mode 100644 index 00000000..354acc1f --- /dev/null +++ b/instance.go @@ -0,0 +1,631 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "context" + "fmt" + "math" + "sync" + "time" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/nonvalidator" + "github.com/ava-labs/simplex/simplex" + "github.com/ava-labs/simplex/wal" + "go.uber.org/zap" +) + +const ( + // tickInterval is the interval at which the instance will call AdvanceTime on the current epoch or non-validator. + tickInterval = time.Millisecond * 100 +) + +type Config struct { + // LastNonSimplexInnerBlock is the last non-simplex inner block that was persisted to storage. + // This is used to determine the current epoch and validator set. + LastNonSimplexInnerBlock metadata.VMBlock + // ParameterConfig is the configuration for the simplex instance. + ParameterConfig ParameterConfig + // PlatformChain is the interface to the P-chain. + PlatformChain PlatformChain + // CryptoOps is the interface to the cryptographic operations needed by the simplex instance. + CryptoOps CryptoOps + // Storage is the interface to the block storage layer for the simplex instance. + Storage Storage + Logger common.Logger + Sender Sender + Broadcaster Broadcaster + WALs []wal.DeletableWAL + VM VM + ID common.NodeID +} + +type MessageHandler interface { +} + +type nodeRole byte + +const ( + nonValidator nodeRole = iota + validator +) + +type epochChange struct { + epochNum uint64 + validators common.Nodes + nodeRole nodeRole +} + +type timeAdvancer interface { + AdvanceTime(t time.Time) +} + +type Instance struct { + Config Config + lock sync.Mutex + cs *CachedStorage + wal *wal.GarbageCollectedWAL + msm *metadata.StateMachine + e *simplex.Epoch + nv *nonvalidator.NonValidator + epochOrNV timeAdvancer + epochChanges chan epochChange + stopCh chan struct{} + epochChangeSupression epochChangeSupression +} + +func (i *Instance) Start(ctx context.Context) error { + // Hold the lock throughout startup to block HandleMessage from being called in between. + i.lock.Lock() + defer i.lock.Unlock() + + i.stopCh = make(chan struct{}) + i.epochChanges = make(chan epochChange) + context.AfterFunc(ctx, i.Stop) + + cachedStorage := NewCachedStorage(i.Config.Storage) + i.cs = cachedStorage + + lastBlock, numBlocks, err := i.lastBlock() + + lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() + genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() + nodes, epochNum, err := constructEpochAndValidatorSet(lastNonSimplexHeight, genesisValidatorSet, numBlocks, ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) + if err != nil { + return fmt.Errorf("error determining latest epoch and validator set: %w", err) + } + + iAmValidator := i.determineValidatorOrNot(nodes) + + if iAmValidator { + if err := i.startValidator(); err != nil { + return fmt.Errorf("error starting validator: %w", err) + } + } else { + if err := i.startNonValidator(epochNum, nodes); err != nil { + return fmt.Errorf("error starting non-validator: %w", err) + } + } + + go i.tick() + go i.listenForEpochChanges() + + return nil +} + +func (i *Instance) startValidator() error { + epochConfig, err := i.createEpochConfig() + if err != nil { + return err + } + + if err := i.startEpoch(epochConfig); err != nil { + return err + } + + return nil +} + +func (i *Instance) startNonValidator(epochNum uint64, validators common.Nodes) error { + config, err := i.createNonValidatorConfig(epochNum, validators) + if err != nil { + return err + } + + nonValidator, err := nonvalidator.NewNonValidator(config) + if err != nil { + return fmt.Errorf("error creating non-validator: %w", err) + } + i.nv = nonValidator + i.epochOrNV = nonValidator + nonValidator.Start() + return nil +} + +func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.Nodes) (nonvalidator.Config, error) { + source, err := simplex.NewRandomSource() + if err != nil { + return nonvalidator.Config{}, err + } + + comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster, epochChangeSupression: &i.epochChangeSupression} + comm.SetValidators(validators) + + epochAwareStorage := &EpochAwareStorage{ + Epoch: epochNum, + Storage: i.Config.Storage, + OnEpochChange: func(epoch uint64, validators common.Nodes) error { + i.epochChangeSupression.setSupression(epoch) // The epoch number is also the sealing block sequence. + i.notifyEpochChange(epoch, validators, nonValidator) + comm.SetValidators(validators) + return nil + }, + } + + // Plant an artificial MSM that just skips verification. + i.msm = &metadata.StateMachine{ + Config: &metadata.Config{ + SkipMSMVerification: true, + }, + } + i.cs.msm = i.msm + + config := nonvalidator.Config{ + ID: i.Config.ID, + RandomSource: source, + Storage: epochAwareStorage, + Comm: comm, + Logger: i.Config.Logger, + StartTime: time.Now(), + SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, + MaxSequenceWindow: nonvalidator.DefaultMaxSequenceWindow, + } + return config, nil +} + +func (i *Instance) notifyEpochChange(epoch uint64, validators common.Nodes, role nodeRole) { + select { + case i.epochChanges <- epochChange{ + epochNum: epoch, + validators: validators, + nodeRole: role, + }: + case <-i.stopCh: + // If the instance is stopped, we don't need to notify about epoch changes. + return + } +} + +func (i *Instance) tick() { + ticker := time.NewTicker(tickInterval) + for { + select { + case now := <-ticker.C: + i.lock.Lock() + timeAdvancer := i.epochOrNV + i.lock.Unlock() + if timeAdvancer != nil { + timeAdvancer.AdvanceTime(now) + } + case <-i.stopCh: + return + } + } +} + +func (i *Instance) Stop() { + i.lock.Lock() + defer i.lock.Unlock() + + select { + case <-i.stopCh: + // Already stopped, do nothing + return + default: + close(i.stopCh) + } + + if i.e != nil { + i.stopValidator() + } +} + +func (i *Instance) stopNonValidator() { + if i.nv != nil { + i.nv.Stop() + i.nv = nil + i.epochOrNV = nil + } +} + +func (i *Instance) stopValidator() { + if i.e != nil { + i.e.Stop() + i.e = nil + i.epochOrNV = nil + } +} + +func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error { + i.lock.Lock() + defer i.lock.Unlock() + + if i.epochChangeSupression.isSupressionActive() { + return nil + } + + switch { + case msg.BlockMessage != nil: + err := i.wireBlockMessage(msg) + if err != nil { + i.Config.Logger.Debug("Error wiring block message", zap.Error(err)) + return nil + } + case msg.ReplicationResponse != nil: + err := i.wireReplicationResponse(msg) + if err != nil { + i.Config.Logger.Debug("Error wiring replication response message", zap.Error(err)) + return nil + } + } + + if i.e != nil { + return i.e.HandleMessage(msg, from) + } + + if i.nv != nil { + return i.nv.HandleMessage(msg, from) + } + return nil +} + +func (i *Instance) wireReplicationResponse(msg *common.Message) error { + resp := msg.ReplicationResponse + if resp.LatestRound != nil && resp.LatestRound.Block != nil { + pb, isParsedBlock := resp.LatestRound.Block.(*ParsedBlock) + if !isParsedBlock { + return fmt.Errorf("expected ParsedBlock, got %T", resp.LatestRound.Block) + } + resp.LatestRound.Block = &cachedBlock{ + cache: i.cs, + ParsedBlock: pb, + } + pb.msm = i.msm + } + if resp.LatestSeq != nil && resp.LatestSeq.Block != nil { + pb, isParsedBlock := resp.LatestSeq.Block.(*ParsedBlock) + if !isParsedBlock { + return fmt.Errorf("expected ParsedBlock, got %T", resp.LatestSeq.Block) + } + resp.LatestSeq.Block = &cachedBlock{ + cache: i.cs, + ParsedBlock: pb, + } + pb.msm = i.msm + } + for j, datum := range resp.Data { + if datum.Block == nil { + continue + } + pb, isParsedBlock := datum.Block.(*ParsedBlock) + if !isParsedBlock { + return fmt.Errorf("expected ParsedBlock, got %T", datum.Block) + } + resp.Data[j].Block = &cachedBlock{ + cache: i.cs, + ParsedBlock: pb, + } + pb.msm = i.msm + } + return nil +} + +func (i *Instance) wireBlockMessage(msg *common.Message) error { + pb, isParsedBlock := msg.BlockMessage.Block.(*ParsedBlock) + if !isParsedBlock { + return fmt.Errorf("expected ParsedBlock, got %T", msg.BlockMessage.Block) + } + msg.BlockMessage.Block = &cachedBlock{ + cache: i.cs, + ParsedBlock: pb, + } + pb.msm = i.msm + return nil +} + +func (i *Instance) listenForEpochChanges() { + for { + select { + case epochChange := <-i.epochChanges: + i.processEpochChange(epochChange) + case <-i.stopCh: + return + } + } +} + +func (i *Instance) processEpochChange(epochChange epochChange) { + switch epochChange.nodeRole { + case nonValidator: + i.transitionEpochNonValidator(epochChange) + case validator: + i.transitionEpochValidator(epochChange) + default: // This should never happen, but we log it just in case. + i.Config.Logger.Fatal("Unknown node role on epoch change", + zap.String("role", fmt.Sprintf("%v", epochChange.nodeRole))) + } +} + +// startEpoch starts a new epoch with the given configuration. +// Must be called under the lock, and assumes that the previous epoch has been stopped (if any). +func (i *Instance) startEpoch(epochConfig simplex.EpochConfig) error { + epoch, err := simplex.NewEpoch(epochConfig) + if err != nil { + return fmt.Errorf("error creating simplex epoch: %w", err) + } + epoch.Epoch = epochConfig.Epoch + i.e = epoch + i.epochOrNV = epoch + + return epoch.Start() +} + +func (i *Instance) lastBlock() (metadata.StateMachineBlock, uint64, error) { + numBlocks := i.Config.Storage.NumBlocks() + if numBlocks == 0 { + return metadata.StateMachineBlock{}, 0, fmt.Errorf("no genesis block found in storage") + } + + lastBlock, _, err := i.Config.Storage.GetBlock(numBlocks - 1) + if err != nil { + return metadata.StateMachineBlock{}, 0, fmt.Errorf("error retrieving last block from storage: %w", err) + } + + return lastBlock, numBlocks, nil +} + +func (i *Instance) determineValidatorOrNot(nodes common.Nodes) bool { + for _, node := range nodes { + if i.Config.ID.Equals(node.Id) { + return true + } + } + return false +} + +func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { + lastBlock, numBlocks, err := i.lastBlock() + + lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() + genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() + nodes, epochNum, err := constructEpochAndValidatorSet(lastNonSimplexHeight, genesisValidatorSet, numBlocks, ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) + if err != nil { + return simplex.EpochConfig{}, err + } + + wal, err := wal.NewGarbageCollectedWAL(i.Config.WALs, i.Config.Storage.CreateWAL, &common.WALRetentionReader{}, i.Config.ParameterConfig.WALMaxEntryCount) + if err != nil { + return simplex.EpochConfig{}, fmt.Errorf("error creating garbage collected wal: %w", err) + } + i.wal = wal + + // We might have crashed right after a sealing block was persisted to storage, + // but before the WAL was garbage collected. + // In that case, we need to garbage collect the WAL to remove all entries from previous epochs. + if err := i.maybeGarbageCollectWAL(lastBlock); err != nil { + return simplex.EpochConfig{}, err + } + + msm, err := metadata.NewStateMachine(&metadata.Config{ + GetTime: time.Now, + MyNodeID: i.Config.ID, + KeyAggregator: i.Config.CryptoOps, + GetValidatorSet: i.Config.PlatformChain.GetValidatorSet, + SignatureVerifier: i.Config.CryptoOps, + PChainProgressListener: i.Config.PlatformChain, + LatestPersistedHeight: i.Config.Storage.NumBlocks(), + MaxBlockBuildingWaitTime: i.Config.ParameterConfig.MaxNetworkDelay, + Logger: i.Config.Logger, + Signer: i.Config.CryptoOps, + GenesisValidatorSet: genesisValidatorSet, + LastNonSimplexBlockPChainHeight: lastNonSimplexHeight, + SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, + BlockBuilder: i.Config.VM, + LastNonSimplexInnerBlock: i.Config.LastNonSimplexInnerBlock, + GetPChainHeightForProposing: i.Config.PlatformChain.GetMinimumHeight, + GetPChainHeightForVerifying: i.Config.PlatformChain.GetCurrentHeight, + AuxiliaryInfoApp: &NoopAuxiliaryInfoApp{}, + ComputeICMEpoch: i.Config.VM.ComputeICMEpoch, + GetBlock: i.cs.RetrieveBlock, + }) + if err != nil { + return simplex.EpochConfig{}, fmt.Errorf("error creating metadata state machine: %w", err) + } + + i.msm = msm + i.cs.msm = msm + + source, err := simplex.NewRandomSource() + if err != nil { + return simplex.EpochConfig{}, err + } + + blockBuilder := &BlockBuilderWaiter{vm: i.Config.VM, msm: msm, epochChangeSupression: &i.epochChangeSupression} + + comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster, epochChangeSupression: &i.epochChangeSupression} + comm.SetValidators(nodes) + + epochAwareStorage := &EpochAwareStorage{ + msm: msm, + Epoch: epochNum, + Storage: i.cs, + OnEpochChange: func(epoch uint64, validators common.Nodes) error { + i.epochChangeSupression.setSupression(epoch) // The epoch number is also the sealing block sequence. + blockBuilder.stop() + comm.SetValidators(validators) + i.notifyEpochChange(epoch, validators, validator) + return nil + }, + } + + epochConfig := simplex.EpochConfig{ + Epoch: epochNum, + ReplicationEnabled: true, + StartTime: time.Now(), + // TODO: For simpicity, we use the same value for all timeouts. If needed we can expand the config. + MaxProposalWait: i.Config.ParameterConfig.MaxNetworkDelay * 2, // 1 proposal + 1 vote + MaxRebroadcastWait: i.Config.ParameterConfig.MaxNetworkDelay * 2, + FinalizeRebroadcastTimeout: i.Config.ParameterConfig.MaxNetworkDelay * 2, + MaxRoundWindow: i.Config.ParameterConfig.MaxRoundWindow, + ID: i.Config.ID, + RandomSource: source, // Seed the random source from crypto/rand + WAL: wal, + Logger: i.Config.Logger, + SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, + QCDeserializer: i.Config.CryptoOps, + Signer: i.Config.CryptoOps, + Verifier: i.Config.CryptoOps, + Storage: epochAwareStorage, + Comm: comm, + BlockBuilder: blockBuilder, + BlockDeserializer: &blockDeserializer{vm: i.Config.VM, msm: msm}, + } + return epochConfig, nil +} + +func (i *Instance) maybeGarbageCollectWAL(lastBlock metadata.StateMachineBlock) error { + if lastBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor != nil { + i.Config.Logger.Info("Last block is a sealing block, garbage collecting all WALs preceding it to start a new epoch") + // We figure out the round number of the latest block and garbage collect all WALs preceding it. + // TODO: We need to test a scenario where an epoch change occurred and then a few notarizations have been persisted to WAL, + // but no block has been finalized. So the WAL contains entries from previous epochs as well as from the current epoch. + // TODO: We need to test a scenario where an epoch change occurred but the node has crashed after notarizing some Telocks. + md, err := common.ProtocolMetadataFromBytes(lastBlock.Metadata.SimplexProtocolMetadata) + if err != nil { + return fmt.Errorf("error parsing protocol metadata from last block: %w", err) + } + if err := i.wal.GarbageCollect(md.Round); err != nil { + return fmt.Errorf("error garbage collecting WALs: %w", err) + } + } + return nil +} + +func (i *Instance) transitionEpochNonValidator(epochChange epochChange) { + i.lock.Lock() + defer i.lock.Unlock() + + // Stop the non-validator before doing anything else, so that we don't process any more messages while we are changing epochs. + i.stopNonValidator() + + i.epochChangeSupression.clearSupression() + + // First, figure out if I'm still a validator. + if i.determineValidatorOrNot(epochChange.validators) { + i.Config.Logger.Info("I am now a validator") + i.startValidator() + return + } + + i.startNonValidator(epochChange.epochNum, epochChange.validators) +} + +func (i *Instance) transitionEpochValidator(epochChange epochChange) { + i.lock.Lock() + defer i.lock.Unlock() + + // Stop the epoch before doing anything else, so that we don't process any more messages while we are changing epochs. + i.stopValidator() + // Wipe out the WALs from the config so we won't try to load them again + i.Config.WALs = nil + // On epoch change, garbage collect the WAL to remove all entries from previous epochs. + if err := i.wal.GarbageCollect(math.MaxUint64); err != nil { + i.Config.Logger.Error("Error garbage collecting epoch config on epoch change", zap.Error(err)) + } + + i.epochChangeSupression.clearSupression() + + // First, figure out if I'm still a validator. + if !i.determineValidatorOrNot(epochChange.validators) { + i.Config.Logger.Info("I am no longer a validator") + i.startNonValidator(epochChange.epochNum, epochChange.validators) + return + } + + config, err := i.createEpochConfig() + if err != nil { + i.Config.Logger.Error("Error creating epoch config on epoch change", zap.Error(err)) + return + } + + if config.Epoch != epochChange.epochNum { + i.Config.Logger.Error("Epoch number mismatch on epoch change", zap.Uint64("expected", epochChange.epochNum), zap.Uint64("actual", config.Epoch)) + return + } + + if err := i.startEpoch(config); err != nil { + i.Config.Logger.Error("Error starting new epoch on epoch change", zap.Error(err)) + } +} + +func constructEpochAndValidatorSet(lastNonSimplexInnerBlockHeight uint64, genesisValidatorSet metadata.NodeBLSMappings, numBlocks uint64, lastBlock ParsedBlock, storage Storage) (common.Nodes, uint64, error) { + epochNum := lastBlock.BlockHeader().Epoch + + var validatorSet metadata.NodeBLSMappings + var nodes common.Nodes + + switch { + // If all we have in the ledger is non-Simplex blocks, load the validator set from genesis + case lastNonSimplexInnerBlockHeight+1 == numBlocks: + validatorSet = genesisValidatorSet + nodes = validatorSetToNodes(genesisValidatorSet) + epochNum = lastNonSimplexInnerBlockHeight + 1 + // If the last block persisted is a sealing block, then we are in the next h. + case lastBlock.SealingBlockInfo() != nil: + epochNum = lastBlock.BlockHeader().Seq + validatorSet = constructValidatorSetFromSealingBlock(lastBlock) + nodes = lastBlock.SealingBlockInfo().ValidatorSet + // Else, we have at least one Simplex block in the ledger, and it's not a sealing block. + default: + // Therefore, the sequence of the sealing block is the h number. + sealingBlockSeq := lastBlock.BlockHeader().Epoch + sealingBlock, _, err := storage.GetBlock(sealingBlockSeq) + if err != nil { + return nil, 0, fmt.Errorf("error retrieving sealing block from storage: %w", err) + } + if sealingBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor == nil { + return nil, 0, fmt.Errorf("expected sealing block at seq %d, but got a non-sealing block", sealingBlockSeq) + } + validatorSet = constructValidatorSetFromSealingBlock(ParsedBlock{StateMachineBlock: sealingBlock}) + nodes = validatorSetToNodes(validatorSet) + } + return nodes, epochNum, nil +} + +func validatorSetToNodes(genesisValidatorSet metadata.NodeBLSMappings) common.Nodes { + var nodes common.Nodes + for _, vdr := range genesisValidatorSet { + nodes = append(nodes, common.Node{ + Id: vdr.NodeID[:], + Weight: vdr.Weight, + PK: vdr.BLSKey, + }) + } + return nodes +} + +func constructValidatorSetFromSealingBlock(lastBlock ParsedBlock) metadata.NodeBLSMappings { + var validatorSet metadata.NodeBLSMappings + vdrs := lastBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor.AggregatedMembership.Members + for _, vdr := range vdrs { + validatorSet = append(validatorSet, metadata.NodeBLSMapping{ + NodeID: vdr.NodeID, + BLSKey: vdr.BLSKey, + Weight: vdr.Weight, + }) + } + return validatorSet +} diff --git a/instance_test.go b/instance_test.go new file mode 100644 index 00000000..09522bd9 --- /dev/null +++ b/instance_test.go @@ -0,0 +1,691 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/asn1" + "encoding/binary" + "fmt" + "sort" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/testutil" + "github.com/ava-labs/simplex/wal" + + "github.com/stretchr/testify/require" +) + +func TestInstance(t *testing.T) { + // One node is a validator at genesis, the other is a non-validator. + // After some blocks, the second (non-validator) node also becomes a validator. + // The test ensures that the second node tracks the chain while the first node expands the chain + // in the first epoch, and that both nodes move to the second epoch and then both are used for consensus together. + const ( + basePChainHeight = uint64(1) + epochChangePChainHeight = uint64(100) + ) + + var id [20]byte + rand.Read(id[:]) + firstNodeID := common.NodeID(id[:]) + + // The peer that joins the validator set in the last epoch. Its ID is chosen + // to differ from the (random) node under test. + var peerID [20]byte + rand.Read(peerID[:]) + secondNodeID := common.NodeID(peerID[:]) + + // Epoch 1 is single-validator + // The last epoch is expanded to two validators. + validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ + basePChainHeight: { + {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, + }, + epochChangePChainHeight: { + {NodeID: id, BLSKey: []byte{0xaa}, Weight: 2}, + {NodeID: peerID, BLSKey: []byte{0xbb}, Weight: 2}, + }, + } + + pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) + cops := &testCryptoOps{} + + genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} + + net := newInMemNetwork(t) + t.Cleanup(net.stop) + + // newInstance builds an Instance sharing the common test dependencies but with + // its own ID, storage and VM. + + // Create the storage for the instances and append the genesis block to each + storage := NewMockStorage(t) + smb := metadata.StateMachineBlock{InnerBlock: genesisBlock} + require.NoError(t, storage.Index(context.Background(), &ParsedBlock{StateMachineBlock: smb}, common.Finalization{})) + + storage2 := NewMockStorage(t) + smb = metadata.StateMachineBlock{InnerBlock: genesisBlock} + require.NoError(t, storage2.Index(context.Background(), &ParsedBlock{StateMachineBlock: smb}, common.Finalization{})) + + // Create the instances and register them to the network + firstInstance := newInstance(t, firstNodeID, storage, net, pChain, cops, genesisBlock) + secondInstance := newInstance(t, secondNodeID, storage2, net, pChain, cops, genesisBlock) + net.register(firstNodeID, firstInstance) + net.register(secondNodeID, secondInstance) + + /// Start the instances + require.NoError(t, firstInstance.Start(t.Context())) + require.NoError(t, secondInstance.Start(t.Context())) + t.Cleanup(firstInstance.Stop) + t.Cleanup(secondInstance.Stop) + + // Epoch 1: wait until the node has committed a series of normal blocks on its own. + const epoch1Target = uint64(5) // genesis(0) + zero block(1) + 3 normal blocks + waitForNumBlocks(t, storage, epoch1Target) + waitForNumBlocks(t, storage2, epoch1Target) + + require.GreaterOrEqual(t, storage.NumBlocks(), epoch1Target) + require.GreaterOrEqual(t, storage2.NumBlocks(), epoch1Target) + + // The validator set in force is the one introduced by the most recent block + // that carries a BlockValidationDescriptor (the zero block in epoch 1). + require.Equal(t, firstInstance.Config.ID, latestValidatorID(t, storage)) + require.Equal(t, firstInstance.Config.ID, latestValidatorID(t, storage2)) + + // Trigger the epoch change: the validator set changes at epochChangePChainHeight, + // growing from one validator to two. + pChain.advanceTo(epochChangePChainHeight) + approval := &metadata.ValidatorSetApproval{ + NodeID: peerID, + PChainHeight: epochChangePChainHeight, + AuxInfoDigest: sha256.Sum256(nil), + Signature: []byte{1, 2, 3}, + } + + // The node seals the epoch once it has a quorum of approvals of the new + // (two-validator) set. With two validators the node's self-approval is no longer + // a quorum and the peer is not running yet, so waitForSealingBlock injects the + // peer's approval on each poll until the sealing block is committed. + // TODO: Implement this capability in production so we won't need to inject approvals in tests. + sealingBlockSeq := waitForSealingBlock(t, firstInstance, approval, storage.NumBlocks()) + waitForNumBlocks(t, storage2, sealingBlockSeq) // Ensure the new validator has replicated the sealing block. + + // With both validators live, the two-validator epoch commits more blocks. + const epoch2Extra = uint64(3) + waitForNumBlocks(t, storage, sealingBlockSeq+epoch2Extra) + + // Confirm the second epoch has the second validator in the sealing block + require.Equal(t, secondInstance.Config.ID, latestValidatorID(t, storage)) +} + +func newInstance(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock) *Instance { + comm := &networkSender{net: net, self: nodeID} + return &Instance{ + Config: Config{ + Logger: testutil.MakeLogger(t, int(nodeID[0])), + ID: nodeID, + VM: newTestVM(), + Storage: storage, + Sender: comm, + Broadcaster: comm, + PlatformChain: pChain, + CryptoOps: cops, + LastNonSimplexInnerBlock: genesisBlock, + ParameterConfig: ParameterConfig{ + MaxNetworkDelay: 500 * time.Millisecond, + MaxRoundWindow: 100, + WALMaxEntryCount: 1024, + }, + }, + } +} + +func latestValidatorID(t *testing.T, storage *MockStorage) common.NodeID { + t.Helper() + num := storage.NumBlocks() + // Iterate backwards and find the latest sealing block (a block with a block validation descriptor) + for seq := int64(num) - 1; seq >= 0; seq-- { + block, ok := storage.blockAt(uint64(seq)) + if !ok { + continue + } + bvd := block.Metadata.SimplexEpochInfo.BlockValidationDescriptor + if bvd != nil { + validatorCount := len(bvd.AggregatedMembership.Members) + return bvd.AggregatedMembership.Members[validatorCount-1].NodeID[:] + } + } + t.Fatalf("no block with a BlockValidationDescriptor found in storage") + return nil +} + +// waitForNumBlocks waits until the given storage has at least targetHeight blocks. +func waitForNumBlocks(t *testing.T, storage *MockStorage, targetHeight uint64) { + t.Helper() + require.Eventually(t, func() bool { + return storage.NumBlocks() >= targetHeight + }, 20*time.Second, 100*time.Millisecond) +} + +// waitForSealingBlock waits until a sealing block (a block carrying a BlockValidationDescriptor with the new weight) +// is committed at or after fromSeq. It periodically injects approvals into the given instance. +// Returns the seq of the sealing block. +func waitForSealingBlock(t *testing.T, inst *Instance, approval *metadata.ValidatorSetApproval, fromSeq uint64) uint64 { + t.Helper() + var result uint64 + storage := inst.Config.Storage.(*MockStorage) + require.Eventually(t, func() bool { + inst.lock.Lock() + msm := inst.msm + inst.lock.Unlock() + if msm != nil { + require.NoError(t, msm.HandleApproval(approval, 1)) + } + + num := storage.NumBlocks() + for seq := fromSeq; seq < num; seq++ { + block, ok := storage.blockAt(seq) + if !ok { + continue + } + bvd := block.Metadata.SimplexEpochInfo.BlockValidationDescriptor + if bvd != nil { + result = seq + return true + } + } + return false + }, 20*time.Second, 100*time.Millisecond) + return result +} + +type testInnerBlock struct { + Height_ uint64 + TS time.Time + Payload []byte +} + +func (b *testInnerBlock) Bytes() ([]byte, error) { + out := make([]byte, 16, 16+len(b.Payload)) + binary.BigEndian.PutUint64(out[0:8], b.Height_) + binary.BigEndian.PutUint64(out[8:16], uint64(b.TS.UnixMilli())) + out = append(out, b.Payload...) + return out, nil +} + +func (b *testInnerBlock) Digest() [32]byte { + bytes, _ := b.Bytes() + return sha256.Sum256(bytes) +} + +func (b *testInnerBlock) Height() uint64 { return b.Height_ } +func (b *testInnerBlock) Timestamp() time.Time { return b.TS } +func (b *testInnerBlock) Verify(context.Context, uint64) error { return nil } + +func parseTestInnerBlock(buff []byte) (*testInnerBlock, error) { + b := &testInnerBlock{} + b.Height_ = binary.BigEndian.Uint64(buff[0:8]) + b.TS = time.UnixMilli(int64(binary.BigEndian.Uint64(buff[8:16]))) + b.Payload = append([]byte(nil), buff[16:]...) + return b, nil +} + +type testVM struct { + nextHeight atomic.Uint64 +} + +func newTestVM() *testVM { + vm := &testVM{} + vm.nextHeight.Store(2) // genesis inner block is height 1 + return vm +} + +func (vm *testVM) BuildBlock(_ context.Context, _ uint64) (metadata.VMBlock, error) { + h := vm.nextHeight.Add(1) - 1 + payload := make([]byte, 8) + binary.BigEndian.PutUint64(payload, h) + return &testInnerBlock{Height_: h, TS: time.Now(), Payload: payload}, nil +} + +func (vm *testVM) WaitForPendingBlock(ctx context.Context) { + select { + case <-ctx.Done(): + case <-time.After(100 * time.Millisecond): + } +} + +func (vm *testVM) ParseBlock(_ context.Context, b []byte) (metadata.VMBlock, error) { + return parseTestInnerBlock(b) +} + +func (vm *testVM) ComputeICMEpoch(input metadata.ICMEpochInput) metadata.ICMEpochInfo { + // ACP-181-style transition (mirrors the msm test helper). + var zero metadata.ICMEpochInfo + if input.ParentEpoch == zero { + return metadata.ICMEpochInfo{ + PChainEpochHeight: input.ParentPChainHeight, + EpochNumber: 1, + EpochStartTime: uint64(input.ParentTimestamp.Unix()), + } + } + endTime := time.Unix(int64(input.ParentEpoch.EpochStartTime), 0).Add(time.Second) + if input.ParentTimestamp.Before(endTime) { + return input.ParentEpoch + } + return metadata.ICMEpochInfo{ + PChainEpochHeight: input.ParentPChainHeight, + EpochNumber: input.ParentEpoch.EpochNumber + 1, + EpochStartTime: uint64(input.ParentTimestamp.Unix()), + } +} + +type testPlatformChain struct { + baseHeight uint64 + validatorSetAtHeight map[uint64]metadata.NodeBLSMappings // height --> validator set + lock sync.Mutex + cond *sync.Cond + height uint64 +} + +func newTestPlatformChain(baseHeight uint64, validatorSetsAtHeight map[uint64]metadata.NodeBLSMappings) *testPlatformChain { + pc := &testPlatformChain{ + baseHeight: baseHeight, + validatorSetAtHeight: validatorSetsAtHeight, + height: baseHeight, + } + pc.cond = sync.NewCond(&pc.lock) + return pc +} + +func (pc *testPlatformChain) advanceTo(h uint64) { + pc.lock.Lock() + defer pc.lock.Unlock() + pc.height = h + pc.cond.Broadcast() // wake any WaitForProgress waiters +} + +func (pc *testPlatformChain) currentHeight() uint64 { + pc.lock.Lock() + defer pc.lock.Unlock() + return pc.height +} + +func (pc *testPlatformChain) validatorSet(height uint64) metadata.NodeBLSMappings { + heights := make([]uint64, 0, len(pc.validatorSetAtHeight)) + for h := range pc.validatorSetAtHeight { + heights = append(heights, h) + } + sort.Slice(heights, func(i, j int) bool { return heights[i] < heights[j] }) + + var lastCheckpoint uint64 + for _, h := range heights { + if h > height { + break + } + lastCheckpoint = h + } + // Return a copy instead of the original slice so the reference won't be used in other goroutines concurrently. + // Since we allocate a nil slice, a new underlying array is allocated and the copy is safe to use concurrently. + src := pc.validatorSetAtHeight[lastCheckpoint] + return append(metadata.NodeBLSMappings(nil), src...) +} + +func (pc *testPlatformChain) GetValidatorSet(height uint64) (metadata.NodeBLSMappings, error) { + return pc.validatorSet(height), nil +} + +func (pc *testPlatformChain) GenesisValidatorSet() metadata.NodeBLSMappings { + return pc.validatorSet(pc.baseHeight) +} + +func (pc *testPlatformChain) GetMinimumHeight(context.Context) (uint64, error) { + return pc.currentHeight(), nil +} + +func (pc *testPlatformChain) GetCurrentHeight(context.Context) (uint64, error) { + return pc.currentHeight(), nil +} + +func (pc *testPlatformChain) WaitForProgress(ctx context.Context, pChainHeight uint64) error { + stop := pc.signalWhenContextFinished(ctx) + defer stop() + + pc.lock.Lock() + defer pc.lock.Unlock() + for pc.height == pChainHeight { + if err := ctx.Err(); err != nil { + return err + } + pc.cond.Wait() + } + return nil +} + +func (pc *testPlatformChain) signalWhenContextFinished(ctx context.Context) func() bool { + stop := context.AfterFunc(ctx, func() { + pc.lock.Lock() + defer pc.lock.Unlock() + pc.cond.Broadcast() + }) + return stop +} + +func (pc *testPlatformChain) LastNonSimplexBlockPChainHeight() uint64 { + return pc.baseHeight +} + +type testCryptoOps struct{} + +func (c *testCryptoOps) Sign(message []byte) ([]byte, error) { + // A deterministic, non-empty placeholder signature. + d := sha256.Sum256(message) + return d[:], nil +} + +func (c *testCryptoOps) AggregateKeys(keys ...[]byte) ([]byte, error) { + var out []byte + for _, k := range keys { + out = append(out, k...) + } + return out, nil +} + +func (c *testCryptoOps) VerifySignature(_ []byte, _ []byte, _ []byte) error { + return nil +} + +func (c *testCryptoOps) CreateSignatureAggregator(nodes []common.Node) common.SignatureAggregator { + return &testutil.TestSignatureAggregator{N: len(nodes)} +} + +func (c *testCryptoOps) DeserializeQuorumCertificate(bytes []byte) (common.QuorumCertificate, error) { + var qc []common.Signature + if _, err := asn1.Unmarshal(bytes, &qc); err != nil { + return nil, err + } + return testutil.TestQC(qc), nil +} + +type MockStorage struct { + t *testing.T + *testutil.InMemStorage + + snapLock sync.Mutex + blocks map[uint64]storedBlock +} + +type storedBlock struct { + rawBlock []byte + fin common.Finalization +} + +func NewMockStorage(t *testing.T) *MockStorage { + return &MockStorage{ + t: t, + InMemStorage: testutil.NewInMemStorage(), + blocks: make(map[uint64]storedBlock), + } +} + +func (m *MockStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { + // We serialized the block so that the original reference isn't shared with other goroutines that may concurrently mutate it. + encoded, err := block.Bytes() + if err != nil { + return err + } + seq := m.InMemStorage.NumBlocks() + m.snapLock.Lock() + m.blocks[seq] = storedBlock{rawBlock: encoded, fin: certificate} + m.snapLock.Unlock() + return m.InMemStorage.Index(ctx, block, certificate) +} + +func (m *MockStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) { + _, f, err := m.InMemStorage.Retrieve(seq) + if err != nil { + return metadata.StateMachineBlock{}, nil, err + } + sb, ok := m.blockAt(seq) + if !ok { + return metadata.StateMachineBlock{}, nil, fmt.Errorf("no snapshot for seq %d", seq) + } + return sb, &f, nil +} + +// blockAt reconstructs an independent copy of the block at seq from its +// stored bytes. Test-only readers use it instead of GetBlock so they never touch +// the instance's live block objects (whose canoto digest cache the instance keeps +// mutating). +func (m *MockStorage) blockAt(seq uint64) (metadata.StateMachineBlock, bool) { + m.snapLock.Lock() + sb, ok := m.blocks[seq] + m.snapLock.Unlock() + if !ok { + return metadata.StateMachineBlock{}, false + } + return m.parseStored(sb.rawBlock), true +} + +func (m *MockStorage) parseStored(encoded []byte) metadata.StateMachineBlock { + raw := &RawBlock{} + require.NoError(m.t, raw.UnmarshalCanoto(encoded)) + var inner metadata.VMBlock + if len(raw.InnerBlockBytes) > 0 { + parsed, err := parseTestInnerBlock(raw.InnerBlockBytes) + require.NoError(m.t, err) + inner = parsed + } + return metadata.StateMachineBlock{InnerBlock: inner, Metadata: raw.Metadata} +} + +func (m *MockStorage) CreateWAL() (wal.DeletableWAL, error) { + return testutil.NewTestWAL(m.t), nil +} + +// --------------------------------------------------------------------------- +// inMemNetwork: Routes messages between Instances. +// Delivery happens on a per-node goroutine rather than inline in Send, +// due to locking. +// --------------------------------------------------------------------------- + +type netMsg struct { + from common.NodeID + msg *common.Message +} + +type netNode struct { + inst *Instance + // in is a buffered inbox drained by the delivery goroutine. The channel itself + // signals that work is available, so no separate wake signal is needed. Sends + // never block (see enqueue); on the rare chance the buffer fills, a dropped + // message costs at most an empty round the epoch recovers from. + in chan netMsg + done chan struct{} + stopped chan struct{} +} + +type inMemNetwork struct { + t *testing.T + lock sync.Mutex + nodes map[string]*netNode +} + +func newInMemNetwork(t *testing.T) *inMemNetwork { + return &inMemNetwork{t: t, nodes: make(map[string]*netNode)} +} + +// register wires inst into the network and starts delivering messages to it. +// Messages that arrive before the epoch exists are dropped by the instance's +// nil-epoch guard, which at worst costs a few empty rounds the epoch recovers from. +func (n *inMemNetwork) register(id common.NodeID, inst *Instance) { + node := &netNode{ + inst: inst, + in: make(chan netMsg, 1024), + done: make(chan struct{}), + stopped: make(chan struct{}), + } + n.lock.Lock() + n.nodes[string(id)] = node + n.lock.Unlock() + go n.deliver(node) +} + +func (n *inMemNetwork) stop() { + n.lock.Lock() + nodes := make([]*netNode, 0, len(n.nodes)) + for _, node := range n.nodes { + nodes = append(nodes, node) + } + n.nodes = make(map[string]*netNode) + n.lock.Unlock() + for _, node := range nodes { + close(node.done) + <-node.stopped + } +} + +func (n *inMemNetwork) enqueue(dest common.NodeID, m netMsg) { + n.lock.Lock() + node := n.nodes[string(dest)] + n.lock.Unlock() + if node == nil { + // Destination not registered; drop. This only happens before an instance + // is registered, never mid-run. + return + } + select { + case node.in <- m: + default: + // Never block the sender (Send runs under the epoch lock). A dropped message + // costs at most an empty round the epoch recovers from. + } +} + +func (n *inMemNetwork) deliver(node *netNode) { + defer close(node.stopped) + for { + select { + case <-node.done: + return + case m := <-node.in: + n.dispatch(node.inst, m) + } + } +} + +func (n *inMemNetwork) dispatch(inst *Instance, m netMsg) { + if err := inst.HandleMessage(m.msg, m.from); err != nil { + n.t.Logf("HandleMessage from %x failed: %v", m.from, err) + } +} + +// toRawBlock re-encodes a verified block into the wire RawBlock the receiving +// instance parses in HandleBlockMessage. +func toRawBlock(t *testing.T, vb common.VerifiedBlock) *RawBlock { + bytes, err := vb.Bytes() + require.NoError(t, err) + raw := &RawBlock{} + require.NoError(t, raw.UnmarshalCanoto(bytes)) + return raw +} + +// reparseBlock reconstructs an independent *ParsedBlock from a verified block's +// wire bytes. Each call yields a fresh object sharing no pointers with the +// sender's live block, so that remaining references to the sender's block don't race with the receiver. +func reparseBlock(t *testing.T, vb common.VerifiedBlock) *ParsedBlock { + raw := toRawBlock(t, vb) + var inner metadata.VMBlock + if len(raw.InnerBlockBytes) > 0 { + parsed, err := parseTestInnerBlock(raw.InnerBlockBytes) + require.NoError(t, err) + inner = parsed + } + return &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{InnerBlock: inner, Metadata: raw.Metadata}, + } +} + +type networkSender struct { + net *inMemNetwork + self common.NodeID +} + +func (s *networkSender) Broadcast(msg *common.Message) { + for _, n := range s.net.nodes { + s.Send(msg, n.inst.Config.ID) + } +} + +func (s *networkSender) Send(msg *common.Message, dest common.NodeID) { + if bytes.Equal(s.self, dest) { + // Do not send to myself + return + } + m := s.createIngressMessage(msg) + s.net.enqueue(dest, m) +} + +// CreateIngressMessage translates a message into the form the receiving instance expects on the wire. +// For example, a VerifiedBlockMessage is re-encoded as a BlockMessage with a RawBlock. +// A VerifiedReplicationResponse is re-encoded as a ReplicationResponse with independent copies of the carried blocks. +func (s *networkSender) createIngressMessage(msg *common.Message) netMsg { + m := netMsg{from: s.self} + switch { + case msg.VerifiedBlockMessage != nil: + m.msg = &common.Message{ + BlockMessage: &common.BlockMessage{ + Vote: msg.VerifiedBlockMessage.Vote, + Block: reparseBlock(s.net.t, msg.VerifiedBlockMessage.VerifiedBlock), + }, + } + case msg.VerifiedReplicationResponse != nil: + m.msg = &common.Message{ReplicationResponse: toReplicationResponse(s.net.t, msg.VerifiedReplicationResponse)} + default: + m.msg = msg + } + return m +} + +// toReplicationResponse translates a VerifiedReplicationResponse (the sender's +// internal form) into the ReplicationResponse a receiver handles on the wire, +// mirroring testutil.TestComm. Each carried block is reconstructed as an +// independent copy so the delivery goroutine never touches the sender's live +// block object (whose canoto digest cache the sender keeps mutating). +func toReplicationResponse(t *testing.T, vrr *common.VerifiedReplicationResponse) *common.ReplicationResponse { + data := make([]common.QuorumRound, 0, len(vrr.Data)) + for _, vqr := range vrr.Data { + data = append(data, verifiedQuorumRoundToQuorumRound(t, vqr)) + } + resp := &common.ReplicationResponse{Data: data} + if vrr.LatestRound != nil { + qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestRound) + resp.LatestRound = &qr + } + if vrr.LatestFinalizedSeq != nil { + qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestFinalizedSeq) + resp.LatestSeq = &qr + } + return resp +} + +func verifiedQuorumRoundToQuorumRound(t *testing.T, vqr common.VerifiedQuorumRound) common.QuorumRound { + qr := common.QuorumRound{ + Notarization: vqr.Notarization, + Finalization: vqr.Finalization, + EmptyNotarization: vqr.EmptyNotarization, + } + if vqr.VerifiedBlock != nil { + qr.Block = reparseBlock(t, vqr.VerifiedBlock) + } + return qr +} diff --git a/msm/build_decision.go b/msm/build_decision.go index 252be698..e17d393d 100644 --- a/msm/build_decision.go +++ b/msm/build_decision.go @@ -35,7 +35,7 @@ type blockBuildingDecider struct { // hasValidatorSetChanged should return whether the validator set has changed since the // P-chain height referenced by the last block in the chain and until the provided P-chain height. hasValidatorSetChanged func(pChainHeight uint64) (bool, NodeBLSMappings, error) - getPChainHeight func() uint64 + getPChainHeight func(ctx context.Context) (uint64, error) } // shouldBuildBlock determines whether we should build a block at the current time, @@ -47,7 +47,10 @@ func (bbd *blockBuildingDecider) shouldBuildBlock( ctx context.Context, ) (blockBuildingDecision, error) { for { - pChainHeight := bbd.getPChainHeight() + pChainHeight, err := bbd.getPChainHeight(ctx) + if err != nil { + return blockBuildingDecision{}, err + } shouldTransitionEpoch, newValidatorSet, err := bbd.hasValidatorSetChanged(pChainHeight) if err != nil { @@ -72,7 +75,11 @@ func (bbd *blockBuildingDecider) shouldBuildBlock( // If the P-chain height changed, re-evaluate again whether we should transition to a new epoch, // or continue waiting to build a block. - if bbd.getPChainHeight() != pChainHeight { + h, err := bbd.getPChainHeight(ctx) + if err != nil { + return blockBuildingDecision{}, err + } + if h != pChainHeight { continue } diff --git a/msm/build_decision_test.go b/msm/build_decision_test.go index 6b34b067..b5400584 100644 --- a/msm/build_decision_test.go +++ b/msm/build_decision_test.go @@ -32,7 +32,7 @@ func TestShouldBuildBlock_VMSignalsBlock(t *testing.T) { }, waitForPendingBlock: func(ctx context.Context) {}, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return false, nil, nil }, - getPChainHeight: func() uint64 { return 100 }, + getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -56,7 +56,7 @@ func TestShouldBuildBlock_ContextCanceled(t *testing.T) { <-ctx.Done() }, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return false, nil, nil }, - getPChainHeight: func() uint64 { return 100 }, + getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, } decision, err := bbd.shouldBuildBlock(ctx) @@ -88,7 +88,7 @@ func TestShouldBuildBlock_PChainHeightChangeTriggersEpochTransition(t *testing.T hasValidatorSetChanged: func(height uint64) (bool, NodeBLSMappings, error) { return height == 200, nil, nil }, - getPChainHeight: func() uint64 { return pChainHeight.Load() }, + getPChainHeight: func(context.Context) (uint64, error) { return pChainHeight.Load(), nil }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -123,7 +123,7 @@ func TestShouldBuildBlock_PChainHeightChangeButNoEpochTransition(t *testing.T) { } }, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return false, nil, nil }, - getPChainHeight: func() uint64 { return pChainHeight.Load() }, + getPChainHeight: func(context.Context) (uint64, error) { return pChainHeight.Load(), nil }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -142,7 +142,7 @@ func TestShouldBuildBlock_EpochTransitionWithVMBlock(t *testing.T) { }, waitForPendingBlock: func(ctx context.Context) {}, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return true, nil, nil }, - getPChainHeight: func() uint64 { return 100 }, + getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -163,7 +163,7 @@ func TestShouldBuildBlock_EpochTransitionWithoutVMBlock(t *testing.T) { <-ctx.Done() }, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return true, nil, nil }, - getPChainHeight: func() uint64 { return 100 }, + getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -188,7 +188,7 @@ func TestShouldBuildBlock_EpochTransitionContextCanceled(t *testing.T) { <-ctx.Done() }, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return true, nil, nil }, - getPChainHeight: func() uint64 { return 100 }, + getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, } decision, err := bbd.shouldBuildBlock(ctx) diff --git a/msm/encoding.go b/msm/encoding.go index 9659e5d6..1cd6ffbe 100644 --- a/msm/encoding.go +++ b/msm/encoding.go @@ -292,12 +292,13 @@ func (nea *NextEpochApprovals) Equals(other *NextEpochApprovals) bool { type NodeBLSMappings []NodeBLSMapping -func (nbms NodeBLSMappings) NodeWeights() common.Nodes { +func (nbms NodeBLSMappings) Nodes() common.Nodes { nodeWeights := make(common.Nodes, len(nbms)) for i, nbm := range nbms { nodeWeights[i] = common.Node{ Id: nbm.NodeID[:], Weight: nbm.Weight, + PK: nbm.BLSKey[:], } } return nodeWeights diff --git a/msm/fake_node_test.go b/msm/fake_node_test.go index be6fc29d..07922e70 100644 --- a/msm/fake_node_test.go +++ b/msm/fake_node_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/ava-labs/simplex/common" + "github.com/ava-labs/simplex/testutil" "github.com/stretchr/testify/require" ) @@ -33,13 +34,15 @@ func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { var pChainHeight atomic.Uint64 pChainHeight.Store(100) node := newFakeNode(t) + myID := [20]byte{1} + node.sm.MyNodeID = myID[:] node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet - node.sm.GetPChainHeightForProposing = func() uint64 { - return pChainHeight.Load() + node.sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { + return pChainHeight.Load(), nil } - node.sm.GetPChainHeightForVerifying = func() uint64 { - return pChainHeight.Load() + node.sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { + return pChainHeight.Load(), nil } node.mempoolEmpty = true node.sm.MaxBlockBuildingWaitTime = 100 * time.Millisecond @@ -86,13 +89,15 @@ func TestFakeNode(t *testing.T) { var pChainHeight atomic.Uint64 pChainHeight.Store(100) node := newFakeNode(t) + myID := [20]byte{1} + node.sm.MyNodeID = myID[:] node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet - node.sm.GetPChainHeightForProposing = func() uint64 { - return pChainHeight.Load() + node.sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { + return pChainHeight.Load(), nil } - node.sm.GetPChainHeightForVerifying = func() uint64 { - return pChainHeight.Load() + node.sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { + return pChainHeight.Load(), nil } // Create some blocks and finalize them, until we reach height 10 @@ -148,14 +153,16 @@ func TestFakeNodeEmptyMempool(t *testing.T) { var pChainHeight uint64 = 100 node := newFakeNode(t) + myID := [20]byte{1} + node.sm.MyNodeID = myID[:] node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.MaxBlockBuildingWaitTime = 100 * time.Millisecond node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet - node.sm.GetPChainHeightForProposing = func() uint64 { - return pChainHeight + node.sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { + return pChainHeight, nil } - node.sm.GetPChainHeightForVerifying = func() uint64 { - return pChainHeight + node.sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { + return pChainHeight, nil } // Create some blocks and finalize them, until we reach height 10 @@ -214,7 +221,7 @@ func TestFakeNodeEmptyMempool(t *testing.T) { } type innerBlock struct { - InnerBlock + testutil.InnerBlock Prev [32]byte } @@ -240,7 +247,11 @@ func (fn *fakeNode) WaitForProgress(ctx context.Context, pChainHeight uint64) er case <-ctx.Done(): return ctx.Err() case <-time.After(10 * time.Millisecond): - if fn.sm.GetPChainHeightForProposing() != pChainHeight { + height, err := fn.sm.GetPChainHeightForProposing(ctx) + if err != nil { + return err + } + if height != pChainHeight { return nil } } @@ -454,8 +465,8 @@ func (fn *fakeNode) BuildBlock(ctx context.Context, _ uint64) (VMBlock, error) { vmBlock := &innerBlock{ Prev: fn.getLastVMBlockDigest(), - InnerBlock: InnerBlock{ - Bytes: randomBuff(10), + InnerBlock: testutil.InnerBlock{ + Content: randomBuff(10), TS: time.Now(), BlockHeight: uint64(count), }, @@ -467,7 +478,7 @@ func (fn *fakeNode) getParentBlock() StateMachineBlock { if len(fn.blocks) > 0 { return fn.blocks[len(fn.blocks)-1].block } - gb := genesisBlock.InnerBlock.(*InnerBlock) + gb := genesisBlock.InnerBlock.(*testutil.InnerBlock) return StateMachineBlock{ InnerBlock: &innerBlock{ InnerBlock: *gb, diff --git a/msm/fuzz_test.go b/msm/fuzz_test.go index de4dc402..11cc4de9 100644 --- a/msm/fuzz_test.go +++ b/msm/fuzz_test.go @@ -105,8 +105,8 @@ func FuzzVerifyBlock(f *testing.F) { // Model the verifier as knowing the P-chain exactly up to the height the block // references — the minimal knowledge required to verify it, mirroring production // where GetPChainHeightForVerifying returns the verifier's latest observed height. - sm.GetPChainHeightForProposing = func() uint64 { return block.Metadata.PChainHeight } - sm.GetPChainHeightForVerifying = func() uint64 { return block.Metadata.PChainHeight } + sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return block.Metadata.PChainHeight, nil } + sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return block.Metadata.PChainHeight, nil } // The unfuzzed block built by the chain MSM must verify. require.NoError(t, sm.VerifyBlock(context.Background(), block), @@ -184,7 +184,7 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, // Use a genesis block anchored at startTime: the zero block carries over the last // non-Simplex block's timestamp, so it must not be ahead of the blocks built after it. - genesis := StateMachineBlock{InnerBlock: &InnerBlock{BlockHeight: 0, TS: startTime, Bytes: []byte{0}}} + genesis := StateMachineBlock{InnerBlock: &InnerBlock{BlockHeight: 0, TS: startTime, bytes: []byte{0}}} currentPChainHeight := pChainHeight1 currentTime := startTime @@ -195,8 +195,8 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, // round (the builder only collects approvals once the aux info history is ready). sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} sm.GetValidatorSet = getValidatorSet - sm.GetPChainHeightForProposing = func() uint64 { return currentPChainHeight } - sm.GetPChainHeightForVerifying = func() uint64 { return currentPChainHeight } + sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return currentPChainHeight, nil } + sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return currentPChainHeight, nil } sm.GetTime = func() time.Time { return currentTime } sm.GenesisValidatorSet = validatorSet1 sm.LastNonSimplexBlockPChainHeight = pChainHeight1 @@ -212,7 +212,7 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, return &InnerBlock{ TS: startTime.Add(time.Duration(h) * time.Millisecond), BlockHeight: h, - Bytes: []byte{byte(h)}, + bytes: []byte{byte(h)}, } } build := func(seq, round, epoch uint64, prev *StateMachineBlock) *StateMachineBlock { @@ -232,14 +232,14 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, // block1: the zero block, finalized so it can later serve as the epoch's reference for // the validator-set change and the sealing-block computation. - tc.blockBuilder.block = nextInner(1) + tc.blockBuilder.Block = nextInner(1) block1 := build(1, 0, 1, nil) addBlock(1, block1, &common.Finalization{}) sm.LatestPersistedHeight = 1 // block2: a normal in-epoch block. currentTime = startTime.Add(2 * time.Millisecond) - tc.blockBuilder.block = nextInner(2) + tc.blockBuilder.Block = nextInner(2) block2 := build(2, 1, 1, block1) addBlock(2, block2, nil) @@ -247,7 +247,7 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, // collect approvals for the next epoch. currentPChainHeight = pChainHeight2 currentTime = startTime.Add(time.Second + 3*time.Millisecond) - tc.blockBuilder.block = nextInner(3) + tc.blockBuilder.Block = nextInner(3) block3 := build(3, 2, 1, block2) addBlock(3, block3, nil) @@ -259,20 +259,20 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, // block4 & block5: collecting-approvals blocks (1/3 then 2/3, not enough to seal). require.NoError(tb, sm.HandleApproval(&ValidatorSetApproval{NodeID: node1, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}, 1)) currentTime = startTime.Add(time.Second + 4*time.Millisecond) - tc.blockBuilder.block = nextInner(4) + tc.blockBuilder.Block = nextInner(4) block4 := build(4, 3, 1, block3) addBlock(4, block4, nil) require.NoError(tb, sm.HandleApproval(&ValidatorSetApproval{NodeID: node2, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}, 2)) currentTime = startTime.Add(time.Second + 5*time.Millisecond) - tc.blockBuilder.block = nextInner(5) + tc.blockBuilder.Block = nextInner(5) block5 := build(5, 4, 1, block4) addBlock(5, block5, nil) // block6: the sealing block (3/3 approvals). Its successor is in stateBuildBlockEpochSealed. require.NoError(tb, sm.HandleApproval(&ValidatorSetApproval{NodeID: node3, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}, 3)) currentTime = startTime.Add(time.Second + 6*time.Millisecond) - tc.blockBuilder.block = nextInner(6) + tc.blockBuilder.Block = nextInner(6) block6 := build(6, 5, 1, block5) require.Equal(tb, stateBuildBlockEpochSealed, block6.Metadata.SimplexEpochInfo.NextState()) // Finalize the sealing block so the epoch transition can proceed. @@ -283,13 +283,13 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, // block7: the first block of the new epoch, built in stateBuildBlockEpochSealed. sealingSeq := uint64(6) currentTime = startTime.Add(time.Second + 7*time.Millisecond) - tc.blockBuilder.block = nextInner(7) + tc.blockBuilder.Block = nextInner(7) block7 := build(7, 6, sealingSeq, block6) addBlock(7, block7, nil) // block8: a normal in-epoch block in the second epoch. currentTime = startTime.Add(time.Second + 8*time.Millisecond) - tc.blockBuilder.block = nextInner(8) + tc.blockBuilder.Block = nextInner(8) block8 := build(8, 7, sealingSeq, block7) addBlock(8, block8, nil) diff --git a/msm/misc.go b/msm/misc.go index c0f13239..920bee8d 100644 --- a/msm/misc.go +++ b/msm/misc.go @@ -49,6 +49,9 @@ type VMBlock interface { // If nil is returned, it is guaranteed that either Accept or Reject will be // called on this block, unless the VM is shut down. Verify(ctx context.Context, pChainHeight uint64) error + + // Bytes returns the byte representation of this block. + Bytes() ([]byte, error) } type bitmask big.Int diff --git a/msm/msm.go b/msm/msm.go index dd831bc8..b0ab2257 100644 --- a/msm/msm.go +++ b/msm/msm.go @@ -194,6 +194,9 @@ type StateMachine struct { // Config contains the dependencies and configuration parameters needed to initialize the StateMachine. type Config struct { + // SkipMSMVerification indicates whether to skip the verification of the state machine transition when verifying blocks, + // and only verify the inner block. This is useful when replicating finalized blocks as a non-validator. + SkipMSMVerification bool // LatestPersistedHeight is the height of the most recently persisted block. LatestPersistedHeight uint64 // MaxBlockBuildingWaitTime is the maximum duration to wait for the VM to build a block @@ -204,9 +207,9 @@ type Config struct { // GetTime returns the current time. GetTime func() time.Time // GetPChainHeightForProposing returns the latest known P-chain height to be used when building a block. - GetPChainHeightForProposing func() uint64 + GetPChainHeightForProposing func(context.Context) (uint64, error) // GetPChainHeightForVerifying returns the latest known P-chain height to be used when verifying a block. - GetPChainHeightForVerifying func() uint64 + GetPChainHeightForVerifying func(context.Context) (uint64, error) // BlockBuilder builds new VM blocks. BlockBuilder BlockBuilder // Logger is used for logging state machine operations. @@ -358,6 +361,14 @@ func (sm *StateMachine) VerifyBlock(ctx context.Context, block *StateMachineBloc return errNilBlock } + if sm.SkipMSMVerification { + // If SkipMSMVerification is true, we only verify the inner block and skip the state machine verification. + if block.InnerBlock == nil { + return nil + } + return block.InnerBlock.Verify(ctx, block.Metadata.ICMEpochInfo.PChainEpochHeight) + } + pmd, err := common.ProtocolMetadataFromBytes(block.Metadata.SimplexProtocolMetadata) if err != nil { return fmt.Errorf("failed to parse ProtocolMetadata: %w", err) @@ -396,7 +407,10 @@ func (sm *StateMachine) verifyNonZeroBlock(ctx context.Context, block, prevBlock return fmt.Errorf("failed to verify timestamp: %w", err) } - currentPChainHeight := sm.GetPChainHeightForVerifying() + currentPChainHeight, err := sm.GetPChainHeightForVerifying(ctx) + if err != nil { + return fmt.Errorf("failed to get current P-chain height for verifying: %w", err) + } prevPChainHeight := prevBlockMD.PChainHeight proposedPChainHeight := block.Metadata.PChainHeight @@ -404,7 +418,7 @@ func (sm *StateMachine) verifyNonZeroBlock(ctx context.Context, block, prevBlock return fmt.Errorf("failed to verify P-chain height: %w", err) } - err := sm.verifyEpochNumber(block) + err = sm.verifyEpochNumber(block) if err != nil { return err } @@ -573,7 +587,7 @@ func (sm *StateMachine) verifyNormalBlock(ctx context.Context, parentBlock State icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, timestamp) - if err := sm.verifyNextPChainRefHeightNormal(parentBlock.Metadata, nextBlock.Metadata.SimplexEpochInfo); err != nil { + if err := sm.verifyNextPChainRefHeightNormal(ctx, parentBlock.Metadata, nextBlock.Metadata.SimplexEpochInfo); err != nil { return fmt.Errorf("failed to verify next P-chain reference height for normal block: %w", err) } newSimplexEpochInfo.NextPChainReferenceHeight = nextBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight @@ -594,7 +608,7 @@ func verifyPChainHeight(proposedPChainHeight uint64, currentPChainHeight uint64, return nil } -func (sm *StateMachine) verifyNextPChainRefHeightNormal(prevMD StateMachineMetadata, next SimplexEpochInfo) error { +func (sm *StateMachine) verifyNextPChainRefHeightNormal(ctx context.Context, prevMD StateMachineMetadata, next SimplexEpochInfo) error { prev := prevMD.SimplexEpochInfo // Next P-chain height can only increase, not decrease. if next.NextPChainReferenceHeight > 0 && prev.PChainReferenceHeight > next.NextPChainReferenceHeight { @@ -625,7 +639,10 @@ func (sm *StateMachine) verifyNextPChainRefHeightNormal(prevMD StateMachineMetad } // Make sure we have reached the next P-chain reference height, otherwise we won't be able to validate it. - pChainHeight := sm.GetPChainHeightForVerifying() + pChainHeight, err := sm.GetPChainHeightForVerifying(ctx) + if err != nil { + return fmt.Errorf("failed to get current P-chain height for verifying: %w", err) + } if pChainHeight < next.NextPChainReferenceHeight { return fmt.Errorf("%w: target %d, current %d", errPChainHeightNotReached, next.NextPChainReferenceHeight, pChainHeight) @@ -668,7 +685,7 @@ func (sm *StateMachine) verifyNextPChainRefHeightNormal(prevMD StateMachineMetad // We cannot reuse verifyNextPChainRefHeightNormal here — the baseline // for the validator-set change check is the new epoch's PChainReferenceHeight, not the parent's, // as in verifyNextPChainRefHeightNormal. -func (sm *StateMachine) verifyNextPChainRefHeightForNewEpoch(expectedEpochInfo SimplexEpochInfo, next SimplexEpochInfo) error { +func (sm *StateMachine) verifyNextPChainRefHeightForNewEpoch(ctx context.Context, expectedEpochInfo SimplexEpochInfo, next SimplexEpochInfo) error { // The first block of the epoch doesn't trigger an epoch change, we're all set. if next.NextPChainReferenceHeight == 0 { return nil @@ -684,7 +701,10 @@ func (sm *StateMachine) verifyNextPChainRefHeightForNewEpoch(expectedEpochInfo S // If we haven't reached this P-chain height yet, we cannot accept the next P-chain reference height, // because there is no way of querying the validator set for the next P-chain reference height. - pChainHeight := sm.GetPChainHeightForVerifying() + pChainHeight, err := sm.GetPChainHeightForVerifying(ctx) + if err != nil { + return fmt.Errorf("failed to get current P-chain height for verifying: %w", err) + } if pChainHeight < next.NextPChainReferenceHeight { return fmt.Errorf("%w: target %d, current %d", errPChainHeightNotReached, next.NextPChainReferenceHeight, pChainHeight) } @@ -734,8 +754,8 @@ func (sm *StateMachine) createBlockBuildingDecider(pChainReferenceHeight uint64) if !currentValidatorSet.Equal(newValidatorSet) { sm.Logger.Debug("Validator set has changed, should transition epoch", - zap.String("currentValidatorSet", fmt.Sprintf("%v", currentValidatorSet.NodeWeights())), - zap.String("newValidatorSet", fmt.Sprintf("%v", newValidatorSet.NodeWeights())), + zap.String("currentValidatorSet", fmt.Sprintf("%v", currentValidatorSet.Nodes())), + zap.String("newValidatorSet", fmt.Sprintf("%v", newValidatorSet.Nodes())), zap.Uint64("currentPChainRefHeight", pChainReferenceHeight), zap.Uint64("newPChainHeight", pChainHeight)) return true, newValidatorSet, nil @@ -966,6 +986,8 @@ func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, pare return err } + sm.maybeInitializeApprovalStore(validators) + newApprovals := nextBlock.Metadata.SimplexEpochInfo.NextEpochApprovals expectedAuxInfo, auxInfoDigest, isAuxInfoReady, err := sm.computeExpectedAuxInfoForApprovalCollection(parentBlock, nextBlock, prevBlockSeq, validators) @@ -1010,7 +1032,7 @@ func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, pare return err } - sigAggr := sm.SignatureAggregatorCreator(validators.NodeWeights()) + sigAggr := sm.SignatureAggregatorCreator(validators.Nodes()) approvals := bitmaskFromBytes(newApprovals.NodeIDs) canSeal := sigAggr.IsQuorum(validators.SelectSubset(approvals)) @@ -1105,7 +1127,7 @@ func computeSimplexEpochInfoForCollectingApprovalsBlock(parentBlock StateMachine func (sm *StateMachine) computeNewApprovals(parentBlock StateMachineBlock, validators NodeBLSMappings, auxInfoDigest [32]byte) (*approvals, error) { prevBlockNextPChainReferenceHeight := parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight - sigAggr := sm.SignatureAggregatorCreator(validators.NodeWeights()) + sigAggr := sm.SignatureAggregatorCreator(validators.Nodes()) // We retrieve approvals that validators have sent us for the next epoch. // These approvals are signed by validators of the next epoch. @@ -1411,13 +1433,16 @@ func (sm *StateMachine) verifyBlockEpochSealed(ctx context.Context, parentBlock // the proposed pchain height and (optional) next pchain reference height, mirroring // what buildBlockOrTransitionEpoch does on the build side. proposedPChainHeight := nextBlock.Metadata.PChainHeight - currentPChainHeight := sm.GetPChainHeightForVerifying() + currentPChainHeight, err := sm.GetPChainHeightForVerifying(ctx) + if err != nil { + return fmt.Errorf("failed to get current P-chain height for verifying: %w", err) + } prevPChainHeight := parentBlock.Metadata.PChainHeight if err := verifyPChainHeight(proposedPChainHeight, currentPChainHeight, prevPChainHeight); err != nil { return fmt.Errorf("failed to verify P-chain height: %w", err) } - if err := sm.verifyNextPChainRefHeightForNewEpoch(newSimplexEpochInfo, nextBlock.Metadata.SimplexEpochInfo); err != nil { + if err := sm.verifyNextPChainRefHeightForNewEpoch(ctx, newSimplexEpochInfo, nextBlock.Metadata.SimplexEpochInfo); err != nil { return fmt.Errorf("failed to verify next P-chain reference height for new epoch block: %w", err) } newSimplexEpochInfo.NextPChainReferenceHeight = nextBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight diff --git a/msm/msm_test.go b/msm/msm_test.go index 8baa3117..053e8834 100644 --- a/msm/msm_test.go +++ b/msm/msm_test.go @@ -145,10 +145,10 @@ func TestMSMBuildAndVerifyBlocksAfterGenesis(t *testing.T) { func TestMSMFirstSimplexBlockAfterPreSimplexBlocks(t *testing.T) { preSimplexParent := StateMachineBlock{ - InnerBlock: &InnerBlock{ + InnerBlock: &testutil.InnerBlock{ TS: time.Now(), BlockHeight: 42, - Bytes: []byte{4, 5, 6}, + Content: []byte{4, 5, 6}, }, // Since the height is 42, this can't be a genesis block, so it must be a // pre-Simplex block. It already participates in an ICM epoch, which the zero @@ -181,10 +181,10 @@ func TestMSMFirstSimplexBlockAfterPreSimplexBlocks(t *testing.T) { sm1.LastNonSimplexInnerBlock = testConfig1.blockStore[42].block.InnerBlock sm2.LastNonSimplexInnerBlock = testConfig1.blockStore[42].block.InnerBlock - testConfig1.blockBuilder.block = &InnerBlock{ + testConfig1.blockBuilder.Block = &testutil.InnerBlock{ TS: time.Now(), BlockHeight: 43, - Bytes: []byte{7, 8, 9}, + Content: []byte{7, 8, 9}, } block, err := sm1.BuildBlock(context.Background(), md, nil) @@ -327,8 +327,8 @@ func TestMSMNormalOp(t *testing.T) { tc.validatorSetRetriever.resultMap = map[uint64]NodeBLSMappings{ newPChainHeight: newValidatorSet, } - sm.GetPChainHeightForProposing = func() uint64 { return newPChainHeight } - sm.GetPChainHeightForVerifying = func() uint64 { return newPChainHeight } + sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return newPChainHeight, nil } + sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return newPChainHeight, nil } }, expectedPChainHeight: newPChainHeight, expectedNextPChainRefHeight: newPChainHeight, @@ -366,10 +366,10 @@ func TestMSMNormalOp(t *testing.T) { _, err = rand.Read(content) require.NoError(t, err) - testConfig1.blockBuilder.block = &InnerBlock{ + testConfig1.blockBuilder.Block = &testutil.InnerBlock{ TS: blockTime, BlockHeight: lastBlock.InnerBlock.Height(), - Bytes: content, + Content: content, } if testCase.setup != nil { @@ -393,10 +393,10 @@ func TestMSMNormalOp(t *testing.T) { require.NoError(t, err) expected := &StateMachineBlock{ - InnerBlock: &InnerBlock{ + InnerBlock: &testutil.InnerBlock{ TS: blockTime, BlockHeight: lastBlock.InnerBlock.Height(), - Bytes: content, + Content: content, }, Metadata: StateMachineMetadata{ SimplexBlacklist: blacklist.Bytes(), @@ -444,28 +444,28 @@ func TestMSMFullEpochLifecycle(t *testing.T) { // extra ICM epoch transition, making the test flaky. startTime := time.Now().Truncate(time.Second) - nextBlock := func(height uint64) *InnerBlock { - return &InnerBlock{ + nextBlock := func(height uint64) *testutil.InnerBlock { + return &testutil.InnerBlock{ TS: startTime.Add(time.Duration(height) * time.Millisecond), BlockHeight: height, - Bytes: []byte{byte(height)}, + Content: []byte{byte(height)}, } } // ----- Step 0: Building on top of genesis or upgrading to Simplex----- genesis := StateMachineBlock{ - InnerBlock: &InnerBlock{ + InnerBlock: &testutil.InnerBlock{ BlockHeight: 0, // Genesis block has height 0 TS: startTime, - Bytes: []byte{0}, + Content: []byte{0}, }, } notGenesis := StateMachineBlock{ - InnerBlock: &InnerBlock{ + InnerBlock: &testutil.InnerBlock{ BlockHeight: 42, TS: startTime, - Bytes: []byte{0}, + Content: []byte{0}, }, } for _, testCase := range []struct { @@ -511,11 +511,11 @@ func TestMSMFullEpochLifecycle(t *testing.T) { // getVerifyingPChainHeight stays ahead of it, modelling the more up-to-date height // the verifier checks against. const pChainHeightLag = uint64(50) - getProposingPChainHeight := func() uint64 { - return currentPChainHeight + getProposingPChainHeight := func(context.Context) (uint64, error) { + return currentPChainHeight, nil } - getVerifyingPChainHeight := func() uint64 { - return currentPChainHeight + pChainHeightLag + getVerifyingPChainHeight := func(context.Context) (uint64, error) { + return currentPChainHeight + pChainHeightLag, nil } // Since we explicitly compare the built block with an expected value, @@ -577,14 +577,14 @@ func TestMSMFullEpochLifecycle(t *testing.T) { // Each one fails the test if it consults the P-chain height function it must not use, // proving that building reads GetPChainHeightForProposing and verifying reads GetPChainHeightForVerifying. sm.GetPChainHeightForProposing = getProposingPChainHeight - sm.GetPChainHeightForVerifying = func() uint64 { + sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { require.FailNow(t, "builder must not use GetPChainHeightForVerifying when proposing") - return 0 + return 0, nil } - smVerify.GetPChainHeightForProposing = func() uint64 { + smVerify.GetPChainHeightForProposing = func(context.Context) (uint64, error) { require.FailNow(t, "verifier must not use GetPChainHeightForProposing when verifying") - return 0 + return 0, nil } smVerify.GetPChainHeightForVerifying = getVerifyingPChainHeight @@ -606,7 +606,7 @@ func TestMSMFullEpochLifecycle(t *testing.T) { aggr := &signatureAggregator{} // ----- Step 1: Build zero epoch block (first simplex block) ----- - tc.blockBuilder.block = nextBlock(1) + tc.blockBuilder.Block = nextBlock(1) md := common.ProtocolMetadata{ Seq: baseSeq + 1, Round: 0, @@ -646,7 +646,7 @@ func TestMSMFullEpochLifecycle(t *testing.T) { // ----- Step 2: Build a normal block (no validator set change) ----- currentTime = startTime.Add(2 * time.Millisecond) - tc.blockBuilder.block = nextBlock(2) + tc.blockBuilder.Block = nextBlock(2) md = common.ProtocolMetadata{Seq: baseSeq + 2, Round: 1, Epoch: testCase.epochNum, Prev: block1.Digest()} block2, err := sm.BuildBlock(context.Background(), md, nil) require.NoError(t, err) @@ -676,7 +676,7 @@ func TestMSMFullEpochLifecycle(t *testing.T) { // block4 (whose parent is block3) sees parentTimestamp >= // epochStart + 1s and transitions ICM to epoch 2. currentTime = startTime.Add(time.Second + 3*time.Millisecond) - tc.blockBuilder.block = nextBlock(3) + tc.blockBuilder.Block = nextBlock(3) md = common.ProtocolMetadata{Seq: baseSeq + 3, Round: 2, Epoch: testCase.epochNum, Prev: block2.Digest()} block3, err := sm.BuildBlock(context.Background(), md, nil) require.NoError(t, err) @@ -715,7 +715,7 @@ func TestMSMFullEpochLifecycle(t *testing.T) { require.NoError(t, err) currentTime = startTime.Add(time.Second + 4*time.Millisecond) - tc.blockBuilder.block = nextBlock(4) + tc.blockBuilder.Block = nextBlock(4) md = common.ProtocolMetadata{Seq: baseSeq + 4, Round: 3, Epoch: testCase.epochNum, Prev: block3.Digest()} block4, err := sm.BuildBlock(context.Background(), md, nil) require.NoError(t, err) @@ -757,7 +757,7 @@ func TestMSMFullEpochLifecycle(t *testing.T) { bitmask = []byte{3} currentTime = startTime.Add(time.Second + 5*time.Millisecond) - tc.blockBuilder.block = nextBlock(5) + tc.blockBuilder.Block = nextBlock(5) md = common.ProtocolMetadata{Seq: baseSeq + 5, Round: 4, Epoch: testCase.epochNum, Prev: block4.Digest()} block5, err := sm.BuildBlock(context.Background(), md, nil) require.NoError(t, err) @@ -799,7 +799,7 @@ func TestMSMFullEpochLifecycle(t *testing.T) { bitmask = []byte{7} currentTime = startTime.Add(time.Second + 6*time.Millisecond) - tc.blockBuilder.block = nextBlock(6) + tc.blockBuilder.Block = nextBlock(6) md = common.ProtocolMetadata{Seq: baseSeq + 6, Round: 5, Epoch: testCase.epochNum, Prev: block5.Digest()} block6, err := sm.BuildBlock(context.Background(), md, nil) require.NoError(t, err) @@ -864,7 +864,7 @@ func TestMSMFullEpochLifecycle(t *testing.T) { subTestCase.setup() - tc.blockBuilder.block = nextBlock(7) + tc.blockBuilder.Block = nextBlock(7) md = common.ProtocolMetadata{Seq: baseSeq + 7, Round: 6, Epoch: testCase.epochNum, Prev: block6.Digest()} // If the sealing block isn't finalized yet, we expect to build a Telock. @@ -1085,7 +1085,7 @@ func TestVerifyNextPChainRefHeightNormal(t *testing.T) { tt.setup(tc) } - err := sm.verifyNextPChainRefHeightNormal(prevMD, tt.next) + err := sm.verifyNextPChainRefHeightNormal(t.Context(), prevMD, tt.next) if tt.err == nil { require.NoError(t, err) return @@ -1487,16 +1487,12 @@ func TestBuildBlockCollectingApprovalsDedupsOwnApprovalAcrossRounds(t *testing.T } tc.validatorSetRetriever.result = validators - // No peer approvals on either round — only the internal optimistic - // self-sign is contributed. - tc.approvalsRetriever.result = nil - // Parent block: epoch transition has started (NextPChainReferenceHeight > 0) // but no approvals have been collected yet. NextState() returns // stateBuildCollectingApprovals. parentSeq := uint64(10) parent := StateMachineBlock{ - InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + InnerBlock: &testutil.InnerBlock{TS: time.Now(), BlockHeight: 1, Content: []byte{0xAA}}, Metadata: StateMachineMetadata{ PChainHeight: 200, SimplexProtocolMetadata: (&common.ProtocolMetadata{ @@ -1513,7 +1509,7 @@ func TestBuildBlockCollectingApprovalsDedupsOwnApprovalAcrossRounds(t *testing.T tc.blockStore[parentSeq] = &outerBlock{block: parent} // ----- Round 1: first collecting-approvals block ----- - tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: 2, Bytes: []byte{0x01}} + tc.blockBuilder.Block = &testutil.InnerBlock{TS: time.Now(), BlockHeight: 2, Content: []byte{0x01}} md1 := common.ProtocolMetadata{Seq: parentSeq + 1, Round: 6, Epoch: 1, Prev: parent.Digest()} block1, err := sm.BuildBlock(context.Background(), md1, nil) require.NoError(t, err) @@ -1529,7 +1525,7 @@ func TestBuildBlockCollectingApprovalsDedupsOwnApprovalAcrossRounds(t *testing.T tc.blockStore[md1.Seq] = &outerBlock{block: *block1} // ----- Round 2: another collecting-approvals block, still no peer approvals ----- - tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: 3, Bytes: []byte{0x02}} + tc.blockBuilder.Block = &testutil.InnerBlock{TS: time.Now(), BlockHeight: 3, Content: []byte{0x02}} md2 := common.ProtocolMetadata{Seq: md1.Seq + 1, Round: 7, Epoch: 1, Prev: block1.Digest()} block2, err := sm.BuildBlock(context.Background(), md2, nil) require.NoError(t, err) @@ -1562,13 +1558,13 @@ func TestVerifyCollectingApprovalsNotReady(t *testing.T) { sm, tc := newStateMachine(t) // Default app (AuxiliaryInfoGenVerifier) for newStateMachine has threshold 2 // so IsSufficient returns false: the aux info is not ready. - sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } - sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } + sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } // Parent block: epoch transition in progress (NextPChainReferenceHeight > 0), // not yet sealed, so NextState() is stateBuildCollectingApprovals. parent := StateMachineBlock{ - InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + InnerBlock: &testutil.InnerBlock{TS: time.Now(), BlockHeight: 1, Content: []byte{0xAA}}, Metadata: StateMachineMetadata{ PChainHeight: nextPChainRefHeight, SimplexProtocolMetadata: (&common.ProtocolMetadata{ @@ -1588,7 +1584,7 @@ func TestVerifyCollectingApprovalsNotReady(t *testing.T) { } build := func(t *testing.T, sm *StateMachine, tc *testConfig, parent StateMachineBlock) *StateMachineBlock { - tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: 2, Bytes: []byte{0x01}} + tc.blockBuilder.Block = &testutil.InnerBlock{TS: time.Now(), BlockHeight: 2, Content: []byte{0x01}} md := common.ProtocolMetadata{Seq: parentSeq + 1, Round: 6, Epoch: 1, Prev: parent.Digest()} block, err := sm.BuildBlock(context.Background(), md, nil) require.NoError(t, err) @@ -1646,8 +1642,8 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { ) sm, tc := newStateMachine(t) - sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } - sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } + sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } // Deterministic votes so the built auxiliary info is predictable. vote1 := []byte("vote-1") @@ -1673,7 +1669,7 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { tc.validatorSetRetriever.result = validators parent := StateMachineBlock{ - InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + InnerBlock: &testutil.InnerBlock{TS: time.Now(), BlockHeight: 1, Content: []byte{0xAA}}, Metadata: StateMachineMetadata{ PChainHeight: nextPChainRefHeight, SimplexProtocolMetadata: (&common.ProtocolMetadata{ @@ -1692,7 +1688,7 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { // build constructs the next collecting block on top of prev, stores it so it can serve // as a parent (and as a back-pointer target for the aux info history), and verifies it. build := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { - tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: seq, Bytes: []byte{byte(seq)}} + tc.blockBuilder.Block = &testutil.InnerBlock{TS: time.Now(), BlockHeight: seq, Content: []byte{byte(seq)}} md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} block, err := sm.BuildBlock(context.Background(), md, nil) require.NoError(t, err) @@ -1771,8 +1767,8 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { ) sm, tc := newStateMachine(t) - sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } - sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } + sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } // threshold 4 so Generate() runs for the first three collecting blocks built on top of the // pre-seeded parent (history not yet sufficient), giving us one "first" and several "later" @@ -1797,7 +1793,7 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { // The parent already carries auxiliary info for this epoch, stamped with VersionID 1. // This is the backward-compatibility precondition: the epoch's VersionID is already set. parent := StateMachineBlock{ - InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + InnerBlock: &testutil.InnerBlock{TS: time.Now(), BlockHeight: 1, Content: []byte{0xAA}}, Metadata: StateMachineMetadata{ PChainHeight: nextPChainRefHeight, SimplexProtocolMetadata: (&common.ProtocolMetadata{ @@ -1821,7 +1817,7 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { // buildAndVerify constructs the next collecting block on top of prev, verifies it, and stores it so it // can serve as the next parent (and as a back-pointer target for the aux info history). buildAndVerify := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { - tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: seq, Bytes: []byte{byte(seq)}} + tc.blockBuilder.Block = &testutil.InnerBlock{TS: time.Now(), BlockHeight: seq, Content: []byte{byte(seq)}} md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} block, err := sm.BuildBlock(context.Background(), md, nil) require.NoError(t, err) diff --git a/msm/util_test.go b/msm/util_test.go index 6a1dc8ea..1f4402b2 100644 --- a/msm/util_test.go +++ b/msm/util_test.go @@ -42,11 +42,15 @@ func init() { type InnerBlock struct { TS time.Time BlockHeight uint64 - Bytes []byte + bytes []byte +} + +func (i *InnerBlock) Bytes() ([]byte, error) { + return i.bytes, nil } func (i *InnerBlock) Digest() [32]byte { - return sha256.Sum256(i.Bytes) + return sha256.Sum256(i.bytes) } func (i *InnerBlock) Height() uint64 { @@ -66,6 +70,10 @@ type fakeVMBlock struct { height uint64 } +func (f *fakeVMBlock) Bytes() ([]byte, error) { + panic("implement me") +} + func (f *fakeVMBlock) Digest() [32]byte { return [32]byte{} } func (f *fakeVMBlock) Height() uint64 { return f.height } func (f *fakeVMBlock) Timestamp() time.Time { return time.Time{} } @@ -92,14 +100,6 @@ func (bs blockStore) getBlock(seq uint64, _ [32]byte) (StateMachineBlock, *commo return blk.block, blk.finalization, nil } -type approvalsRetriever struct { - result ValidatorSetApprovals -} - -func (a approvalsRetriever) Approvals() ValidatorSetApprovals { - return a.result -} - type signer struct { } @@ -206,19 +206,6 @@ func (n *noOpPChainListener) WaitForProgress(ctx context.Context, _ uint64) erro return ctx.Err() } -type blockBuilder struct { - block VMBlock - err error -} - -func (bb *blockBuilder) WaitForPendingBlock(_ context.Context) { - // Block is always ready in tests. -} - -func (bb *blockBuilder) BuildBlock(_ context.Context, _ uint64) (VMBlock, error) { - return bb.block, bb.err -} - type validatorSetRetriever struct { result NodeBLSMappings resultMap map[uint64]NodeBLSMappings @@ -247,21 +234,13 @@ func (ka *keyAggregator) AggregateKeys(keys ...[]byte) ([]byte, error) { var ( genesisBlock = StateMachineBlock{ // Genesis block metadata has all zero values - InnerBlock: &InnerBlock{ - TS: time.Now(), - Bytes: []byte{1, 2, 3}, + InnerBlock: &testutil.InnerBlock{ + TS: time.Now(), + Content: []byte{1, 2, 3}, }, } ) -type dynamicApprovalsRetriever struct { - approvals *ValidatorSetApprovals -} - -func (d *dynamicApprovalsRetriever) Approvals() ValidatorSetApprovals { - return *d.approvals -} - func makeChain(t *testing.T, simplexStartHeight uint64, endHeight uint64) []StateMachineBlock { startTime := time.Now().Add(-time.Duration(endHeight+2) * time.Second) blocks := make([]StateMachineBlock, 0, endHeight+1) @@ -301,7 +280,7 @@ func makeNormalSimplexBlock(t *testing.T, index int, blocks []StateMachineBlock, InnerBlock: &InnerBlock{ TS: start.Add(time.Duration(h) * time.Second), BlockHeight: h, - Bytes: []byte{1, 2, 3}, + bytes: []byte{1, 2, 3}, }, Metadata: StateMachineMetadata{ PChainHeight: 100, @@ -330,21 +309,32 @@ func makeNonSimplexBlock(t *testing.T, startHeight uint64, start time.Time, h ui InnerBlock: &InnerBlock{ TS: start.Add(time.Duration(h-startHeight) * time.Second), BlockHeight: h, - Bytes: []byte{1, 2, 3}, + bytes: []byte{1, 2, 3}, }, } } type testConfig struct { blockStore blockStore - approvalsRetriever approvalsRetriever signatureVerifier signatureVerifier signatureAggregator signatureAggregator - blockBuilder blockBuilder + blockBuilder mockBlockBuilder keyAggregator keyAggregator validatorSetRetriever validatorSetRetriever } +// mockBlockBuilder is a test BlockBuilder whose BuildBlock returns a preconfigured block/error. +type mockBlockBuilder struct { + Block VMBlock + Err error +} + +func (bb *mockBlockBuilder) BuildBlock(context.Context, uint64) (VMBlock, error) { + return bb.Block, bb.Err +} + +func (bb *mockBlockBuilder) WaitForPendingBlock(context.Context) {} + func newStateMachine(t *testing.T) (*StateMachine, *testConfig) { return newStateMachineWithLogger(t, testutil.MakeLogger(t)) } @@ -373,11 +363,11 @@ func newStateMachineWithLogger(tb testing.TB, logger common.Logger) (*StateMachi SignatureAggregatorCreator: newSignatureAggregatorCreator(), BlockBuilder: &testConfig.blockBuilder, KeyAggregator: &testConfig.keyAggregator, - GetPChainHeightForProposing: func() uint64 { - return 100 + GetPChainHeightForProposing: func(context.Context) (uint64, error) { + return 100, nil }, - GetPChainHeightForVerifying: func() uint64 { - return 100 + GetPChainHeightForVerifying: func(context.Context) (uint64, error) { + return 100, nil }, GetValidatorSet: testConfig.validatorSetRetriever.getValidatorSet, PChainProgressListener: &noOpPChainListener{}, diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index e890710e..d198b6a4 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -16,6 +16,10 @@ import ( "go.uber.org/zap" ) +const ( + DefaultMaxSequenceWindow = 100 +) + type finalizedSeq struct { block common.Block finalization *common.Finalization @@ -144,6 +148,8 @@ func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) er return nil } + n.Config.Logger.Debug("Received a message", zap.Any("Message", msg), zap.Stringer("from", from)) + if n.haltedError != nil { return n.haltedError } diff --git a/testutil/block.go b/testutil/block.go index 7d067870..8df92e5a 100644 --- a/testutil/block.go +++ b/testutil/block.go @@ -9,6 +9,7 @@ import ( "crypto/sha256" "encoding/asn1" "fmt" + "time" "github.com/ava-labs/simplex/common" ) @@ -149,3 +150,29 @@ func (b *BlockDeserializer) DeserializeBlock(ctx context.Context, buff []byte) ( return &tb, nil } + +type InnerBlock struct { + TS time.Time + BlockHeight uint64 + Content []byte +} + +func (i *InnerBlock) Bytes() ([]byte, error) { + return i.Content, nil +} + +func (i *InnerBlock) Digest() [32]byte { + return sha256.Sum256(i.Content) +} + +func (i *InnerBlock) Height() uint64 { + return i.BlockHeight +} + +func (i *InnerBlock) Timestamp() time.Time { + return i.TS +} + +func (i *InnerBlock) Verify(_ context.Context, _ uint64) error { + return nil +} diff --git a/testutil/util.go b/testutil/util.go index 045d4e88..b4694d86 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -30,7 +30,7 @@ func DefaultTestNodeEpochConfig(t *testing.T, nodeID common.NodeID, comm common. ID: nodeID, Signer: &TestSigner{}, WAL: wal, - Verifier: &testVerifier{}, + Verifier: &TestVerifier{}, Storage: storage, BlockBuilder: bb, SignatureAggregatorCreator: func(weights []common.Node) common.SignatureAggregator { @@ -184,14 +184,14 @@ func (t *TestSigner) Sign([]byte) ([]byte, error) { return []byte{1, 2, 3}, nil } -type testVerifier struct { +type TestVerifier struct { } -func (t *testVerifier) VerifyBlock(common.VerifiedBlock) error { +func (t *TestVerifier) VerifyBlock(common.VerifiedBlock) error { return nil } -func (t *testVerifier) Verify(_ []byte, _ []byte, pk []byte) error { +func (t *TestVerifier) VerifySignature(_ []byte, _ []byte, pk []byte) error { return nil } From 3d2625bf30e12c42cf095e4fd797d289070a0970 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Thu, 9 Jul 2026 17:08:17 +0200 Subject: [PATCH 02/14] Revert P-chain height getters function signature change Signed-off-by: Yacov Manevich --- config.go | 4 ++-- instance_test.go | 8 ++++---- msm/build_decision.go | 12 +++--------- msm/build_decision_test.go | 14 +++++++------- msm/fake_node_test.go | 29 +++++++++++++---------------- msm/fuzz_test.go | 8 ++++---- msm/msm.go | 27 +++++++-------------------- msm/msm_test.go | 32 ++++++++++++++++---------------- msm/util_test.go | 8 ++++---- 9 files changed, 60 insertions(+), 82 deletions(-) diff --git a/config.go b/config.go index b5f9eb99..6a0d3fbe 100644 --- a/config.go +++ b/config.go @@ -27,9 +27,9 @@ type PlatformChain interface { // GenesisValidatorSet returns the first ever validator set for this network. GenesisValidatorSet() metadata.NodeBLSMappings // GetMinimumHeight returns the minimum height of the block still in the proposal window. - GetMinimumHeight(context.Context) (uint64, error) + GetMinimumHeight() uint64 // GetCurrentHeight returns the current height of the P-chain. - GetCurrentHeight(context.Context) (uint64, error) + GetCurrentHeight() uint64 // WaitForProgress should block until either the context is cancelled, or the P-chain height has increased from the provided pChainHeight. WaitForProgress(ctx context.Context, pChainHeight uint64) error // LastNonSimplexBlockPChainHeight returns the P-chain height of the last non-simplex block in the chain. diff --git a/instance_test.go b/instance_test.go index 09522bd9..c196f624 100644 --- a/instance_test.go +++ b/instance_test.go @@ -348,12 +348,12 @@ func (pc *testPlatformChain) GenesisValidatorSet() metadata.NodeBLSMappings { return pc.validatorSet(pc.baseHeight) } -func (pc *testPlatformChain) GetMinimumHeight(context.Context) (uint64, error) { - return pc.currentHeight(), nil +func (pc *testPlatformChain) GetMinimumHeight() uint64 { + return pc.currentHeight() } -func (pc *testPlatformChain) GetCurrentHeight(context.Context) (uint64, error) { - return pc.currentHeight(), nil +func (pc *testPlatformChain) GetCurrentHeight() uint64 { + return pc.currentHeight() } func (pc *testPlatformChain) WaitForProgress(ctx context.Context, pChainHeight uint64) error { diff --git a/msm/build_decision.go b/msm/build_decision.go index e17d393d..e3dbe672 100644 --- a/msm/build_decision.go +++ b/msm/build_decision.go @@ -35,7 +35,7 @@ type blockBuildingDecider struct { // hasValidatorSetChanged should return whether the validator set has changed since the // P-chain height referenced by the last block in the chain and until the provided P-chain height. hasValidatorSetChanged func(pChainHeight uint64) (bool, NodeBLSMappings, error) - getPChainHeight func(ctx context.Context) (uint64, error) + getPChainHeight func() uint64 } // shouldBuildBlock determines whether we should build a block at the current time, @@ -47,10 +47,7 @@ func (bbd *blockBuildingDecider) shouldBuildBlock( ctx context.Context, ) (blockBuildingDecision, error) { for { - pChainHeight, err := bbd.getPChainHeight(ctx) - if err != nil { - return blockBuildingDecision{}, err - } + pChainHeight := bbd.getPChainHeight() shouldTransitionEpoch, newValidatorSet, err := bbd.hasValidatorSetChanged(pChainHeight) if err != nil { @@ -75,10 +72,7 @@ func (bbd *blockBuildingDecider) shouldBuildBlock( // If the P-chain height changed, re-evaluate again whether we should transition to a new epoch, // or continue waiting to build a block. - h, err := bbd.getPChainHeight(ctx) - if err != nil { - return blockBuildingDecision{}, err - } + h := bbd.getPChainHeight() if h != pChainHeight { continue } diff --git a/msm/build_decision_test.go b/msm/build_decision_test.go index b5400584..6b34b067 100644 --- a/msm/build_decision_test.go +++ b/msm/build_decision_test.go @@ -32,7 +32,7 @@ func TestShouldBuildBlock_VMSignalsBlock(t *testing.T) { }, waitForPendingBlock: func(ctx context.Context) {}, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return false, nil, nil }, - getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, + getPChainHeight: func() uint64 { return 100 }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -56,7 +56,7 @@ func TestShouldBuildBlock_ContextCanceled(t *testing.T) { <-ctx.Done() }, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return false, nil, nil }, - getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, + getPChainHeight: func() uint64 { return 100 }, } decision, err := bbd.shouldBuildBlock(ctx) @@ -88,7 +88,7 @@ func TestShouldBuildBlock_PChainHeightChangeTriggersEpochTransition(t *testing.T hasValidatorSetChanged: func(height uint64) (bool, NodeBLSMappings, error) { return height == 200, nil, nil }, - getPChainHeight: func(context.Context) (uint64, error) { return pChainHeight.Load(), nil }, + getPChainHeight: func() uint64 { return pChainHeight.Load() }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -123,7 +123,7 @@ func TestShouldBuildBlock_PChainHeightChangeButNoEpochTransition(t *testing.T) { } }, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return false, nil, nil }, - getPChainHeight: func(context.Context) (uint64, error) { return pChainHeight.Load(), nil }, + getPChainHeight: func() uint64 { return pChainHeight.Load() }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -142,7 +142,7 @@ func TestShouldBuildBlock_EpochTransitionWithVMBlock(t *testing.T) { }, waitForPendingBlock: func(ctx context.Context) {}, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return true, nil, nil }, - getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, + getPChainHeight: func() uint64 { return 100 }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -163,7 +163,7 @@ func TestShouldBuildBlock_EpochTransitionWithoutVMBlock(t *testing.T) { <-ctx.Done() }, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return true, nil, nil }, - getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, + getPChainHeight: func() uint64 { return 100 }, } decision, err := bbd.shouldBuildBlock(t.Context()) @@ -188,7 +188,7 @@ func TestShouldBuildBlock_EpochTransitionContextCanceled(t *testing.T) { <-ctx.Done() }, hasValidatorSetChanged: func(uint64) (bool, NodeBLSMappings, error) { return true, nil, nil }, - getPChainHeight: func(context.Context) (uint64, error) { return 100, nil }, + getPChainHeight: func() uint64 { return 100 }, } decision, err := bbd.shouldBuildBlock(ctx) diff --git a/msm/fake_node_test.go b/msm/fake_node_test.go index 07922e70..f81edd10 100644 --- a/msm/fake_node_test.go +++ b/msm/fake_node_test.go @@ -38,11 +38,11 @@ func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { node.sm.MyNodeID = myID[:] node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet - node.sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { - return pChainHeight.Load(), nil + node.sm.GetPChainHeightForProposing = func() uint64 { + return pChainHeight.Load() } - node.sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { - return pChainHeight.Load(), nil + node.sm.GetPChainHeightForVerifying = func() uint64 { + return pChainHeight.Load() } node.mempoolEmpty = true node.sm.MaxBlockBuildingWaitTime = 100 * time.Millisecond @@ -93,11 +93,11 @@ func TestFakeNode(t *testing.T) { node.sm.MyNodeID = myID[:] node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet - node.sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { - return pChainHeight.Load(), nil + node.sm.GetPChainHeightForProposing = func() uint64 { + return pChainHeight.Load() } - node.sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { - return pChainHeight.Load(), nil + node.sm.GetPChainHeightForVerifying = func() uint64 { + return pChainHeight.Load() } // Create some blocks and finalize them, until we reach height 10 @@ -158,11 +158,11 @@ func TestFakeNodeEmptyMempool(t *testing.T) { node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.MaxBlockBuildingWaitTime = 100 * time.Millisecond node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet - node.sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { - return pChainHeight, nil + node.sm.GetPChainHeightForProposing = func() uint64 { + return pChainHeight } - node.sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { - return pChainHeight, nil + node.sm.GetPChainHeightForVerifying = func() uint64 { + return pChainHeight } // Create some blocks and finalize them, until we reach height 10 @@ -247,10 +247,7 @@ func (fn *fakeNode) WaitForProgress(ctx context.Context, pChainHeight uint64) er case <-ctx.Done(): return ctx.Err() case <-time.After(10 * time.Millisecond): - height, err := fn.sm.GetPChainHeightForProposing(ctx) - if err != nil { - return err - } + height := fn.sm.GetPChainHeightForProposing() if height != pChainHeight { return nil } diff --git a/msm/fuzz_test.go b/msm/fuzz_test.go index 11cc4de9..d79dbeb9 100644 --- a/msm/fuzz_test.go +++ b/msm/fuzz_test.go @@ -105,8 +105,8 @@ func FuzzVerifyBlock(f *testing.F) { // Model the verifier as knowing the P-chain exactly up to the height the block // references — the minimal knowledge required to verify it, mirroring production // where GetPChainHeightForVerifying returns the verifier's latest observed height. - sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return block.Metadata.PChainHeight, nil } - sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return block.Metadata.PChainHeight, nil } + sm.GetPChainHeightForProposing = func() uint64 { return block.Metadata.PChainHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return block.Metadata.PChainHeight } // The unfuzzed block built by the chain MSM must verify. require.NoError(t, sm.VerifyBlock(context.Background(), block), @@ -195,8 +195,8 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, // round (the builder only collects approvals once the aux info history is ready). sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} sm.GetValidatorSet = getValidatorSet - sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return currentPChainHeight, nil } - sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return currentPChainHeight, nil } + sm.GetPChainHeightForProposing = func() uint64 { return currentPChainHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return currentPChainHeight } sm.GetTime = func() time.Time { return currentTime } sm.GenesisValidatorSet = validatorSet1 sm.LastNonSimplexBlockPChainHeight = pChainHeight1 diff --git a/msm/msm.go b/msm/msm.go index b0ab2257..0a57acc1 100644 --- a/msm/msm.go +++ b/msm/msm.go @@ -207,9 +207,9 @@ type Config struct { // GetTime returns the current time. GetTime func() time.Time // GetPChainHeightForProposing returns the latest known P-chain height to be used when building a block. - GetPChainHeightForProposing func(context.Context) (uint64, error) + GetPChainHeightForProposing func() uint64 // GetPChainHeightForVerifying returns the latest known P-chain height to be used when verifying a block. - GetPChainHeightForVerifying func(context.Context) (uint64, error) + GetPChainHeightForVerifying func() uint64 // BlockBuilder builds new VM blocks. BlockBuilder BlockBuilder // Logger is used for logging state machine operations. @@ -407,10 +407,7 @@ func (sm *StateMachine) verifyNonZeroBlock(ctx context.Context, block, prevBlock return fmt.Errorf("failed to verify timestamp: %w", err) } - currentPChainHeight, err := sm.GetPChainHeightForVerifying(ctx) - if err != nil { - return fmt.Errorf("failed to get current P-chain height for verifying: %w", err) - } + currentPChainHeight := sm.GetPChainHeightForVerifying() prevPChainHeight := prevBlockMD.PChainHeight proposedPChainHeight := block.Metadata.PChainHeight @@ -418,8 +415,7 @@ func (sm *StateMachine) verifyNonZeroBlock(ctx context.Context, block, prevBlock return fmt.Errorf("failed to verify P-chain height: %w", err) } - err = sm.verifyEpochNumber(block) - if err != nil { + if err := sm.verifyEpochNumber(block); err != nil { return err } @@ -639,10 +635,7 @@ func (sm *StateMachine) verifyNextPChainRefHeightNormal(ctx context.Context, pre } // Make sure we have reached the next P-chain reference height, otherwise we won't be able to validate it. - pChainHeight, err := sm.GetPChainHeightForVerifying(ctx) - if err != nil { - return fmt.Errorf("failed to get current P-chain height for verifying: %w", err) - } + pChainHeight := sm.GetPChainHeightForVerifying() if pChainHeight < next.NextPChainReferenceHeight { return fmt.Errorf("%w: target %d, current %d", errPChainHeightNotReached, next.NextPChainReferenceHeight, pChainHeight) @@ -701,10 +694,7 @@ func (sm *StateMachine) verifyNextPChainRefHeightForNewEpoch(ctx context.Context // If we haven't reached this P-chain height yet, we cannot accept the next P-chain reference height, // because there is no way of querying the validator set for the next P-chain reference height. - pChainHeight, err := sm.GetPChainHeightForVerifying(ctx) - if err != nil { - return fmt.Errorf("failed to get current P-chain height for verifying: %w", err) - } + pChainHeight := sm.GetPChainHeightForVerifying() if pChainHeight < next.NextPChainReferenceHeight { return fmt.Errorf("%w: target %d, current %d", errPChainHeightNotReached, next.NextPChainReferenceHeight, pChainHeight) } @@ -1433,10 +1423,7 @@ func (sm *StateMachine) verifyBlockEpochSealed(ctx context.Context, parentBlock // the proposed pchain height and (optional) next pchain reference height, mirroring // what buildBlockOrTransitionEpoch does on the build side. proposedPChainHeight := nextBlock.Metadata.PChainHeight - currentPChainHeight, err := sm.GetPChainHeightForVerifying(ctx) - if err != nil { - return fmt.Errorf("failed to get current P-chain height for verifying: %w", err) - } + currentPChainHeight := sm.GetPChainHeightForVerifying() prevPChainHeight := parentBlock.Metadata.PChainHeight if err := verifyPChainHeight(proposedPChainHeight, currentPChainHeight, prevPChainHeight); err != nil { return fmt.Errorf("failed to verify P-chain height: %w", err) diff --git a/msm/msm_test.go b/msm/msm_test.go index 053e8834..1e090b73 100644 --- a/msm/msm_test.go +++ b/msm/msm_test.go @@ -327,8 +327,8 @@ func TestMSMNormalOp(t *testing.T) { tc.validatorSetRetriever.resultMap = map[uint64]NodeBLSMappings{ newPChainHeight: newValidatorSet, } - sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return newPChainHeight, nil } - sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return newPChainHeight, nil } + sm.GetPChainHeightForProposing = func() uint64 { return newPChainHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return newPChainHeight } }, expectedPChainHeight: newPChainHeight, expectedNextPChainRefHeight: newPChainHeight, @@ -511,11 +511,11 @@ func TestMSMFullEpochLifecycle(t *testing.T) { // getVerifyingPChainHeight stays ahead of it, modelling the more up-to-date height // the verifier checks against. const pChainHeightLag = uint64(50) - getProposingPChainHeight := func(context.Context) (uint64, error) { - return currentPChainHeight, nil + getProposingPChainHeight := func() uint64 { + return currentPChainHeight } - getVerifyingPChainHeight := func(context.Context) (uint64, error) { - return currentPChainHeight + pChainHeightLag, nil + getVerifyingPChainHeight := func() uint64 { + return currentPChainHeight + pChainHeightLag } // Since we explicitly compare the built block with an expected value, @@ -577,14 +577,14 @@ func TestMSMFullEpochLifecycle(t *testing.T) { // Each one fails the test if it consults the P-chain height function it must not use, // proving that building reads GetPChainHeightForProposing and verifying reads GetPChainHeightForVerifying. sm.GetPChainHeightForProposing = getProposingPChainHeight - sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { + sm.GetPChainHeightForVerifying = func() uint64 { require.FailNow(t, "builder must not use GetPChainHeightForVerifying when proposing") - return 0, nil + return 0 } - smVerify.GetPChainHeightForProposing = func(context.Context) (uint64, error) { + smVerify.GetPChainHeightForProposing = func() uint64 { require.FailNow(t, "verifier must not use GetPChainHeightForProposing when verifying") - return 0, nil + return 0 } smVerify.GetPChainHeightForVerifying = getVerifyingPChainHeight @@ -1558,8 +1558,8 @@ func TestVerifyCollectingApprovalsNotReady(t *testing.T) { sm, tc := newStateMachine(t) // Default app (AuxiliaryInfoGenVerifier) for newStateMachine has threshold 2 // so IsSufficient returns false: the aux info is not ready. - sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } - sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } // Parent block: epoch transition in progress (NextPChainReferenceHeight > 0), // not yet sealed, so NextState() is stateBuildCollectingApprovals. @@ -1642,8 +1642,8 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { ) sm, tc := newStateMachine(t) - sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } - sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } // Deterministic votes so the built auxiliary info is predictable. vote1 := []byte("vote-1") @@ -1767,8 +1767,8 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { ) sm, tc := newStateMachine(t) - sm.GetPChainHeightForVerifying = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } - sm.GetPChainHeightForProposing = func(context.Context) (uint64, error) { return nextPChainRefHeight, nil } + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } // threshold 4 so Generate() runs for the first three collecting blocks built on top of the // pre-seeded parent (history not yet sufficient), giving us one "first" and several "later" diff --git a/msm/util_test.go b/msm/util_test.go index 1f4402b2..4388f359 100644 --- a/msm/util_test.go +++ b/msm/util_test.go @@ -363,11 +363,11 @@ func newStateMachineWithLogger(tb testing.TB, logger common.Logger) (*StateMachi SignatureAggregatorCreator: newSignatureAggregatorCreator(), BlockBuilder: &testConfig.blockBuilder, KeyAggregator: &testConfig.keyAggregator, - GetPChainHeightForProposing: func(context.Context) (uint64, error) { - return 100, nil + GetPChainHeightForProposing: func() uint64 { + return 100 }, - GetPChainHeightForVerifying: func(context.Context) (uint64, error) { - return 100, nil + GetPChainHeightForVerifying: func() uint64 { + return 100 }, GetValidatorSet: testConfig.validatorSetRetriever.getValidatorSet, PChainProgressListener: &noOpPChainListener{}, From fcef0a994373a9c34c55c2f17c4642d6d6676a09 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Thu, 9 Jul 2026 17:27:06 +0200 Subject: [PATCH 03/14] Remove artificial silence to sealed epoch Signed-off-by: Yacov Manevich --- adapters.go | 66 ----------------------------------------------------- instance.go | 13 ++--------- 2 files changed, 2 insertions(+), 77 deletions(-) diff --git a/adapters.go b/adapters.go index 88c6f689..04182ad4 100644 --- a/adapters.go +++ b/adapters.go @@ -13,47 +13,7 @@ import ( metadata "github.com/ava-labs/simplex/msm" ) -// epochChangeSupression is used to suppress sending messages during an epoch change or using the VM. -// The motivation is that an epoch change occurrs during indexing a block, and we don't want any further -// external side effects to take place before the epoch change finishes. -// We therefore drop all messages and cancel VM operations such as block building, -// and delay VM transaction listening until the epoch change is complete. -type epochChangeSupression struct { - lock sync.RWMutex - sealingBlockSeq uint64 - active bool -} - -func (ecs *epochChangeSupression) isSupressionActive() bool { - ecs.lock.RLock() - defer ecs.lock.RUnlock() - return ecs.active -} - -func (ecs *epochChangeSupression) sendProhibited(seq uint64) bool { - ecs.lock.RLock() - defer ecs.lock.RUnlock() - if !ecs.active { - return false - } - return seq > ecs.sealingBlockSeq -} - -func (ecs *epochChangeSupression) setSupression(sealingBlockSeq uint64) { - ecs.lock.Lock() - defer ecs.lock.Unlock() - ecs.sealingBlockSeq = sealingBlockSeq - ecs.active = true -} - -func (ecs *epochChangeSupression) clearSupression() { - ecs.lock.Lock() - defer ecs.lock.Unlock() - ecs.active = false -} - type Communication struct { - epochChangeSupression *epochChangeSupression nodes atomic.Value // common.Nodes Sender Broadcaster @@ -71,22 +31,6 @@ func (c *Communication) Validators() common.Nodes { return nodes } -func (c *Communication) Broadcast(msg *common.Message) { - if c.epochChangeSupression.sendProhibited(msg.Seq()) { - return - } - - c.Broadcaster.Broadcast(msg) -} - -func (c *Communication) Send(msg *common.Message, destination common.NodeID) { - if c.epochChangeSupression.sendProhibited(msg.Seq()) { - return - } - - c.Sender.Send(msg, destination) -} - // EpochAwareStorage is a wrapper around Storage that is aware of epoch changes. // Upon an epoch change, it will ignore blocks from previous epochs // and will call the OnEpochChange callback when a new epoch is detected. @@ -242,7 +186,6 @@ func (n *NoopAuxiliaryInfoApp) DefaultVersionID() metadata.VersionID { } type BlockBuilderWaiter struct { - epochChangeSupression *epochChangeSupression lock sync.Mutex cancel context.CancelFunc msm *metadata.StateMachine @@ -259,11 +202,6 @@ func (bw *BlockBuilderWaiter) stop() { } func (bw *BlockBuilderWaiter) WaitForPendingBlock(ctx context.Context) { - if bw.epochChangeSupression.isSupressionActive() { - <-ctx.Done() // We wait for the context to be cancelled once the epoch is tore down. - return - } - bw.lock.Lock() if bw.cancel != nil { bw.cancel() @@ -276,10 +214,6 @@ func (bw *BlockBuilderWaiter) WaitForPendingBlock(ctx context.Context) { } func (bw *BlockBuilderWaiter) BuildBlock(ctx context.Context, metadata common.ProtocolMetadata, blacklist common.Blacklist) (common.VerifiedBlock, bool) { - if bw.epochChangeSupression.isSupressionActive() { - return nil, false - } - block, err := bw.msm.BuildBlock(ctx, metadata, &blacklist) if err != nil { return nil, false diff --git a/instance.go b/instance.go index 354acc1f..8e583417 100644 --- a/instance.go +++ b/instance.go @@ -74,7 +74,6 @@ type Instance struct { epochOrNV timeAdvancer epochChanges chan epochChange stopCh chan struct{} - epochChangeSupression epochChangeSupression } func (i *Instance) Start(ctx context.Context) error { @@ -151,14 +150,13 @@ func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.N return nonvalidator.Config{}, err } - comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster, epochChangeSupression: &i.epochChangeSupression} + comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster} comm.SetValidators(validators) epochAwareStorage := &EpochAwareStorage{ Epoch: epochNum, Storage: i.Config.Storage, OnEpochChange: func(epoch uint64, validators common.Nodes) error { - i.epochChangeSupression.setSupression(epoch) // The epoch number is also the sealing block sequence. i.notifyEpochChange(epoch, validators, nonValidator) comm.SetValidators(validators) return nil @@ -253,10 +251,6 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error i.lock.Lock() defer i.lock.Unlock() - if i.epochChangeSupression.isSupressionActive() { - return nil - } - switch { case msg.BlockMessage != nil: err := i.wireBlockMessage(msg) @@ -453,7 +447,7 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { return simplex.EpochConfig{}, err } - blockBuilder := &BlockBuilderWaiter{vm: i.Config.VM, msm: msm, epochChangeSupression: &i.epochChangeSupression} + blockBuilder := &BlockBuilderWaiter{vm: i.Config.VM, msm: msm} comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster, epochChangeSupression: &i.epochChangeSupression} comm.SetValidators(nodes) @@ -463,7 +457,6 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { Epoch: epochNum, Storage: i.cs, OnEpochChange: func(epoch uint64, validators common.Nodes) error { - i.epochChangeSupression.setSupression(epoch) // The epoch number is also the sealing block sequence. blockBuilder.stop() comm.SetValidators(validators) i.notifyEpochChange(epoch, validators, validator) @@ -521,8 +514,6 @@ func (i *Instance) transitionEpochNonValidator(epochChange epochChange) { // Stop the non-validator before doing anything else, so that we don't process any more messages while we are changing epochs. i.stopNonValidator() - i.epochChangeSupression.clearSupression() - // First, figure out if I'm still a validator. if i.determineValidatorOrNot(epochChange.validators) { i.Config.Logger.Info("I am now a validator") From 9924691a2d182ad8b5d0f62ee3e6867a79b296c9 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Thu, 9 Jul 2026 17:55:58 +0200 Subject: [PATCH 04/14] Rename structs used in tests Signed-off-by: Yacov Manevich --- adapters_test.go | 61 ------------------------------------------------ instance.go | 4 +--- msm/util_test.go | 10 ++++---- 3 files changed, 6 insertions(+), 69 deletions(-) delete mode 100644 adapters_test.go diff --git a/adapters_test.go b/adapters_test.go deleted file mode 100644 index 117adf9c..00000000 --- a/adapters_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package simplex - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestEpochChangeSupression(t *testing.T) { - t.Run("Inactive by default", func(t *testing.T) { - var ecs epochChangeSupression - require.False(t, ecs.isSupressionActive()) - // When inactive, nothing is prohibited regardless of sequence. - require.False(t, ecs.sendProhibited(0)) - require.False(t, ecs.sendProhibited(100)) - }) - - t.Run("setSupression activates and prohibits sequences after the sealing block", func(t *testing.T) { - var ecs epochChangeSupression - const sealingBlockSeq = uint64(10) - ecs.setSupression(sealingBlockSeq) - - require.True(t, ecs.isSupressionActive()) - - // Sequences up to and including the sealing block are allowed. - require.False(t, ecs.sendProhibited(sealingBlockSeq-1)) - require.False(t, ecs.sendProhibited(sealingBlockSeq)) - - // Sequences after the sealing block are prohibited. - require.True(t, ecs.sendProhibited(sealingBlockSeq+1)) - require.True(t, ecs.sendProhibited(sealingBlockSeq+100)) - }) - - t.Run("clearSupression deactivates and allows all sequences", func(t *testing.T) { - var ecs epochChangeSupression - const sealingBlockSeq = uint64(10) - ecs.setSupression(sealingBlockSeq) - require.True(t, ecs.sendProhibited(sealingBlockSeq+1)) - - ecs.clearSupression() - - require.False(t, ecs.isSupressionActive()) - // Once cleared, previously prohibited sequences are allowed again. - require.False(t, ecs.sendProhibited(sealingBlockSeq+1)) - }) - - t.Run("setSupression overwrites the previous sealing block sequence", func(t *testing.T) { - var ecs epochChangeSupression - ecs.setSupression(5) - require.True(t, ecs.sendProhibited(8)) - - ecs.setSupression(10) - require.True(t, ecs.isSupressionActive()) - // The new, higher sealing block sequence now permits what was previously prohibited. - require.False(t, ecs.sendProhibited(8)) - require.True(t, ecs.sendProhibited(11)) - }) -} diff --git a/instance.go b/instance.go index 8e583417..92d28b69 100644 --- a/instance.go +++ b/instance.go @@ -449,7 +449,7 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { blockBuilder := &BlockBuilderWaiter{vm: i.Config.VM, msm: msm} - comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster, epochChangeSupression: &i.epochChangeSupression} + comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster} comm.SetValidators(nodes) epochAwareStorage := &EpochAwareStorage{ @@ -537,8 +537,6 @@ func (i *Instance) transitionEpochValidator(epochChange epochChange) { i.Config.Logger.Error("Error garbage collecting epoch config on epoch change", zap.Error(err)) } - i.epochChangeSupression.clearSupression() - // First, figure out if I'm still a validator. if !i.determineValidatorOrNot(epochChange.validators) { i.Config.Logger.Info("I am no longer a validator") diff --git a/msm/util_test.go b/msm/util_test.go index 4388f359..911fb017 100644 --- a/msm/util_test.go +++ b/msm/util_test.go @@ -318,22 +318,22 @@ type testConfig struct { blockStore blockStore signatureVerifier signatureVerifier signatureAggregator signatureAggregator - blockBuilder mockBlockBuilder + blockBuilder bb keyAggregator keyAggregator validatorSetRetriever validatorSetRetriever } -// mockBlockBuilder is a test BlockBuilder whose BuildBlock returns a preconfigured block/error. -type mockBlockBuilder struct { +// bb is a test BlockBuilder whose BuildBlock returns a preconfigured block/error. +type bb struct { Block VMBlock Err error } -func (bb *mockBlockBuilder) BuildBlock(context.Context, uint64) (VMBlock, error) { +func (bb *bb) BuildBlock(context.Context, uint64) (VMBlock, error) { return bb.Block, bb.Err } -func (bb *mockBlockBuilder) WaitForPendingBlock(context.Context) {} +func (bb *bb) WaitForPendingBlock(context.Context) {} func newStateMachine(t *testing.T) (*StateMachine, *testConfig) { return newStateMachineWithLogger(t, testutil.MakeLogger(t)) From e6b32cab6703380c31ef44ecba31a21bdfbac361 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Thu, 9 Jul 2026 18:00:09 +0200 Subject: [PATCH 05/14] gofmt Signed-off-by: Yacov Manevich --- adapters.go | 10 +++++----- instance.go | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/adapters.go b/adapters.go index 04182ad4..40621a15 100644 --- a/adapters.go +++ b/adapters.go @@ -14,7 +14,7 @@ import ( ) type Communication struct { - nodes atomic.Value // common.Nodes + nodes atomic.Value // common.Nodes Sender Broadcaster } @@ -186,10 +186,10 @@ func (n *NoopAuxiliaryInfoApp) DefaultVersionID() metadata.VersionID { } type BlockBuilderWaiter struct { - lock sync.Mutex - cancel context.CancelFunc - msm *metadata.StateMachine - vm VM + lock sync.Mutex + cancel context.CancelFunc + msm *metadata.StateMachine + vm VM } func (bw *BlockBuilderWaiter) stop() { diff --git a/instance.go b/instance.go index 92d28b69..a9832111 100644 --- a/instance.go +++ b/instance.go @@ -64,16 +64,16 @@ type timeAdvancer interface { } type Instance struct { - Config Config - lock sync.Mutex - cs *CachedStorage - wal *wal.GarbageCollectedWAL - msm *metadata.StateMachine - e *simplex.Epoch - nv *nonvalidator.NonValidator - epochOrNV timeAdvancer - epochChanges chan epochChange - stopCh chan struct{} + Config Config + lock sync.Mutex + cs *CachedStorage + wal *wal.GarbageCollectedWAL + msm *metadata.StateMachine + e *simplex.Epoch + nv *nonvalidator.NonValidator + epochOrNV timeAdvancer + epochChanges chan epochChange + stopCh chan struct{} } func (i *Instance) Start(ctx context.Context) error { From baadba5b871479b42b757b86ff1f279ac5cb50ee Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Thu, 9 Jul 2026 21:39:11 +0200 Subject: [PATCH 06/14] Address code review comments Signed-off-by: Yacov Manevich --- instance.go | 96 ++++++++++++++++++----------------- nonvalidator/non_validator.go | 4 -- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/instance.go b/instance.go index a9832111..4c487ca9 100644 --- a/instance.go +++ b/instance.go @@ -82,7 +82,7 @@ func (i *Instance) Start(ctx context.Context) error { defer i.lock.Unlock() i.stopCh = make(chan struct{}) - i.epochChanges = make(chan epochChange) + i.epochChanges = make(chan epochChange, 1) context.AfterFunc(ctx, i.Stop) cachedStorage := NewCachedStorage(i.Config.Storage) @@ -97,17 +97,7 @@ func (i *Instance) Start(ctx context.Context) error { return fmt.Errorf("error determining latest epoch and validator set: %w", err) } - iAmValidator := i.determineValidatorOrNot(nodes) - - if iAmValidator { - if err := i.startValidator(); err != nil { - return fmt.Errorf("error starting validator: %w", err) - } - } else { - if err := i.startNonValidator(epochNum, nodes); err != nil { - return fmt.Errorf("error starting non-validator: %w", err) - } - } + i.startValidatorOrNonValidator(nodes, epochNum) go i.tick() go i.listenForEpochChanges() @@ -157,8 +147,19 @@ func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.N Epoch: epochNum, Storage: i.Config.Storage, OnEpochChange: func(epoch uint64, validators common.Nodes) error { - i.notifyEpochChange(epoch, validators, nonValidator) + height := i.Config.PlatformChain.GetCurrentHeight() + vdrs, err := i.Config.PlatformChain.GetValidatorSet(height) + if err != nil { + i.Config.Logger.Error("error getting validator set", zap.Error(err)) + return fmt.Errorf("error getting validator set from platform chain: %w", err) + } comm.SetValidators(validators) + if i.iAmValidator(vdrs.Nodes()) { + i.notifyEpochChange(epoch, validators, nonValidator) + } else { + i.Config.Logger.Debug("I am still a non-validator at the tip of the P-chain, skipping role change", + zap.Uint64("height", height)) + } return nil }, } @@ -179,7 +180,7 @@ func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.N Logger: i.Config.Logger, StartTime: time.Now(), SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, - MaxSequenceWindow: nonvalidator.DefaultMaxSequenceWindow, + MaxSequenceWindow: simplex.DefaultMaxRoundWindow, } return config, nil } @@ -344,9 +345,15 @@ func (i *Instance) listenForEpochChanges() { func (i *Instance) processEpochChange(epochChange epochChange) { switch epochChange.nodeRole { case nonValidator: - i.transitionEpochNonValidator(epochChange) + if err := i.transitionEpochNonValidator(epochChange); err != nil { + i.Config.Logger.Error("Error transitioning epoch for non-validator", zap.Error(err)) + i.Stop() + } case validator: - i.transitionEpochValidator(epochChange) + if err := i.transitionEpochValidator(epochChange); err != nil { + i.Config.Logger.Error("Error transitioning epoch for validator", zap.Error(err)) + i.Stop() + } default: // This should never happen, but we log it just in case. i.Config.Logger.Fatal("Unknown node role on epoch change", zap.String("role", fmt.Sprintf("%v", epochChange.nodeRole))) @@ -381,7 +388,7 @@ func (i *Instance) lastBlock() (metadata.StateMachineBlock, uint64, error) { return lastBlock, numBlocks, nil } -func (i *Instance) determineValidatorOrNot(nodes common.Nodes) bool { +func (i *Instance) iAmValidator(nodes common.Nodes) bool { for _, node := range nodes { if i.Config.ID.Equals(node.Id) { return true @@ -392,6 +399,9 @@ func (i *Instance) determineValidatorOrNot(nodes common.Nodes) bool { func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { lastBlock, numBlocks, err := i.lastBlock() + if err != nil { + return simplex.EpochConfig{}, err + } lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() @@ -507,24 +517,38 @@ func (i *Instance) maybeGarbageCollectWAL(lastBlock metadata.StateMachineBlock) return nil } -func (i *Instance) transitionEpochNonValidator(epochChange epochChange) { +func (i *Instance) transitionEpochNonValidator(epochChange epochChange) error { i.lock.Lock() defer i.lock.Unlock() + if !i.iAmValidator(epochChange.validators) { + i.Config.Logger.Debug("Skipping restarting a non-validator because I am not a validator yet") + return nil + } + // Stop the non-validator before doing anything else, so that we don't process any more messages while we are changing epochs. i.stopNonValidator() - // First, figure out if I'm still a validator. - if i.determineValidatorOrNot(epochChange.validators) { - i.Config.Logger.Info("I am now a validator") - i.startValidator() - return + return i.startValidatorOrNonValidator(epochChange.validators, epochChange.epochNum) +} + +func (i *Instance) startValidatorOrNonValidator(validators common.Nodes, epoch uint64) error { + if i.iAmValidator(validators) { + if err := i.startValidator(); err != nil { + i.Config.Logger.Error("Error starting validator on epoch change", zap.Error(err)) + return err + } + return nil } - i.startNonValidator(epochChange.epochNum, epochChange.validators) + if err := i.startNonValidator(epoch, validators); err != nil { + i.Config.Logger.Error("Error starting non-validator on epoch change", zap.Error(err)) + return err + } + return nil } -func (i *Instance) transitionEpochValidator(epochChange epochChange) { +func (i *Instance) transitionEpochValidator(epochChange epochChange) error { i.lock.Lock() defer i.lock.Unlock() @@ -537,27 +561,7 @@ func (i *Instance) transitionEpochValidator(epochChange epochChange) { i.Config.Logger.Error("Error garbage collecting epoch config on epoch change", zap.Error(err)) } - // First, figure out if I'm still a validator. - if !i.determineValidatorOrNot(epochChange.validators) { - i.Config.Logger.Info("I am no longer a validator") - i.startNonValidator(epochChange.epochNum, epochChange.validators) - return - } - - config, err := i.createEpochConfig() - if err != nil { - i.Config.Logger.Error("Error creating epoch config on epoch change", zap.Error(err)) - return - } - - if config.Epoch != epochChange.epochNum { - i.Config.Logger.Error("Epoch number mismatch on epoch change", zap.Uint64("expected", epochChange.epochNum), zap.Uint64("actual", config.Epoch)) - return - } - - if err := i.startEpoch(config); err != nil { - i.Config.Logger.Error("Error starting new epoch on epoch change", zap.Error(err)) - } + return i.startValidatorOrNonValidator(epochChange.validators, epochChange.epochNum) } func constructEpochAndValidatorSet(lastNonSimplexInnerBlockHeight uint64, genesisValidatorSet metadata.NodeBLSMappings, numBlocks uint64, lastBlock ParsedBlock, storage Storage) (common.Nodes, uint64, error) { diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index d198b6a4..82f36ee8 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -16,10 +16,6 @@ import ( "go.uber.org/zap" ) -const ( - DefaultMaxSequenceWindow = 100 -) - type finalizedSeq struct { block common.Block finalization *common.Finalization From f33677252192aa94d19debb3e7feee538b9592b3 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Thu, 9 Jul 2026 22:02:05 +0200 Subject: [PATCH 07/14] Address code review comments II Signed-off-by: Yacov Manevich --- adapters.go | 12 ++++++------ config.go | 25 ------------------------- instance.go | 27 +++++++++++++++------------ instance_test.go | 1 + 4 files changed, 22 insertions(+), 43 deletions(-) diff --git a/adapters.go b/adapters.go index 40621a15..d61d3fa6 100644 --- a/adapters.go +++ b/adapters.go @@ -33,12 +33,12 @@ func (c *Communication) Validators() common.Nodes { // EpochAwareStorage is a wrapper around Storage that is aware of epoch changes. // Upon an epoch change, it will ignore blocks from previous epochs -// and will call the OnEpochChange callback when a new epoch is detected. +// and will call the onEpochChange callback when a new epoch is detected. type EpochAwareStorage struct { msm *metadata.StateMachine - OnEpochChange func(seq uint64, validators common.Nodes) error + onEpochChange func(seq uint64, validators common.Nodes) error Storage - Epoch uint64 + epoch uint64 } func (e *EpochAwareStorage) Retrieve(seq uint64) (common.VerifiedBlock, common.Finalization, error) { @@ -54,7 +54,7 @@ func (e *EpochAwareStorage) Retrieve(seq uint64) (common.VerifiedBlock, common.F } func (e *EpochAwareStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { - if block.BlockHeader().Epoch < e.Epoch { + if block.BlockHeader().Epoch < e.epoch { // This is a Telock from a previous h, so we ignore it and do not index it. return nil } @@ -62,11 +62,11 @@ func (e *EpochAwareStorage) Index(ctx context.Context, block common.VerifiedBloc return err } if block.SealingBlockInfo() != nil { - if err := e.OnEpochChange(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet); err != nil { + if err := e.onEpochChange(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet); err != nil { return err } // We are now in a new h, so we update the h number to prevent indexing Telocks from the previous h. - e.Epoch = block.BlockHeader().Seq + e.epoch = block.BlockHeader().Seq } return nil } diff --git a/config.go b/config.go index 6a0d3fbe..a212db7c 100644 --- a/config.go +++ b/config.go @@ -9,7 +9,6 @@ import ( "github.com/ava-labs/simplex/common" metadata "github.com/ava-labs/simplex/msm" - "github.com/ava-labs/simplex/wal" ) type ParameterConfig struct { @@ -72,9 +71,6 @@ type Storage interface { // Index indexes the given block and finalization in the storage. Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error - - // CreateWAL creates a new Write-Ahead Log (WAL). - CreateWAL() (wal.DeletableWAL, error) } type CryptoOps interface { @@ -84,24 +80,3 @@ type CryptoOps interface { CreateSignatureAggregator([]common.Node) common.SignatureAggregator DeserializeQuorumCertificate(bytes []byte) (common.QuorumCertificate, error) } - -type MSMConfig struct { - LastNonSimplexInnerBlock metadata.VMBlock - ComputeICMEpoch metadata.ICMEpochTransition -} - -type EpochParams struct { - ID common.NodeID - Sender Sender - Storage Storage - QCDeserializer common.QCDeserializer - BlockDeserializer common.BlockDeserializer - Verifier common.SignatureVerifier - MaxProposalWait time.Duration - MaxRoundWindow uint64 - MaxRebroadcastWait time.Duration - FinalizeRebroadcastTimeout time.Duration - MaxWALSize int - WALCreator wal.Creator - WALs []wal.DeletableWAL -} diff --git a/instance.go b/instance.go index 4c487ca9..70d8701d 100644 --- a/instance.go +++ b/instance.go @@ -31,16 +31,19 @@ type Config struct { ParameterConfig ParameterConfig // PlatformChain is the interface to the P-chain. PlatformChain PlatformChain + // Broadcaster is the interface to broadcast messages to other nodes in the network. + Broadcaster Broadcaster // CryptoOps is the interface to the cryptographic operations needed by the simplex instance. CryptoOps CryptoOps + // WalCreator is the interface to create new write-ahead logs for the simplex instance. + WalCreator wal.Creator // Storage is the interface to the block storage layer for the simplex instance. - Storage Storage - Logger common.Logger - Sender Sender - Broadcaster Broadcaster - WALs []wal.DeletableWAL - VM VM - ID common.NodeID + Storage Storage + Logger common.Logger + Sender Sender + WALs []wal.DeletableWAL + VM VM + ID common.NodeID } type MessageHandler interface { @@ -144,9 +147,9 @@ func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.N comm.SetValidators(validators) epochAwareStorage := &EpochAwareStorage{ - Epoch: epochNum, + epoch: epochNum, Storage: i.Config.Storage, - OnEpochChange: func(epoch uint64, validators common.Nodes) error { + onEpochChange: func(epoch uint64, validators common.Nodes) error { height := i.Config.PlatformChain.GetCurrentHeight() vdrs, err := i.Config.PlatformChain.GetValidatorSet(height) if err != nil { @@ -410,7 +413,7 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { return simplex.EpochConfig{}, err } - wal, err := wal.NewGarbageCollectedWAL(i.Config.WALs, i.Config.Storage.CreateWAL, &common.WALRetentionReader{}, i.Config.ParameterConfig.WALMaxEntryCount) + wal, err := wal.NewGarbageCollectedWAL(i.Config.WALs, i.Config.WalCreator, &common.WALRetentionReader{}, i.Config.ParameterConfig.WALMaxEntryCount) if err != nil { return simplex.EpochConfig{}, fmt.Errorf("error creating garbage collected wal: %w", err) } @@ -464,9 +467,9 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { epochAwareStorage := &EpochAwareStorage{ msm: msm, - Epoch: epochNum, + epoch: epochNum, Storage: i.cs, - OnEpochChange: func(epoch uint64, validators common.Nodes) error { + onEpochChange: func(epoch uint64, validators common.Nodes) error { blockBuilder.stop() comm.SetValidators(validators) i.notifyEpochChange(epoch, validators, validator) diff --git a/instance_test.go b/instance_test.go index c196f624..1e474bb6 100644 --- a/instance_test.go +++ b/instance_test.go @@ -141,6 +141,7 @@ func newInstance(t *testing.T, nodeID common.NodeID, storage *MockStorage, net * PlatformChain: pChain, CryptoOps: cops, LastNonSimplexInnerBlock: genesisBlock, + WalCreator: storage.CreateWAL, ParameterConfig: ParameterConfig{ MaxNetworkDelay: 500 * time.Millisecond, MaxRoundWindow: 100, From 6f52dc3809b489a50368bf9d5d642ddf50bfc568 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Fri, 10 Jul 2026 21:24:55 +0200 Subject: [PATCH 08/14] Address code review comments III Signed-off-by: Yacov Manevich --- common/msg.go | 25 ------------------------- config.go | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/common/msg.go b/common/msg.go index 539a7f90..1c287b11 100644 --- a/common/msg.go +++ b/common/msg.go @@ -40,31 +40,6 @@ func (m *Message) IsReplicationMessage() bool { } } -func (m *Message) Seq() uint64 { - switch { - case m.BlockMessage != nil: - return m.BlockMessage.Block.BlockHeader().Seq - case m.VerifiedBlockMessage != nil: - return m.VerifiedBlockMessage.VerifiedBlock.BlockHeader().Seq - case m.EmptyNotarization != nil: - // Empty notarizations have no sequence, only a round. - return m.EmptyNotarization.Vote.Round - case m.VoteMessage != nil: - return m.VoteMessage.Vote.Seq - case m.EmptyVoteMessage != nil: - // Empty votes have no sequence, only a round. - return m.EmptyVoteMessage.Vote.Round - case m.Notarization != nil: - return m.Notarization.Vote.Seq - case m.FinalizeVote != nil: - return m.FinalizeVote.Finalization.Seq - case m.Finalization != nil: - return m.Finalization.Finalization.Seq - default: - return 0 - } -} - type EmptyVoteMetadata struct { Round uint64 Epoch uint64 diff --git a/config.go b/config.go index a212db7c..2a7f696d 100644 --- a/config.go +++ b/config.go @@ -14,7 +14,7 @@ import ( type ParameterConfig struct { // WalMaxEntryCount is the maximum number of entries in the write-ahead log before it is closed. WALMaxEntryCount int - // MaxnetworkDelay is the assumed upper bound on the network delay for messages to be delivered. + // MaxNetworkDelay is the assumed upper bound on the network delay for messages to be delivered. MaxNetworkDelay time.Duration // MaxRoundWindow is the maximum number of rounds that can be stored in memory. MaxRoundWindow uint64 From 21b4bbf0f558d3af61323a57d536681d537769ec Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Fri, 10 Jul 2026 22:17:18 +0200 Subject: [PATCH 09/14] Add test that bootstraps a non validator that becomes a validator only after bootstrapping Signed-off-by: Yacov Manevich --- instance_test.go | 188 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 187 insertions(+), 1 deletion(-) diff --git a/instance_test.go b/instance_test.go index 1e474bb6..e76ed6aa 100644 --- a/instance_test.go +++ b/instance_test.go @@ -12,6 +12,7 @@ import ( "encoding/binary" "fmt" "sort" + "strings" "sync" "sync/atomic" "testing" @@ -23,9 +24,10 @@ import ( "github.com/ava-labs/simplex/wal" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" ) -func TestInstance(t *testing.T) { +func TestInstanceMixedNodeType(t *testing.T) { // One node is a validator at genesis, the other is a non-validator. // After some blocks, the second (non-validator) node also becomes a validator. // The test ensures that the second node tracks the chain while the first node expands the chain @@ -128,6 +130,190 @@ func TestInstance(t *testing.T) { require.Equal(t, secondInstance.Config.ID, latestValidatorID(t, storage)) } +func TestInstanceNonValidatorBootstraps(t *testing.T) { + // One node is a validator and progresses the chain by building blocks, + // and its weight changes while the chain progresses in 3 different P-chain epoch heights. + // Then, we add another node which is a non-validator. + // The node should bootstrap the chain but without shutting down the non-validator instance, + // and the test should detect the log entry "I am still a non-validator at the tip of the P-chain, skipping role change" + // being printed several times until the non-validator node bootstraps. + // Later on, the non-validator becomes a validator. + const ( + basePChainHeight = uint64(1) + secondEpochP = uint64(100) + thirdEpochP = uint64(200) + joinEpochP = uint64(300) + ) + + var id [20]byte + rand.Read(id[:]) + validatorNodeID := common.NodeID(id[:]) + + // The node that joins later, first as a non-validator and eventually as a validator. + var nv [20]byte + rand.Read(nv[:]) + nonValidatorNodeID := common.NodeID(nv[:]) + + // The lone validator's weight changes at three different P-chain heights, sealing an + // epoch on each change. Because it remains the sole validator throughout, its own + // approval is a quorum and every epoch seals without any other node's participation. + // The last checkpoint (joinEpochP) grows the set to two validators, admitting the peer. + validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ + basePChainHeight: { + {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, + }, + secondEpochP: { + {NodeID: id, BLSKey: []byte{0xaa}, Weight: 2}, + }, + thirdEpochP: { + {NodeID: id, BLSKey: []byte{0xaa}, Weight: 3}, + }, + joinEpochP: { + {NodeID: id, BLSKey: []byte{0xaa}, Weight: 3}, + {NodeID: nv, BLSKey: []byte{0xbb}, Weight: 1}, + }, + } + + pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) + cops := &testCryptoOps{} + + genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} + + net := newInMemNetwork(t) + t.Cleanup(net.stop) + + // Both storages start with only the genesis block. + storage := NewMockStorage(t) + smb := metadata.StateMachineBlock{InnerBlock: genesisBlock} + require.NoError(t, storage.Index(context.Background(), &ParsedBlock{StateMachineBlock: smb}, common.Finalization{})) + + storage2 := NewMockStorage(t) + smb = metadata.StateMachineBlock{InnerBlock: genesisBlock} + require.NoError(t, storage2.Index(context.Background(), &ParsedBlock{StateMachineBlock: smb}, common.Finalization{})) + + validatorInstance := newInstance(t, validatorNodeID, storage, net, pChain, cops, genesisBlock) + nonValidatorInstance := newInstance(t, nonValidatorNodeID, storage2, net, pChain, cops, genesisBlock) + + // Count how many times the non-validator reports that it is still not a validator at the + // tip of the P-chain while it replicates across the sealed epochs. + var stillNonValidatorLogs atomic.Uint64 + // transitioned is closed when the node starts a Simplex epoch, i.e. becomes a validator. + // The node only ever starts an epoch here as part of its non-validator -> validator + // transition. + transitioned := make(chan struct{}) + nonValidatorInstance.Config.Logger.(*testutil.TestLogger).Intercept(func(entry zapcore.Entry) error { + if strings.Contains(entry.Message, "I am still a non-validator at the tip of the P-chain, skipping role change") { + stillNonValidatorLogs.Add(1) + } + if strings.Contains(entry.Message, "Starting Simplex Epoch") { + select { + case <-transitioned: + default: + close(transitioned) + } + } + return nil + }) + + // Only the validator is running at first; it builds and seals the chain on its own. + net.register(validatorNodeID, validatorInstance) + require.NoError(t, validatorInstance.Start(t.Context())) + t.Cleanup(validatorInstance.Stop) + + // Epoch 1: wait until the validator has committed a series of blocks on its own. + waitForNumBlocks(t, storage, 5) // genesis(0) + zero block(1) + a few normal blocks + + // Drive two more epoch transitions by changing the validator's weight. Each change seals + // an epoch (and produces a sealing block) without any other node, since the validator's + // own approval is a quorum of the single-node set. + pChain.advanceTo(secondEpochP) + waitForSealingBlockCount(t, storage, 2) + + pChain.advanceTo(thirdEpochP) + waitForSealingBlockCount(t, storage, 3) + + // Let the third epoch grow a few normal blocks before the non validator joins, so bootstrap has to + // replicate past the sealing blocks and into ordinary blocks. + waitForNumBlocks(t, storage, storage.NumBlocks()+3) + + // The new node joins as a non-validator (it is absent from the validator set at the current + // P-chain tip) and bootstraps the chain from the validator. + net.register(nonValidatorNodeID, nonValidatorInstance) + require.NoError(t, nonValidatorInstance.Start(t.Context())) + t.Cleanup(nonValidatorInstance.Stop) + + // The non-validator replicates every sealed epoch. It stays a non-validator throughout, + // so on each sealing block it logs that it is still a non-validator at the tip. + bootstrapTarget := storage.NumBlocks() + waitForNumBlocks(t, storage2, bootstrapTarget) + + // The "still a non-validator" message was printed several times (once per sealed epoch it + // replicated through) while it caught up. + require.Eventually(t, func() bool { + return stillNonValidatorLogs.Load() >= 3 + }, 20*time.Second, 100*time.Millisecond) + + // Now grow the validator set to include the peer at the P-chain tip. + pChain.advanceTo(joinEpochP) + approval := &metadata.ValidatorSetApproval{ + NodeID: nv, + PChainHeight: joinEpochP, + AuxInfoDigest: sha256.Sum256(nil), + Signature: []byte{1, 2, 3}, + } + + // With two validators the validator's self-approval is no longer a quorum and the peer is + // still a non-validator, so we inject the peer's approval until the sealing block commits. + // TODO: Implement this capability in production so we won't need to inject approvals in tests. + sealingBlockSeq := waitForSealingBlock(t, validatorInstance, approval, storage.NumBlocks()) + waitForNumBlocks(t, storage2, sealingBlockSeq) + + // Once the non-validator replicates the sealing block that admits it, it detects that it is + // now a validator at the tip and transitions from non-validator to validator. + select { + case <-transitioned: + case <-time.After(20 * time.Second): + t.Fatal("non-validator did not transition to validator") + } + + // The newly promoted validator now participates in extending the chain. + require.Equal(t, nonValidatorInstance.Config.ID, latestValidatorID(t, storage)) + + // With both validators live, the two-validator epoch keeps committing blocks, and both + // nodes replicate them together. This confirms the promoted node contributes to consensus + // rather than merely tracking the chain. + const twoValidatorExtra = uint64(3) + extendedTarget := sealingBlockSeq + twoValidatorExtra + waitForNumBlocks(t, storage, extendedTarget) + waitForNumBlocks(t, storage2, extendedTarget) +} + +// countSealingBlocks returns the number of sealing blocks (blocks carrying a +// BlockValidationDescriptor) currently in storage. +func countSealingBlocks(t *testing.T, storage *MockStorage) int { + t.Helper() + count := 0 + num := storage.NumBlocks() + for seq := uint64(0); seq < num; seq++ { + block, ok := storage.blockAt(seq) + if !ok { + continue + } + if block.Metadata.SimplexEpochInfo.BlockValidationDescriptor != nil { + count++ + } + } + return count +} + +// waitForSealingBlockCount waits until storage holds at least target sealing blocks. +func waitForSealingBlockCount(t *testing.T, storage *MockStorage, target int) { + t.Helper() + require.Eventually(t, func() bool { + return countSealingBlocks(t, storage) >= target + }, 20*time.Second, 100*time.Millisecond) +} + func newInstance(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock) *Instance { comm := &networkSender{net: net, self: nodeID} return &Instance{ From 7006400c61a07773e3c57293712bf802a5db20ff Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Mon, 13 Jul 2026 00:37:05 +0200 Subject: [PATCH 10/14] Address code review comments IV Signed-off-by: Yacov Manevich --- adapters.go | 12 ++--- instance.go | 121 +++++++++++++++++++++--------------------- instance_test.go | 33 ++++++------ msm/build_decision.go | 3 +- msm/fake_node_test.go | 2 +- msm/msm.go | 2 +- msm/msm_test.go | 2 +- msm/util_test.go | 2 +- 8 files changed, 87 insertions(+), 90 deletions(-) diff --git a/adapters.go b/adapters.go index d61d3fa6..b52c29a5 100644 --- a/adapters.go +++ b/adapters.go @@ -55,7 +55,7 @@ func (e *EpochAwareStorage) Retrieve(seq uint64) (common.VerifiedBlock, common.F func (e *EpochAwareStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { if block.BlockHeader().Epoch < e.epoch { - // This is a Telock from a previous h, so we ignore it and do not index it. + // This is a Telock from a previous epoch, so we ignore it and do not index it. return nil } if err := e.Storage.Index(ctx, block, certificate); err != nil { @@ -65,7 +65,7 @@ func (e *EpochAwareStorage) Index(ctx context.Context, block common.VerifiedBloc if err := e.onEpochChange(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet); err != nil { return err } - // We are now in a new h, so we update the h number to prevent indexing Telocks from the previous h. + // We are now in a new epoch, so we update the epoch number to prevent indexing Telocks from the previous epoch. e.epoch = block.BlockHeader().Seq } return nil @@ -91,17 +91,17 @@ type CachedStorage struct { msm *metadata.StateMachine lock sync.RWMutex Storage - cache map[[32]byte]cachedBlock + cache map[common.Digest]cachedBlock } func NewCachedStorage(storage Storage) *CachedStorage { return &CachedStorage{ Storage: storage, - cache: make(map[[32]byte]cachedBlock), + cache: make(map[common.Digest]cachedBlock), } } -func (cs *CachedStorage) RetrieveBlock(seq uint64, digest [32]byte) (metadata.StateMachineBlock, *common.Finalization, error) { +func (cs *CachedStorage) RetrieveBlock(seq uint64, digest common.Digest) (metadata.StateMachineBlock, *common.Finalization, error) { block, finalization, err := cs.Retrieve(seq, digest) if err != nil { return metadata.StateMachineBlock{}, nil, err @@ -110,7 +110,7 @@ func (cs *CachedStorage) RetrieveBlock(seq uint64, digest [32]byte) (metadata.St return block.(*ParsedBlock).StateMachineBlock, finalization, nil } -func (cs *CachedStorage) Retrieve(seq uint64, digest [32]byte) (common.VerifiedBlock, *common.Finalization, error) { +func (cs *CachedStorage) Retrieve(seq uint64, digest common.Digest) (common.VerifiedBlock, *common.Finalization, error) { cs.lock.RLock() item, exists := cs.cache[digest] if exists { diff --git a/instance.go b/instance.go index 70d8701d..c98f7a25 100644 --- a/instance.go +++ b/instance.go @@ -46,9 +46,6 @@ type Config struct { ID common.NodeID } -type MessageHandler interface { -} - type nodeRole byte const ( @@ -79,19 +76,26 @@ type Instance struct { stopCh chan struct{} } +func NewInstance(config Config) *Instance { + return &Instance{ + Config: config, + stopCh: make(chan struct{}), + cs: NewCachedStorage(config.Storage), + epochChanges: make(chan epochChange, 1), + } +} + func (i *Instance) Start(ctx context.Context) error { // Hold the lock throughout startup to block HandleMessage from being called in between. i.lock.Lock() defer i.lock.Unlock() - i.stopCh = make(chan struct{}) - i.epochChanges = make(chan epochChange, 1) context.AfterFunc(ctx, i.Stop) - cachedStorage := NewCachedStorage(i.Config.Storage) - i.cs = cachedStorage - lastBlock, numBlocks, err := i.lastBlock() + if err != nil { + return fmt.Errorf("error retrieving last block: %w", err) + } lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() @@ -100,7 +104,9 @@ func (i *Instance) Start(ctx context.Context) error { return fmt.Errorf("error determining latest epoch and validator set: %w", err) } - i.startValidatorOrNonValidator(nodes, epochNum) + if err := i.startAtEpoch(nodes, epochNum); err != nil { + return fmt.Errorf("error starting instance at epoch %d: %w", epochNum, err) + } go i.tick() go i.listenForEpochChanges() @@ -113,12 +119,7 @@ func (i *Instance) startValidator() error { if err != nil { return err } - - if err := i.startEpoch(epochConfig); err != nil { - return err - } - - return nil + return i.startEpoch(epochConfig) } func (i *Instance) startNonValidator(epochNum uint64, validators common.Nodes) error { @@ -230,9 +231,8 @@ func (i *Instance) Stop() { close(i.stopCh) } - if i.e != nil { - i.stopValidator() - } + i.stopValidator() + i.stopNonValidator() } func (i *Instance) stopNonValidator() { @@ -255,6 +255,8 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error i.lock.Lock() defer i.lock.Unlock() + // We need to artificially wire the MSM and the cache to the block, + // in order to intercept the Verify() call. switch { case msg.BlockMessage != nil: err := i.wireBlockMessage(msg) @@ -283,54 +285,51 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error func (i *Instance) wireReplicationResponse(msg *common.Message) error { resp := msg.ReplicationResponse if resp.LatestRound != nil && resp.LatestRound.Block != nil { - pb, isParsedBlock := resp.LatestRound.Block.(*ParsedBlock) - if !isParsedBlock { - return fmt.Errorf("expected ParsedBlock, got %T", resp.LatestRound.Block) - } - resp.LatestRound.Block = &cachedBlock{ - cache: i.cs, - ParsedBlock: pb, + block, err := i.wireBlock(resp.LatestRound.Block) + if err != nil { + return err } - pb.msm = i.msm + resp.LatestRound.Block = block } if resp.LatestSeq != nil && resp.LatestSeq.Block != nil { - pb, isParsedBlock := resp.LatestSeq.Block.(*ParsedBlock) - if !isParsedBlock { - return fmt.Errorf("expected ParsedBlock, got %T", resp.LatestSeq.Block) - } - resp.LatestSeq.Block = &cachedBlock{ - cache: i.cs, - ParsedBlock: pb, + block, err := i.wireBlock(resp.LatestSeq.Block) + if err != nil { + return err } - pb.msm = i.msm + resp.LatestSeq.Block = block } for j, datum := range resp.Data { if datum.Block == nil { continue } - pb, isParsedBlock := datum.Block.(*ParsedBlock) - if !isParsedBlock { - return fmt.Errorf("expected ParsedBlock, got %T", datum.Block) - } - resp.Data[j].Block = &cachedBlock{ - cache: i.cs, - ParsedBlock: pb, + block, err := i.wireBlock(datum.Block) + if err != nil { + return err } - pb.msm = i.msm + resp.Data[j].Block = block } return nil } -func (i *Instance) wireBlockMessage(msg *common.Message) error { - pb, isParsedBlock := msg.BlockMessage.Block.(*ParsedBlock) +func (i *Instance) wireBlock(block common.Block) (common.Block, error) { + pb, isParsedBlock := block.(*ParsedBlock) if !isParsedBlock { - return fmt.Errorf("expected ParsedBlock, got %T", msg.BlockMessage.Block) + return nil, fmt.Errorf("expected ParsedBlock, got %T", block) } - msg.BlockMessage.Block = &cachedBlock{ + block = &cachedBlock{ cache: i.cs, ParsedBlock: pb, } pb.msm = i.msm + return block, nil +} + +func (i *Instance) wireBlockMessage(msg *common.Message) error { + block, err := i.wireBlock(msg.BlockMessage.Block) + if err != nil { + return err + } + msg.BlockMessage.Block = block return nil } @@ -346,20 +345,20 @@ func (i *Instance) listenForEpochChanges() { } func (i *Instance) processEpochChange(epochChange epochChange) { + var err error switch epochChange.nodeRole { case nonValidator: - if err := i.transitionEpochNonValidator(epochChange); err != nil { - i.Config.Logger.Error("Error transitioning epoch for non-validator", zap.Error(err)) - i.Stop() - } + err = i.transitionEpochNonValidator(epochChange) case validator: - if err := i.transitionEpochValidator(epochChange); err != nil { - i.Config.Logger.Error("Error transitioning epoch for validator", zap.Error(err)) - i.Stop() - } + err = i.transitionEpochValidator(epochChange) default: // This should never happen, but we log it just in case. i.Config.Logger.Fatal("Unknown node role on epoch change", zap.String("role", fmt.Sprintf("%v", epochChange.nodeRole))) + return + } + if err != nil { + i.Config.Logger.Error("Error transitioning epoch", zap.Uint8("role", uint8(epochChange.nodeRole)), zap.Error(err)) + i.Stop() } } @@ -532,10 +531,10 @@ func (i *Instance) transitionEpochNonValidator(epochChange epochChange) error { // Stop the non-validator before doing anything else, so that we don't process any more messages while we are changing epochs. i.stopNonValidator() - return i.startValidatorOrNonValidator(epochChange.validators, epochChange.epochNum) + return i.startAtEpoch(epochChange.validators, epochChange.epochNum) } -func (i *Instance) startValidatorOrNonValidator(validators common.Nodes, epoch uint64) error { +func (i *Instance) startAtEpoch(validators common.Nodes, epoch uint64) error { if i.iAmValidator(validators) { if err := i.startValidator(); err != nil { i.Config.Logger.Error("Error starting validator on epoch change", zap.Error(err)) @@ -564,7 +563,7 @@ func (i *Instance) transitionEpochValidator(epochChange epochChange) error { i.Config.Logger.Error("Error garbage collecting epoch config on epoch change", zap.Error(err)) } - return i.startValidatorOrNonValidator(epochChange.validators, epochChange.epochNum) + return i.startAtEpoch(epochChange.validators, epochChange.epochNum) } func constructEpochAndValidatorSet(lastNonSimplexInnerBlockHeight uint64, genesisValidatorSet metadata.NodeBLSMappings, numBlocks uint64, lastBlock ParsedBlock, storage Storage) (common.Nodes, uint64, error) { @@ -579,14 +578,14 @@ func constructEpochAndValidatorSet(lastNonSimplexInnerBlockHeight uint64, genesi validatorSet = genesisValidatorSet nodes = validatorSetToNodes(genesisValidatorSet) epochNum = lastNonSimplexInnerBlockHeight + 1 - // If the last block persisted is a sealing block, then we are in the next h. + // If the last block persisted is a sealing block, then we are in the next epoch. case lastBlock.SealingBlockInfo() != nil: epochNum = lastBlock.BlockHeader().Seq validatorSet = constructValidatorSetFromSealingBlock(lastBlock) nodes = lastBlock.SealingBlockInfo().ValidatorSet // Else, we have at least one Simplex block in the ledger, and it's not a sealing block. default: - // Therefore, the sequence of the sealing block is the h number. + // Therefore, the sequence of the sealing block is the epoch number. sealingBlockSeq := lastBlock.BlockHeader().Epoch sealingBlock, _, err := storage.GetBlock(sealingBlockSeq) if err != nil { @@ -601,9 +600,9 @@ func constructEpochAndValidatorSet(lastNonSimplexInnerBlockHeight uint64, genesi return nodes, epochNum, nil } -func validatorSetToNodes(genesisValidatorSet metadata.NodeBLSMappings) common.Nodes { +func validatorSetToNodes(validatorSet metadata.NodeBLSMappings) common.Nodes { var nodes common.Nodes - for _, vdr := range genesisValidatorSet { + for _, vdr := range validatorSet { nodes = append(nodes, common.Node{ Id: vdr.NodeID[:], Weight: vdr.Weight, diff --git a/instance_test.go b/instance_test.go index e76ed6aa..94cd8dac 100644 --- a/instance_test.go +++ b/instance_test.go @@ -316,25 +316,24 @@ func waitForSealingBlockCount(t *testing.T, storage *MockStorage, target int) { func newInstance(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock) *Instance { comm := &networkSender{net: net, self: nodeID} - return &Instance{ - Config: Config{ - Logger: testutil.MakeLogger(t, int(nodeID[0])), - ID: nodeID, - VM: newTestVM(), - Storage: storage, - Sender: comm, - Broadcaster: comm, - PlatformChain: pChain, - CryptoOps: cops, - LastNonSimplexInnerBlock: genesisBlock, - WalCreator: storage.CreateWAL, - ParameterConfig: ParameterConfig{ - MaxNetworkDelay: 500 * time.Millisecond, - MaxRoundWindow: 100, - WALMaxEntryCount: 1024, - }, + config := Config{ + Logger: testutil.MakeLogger(t, int(nodeID[0])), + ID: nodeID, + VM: newTestVM(), + Storage: storage, + Sender: comm, + Broadcaster: comm, + PlatformChain: pChain, + CryptoOps: cops, + LastNonSimplexInnerBlock: genesisBlock, + WalCreator: storage.CreateWAL, + ParameterConfig: ParameterConfig{ + MaxNetworkDelay: 500 * time.Millisecond, + MaxRoundWindow: 100, + WALMaxEntryCount: 1024, }, } + return NewInstance(config) } func latestValidatorID(t *testing.T, storage *MockStorage) common.NodeID { diff --git a/msm/build_decision.go b/msm/build_decision.go index e3dbe672..252be698 100644 --- a/msm/build_decision.go +++ b/msm/build_decision.go @@ -72,8 +72,7 @@ func (bbd *blockBuildingDecider) shouldBuildBlock( // If the P-chain height changed, re-evaluate again whether we should transition to a new epoch, // or continue waiting to build a block. - h := bbd.getPChainHeight() - if h != pChainHeight { + if bbd.getPChainHeight() != pChainHeight { continue } diff --git a/msm/fake_node_test.go b/msm/fake_node_test.go index f81edd10..d93b2352 100644 --- a/msm/fake_node_test.go +++ b/msm/fake_node_test.go @@ -274,7 +274,7 @@ func newFakeNode(t *testing.T) *fakeNode { fn.sm.BlockBuilder = fn fn.sm.PChainProgressListener = fn - fn.sm.GetBlock = func(seq uint64, digest [32]byte) (StateMachineBlock, *common.Finalization, error) { + fn.sm.GetBlock = func(seq uint64, digest common.Digest) (StateMachineBlock, *common.Finalization, error) { if seq == 0 { return genesisBlock, nil, nil } diff --git a/msm/msm.go b/msm/msm.go index 0a57acc1..fb81ffb1 100644 --- a/msm/msm.go +++ b/msm/msm.go @@ -149,7 +149,7 @@ type ValidatorSetRetriever func(pChainHeight uint64) (NodeBLSMappings, error) // BlockRetriever retrieves a block and its finalization status given the block's sequence number and expected digest. // If the block cannot be found it returns ErrBlockNotFound. // If an error occurs during retrieval, it returns a non-nil error. -type BlockRetriever func(seq uint64, digest [32]byte) (StateMachineBlock, *common.Finalization, error) +type BlockRetriever func(seq uint64, digest common.Digest) (StateMachineBlock, *common.Finalization, error) // BlockBuilder builds a new VM block with the given observed P-chain height. type BlockBuilder interface { diff --git a/msm/msm_test.go b/msm/msm_test.go index 1e090b73..43d9d267 100644 --- a/msm/msm_test.go +++ b/msm/msm_test.go @@ -1934,7 +1934,7 @@ func TestCollectAuxiliaryInfo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - getBlock := func(seq uint64, _ [32]byte) (StateMachineBlock, *common.Finalization, error) { + getBlock := func(seq uint64, _ common.Digest) (StateMachineBlock, *common.Finalization, error) { if tt.getBlockErr != nil { return StateMachineBlock{}, nil, tt.getBlockErr } diff --git a/msm/util_test.go b/msm/util_test.go index 911fb017..e2d8f975 100644 --- a/msm/util_test.go +++ b/msm/util_test.go @@ -92,7 +92,7 @@ func (bs blockStore) clone() blockStore { return newStore } -func (bs blockStore) getBlock(seq uint64, _ [32]byte) (StateMachineBlock, *common.Finalization, error) { +func (bs blockStore) getBlock(seq uint64, _ common.Digest) (StateMachineBlock, *common.Finalization, error) { blk, exits := bs[seq] if !exits { return StateMachineBlock{}, nil, fmt.Errorf("%w: block %d not found", common.ErrBlockNotFound, seq) From 9617211e33a30353846de1060562389892569552 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Mon, 13 Jul 2026 19:20:39 +0200 Subject: [PATCH 11/14] Add test for constructEpochAndValidatorSet Signed-off-by: Yacov Manevich --- instance.go | 12 +++- instance_test.go | 148 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 154 insertions(+), 6 deletions(-) diff --git a/instance.go b/instance.go index c98f7a25..600e5363 100644 --- a/instance.go +++ b/instance.go @@ -99,7 +99,7 @@ func (i *Instance) Start(ctx context.Context) error { lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() - nodes, epochNum, err := constructEpochAndValidatorSet(lastNonSimplexHeight, genesisValidatorSet, numBlocks, ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) + nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) if err != nil { return fmt.Errorf("error determining latest epoch and validator set: %w", err) } @@ -407,7 +407,7 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() - nodes, epochNum, err := constructEpochAndValidatorSet(lastNonSimplexHeight, genesisValidatorSet, numBlocks, ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) + nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) if err != nil { return simplex.EpochConfig{}, err } @@ -566,7 +566,7 @@ func (i *Instance) transitionEpochValidator(epochChange epochChange) error { return i.startAtEpoch(epochChange.validators, epochChange.epochNum) } -func constructEpochAndValidatorSet(lastNonSimplexInnerBlockHeight uint64, genesisValidatorSet metadata.NodeBLSMappings, numBlocks uint64, lastBlock ParsedBlock, storage Storage) (common.Nodes, uint64, error) { +func constructEpochAndValidatorSet(logger common.Logger, lastNonSimplexInnerBlockHeight uint64, genesisValidatorSet metadata.NodeBLSMappings, numBlocks uint64, lastBlock ParsedBlock, storage Storage) (common.Nodes, uint64, error) { epochNum := lastBlock.BlockHeader().Epoch var validatorSet metadata.NodeBLSMappings @@ -578,11 +578,15 @@ func constructEpochAndValidatorSet(lastNonSimplexInnerBlockHeight uint64, genesi validatorSet = genesisValidatorSet nodes = validatorSetToNodes(genesisValidatorSet) epochNum = lastNonSimplexInnerBlockHeight + 1 + logger.Debug("Determined epoch and validator set from genesis (ledger holds only non-Simplex blocks)", + zap.Uint64("epoch", epochNum)) // If the last block persisted is a sealing block, then we are in the next epoch. case lastBlock.SealingBlockInfo() != nil: epochNum = lastBlock.BlockHeader().Seq validatorSet = constructValidatorSetFromSealingBlock(lastBlock) nodes = lastBlock.SealingBlockInfo().ValidatorSet + logger.Debug("Determined epoch and validator set from sealing block at tip", + zap.Uint64("epoch", epochNum)) // Else, we have at least one Simplex block in the ledger, and it's not a sealing block. default: // Therefore, the sequence of the sealing block is the epoch number. @@ -596,6 +600,8 @@ func constructEpochAndValidatorSet(lastNonSimplexInnerBlockHeight uint64, genesi } validatorSet = constructValidatorSetFromSealingBlock(ParsedBlock{StateMachineBlock: sealingBlock}) nodes = validatorSetToNodes(validatorSet) + logger.Debug("Determined epoch and validator set from sealing block in storage", + zap.Uint64("epoch", epochNum), zap.Uint64("sealingBlockSeq", sealingBlockSeq)) } return nodes, epochNum, nil } diff --git a/instance_test.go b/instance_test.go index 94cd8dac..bbb2c7dd 100644 --- a/instance_test.go +++ b/instance_test.go @@ -288,6 +288,114 @@ func TestInstanceNonValidatorBootstraps(t *testing.T) { waitForNumBlocks(t, storage2, extendedTarget) } +func TestInstanceRestartAcrossEpochs(t *testing.T) { + // Restart a single validator at three different points in its lifecycle so that, + // on each (re)start, constructEpochAndValidatorSet takes a different branch of + // its switch: + // + // - Cold boot, ledger holds only the genesis (non-Simplex) block -> "genesis" branch. + // - Restart when the tip is a sealing block -> "sealing block at tip" branch. + // - Restart mid-epoch, when the tip is an ordinary Simplex block -> "sealing block in storage" branch. + // + const basePChainHeight = uint64(1) + + var id [20]byte + rand.Read(id[:]) + nodeID := common.NodeID(id[:]) + + validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ + basePChainHeight: { + {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, + }, + } + + pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) + cops := &testCryptoOps{} + genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} + + net := newInMemNetwork(t) + t.Cleanup(net.stop) + + storage := NewMockStorage(t) + smb := metadata.StateMachineBlock{InnerBlock: genesisBlock} + require.NoError(t, storage.Index(context.Background(), &ParsedBlock{StateMachineBlock: smb}, common.Finalization{})) + + vm := newTestVM() + + const ( + logEpochFromGenesis = "Determined epoch and validator set from genesis (ledger holds only non-Simplex blocks)" + logEpochFromSealingTip = "Determined epoch and validator set from sealing block at tip" + logEpochFromSealingStorage = "Determined epoch and validator set from sealing block in storage" + ) + + // lastEpochBranch holds the full debug message constructEpochAndValidatorSet + // logs, identifying which branch of its switch the latest (re)start took. It is + // written synchronously during Start, but also from the epoch-change goroutine, + // so an atomic guards it. + var lastEpochBranch atomic.Pointer[string] + + // start (re)creates an instance over the same storage/network/VM. The log + // interceptor, installed before Start, records which branch startup took. + start := func() *Instance { + inst := newInstanceWithVM(t, nodeID, storage, net, pChain, cops, genesisBlock, vm) + inst.Config.Logger.(*testutil.TestLogger).Intercept(func(entry zapcore.Entry) error { + switch entry.Message { + case logEpochFromGenesis, logEpochFromSealingTip, logEpochFromSealingStorage: + msg := entry.Message + lastEpochBranch.Store(&msg) + } + return nil + }) + net.register(nodeID, inst) + require.NoError(t, inst.Start(t.Context())) + return inst + } + + // Pause block production before the node even starts: only protocol blocks (the + // zero block, the epoch transition and its sealing block) get built, and the + // chain stops at the sealing block since no ordinary block can be built on top. + vm.pause() + + // --- Case 1: cold boot, ledger holds only the genesis block. --- + inst := start() + require.Equal(t, logEpochFromGenesis, *lastEpochBranch.Load()) + + // --- Case 2: restart when the tip is a sealing block. --- + // countSealingBlocks == 2: the zero block plus the sealing block of the initial + // epoch transition. With the VM paused, that sealing block stays the tip. + waitForSealingBlockCount(t, storage, 2) + requireTipIsSealing(t, storage, true) + + inst.Stop() + inst = start() + require.Equal(t, logEpochFromSealingTip, *lastEpochBranch.Load()) + + // --- Case 3: restart mid-epoch, tip is an ordinary Simplex block. --- + // Resume production; the node extends the new epoch with ordinary blocks. + vm.resume() + waitForNumBlocks(t, storage, storage.NumBlocks()+3) + requireTipIsSealing(t, storage, false) + + inst.Stop() + inst = start() + t.Cleanup(inst.Stop) + require.Equal(t, logEpochFromSealingStorage, *lastEpochBranch.Load()) + + // The restarted node keeps extending the chain. + waitForNumBlocks(t, storage, storage.NumBlocks()+2) +} + +// requireTipIsSealing asserts whether the last block in storage is a sealing block. +func requireTipIsSealing(t *testing.T, storage *MockStorage, want bool) { + t.Helper() + num := storage.NumBlocks() + require.Positive(t, num) + block, ok := storage.blockAt(num - 1) + require.True(t, ok) + isSealing := block.Metadata.SimplexEpochInfo.BlockValidationDescriptor != nil + require.Equal(t, want, isSealing) +} + // countSealingBlocks returns the number of sealing blocks (blocks carrying a // BlockValidationDescriptor) currently in storage. func countSealingBlocks(t *testing.T, storage *MockStorage) int { @@ -315,11 +423,17 @@ func waitForSealingBlockCount(t *testing.T, storage *MockStorage, target int) { } func newInstance(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock) *Instance { + return newInstanceWithVM(t, nodeID, storage, net, pChain, cops, genesisBlock, newTestVM()) +} + +// newInstanceWithVM is like newInstance but uses a caller-supplied VM, so a test +// can share one controllable VM across restarts of the same node. +func newInstanceWithVM(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock, vm VM) *Instance { comm := &networkSender{net: net, self: nodeID} config := Config{ Logger: testutil.MakeLogger(t, int(nodeID[0])), ID: nodeID, - VM: newTestVM(), + VM: vm, Storage: storage, Sender: comm, Broadcaster: comm, @@ -428,6 +542,13 @@ func parseTestInnerBlock(buff []byte) (*testInnerBlock, error) { type testVM struct { nextHeight atomic.Uint64 + // When paused, the VM behaves as a chain with no pending transactions: + // WaitForPendingBlock and BuildBlock block until their context expires, so the + // epoch stops producing ordinary blocks. The epoch-transition and sealing + // machinery, which builds its block once the inner build times out, still runs — + // so pausing before an epoch change leaves the sealing block at the tip with + // nothing built on top. Lets a test pin the chain tip without touching storage. + paused atomic.Bool } func newTestVM() *testVM { @@ -436,7 +557,14 @@ func newTestVM() *testVM { return vm } -func (vm *testVM) BuildBlock(_ context.Context, _ uint64) (metadata.VMBlock, error) { +func (vm *testVM) pause() { vm.paused.Store(true) } +func (vm *testVM) resume() { vm.paused.Store(false) } + +func (vm *testVM) BuildBlock(ctx context.Context, _ uint64) (metadata.VMBlock, error) { + if vm.paused.Load() { + <-ctx.Done() // let the caller's impatient build time out + return nil, ctx.Err() + } h := vm.nextHeight.Add(1) - 1 payload := make([]byte, 8) binary.BigEndian.PutUint64(payload, h) @@ -444,6 +572,10 @@ func (vm *testVM) BuildBlock(_ context.Context, _ uint64) (metadata.VMBlock, err } func (vm *testVM) WaitForPendingBlock(ctx context.Context) { + if vm.paused.Load() { + <-ctx.Done() // no pending block while paused + return + } select { case <-ctx.Done(): case <-time.After(100 * time.Millisecond): @@ -721,8 +853,18 @@ func (n *inMemNetwork) register(id common.NodeID, inst *Instance) { stopped: make(chan struct{}), } n.lock.Lock() + defer n.lock.Unlock() + + // If an instance was previously registered under this id (e.g. a restart replacing + // the node), stop its delivery goroutine before swapping in the new one. + old := n.nodes[string(id)] + if old != nil { + close(old.done) + <-old.stopped + } + n.nodes[string(id)] = node - n.lock.Unlock() + go n.deliver(node) } From ea315d67254d56d770cc4a1fc35cdbf59b75a8e2 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Mon, 13 Jul 2026 22:50:56 +0200 Subject: [PATCH 12/14] Add back prev sealing block seq to ParsedBlock Signed-off-by: Yacov Manevich --- adapters.go | 3 ++- external.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/adapters.go b/adapters.go index b52c29a5..c98dbb79 100644 --- a/adapters.go +++ b/adapters.go @@ -61,7 +61,8 @@ func (e *EpochAwareStorage) Index(ctx context.Context, block common.VerifiedBloc if err := e.Storage.Index(ctx, block, certificate); err != nil { return err } - if block.SealingBlockInfo() != nil { + // This is a sealing block, and it is not the zero block + if block.SealingBlockInfo() != nil && block.SealingBlockInfo().PrevSealingBlockHash != [32]byte{} { if err := e.onEpochChange(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet); err != nil { return err } diff --git a/external.go b/external.go index 3ceb0f18..e1c3a0fb 100644 --- a/external.go +++ b/external.go @@ -88,5 +88,6 @@ func (p *ParsedBlock) SealingBlockInfo() *common.SealingBlockInfo { return &common.SealingBlockInfo{ ValidatorSet: nodes, + PrevSealingBlockHash: p.Metadata.SimplexEpochInfo.PrevSealingBlockHash, } } From 3f09ad3f42e6158148bd4ca1e462840e367cb08a Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Mon, 13 Jul 2026 22:59:21 +0200 Subject: [PATCH 13/14] Skip transitoning epoch if in the process of stopping Signed-off-by: Yacov Manevich --- instance.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/instance.go b/instance.go index 600e5363..231256c8 100644 --- a/instance.go +++ b/instance.go @@ -219,6 +219,15 @@ func (i *Instance) tick() { } } +func (i *Instance) isStopped() bool { + select { + case <-i.stopCh: + return true + default: + return false + } +} + func (i *Instance) Stop() { i.lock.Lock() defer i.lock.Unlock() @@ -523,6 +532,11 @@ func (i *Instance) transitionEpochNonValidator(epochChange epochChange) error { i.lock.Lock() defer i.lock.Unlock() + if i.isStopped() { + i.Config.Logger.Info("instance is already stopped, skipping epoch change") + return nil + } + if !i.iAmValidator(epochChange.validators) { i.Config.Logger.Debug("Skipping restarting a non-validator because I am not a validator yet") return nil From ab789cd9772e73ed98e3fad2fa53dca1a3e8ae23 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Mon, 13 Jul 2026 23:01:25 +0200 Subject: [PATCH 14/14] go fmt Signed-off-by: Yacov Manevich --- adapters.go | 2 +- external.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapters.go b/adapters.go index c98dbb79..12ae35ef 100644 --- a/adapters.go +++ b/adapters.go @@ -62,7 +62,7 @@ func (e *EpochAwareStorage) Index(ctx context.Context, block common.VerifiedBloc return err } // This is a sealing block, and it is not the zero block - if block.SealingBlockInfo() != nil && block.SealingBlockInfo().PrevSealingBlockHash != [32]byte{} { + if block.SealingBlockInfo() != nil && block.SealingBlockInfo().PrevSealingBlockHash != [32]byte{} { if err := e.onEpochChange(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet); err != nil { return err } diff --git a/external.go b/external.go index e1c3a0fb..ad19ea40 100644 --- a/external.go +++ b/external.go @@ -87,7 +87,7 @@ func (p *ParsedBlock) SealingBlockInfo() *common.SealingBlockInfo { } return &common.SealingBlockInfo{ - ValidatorSet: nodes, + ValidatorSet: nodes, PrevSealingBlockHash: p.Metadata.SimplexEpochInfo.PrevSealingBlockHash, } }