From 485e05569ab6fd56bf189d422da1a9ff9e595e65 Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 26 Jul 2026 11:37:29 +0800 Subject: [PATCH 01/31] sandboxd: let the node's root token drive the sandbox lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hibernate, fork, promote and checkpoint all resolved the sandbox through its per-sandbox token, so only the client that made the claim could ever use them. An aggregated control plane holds the fleet api_token and nothing else: keeping one secret per sandbox would turn its O(nodes) storage into O(sandboxes), the property the whole design rests on. Add operator variants that resolve by id, authorized by the node's root api_token before the call — the pattern ReleaseOperator already established, so release is no longer the one verb a control plane can reach. A tenant token still takes the per-sandbox path unchanged. Also add an explicit wake verb: waking was previously a side effect of opening an agent connection, which left a control plane no way to resume a sandbox it was not about to talk to. And a single-sandbox read, so a reconcile does not have to scan the whole-node listing. --- sandboxd/pool/checkpoint.go | 65 ++++++++++- sandboxd/pool/fork.go | 24 +++- sandboxd/pool/operator.go | 79 +++++++++++++ sandboxd/pool/template.go | 23 +++- sandboxd/server/server.go | 133 ++++++++++++++++++++-- sandboxd/server/server_test.go | 201 ++++++++++++++++++++++++++++++++- 6 files changed, 504 insertions(+), 21 deletions(-) create mode 100644 sandboxd/pool/operator.go diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 1065613..55b7232 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -26,13 +26,29 @@ var ( // clone that exact state, and a source can be checkpointed again — a tree. // tenant attributes the record; empty means the operator (root). func (m *Manager) Checkpoint(ctx context.Context, id, token, name, tenant string) (types.Checkpoint, error) { - if name != "" && !types.NameRe.MatchString(name) { - return types.Checkpoint{}, fmt.Errorf("%w: %q must match %s", ErrBadName, name, types.NameRe) - } sb, ok := m.claim(id, token) if !ok { return types.Checkpoint{}, ErrUnknownSandbox } + return m.checkpointResolved(ctx, sb, name, tenant) +} + +// CheckpointOperator checkpoints a sandbox by id without a per-sandbox token. +// It is the operator (root) path, authorized by the node's root api_token +// before the call — mirroring ReleaseOperator. +func (m *Manager) CheckpointOperator(ctx context.Context, id, name, tenant string) (types.Checkpoint, error) { + sb, ok := m.byID(id) + if !ok { + return types.Checkpoint{}, ErrUnknownSandbox + } + return m.checkpointResolved(ctx, sb, name, tenant) +} + +// checkpointResolved is Checkpoint's body once the source claim is resolved. +func (m *Manager) checkpointResolved(ctx context.Context, sb *types.Sandbox, name, tenant string) (types.Checkpoint, error) { + if name != "" && !types.NameRe.MatchString(name) { + return types.Checkpoint{}, fmt.Errorf("%w: %q must match %s", ErrBadName, name, types.NameRe) + } if !sb.Key.Capturable() { return types.Checkpoint{}, ErrNoEgressFork } @@ -259,3 +275,46 @@ func parseCheckpoint(raw []byte) (types.Checkpoint, error) { } return ckpt, nil } + +// CheckpointIDs lists this node's checkpoint ids for the mesh to gossip, so a +// peer can resolve a branch of a checkpoint it does not hold to the node that +// does. Archive records are excluded: they are lifecycle-internal wake images, +// never a branch target. Errors are swallowed to nil — gossip is best-effort +// and a transient store read must not take the tick down. +func (m *Manager) CheckpointIDs() []string { + ckpts, err := m.Checkpoints(context.Background(), "") + if err != nil { + return nil + } + ids := make([]string, 0, len(ckpts)) + for _, c := range ckpts { + if !c.Archive { + ids = append(ids, c.ID) + } + } + slices.Sort(ids) + return ids +} + +// FetchCheckpoint materializes a checkpoint's export for a peer transfer, +// returning the local directory, its meta, and the release to call when the +// copy is done. It is the read half of the peer-heal path; the id is validated +// against the store's namespace before any disk is touched. +func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, []byte, func(), error) { + if _, err := m.loadCheckpoint(ctx, ckptID); err != nil { + return "", nil, nil, err + } + l := m.recLock(ckptID) + l.RLock() + dir, meta, release, err := m.ckpts.Fetch(ctx, ckptID) + if err != nil { + l.RUnlock() + if errors.Is(err, store.ErrNotFound) { + return "", nil, nil, ErrUnknownCheckpoint + } + return "", nil, nil, fmt.Errorf("fetch checkpoint: %w", err) + } + // The read lock spans the transfer: a delete must not pull the export out + // from under a stream already writing it to a peer. + return dir, meta, func() { release(); l.RUnlock() }, nil +} diff --git a/sandboxd/pool/fork.go b/sandboxd/pool/fork.go index 28ae28e..20531e1 100644 --- a/sandboxd/pool/fork.go +++ b/sandboxd/pool/fork.go @@ -18,13 +18,31 @@ import ( // and count against its quota. All-or-nothing: any child failing destroys // the ones already built, so an error means no child survived. func (m *Manager) Fork(ctx context.Context, id, token string, count int, ttl time.Duration) ([]*types.Sandbox, error) { - if count < 1 || count > m.maxFork { - return nil, fmt.Errorf("%w: %d not in 1..%d", ErrBadCount, count, m.maxFork) - } sb, ok := m.claim(id, token) if !ok { return nil, ErrUnknownSandbox } + return m.forkResolved(ctx, sb, count, ttl) +} + +// ForkOperator forks a sandbox by id without a per-sandbox token. It is the +// operator (root) path: the server authorizes it by the node's root api_token +// before calling, so no token check happens here — mirroring ReleaseOperator. +// A tenant token never reaches this method (see server.go). +func (m *Manager) ForkOperator(ctx context.Context, id string, count int, ttl time.Duration) ([]*types.Sandbox, error) { + sb, ok := m.byID(id) + if !ok { + return nil, ErrUnknownSandbox + } + return m.forkResolved(ctx, sb, count, ttl) +} + +// forkResolved is Fork's body once the source claim is resolved. +func (m *Manager) forkResolved(ctx context.Context, sb *types.Sandbox, count int, ttl time.Duration) ([]*types.Sandbox, error) { + id := sb.ID + if count < 1 || count > m.maxFork { + return nil, fmt.Errorf("%w: %d not in 1..%d", ErrBadCount, count, m.maxFork) + } if !sb.Key.Capturable() { return nil, ErrNoEgressFork } diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go new file mode 100644 index 0000000..9dbefb5 --- /dev/null +++ b/sandboxd/pool/operator.go @@ -0,0 +1,79 @@ +package pool + +import ( + "context" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +// The operator paths let a control plane holding the node's root api_token +// drive a sandbox's lifecycle without its per-sandbox token. They exist +// because a stateless aggregated control plane cannot keep one secret per +// sandbox without turning O(nodes) storage into O(sandboxes) — the property +// the whole design rests on. ReleaseOperator established the pattern; these +// follow it exactly: the server authorizes by root api_token before calling, +// so no token check happens here, and a tenant token never reaches them. + +// byID resolves a live claim by id alone, with no ownership proof. +func (m *Manager) byID(id string) (*types.Sandbox, bool) { + m.mu.Lock() + defer m.mu.Unlock() + sb := m.claimed[id] + return sb, sb != nil +} + +// Sandbox reports one live claim's summary, the single-sandbox read the +// whole-node listing otherwise forces a caller to scan for. +func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { + sb, ok := m.byID(id) + if !ok { + return SandboxSummary{}, false + } + return SandboxSummary{ + ID: sb.ID, Key: sb.Key, Deadline: sb.Deadline, + Hibernated: sb.HibernateSnap != "", Archived: sb.ArchiveCk != "", + FromCheckpoint: sb.FromCheckpoint, ClaimRef: sb.ClaimRef, + }, true +} + +// HibernateOperator hibernates a sandbox by id without a per-sandbox token. +func (m *Manager) HibernateOperator(ctx context.Context, id string) error { + sb, ok := m.byID(id) + if !ok { + return ErrUnknownSandbox + } + sb.Transition.Lock() + defer sb.Transition.Unlock() + return m.hibernateLocked(ctx, sb) +} + +// Wake restores a hibernated sandbox and leaves it running, the explicit +// counterpart to Hibernate. Waking is otherwise only a side effect of opening +// an agent connection, which gives a control plane no way to resume a sandbox +// it is not about to talk to. Idempotent on an already-running sandbox. +func (m *Manager) Wake(ctx context.Context, id, token string) error { + sb, ok := m.claim(id, token) + if !ok { + return ErrUnknownSandbox + } + return m.wake(ctx, sb) +} + +// WakeOperator wakes a sandbox by id without a per-sandbox token. +func (m *Manager) WakeOperator(ctx context.Context, id string) error { + sb, ok := m.byID(id) + if !ok { + return ErrUnknownSandbox + } + return m.wake(ctx, sb) +} + +// wake is the body shared by both wake entry points. It reuses the relay's +// resolve path, which restores through cocoon's mmap fast path (~55 ms) and +// queues concurrent wakes on the transition lock, then discards the resolved +// socket: the caller wants the VM running, not a connection to it. +func (m *Manager) wake(ctx context.Context, sb *types.Sandbox) error { + sb.Touch() + _, err := m.wakeResolved(ctx, sb) + return err +} diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 4a8570a..d86a1ad 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -29,13 +29,30 @@ type templateRecord struct { // same name replaces it, and the caller owns its lifecycle (DeleteTemplate). // tenant attributes the record; empty means the operator (root). func (m *Manager) Promote(ctx context.Context, id, token, template, tenant string) (types.PoolKey, error) { - if !types.NameRe.MatchString(template) { - return types.PoolKey{}, fmt.Errorf("%w: template %q must match %s", ErrBadKey, template, types.NameRe) - } sb, ok := m.claim(id, token) if !ok { return types.PoolKey{}, ErrUnknownSandbox } + return m.promoteResolved(ctx, sb, template, tenant) +} + +// PromoteOperator promotes a sandbox by id without a per-sandbox token. It is +// the operator (root) path, authorized by the node's root api_token before the +// call — mirroring ReleaseOperator. +func (m *Manager) PromoteOperator(ctx context.Context, id, template, tenant string) (types.PoolKey, error) { + sb, ok := m.byID(id) + if !ok { + return types.PoolKey{}, ErrUnknownSandbox + } + return m.promoteResolved(ctx, sb, template, tenant) +} + +// promoteResolved is Promote's body once the source claim is resolved. +func (m *Manager) promoteResolved(ctx context.Context, sb *types.Sandbox, template, tenant string) (types.PoolKey, error) { + id := sb.ID + if !types.NameRe.MatchString(template) { + return types.PoolKey{}, fmt.Errorf("%w: template %q must match %s", ErrBadKey, template, types.NameRe) + } if !sb.Key.Capturable() { return types.PoolKey{}, ErrNoEgressFork } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 2430283..c569b63 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -21,6 +21,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/store/peer" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -61,17 +62,26 @@ type Manager interface { Release(ctx context.Context, id, token string) error ReleaseOperator(ctx context.Context, id string) error Hibernate(ctx context.Context, id, token string) error + HibernateOperator(ctx context.Context, id string) error + Wake(ctx context.Context, id, token string) error + WakeOperator(ctx context.Context, id string) error Fork(ctx context.Context, id, token string, count int, ttl time.Duration) ([]*types.Sandbox, error) + ForkOperator(ctx context.Context, id string, count int, ttl time.Duration) ([]*types.Sandbox, error) Promote(ctx context.Context, id, token, template, tenant string) (types.PoolKey, error) + PromoteOperator(ctx context.Context, id, template, tenant string) (types.PoolKey, error) DeleteTemplate(ctx context.Context, key types.PoolKey, tenant string) error Checkpoint(ctx context.Context, id, token, name, tenant string) (types.Checkpoint, error) + CheckpointOperator(ctx context.Context, id, name, tenant string) (types.Checkpoint, error) Counters() pool.Counters TenantClaims() map[string]int Sandboxes() []pool.SandboxSummary + Sandbox(id string) (pool.SandboxSummary, bool) + Stats(id string) (pool.SandboxStats, bool) Audit(ctx context.Context, id string, line []byte) AuditEnabled() bool ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) Checkpoints(ctx context.Context, tenant string) ([]types.Checkpoint, error) + FetchCheckpoint(ctx context.Context, ckptID string) (dir string, meta []byte, release func(), err error) DeleteCheckpoint(ctx context.Context, ckptID, tenant string) error ClaimDeadline(id, token string) (time.Time, error) HasGolden(ctx context.Context, key types.PoolKey) bool @@ -93,6 +103,7 @@ type Dialer interface { type Placer interface { Candidates(keyHash string) []string TemplateOwners(keyHash string) []string + CheckpointOwners(ckptID string) []string PeerAddrs() []string ConfigMismatches() int } @@ -157,7 +168,10 @@ func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/claim", s.requireToken(s.handleClaim)) mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleRelease) - mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) + mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate, s.mgr.HibernateOperator)) + mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake, s.mgr.WakeOperator)) + mux.HandleFunc("GET /v1/sandboxes/{id}", s.requireRoot(s.handleSandbox)) + mux.HandleFunc("GET /v1/sandboxes/{id}/stats", s.requireRoot(s.handleSandboxStats)) // Fork and promote create node resources, so they take the same token // class as a claim; the source sandbox's token rides in the body as the // ownership proof. @@ -167,6 +181,9 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /v1/sandboxes/{id}/checkpoint", s.requireToken(s.handleCheckpoint)) mux.HandleFunc("POST /v1/checkpoints/{id}/claim", s.requireToken(s.handleClaimCheckpoint)) mux.HandleFunc("GET /v1/checkpoints", s.requireToken(s.handleListCheckpoints)) + // The peer-transfer half of the snapshot placement design: a node that + // cannot redirect a branch pulls the record from an owner through this. + mux.HandleFunc("GET /v1/checkpoints/{id}/blob", s.requireToken(s.handleCheckpointBlob)) mux.HandleFunc("DELETE /v1/checkpoints/{id}", s.requireToken(s.handleDeleteCheckpoint)) mux.HandleFunc("DELETE /v1/templates", s.requireToken(s.handleDeleteTemplate)) mux.HandleFunc("PUT /v1/pools", s.requireRoot(s.handlePutPools)) @@ -257,16 +274,51 @@ func (s *Server) handleRelease(w http.ResponseWriter, r *http.Request) { } } -// handleSandboxVerb adapts a sandbox-scoped manager call (hibernate) to HTTP: -// per-sandbox bearer auth, 404 on unknown, 204 on success. -func (s *Server) handleSandboxVerb(verb string, do func(ctx context.Context, id, token string) error) http.HandlerFunc { +// handleSandbox reports one live claim, so a control plane can read a single +// sandbox instead of scanning the whole-node listing on every reconcile. +func (s *Server) handleSandbox(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + sb, ok := s.mgr.Sandbox(id) + if !ok { + writeErr(w, http.StatusNotFound, "unknown sandbox") + return + } + writeJSON(w, http.StatusOK, sb) +} + +// handleSandboxStats reports one sandbox's resource usage. It is the only +// per-sandbox usage surface: /metrics is node- and pool-scoped by design. +func (s *Server) handleSandboxStats(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + st, ok := s.mgr.Stats(id) + if !ok { + writeErr(w, http.StatusNotFound, "unknown sandbox") + return + } + writeJSON(w, http.StatusOK, st) +} + +// handleSandboxVerb adapts a sandbox-scoped manager call (hibernate, wake) to +// HTTP: per-sandbox bearer auth, 404 on unknown, 204 on success. The node's +// root api_token takes the operator path instead, exactly as release does, so +// a control plane holding only the fleet token can drive the lifecycle. +func (s *Server) handleSandboxVerb( + verb string, + do func(ctx context.Context, id, token string) error, + operator func(ctx context.Context, id string) error, +) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) if !ok { return } id := r.PathValue("id") - err := do(r.Context(), id, token) + var err error + if s.isRootToken(token) { + err = operator(r.Context(), id) + } else { + err = do(r.Context(), id, token) + } writeResult(w, r, verb, id, verb+" failed", err, func() { w.WriteHeader(http.StatusNoContent) }) @@ -281,7 +333,16 @@ func (s *Server) handleFork(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - children, err := s.mgr.Fork(r.Context(), id, req.Token, req.Count, req.TTL()) + // An operator (root) caller proves authority with the node api_token in the + // header and carries no per-sandbox token; a tenant caller must still + // present the sandbox's own token as ownership proof. + var children []*types.Sandbox + var err error + if req.Token == "" && s.rootRequest(r) { + children, err = s.mgr.ForkOperator(r.Context(), id, req.Count, req.TTL()) + } else { + children, err = s.mgr.Fork(r.Context(), id, req.Token, req.Count, req.TTL()) + } writeResult(w, r, "fork", id, "fork failed", err, func() { resp := types.ForkResponse{Children: make([]types.ClaimResponse, len(children))} for i, c := range children { @@ -298,7 +359,13 @@ func (s *Server) handlePromote(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - key, err := s.mgr.Promote(r.Context(), id, req.Token, req.Template, tenantFrom(r.Context())) + var key types.PoolKey + var err error + if req.Token == "" && s.rootRequest(r) { + key, err = s.mgr.PromoteOperator(r.Context(), id, req.Template, tenantFrom(r.Context())) + } else { + key, err = s.mgr.Promote(r.Context(), id, req.Token, req.Template, tenantFrom(r.Context())) + } writeResult(w, r, "promote", id, "promote failed", err, func() { writeJSON(w, http.StatusOK, types.PromoteResponse{Key: key}) }) @@ -311,7 +378,13 @@ func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - ckpt, err := s.mgr.Checkpoint(r.Context(), id, req.Token, req.Name, tenantFrom(r.Context())) + var ckpt types.Checkpoint + var err error + if req.Token == "" && s.rootRequest(r) { + ckpt, err = s.mgr.CheckpointOperator(r.Context(), id, req.Name, tenantFrom(r.Context())) + } else { + ckpt, err = s.mgr.Checkpoint(r.Context(), id, req.Token, req.Name, tenantFrom(r.Context())) + } writeResult(w, r, "checkpoint", id, "checkpoint failed", err, func() { writeJSON(w, http.StatusOK, types.CheckpointResponse{Checkpoint: ckpt}) }) @@ -325,11 +398,48 @@ func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { } ckptID := r.PathValue("id") sb, err := s.mgr.ClaimCheckpoint(r.Context(), ckptID, req.TTL(), tenantFrom(r.Context())) + // Checkpoints are node-local. When this node does not hold the record, a + // peer that gossiped it does: redirect the branch there rather than failing, + // so the clone still runs on the node whose disk already has the data (its + // local reflink fast path). A no_redirect retry must resolve locally, or two + // nodes would bounce the request between them. + if errors.Is(err, pool.ErrUnknownCheckpoint) && s.placer != nil && !req.NoRedirect && + writeRedirect(w, s.placer.CheckpointOwners(ckptID)) { + return + } writeResult(w, r, "claim checkpoint", ckptID, "provisioning failed", err, func() { writeJSON(w, http.StatusOK, s.claimResponse(sb)) }) } +// handleCheckpointBlob streams a checkpoint record (its export directory and +// meta.json) as a tar, so a peer that must serve a branch locally can pull it. +// It is a read of an immutable record: the same api/tenant token class that +// may branch a checkpoint may also copy one. +func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { + ckptID := r.PathValue("id") + dir, meta, release, err := s.mgr.FetchCheckpoint(r.Context(), ckptID) + if err != nil { + if errors.Is(err, pool.ErrUnknownCheckpoint) { + writeErr(w, http.StatusNotFound, "unknown checkpoint") + return + } + log.WithFunc("server.handleCheckpointBlob").Error(r.Context(), err, "fetch checkpoint") + writeErr(w, http.StatusInternalServerError, "fetch checkpoint failed") + return + } + defer release() + + w.Header().Set("Content-Type", "application/x-tar") + // The status is committed before the walk begins, so a mid-stream failure + // can only truncate the tar — the reader detects that as a short archive + // and treats the pull as failed, which is the honest outcome. + w.WriteHeader(http.StatusOK) + if err := peer.TarRecord(dir, meta, w); err != nil { + log.WithFunc("server.handleCheckpointBlob").Error(r.Context(), err, "stream checkpoint") + } +} + // handleListCheckpoints lists this node's checkpoints, newest first — a // tenant caller sees only its own records. func (s *Server) handleListCheckpoints(w http.ResponseWriter, r *http.Request) { @@ -469,6 +579,13 @@ func (s *Server) requireRoot(next http.HandlerFunc) http.HandlerFunc { // operator credential). It mirrors resolveScope's root branch: an unset api // token matches nothing, so an open node grants no operator elevation on the // per-sandbox release path. Tenant tokens never match — they live in s.tenants. +// rootRequest reports whether this request presented the node's root +// api_token, the credential that authorizes the operator lifecycle paths. +func (s *Server) rootRequest(r *http.Request) bool { + token, ok := bearerToken(r) + return ok && s.isRootToken(token) +} + func (s *Server) isRootToken(token string) bool { return s.apiToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.apiToken)) == 1 } diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index c382525..d02ffd4 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -9,12 +9,15 @@ import ( "net" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "time" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/store/peer" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -957,11 +960,13 @@ func newTenantTestServer(t *testing.T, apiToken string, tenants []config.TenantS // the claim hook stands in for the provision result. Tenant-scoped methods // record the tenant they were handed in gotTenant. type fakeManager struct { + ckptDir string claim func(ctx context.Context, key types.PoolKey, ttl time.Duration) (*types.Sandbox, error) release func(id, token string) error releaseOp func(id string) error socket func(id, token string) (string, error) hibernate func(id, token string) error + wake func(id, token string) error fork func(id, token string, count int, ttl time.Duration) ([]*types.Sandbox, error) promote func(id, token, template string) error @@ -1079,6 +1084,41 @@ func (f *fakeManager) TenantClaims() map[string]int { return f.tenantClaims } func (f *fakeManager) Sandboxes() []pool.SandboxSummary { return nil } +func (f *fakeManager) Sandbox(string) (pool.SandboxSummary, bool) { + return pool.SandboxSummary{}, false +} + +func (f *fakeManager) Stats(string) (pool.SandboxStats, bool) { return pool.SandboxStats{}, false } + +// The operator variants take no per-sandbox token; the fake records them +// through the same hooks with an empty token, mirroring releaseOp. +func (f *fakeManager) HibernateOperator(ctx context.Context, id string) error { + return f.Hibernate(ctx, id, "") +} + +func (f *fakeManager) Wake(_ context.Context, id, token string) error { + if f.wake == nil { + return nil + } + return f.wake(id, token) +} + +func (f *fakeManager) WakeOperator(ctx context.Context, id string) error { + return f.Wake(ctx, id, "") +} + +func (f *fakeManager) ForkOperator(ctx context.Context, id string, count int, ttl time.Duration) ([]*types.Sandbox, error) { + return f.Fork(ctx, id, "", count, ttl) +} + +func (f *fakeManager) PromoteOperator(ctx context.Context, id, template, tenant string) (types.PoolKey, error) { + return f.Promote(ctx, id, "", template, tenant) +} + +func (f *fakeManager) CheckpointOperator(ctx context.Context, id, name, tenant string) (types.Checkpoint, error) { + return f.Checkpoint(ctx, id, "", name, tenant) +} + func (f *fakeManager) Audit(_ context.Context, id string, line []byte) { if f.audited != nil { f.audited(id, line) @@ -1144,11 +1184,164 @@ func (f *fakeDialer) DialSilkd(ctx context.Context, sock string) (net.Conn, erro } type fakePlacer struct { - addrs []string - owners []string + ckptOwners []string + addrs []string + owners []string } func (f *fakePlacer) Candidates(string) []string { return f.addrs } func (f *fakePlacer) TemplateOwners(string) []string { return f.owners } -func (f *fakePlacer) PeerAddrs() []string { return f.addrs } -func (f *fakePlacer) ConfigMismatches() int { return 0 } + +func (f *fakePlacer) CheckpointOwners(string) []string { return f.ckptOwners } +func (f *fakePlacer) PeerAddrs() []string { return f.addrs } +func (f *fakePlacer) ConfigMismatches() int { return 0 } + +// FetchCheckpoint serves the peer-transfer read; the fake reports every record +// missing unless a test supplies a directory. +func (f *fakeManager) FetchCheckpoint(_ context.Context, _ string) (string, []byte, func(), error) { + if f.ckptDir == "" { + return "", nil, nil, pool.ErrUnknownCheckpoint + } + return f.ckptDir, []byte(`{"id":"ck_00000000000000aa"}`), func() {}, nil +} + +// newPlacerTestServer builds a server with a mesh placer, which the shared +// helper deliberately leaves nil (most tests are single-node). +func newPlacerTestServer(t *testing.T, apiToken string, mgr Manager, placer Placer) *httptest.Server { + t.Helper() + srv := New(apiToken, nil, "node:7777", mgr, &fakeDialer{}, placer, nil) + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + return ts +} + +func postJSON(t *testing.T, url, token, body string) *http.Response { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, url, strings.NewReader(body)) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Authorization", "Bearer "+token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + return resp +} + +// TestCheckpointClaimRedirectsToOwner is L2: checkpoints are node-local, so a +// branch that lands on a node without the record must be pointed at one that +// has it — the clone then runs on the node whose disk already holds the data, +// on its local reflink fast path. Failing here instead would make "snapshot on +// A, branch from B" simply not work. +func TestCheckpointClaimRedirectsToOwner(t *testing.T) { + mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", mgr, placer) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 carrying a redirect (the mesh redirect is a 200, not a 3xx)", resp.StatusCode) + } + var got types.ClaimResponse + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if len(got.Redirect) != 1 || got.Redirect[0] != "owner-a:7777" { + t.Errorf("redirect = %v, want [owner-a:7777]", got.Redirect) + } + if got.ID != "" { + t.Errorf("id = %q, want empty: a redirect and a delivered sandbox are mutually exclusive", got.ID) + } +} + +// TestCheckpointClaimNoRedirectResolvesLocally: the retry at a redirect target +// must resolve there or two nodes would bounce the request between them. +func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { + placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", &fakeManager{}, placer) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{"no_redirect":true}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404: a no_redirect retry must not bounce again", resp.StatusCode) + } +} + +// TestCheckpointClaimNoOwnersIs404: nothing gossiped the record, so a miss is +// an honest miss rather than a redirect to nowhere. +func TestCheckpointClaimNoOwnersIs404(t *testing.T) { + ts := newPlacerTestServer(t, "sekret", &fakeManager{}, &fakePlacer{}) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404", resp.StatusCode) + } +} + +// TestCheckpointBlobUnknownIs404 lets a puller move on to the next owner +// instead of treating a stale gossip entry as a transfer failure. +func TestCheckpointBlobUnknownIs404(t *testing.T) { + ts := newTestServer(t, "sekret", &fakeManager{}, nil) + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, + ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", nil) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Authorization", "Bearer sekret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404 so the puller tries the next owner", resp.StatusCode) + } +} + +// TestCheckpointBlobStreamsRecord is the serving half of L3: the record leaves +// as a tar the peer can unpack, with its export contents intact. +func TestCheckpointBlobStreamsRecord(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "mem"), []byte("guest-pages"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + mgr := &fakeManager{ckptDir: src} + ts := newTestServer(t, "sekret", mgr, nil) + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, + ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", nil) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Authorization", "Bearer sekret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + dst := t.TempDir() + if err := peer.Untar(resp.Body, dst); err != nil { + t.Fatalf("Untar the streamed record: %v", err) + } + // The blob is a whole record: meta.json at the root plus the export under + // export/. Streaming only the export produced a directory that published + // cleanly and then failed every read as "unknown checkpoint". + if got, err := os.ReadFile(filepath.Join(dst, "export", "mem")); err != nil || string(got) != "guest-pages" { + t.Fatalf("streamed export/mem = %q, %v; want the record's bytes", got, err) + } + if _, err := os.Stat(filepath.Join(dst, "meta.json")); err != nil { + t.Fatalf("meta.json missing from the streamed record: %v", err) + } +} From d9ff16975d6c46a853fac596ccdeb2a993f37fde Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 26 Jul 2026 11:37:29 +0800 Subject: [PATCH 02/31] sandboxd: gossip checkpoint ownership so a branch can find its node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checkpoints are node-local, so a branch issued to a node that does not hold the record simply failed — "snapshot on A, branch from B" did not work at all. Gossip the checkpoint ids each node holds, alongside the warm counts and template hashes already published, and answer a branch this node cannot serve with a redirect to one that can: the same 200 + redirect contract a warm-miss claim uses. The record does not move, so the clone still runs on the node whose disk already has the data, on its local reflink fast path — cross-node correctness for one extra round trip and zero bytes transferred. CheckpointClaimRequest gains no_redirect so the retry at the target resolves locally instead of bouncing between two nodes. --- sandboxd/main.go | 7 +++-- sandboxd/mesh/mesh.go | 33 +++++++++++++++++-- sandboxd/mesh/mesh_test.go | 63 ++++++++++++++++++++++++++++++++++--- sandboxd/mesh/state_test.go | 8 ++--- sandboxd/types/api.go | 5 +++ 5 files changed, 102 insertions(+), 14 deletions(-) diff --git a/sandboxd/main.go b/sandboxd/main.go index de95c7a..960f0ee 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -101,7 +101,10 @@ func main() { } defer func() { _ = msh.Shutdown() }() placer = msh - mgr.SetTemplateNotifier(func() { msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes()) }) + mgr.SetTemplateNotifier(func() { msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs()) }) + // Peer healing needs the mesh's gossiped view to resolve who holds a + // record, so it is wired here rather than at store construction. + mgr.WithPeerHeal(cfg.CheckpointPeerHeal, msh.CheckpointOwners, cfg.APIToken) go gossipNodeState(ctx, msh, mgr) logger.Infof(ctx, "mesh %s joined (%d seeds)", cmp.Or(cfg.Mesh.NodeID, cfg.Mesh.Bind), len(cfg.Mesh.Join)) } @@ -194,7 +197,7 @@ func gossipNodeState(ctx context.Context, msh *mesh.Mesh, mgr *pool.Manager) { case <-ctx.Done(): return case <-t.C: - msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes()) + msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs()) } } } diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index b6c028d..a9d694a 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -33,7 +33,13 @@ type NodeState struct { Epoch uint64 `json:"epoch"` Pools map[string]int `json:"pools"` // PoolKey hash → warm count Templates []string `json:"templates,omitempty"` // promoted-template key hashes on disk - Digest string `json:"digest,omitempty"` // cluster-invariant config digest + // Checkpoints lists the checkpoint ids this node holds. Checkpoints are + // node-local like templates, so a resume or branch of one this node does + // not hold is answered with a redirect to a node that does — the placement + // moves to the data instead of the data moving to the placement, which is + // what keeps the clone on its local reflink fast path. + Checkpoints []string `json:"checkpoints,omitempty"` + Digest string `json:"digest,omitempty"` // cluster-invariant config digest } // Mesh is the node's view of the cluster and its own gossiped state. @@ -103,11 +109,12 @@ func (m *Mesh) Join(seeds []string) error { // set, bumping the epoch so peers adopt the new view. An unchanged view is // not republished — the periodic tick would otherwise gossip a fresh epoch // every second for nothing. -func (m *Mesh) UpdateSelf(pools map[string]int, templates []string) { +func (m *Mesh) UpdateSelf(pools map[string]int, templates, checkpoints []string) { m.updateMu.Lock() defer m.updateMu.Unlock() m.mu.Lock() - if maps.Equal(m.self.Pools, pools) && slices.Equal(m.self.Templates, templates) { + if maps.Equal(m.self.Pools, pools) && slices.Equal(m.self.Templates, templates) && + slices.Equal(m.self.Checkpoints, checkpoints) { m.mu.Unlock() return } @@ -124,6 +131,7 @@ func (m *Mesh) UpdateSelf(pools map[string]int, templates []string) { m.self.Epoch = epoch m.self.Pools = pools m.self.Templates = templates + m.self.Checkpoints = checkpoints m.view[m.self.NodeID] = m.self m.mu.Unlock() } @@ -215,6 +223,25 @@ func (m *Mesh) TemplateOwners(keyHash string) []string { return owners } +// CheckpointOwners returns up to two peer addresses whose gossiped checkpoint +// set contains ckptID — the redirect targets for a branch of a checkpoint this +// node does not hold. Self is excluded: the caller has already checked its own +// store. Mirrors TemplateOwners; both answer "who has this immutable record". +func (m *Mesh) CheckpointOwners(ckptID string) []string { + m.mu.Lock() + var owners []string + for id, st := range m.view { + if id != m.self.NodeID && slices.Contains(st.Checkpoints, ckptID) { + owners = append(owners, st.Addr) + } + } + m.mu.Unlock() + if len(owners) > 2 { + owners = owners[:2] + } + return owners +} + // Members returns the current cluster view (self included). func (m *Mesh) Members() []NodeState { m.mu.Lock() diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index 60e0b7a..11968ba 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -31,7 +31,7 @@ func TestMergeKeepsHigherEpoch(t *testing.T) { func TestMergeNeverOverwritesSelf(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(map[string]int{"k": 3}, nil) + m.UpdateSelf(map[string]int{"k": 3}, nil, nil) // A peer claiming to be "a" must not clobber our authoritative self entry. m.merge([]NodeState{{NodeID: "a", Addr: "evil:9999", Epoch: 999, Pools: map[string]int{"k": 0}}}) @@ -44,7 +44,7 @@ func TestMergeNeverOverwritesSelf(t *testing.T) { func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(map[string]int{"k": 5}, nil) // self has warm, but is never a candidate + m.UpdateSelf(map[string]int{"k": 5}, nil, nil) // self has warm, but is never a candidate m.merge([]NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 0}}, // no warm @@ -62,7 +62,7 @@ func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { func TestTemplateOwnersExcludeSelfAndUnknown(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(nil, []string{"tpl"}) // self holds it, but is never an owner candidate + m.UpdateSelf(nil, []string{"tpl"}, nil) // self holds it, but is never an owner candidate m.merge([]NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Templates: []string{"tpl", "other"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Templates: []string{"other"}}, @@ -87,7 +87,7 @@ func TestForgetPrunesDeadNode(t *testing.T) { t.Errorf("candidates after forget %v, want nil (b pruned)", got) } // forgetting self is a no-op. - m.UpdateSelf(map[string]int{"k": 1}, nil) + m.UpdateSelf(map[string]int{"k": 1}, nil, nil) m.forget("a") if len(m.Members()) != 1 { t.Error("forget removed self") @@ -121,7 +121,7 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { if err := b.mesh.Join([]string{a.addr}); err != nil { t.Fatalf("join: %v", err) } - a.mesh.UpdateSelf(map[string]int{"kk": 4}, []string{"tpl-hash"}) + a.mesh.UpdateSelf(map[string]int{"kk": 4}, []string{"tpl-hash"}, nil) // Push/pull sync propagates a's warm counts and template set to b within // a few intervals. @@ -169,3 +169,56 @@ func startNode(t *testing.T, host string, port int, id string) *node { t.Cleanup(func() { _ = m.Shutdown() }) return &node{mesh: m, addr: fmt.Sprintf("%s:%d", host, m.ml.LocalNode().Port)} } + +// TestCheckpointOwnersExcludeSelfAndUnknown mirrors the template case: the +// caller has already checked its own store, so self is never an answer — a +// redirect to ourselves would bounce the request in place. +func TestCheckpointOwnersExcludeSelfAndUnknown(t *testing.T) { + m := newTestMesh(t, "a") + m.UpdateSelf(nil, nil, []string{"ck_00000000000000aa"}) // self holds it, still not an owner candidate + m.merge([]NodeState{ + {NodeID: "b", Addr: "b:7777", Epoch: 1, Checkpoints: []string{"ck_00000000000000aa", "ck_00000000000000bb"}}, + {NodeID: "c", Addr: "c:7777", Epoch: 1, Checkpoints: []string{"ck_00000000000000bb"}}, + }) + + if owners := m.CheckpointOwners("ck_00000000000000aa"); len(owners) != 1 || owners[0] != "b:7777" { + t.Errorf("owners %v, want [b:7777]", owners) + } + if owners := m.CheckpointOwners("ck_deadbeefdeadbeef"); owners != nil { + t.Errorf("owners for absent checkpoint %v, want nil", owners) + } +} + +// TestCheckpointOwnersTruncates keeps a redirect answer small: the client only +// needs somewhere to retry, not the whole fleet. +func TestCheckpointOwnersTruncates(t *testing.T) { + m := newTestMesh(t, "a") + id := "ck_00000000000000aa" + m.merge([]NodeState{ + {NodeID: "b", Addr: "b:7777", Epoch: 1, Checkpoints: []string{id}}, + {NodeID: "c", Addr: "c:7777", Epoch: 1, Checkpoints: []string{id}}, + {NodeID: "d", Addr: "d:7777", Epoch: 1, Checkpoints: []string{id}}, + }) + + if owners := m.CheckpointOwners(id); len(owners) != 2 { + t.Errorf("owners = %v, want at most 2", owners) + } +} + +// TestUpdateSelfGossipsCheckpoints guards the wiring: a checkpoint set that +// never reaches NodeState is a redirect that never happens, which degrades L2 +// into "branch fails on the wrong node" without any visible error. +func TestUpdateSelfGossipsCheckpoints(t *testing.T) { + m := newTestMesh(t, "a") + m.UpdateSelf(nil, nil, []string{"ck_00000000000000aa"}) + + var self NodeState + for _, st := range m.Members() { + if st.NodeID == "a" { + self = st + } + } + if len(self.Checkpoints) != 1 || self.Checkpoints[0] != "ck_00000000000000aa" { + t.Fatalf("self.Checkpoints = %v, want the record to be gossiped", self.Checkpoints) + } +} diff --git a/sandboxd/mesh/state_test.go b/sandboxd/mesh/state_test.go index c0fa193..a37921c 100644 --- a/sandboxd/mesh/state_test.go +++ b/sandboxd/mesh/state_test.go @@ -52,7 +52,7 @@ func TestUpdateSelfPersistFailKeepsOldState(t *testing.T) { before := m.self.Epoch // A non-existent dir makes the durable write fail: the epoch must not publish. m.epochPath = filepath.Join(dir, "gone", "mesh-epoch") - m.UpdateSelf(map[string]int{"k": 1}, nil) + m.UpdateSelf(map[string]int{"k": 1}, nil, nil) if m.self.Epoch != before { t.Errorf("self epoch advanced to %d on a failed persist, want %d held", m.self.Epoch, before) } @@ -65,7 +65,7 @@ func TestUpdateSelfPersistsEpoch(t *testing.T) { dir := t.TempDir() m := newBoundMesh(t, dir) before := m.self.Epoch - m.UpdateSelf(map[string]int{"k": 1}, nil) + m.UpdateSelf(map[string]int{"k": 1}, nil, nil) if got := loadEpoch(filepath.Join(dir, "mesh-epoch")); got <= before { t.Errorf("persisted epoch %d did not advance past the seed %d", got, before) } @@ -78,8 +78,8 @@ func TestUpdateSelfConcurrentDropsNothing(t *testing.T) { a := map[string]int{"a": i + 1} b := map[string]int{"b": i + 1} var wg sync.WaitGroup - wg.Go(func() { m.UpdateSelf(a, nil) }) - wg.Go(func() { m.UpdateSelf(b, nil) }) + wg.Go(func() { m.UpdateSelf(a, nil, nil) }) + wg.Go(func() { m.UpdateSelf(b, nil, nil) }) wg.Wait() // Serialized updates with distinct payloads must both land: the loser // of the old TOCTOU race silently dropped its payload and one bump. diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index ce2306f..42be43b 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -84,6 +84,11 @@ type CheckpointResponse struct { // TTLSeconds semantics match a claim's; zero means the server default. type CheckpointClaimRequest struct { TTLSeconds int `json:"ttl_seconds,omitempty"` + // NoRedirect is set by a client retrying at a redirect target: checkpoints + // are node-local, so a branch of a record this node lacks is answered with + // the owning peer's address. The retry must resolve locally or two nodes + // would bounce it between them. + NoRedirect bool `json:"no_redirect,omitempty"` } // TTL converts the requested lease. From 6f7f22c97fdab2e22b1999dd3cbfe65b0b2f7249 Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 26 Jul 2026 11:37:45 +0800 Subject: [PATCH 03/31] sandboxd: pull a checkpoint from a peer when no owner can serve the branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redirect answers the common cross-node case without moving data, but it cannot answer when the owning node is gone, draining, or full — the choice there is between paying one transfer and failing the request. Add an opt-in peer store backend (checkpoint_peer_heal) that pulls the record from a node that gossiped it, publishes it locally through the same atomic staging path a local capture takes, and serves the branch from there. The cost is paid once: the node then gossips the record itself and serves later branches at local speed. The transport is sandboxd's own, deliberately not cocoon's mover — that one is bound to the engine's VM store layout, and reusing it would tie a checkpoint's mobility to the engine's release cycle. This moves exactly one thing, a store record (export/ plus meta.json), as a tar between two nodes that already authenticate with the fleet token. Entry names are validated against traversal and the stream is size-capped: an authenticated peer is still not allowed to write outside the destination this node chose. Also add a per-sandbox stats read. It is the only per-sandbox usage surface — /metrics is node- and pool-scoped by design — and it reports whether memory was measurable at all, so a hibernated sandbox reads as "not measured" rather than as idle. --- sandboxd/config/config.go | 10 + sandboxd/pool/pool.go | 13 ++ sandboxd/pool/stats.go | 121 ++++++++++++ sandboxd/store/peer/peer.go | 137 +++++++++++++ sandboxd/store/peer/peer_test.go | 186 +++++++++++++++++ sandboxd/store/peer/transport.go | 274 ++++++++++++++++++++++++++ sandboxd/store/peer/transport_test.go | 260 ++++++++++++++++++++++++ 7 files changed, 1001 insertions(+) create mode 100644 sandboxd/pool/stats.go create mode 100644 sandboxd/store/peer/peer.go create mode 100644 sandboxd/store/peer/peer_test.go create mode 100644 sandboxd/store/peer/transport.go create mode 100644 sandboxd/store/peer/transport_test.go diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 496c3a1..fcb59ff 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -225,6 +225,16 @@ type Config struct { // storage (credentials from the AWS chain, never this file). CheckpointStore *StoreConfig `json:"checkpoint_store,omitempty"` + // CheckpointPeerHeal lets a node pull a checkpoint it does not hold from a + // node that gossiped it, instead of failing the branch. It is the last + // tier of the snapshot placement design: placement normally follows the + // data (a branch is redirected to the owning node and clones on its local + // reflink fast path), and this moves the data only when no owner can serve + // — the owner is gone, draining, or full. Requires a mesh; ignored without + // one. Off by default: it trades a transfer for availability, and that is + // the operator's call. + CheckpointPeerHeal bool `json:"checkpoint_peer_heal,omitempty"` + // CheckpointTTLHours ages out checkpoints (0 = keep forever); the // sweep runs hourly and on startup. CheckpointTTLHours int `json:"checkpoint_ttl_hours,omitempty"` diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index c09c0ff..4b2807a 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -28,6 +28,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/netfilter" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/store/dir" + "github.com/cocoonstack/sandbox/sandboxd/store/peer" "github.com/cocoonstack/sandbox/sandboxd/store/s3" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -553,6 +554,18 @@ func newStoreView(ctx context.Context, cfg *config.Config, staging string, idRe return dir.New(cmp.Or(cfg.CheckpointDir, filepath.Join(cfg.DataDir, "checkpoints")), idRe) } +// WithPeerHeal wraps the manager's checkpoint store so a record this node does +// not hold is pulled from a node that gossiped it. It is wired after the mesh +// exists (the owners resolver is the mesh's view), and is a no-op unless +// checkpoint_peer_heal is set — a shared backend (s3, a FUSE mount) already +// resolves every record from every node and needs no healing. +func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { + if !enabled || owners == nil { + return + } + m.ckpts = peer.New(m.ckpts, owners, &peer.HTTPPuller{Token: token}) +} + func dirExists(path string) bool { fi, err := os.Stat(path) return err == nil && fi.IsDir() diff --git a/sandboxd/pool/stats.go b/sandboxd/pool/stats.go new file mode 100644 index 0000000..add8b9f --- /dev/null +++ b/sandboxd/pool/stats.go @@ -0,0 +1,121 @@ +package pool + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +// SandboxStats is one sandbox's resource usage. The declared fields (CPUCount, +// MemTotalBytes) come from the size tier, which is what the VM was actually +// booted with, so they are authoritative. The measured field (MemUsedBytes) is +// the host VMM process's resident set — for a microVM that is the guest's +// backing memory plus a small VMM overhead, and it is the only usage signal +// available without a guest agent that reports its own /proc. +// +// MemUsedBytes is zero when the VMM process cannot be found (a hibernated +// sandbox has no process at all); callers must treat zero as "not measured" +// rather than "idle". +type SandboxStats struct { + ID string `json:"id"` + CPUCount int `json:"cpu_count"` + MemTotalBytes int64 `json:"mem_total_bytes"` + MemUsedBytes int64 `json:"mem_used_bytes"` + Hibernated bool `json:"hibernated"` + MeasuredAt time.Time `json:"measured_at"` + MemUsedMeasured bool `json:"mem_used_measured"` +} + +// Stats reports one live claim's resource usage. +func (m *Manager) Stats(id string) (SandboxStats, bool) { + sb, ok := m.byID(id) + if !ok { + return SandboxStats{}, false + } + spec, _ := sb.Key.Size.Spec() + st := SandboxStats{ + ID: sb.ID, + CPUCount: spec.CPU, + MemTotalBytes: parseMemSize(spec.Memory), + Hibernated: sb.HibernateSnap != "", + MeasuredAt: time.Now().UTC(), + } + if !st.Hibernated { + if rss, ok := vmmResidentBytes(sb.VMName); ok { + st.MemUsedBytes, st.MemUsedMeasured = rss, true + } + } + return st, true +} + +// parseMemSize converts a size tier's memory string ("512M", "8G") to bytes. +func parseMemSize(s string) int64 { + if s == "" { + return 0 + } + mult := int64(1) + switch s[len(s)-1] { + case 'K', 'k': + mult, s = 1<<10, s[:len(s)-1] + case 'M', 'm': + mult, s = 1<<20, s[:len(s)-1] + case 'G', 'g': + mult, s = 1<<30, s[:len(s)-1] + } + n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + if err != nil { + return 0 + } + return n * mult +} + +// vmmResidentBytes finds the hypervisor process serving vmName and reports its +// resident set. The VM name appears in the VMM's argv (its run directory), so +// the scan matches on that; it is O(processes) and therefore meant for a +// single-sandbox read, never a whole-fleet sweep. +func vmmResidentBytes(vmName string) (int64, bool) { + if vmName == "" { + return 0, false + } + entries, err := os.ReadDir("/proc") + if err != nil { + return 0, false + } + for _, e := range entries { + if !e.IsDir() { + continue + } + pid := e.Name() + if pid[0] < '0' || pid[0] > '9' { + continue + } + cmdline, err := os.ReadFile(filepath.Join("/proc", pid, "cmdline")) + if err != nil || !strings.Contains(string(cmdline), vmName) { + continue + } + if rss, ok := residentBytes(pid); ok { + return rss, true + } + } + return 0, false +} + +// residentBytes reads a process's resident set from /proc//statm, whose +// second field is the resident page count. +func residentBytes(pid string) (int64, bool) { + b, err := os.ReadFile(filepath.Join("/proc", pid, "statm")) + if err != nil { + return 0, false + } + fields := strings.Fields(string(b)) + if len(fields) < 2 { + return 0, false + } + pages, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0, false + } + return pages * int64(os.Getpagesize()), true +} diff --git a/sandboxd/store/peer/peer.go b/sandboxd/store/peer/peer.go new file mode 100644 index 0000000..6f10082 --- /dev/null +++ b/sandboxd/store/peer/peer.go @@ -0,0 +1,137 @@ +package peer + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/cocoonstack/sandbox/sandboxd/store" +) + +// Owners resolves a record id to the peer addresses that hold it. It is the +// mesh's gossiped view (Mesh.CheckpointOwners), injected so the store stays +// testable without a live cluster. +type Owners func(id string) []string + +// Store is a node-local record store that can heal a miss from a peer. +// +// It is the third tier of the snapshot placement design, and deliberately the +// last resort. The first two tiers never move data: a branch normally runs on +// the node that already holds the record, on its local reflink fast path, and +// a request that lands elsewhere is redirected there. This tier exists for the +// case redirect cannot answer — the owning node is gone, draining, or full — +// where the choice is between paying one transfer and failing the request. +// +// A pull publishes the record into the local store, so the cost is paid once: +// afterwards this node is itself an owner, gossips the record, and serves +// branches of it locally like any other. +type Store struct { + // local is the node's own store; every operation but a Fetch/ReadMeta miss + // is served entirely by it. + local store.Store + // owners resolves who to pull from. Nil disables healing (the store then + // behaves exactly like its local backend). + owners Owners + // puller moves the bytes. + puller Puller +} + +var _ store.Store = (*Store)(nil) + +// New wraps local with peer healing. A nil owners or puller leaves the wrapper +// inert — it degrades to the local backend rather than failing, so a node with +// no mesh keeps working. +func New(local store.Store, owners Owners, puller Puller) *Store { + return &Store{local: local, owners: owners, puller: puller} +} + +func (s *Store) Stage(id string) (string, error) { return s.local.Stage(id) } + +func (s *Store) Publish(ctx context.Context, staging, id string) error { + return s.local.Publish(ctx, staging, id) +} + +func (s *Store) Delete(ctx context.Context, id string) error { return s.local.Delete(ctx, id) } + +func (s *Store) SweepStaging() error { return s.local.SweepStaging() } + +// Metas lists local records only. A cluster-wide listing is the control +// plane's job (it already scatter-gathers every node); having each node +// answer for its peers would return the same record N times. +func (s *Store) Metas(ctx context.Context) ([][]byte, error) { return s.local.Metas(ctx) } + +// Fetch serves the record locally, healing from a peer on a miss. +func (s *Store) Fetch(ctx context.Context, id string) (string, []byte, func(), error) { + dir, meta, release, err := s.local.Fetch(ctx, id) + if !errors.Is(err, store.ErrNotFound) { + return dir, meta, release, err + } + if err := s.heal(ctx, id); err != nil { + return "", nil, nil, err + } + return s.local.Fetch(ctx, id) +} + +// ReadMeta reads the record's metadata, healing from a peer on a miss. +func (s *Store) ReadMeta(ctx context.Context, id string) ([]byte, error) { + meta, err := s.local.ReadMeta(ctx, id) + if !errors.Is(err, store.ErrNotFound) { + return meta, err + } + if err := s.heal(ctx, id); err != nil { + return nil, err + } + return s.local.ReadMeta(ctx, id) +} + +// heal pulls id from the first peer that serves it and publishes it locally. +// Returns store.ErrNotFound when no peer has it, so a miss stays a miss and +// the caller's existing not-found handling is unchanged. +func (s *Store) heal(ctx context.Context, id string) error { + if s.owners == nil || s.puller == nil { + return store.ErrNotFound + } + addrs := s.owners(id) + if len(addrs) == 0 { + return store.ErrNotFound + } + // A started pull must finish even if the requesting client hangs up: + // abandoning it halfway would leave staging to the sweeper and make the + // next branch pay the whole transfer again. + ctx = context.WithoutCancel(ctx) + + var errs []error + for _, addr := range addrs { + if err := s.pullFrom(ctx, addr, id); err != nil { + if !errors.Is(err, ErrNotFound) { + errs = append(errs, fmt.Errorf("peer %s: %w", addr, err)) + } + continue + } + return nil + } + if len(errs) > 0 { + return fmt.Errorf("heal %s from %d peer(s): %w", id, len(addrs), errors.Join(errs...)) + } + // Every owner answered "not found": the gossiped view was stale. + return store.ErrNotFound +} + +// pullFrom stages a peer's copy and publishes it, so the record becomes local +// through the same atomic path a locally-created one takes. +func (s *Store) pullFrom(ctx context.Context, addr, id string) error { + staging, err := s.local.Stage(id) + if err != nil { + return fmt.Errorf("stage: %w", err) + } + defer func() { _ = os.RemoveAll(staging) }() + + if err := s.puller.Pull(ctx, addr, id, staging); err != nil { + return err + } + if err := s.local.Publish(ctx, staging, id); err != nil { + return fmt.Errorf("publish pulled record: %w", err) + } + return nil +} diff --git a/sandboxd/store/peer/peer_test.go b/sandboxd/store/peer/peer_test.go new file mode 100644 index 0000000..6aa216b --- /dev/null +++ b/sandboxd/store/peer/peer_test.go @@ -0,0 +1,186 @@ +package peer + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/store/dir" + "github.com/cocoonstack/sandbox/sandboxd/store/storetest" +) + +const testID = "ck_00000000000000aa" + +func localStore(t *testing.T) *dir.Store { + t.Helper() + st, err := dir.New(t.TempDir(), store.CheckpointIDRe) + if err != nil { + t.Fatalf("dir.New: %v", err) + } + return st +} + +// fakePuller serves records from an in-memory table, recording who was asked. +type fakePuller struct { + records map[string]map[string]string // addr → file → content + asked []string + failAddr string +} + +func (p *fakePuller) Pull(_ context.Context, addr, _ string, dst string) error { + p.asked = append(p.asked, addr) + if addr == p.failAddr { + return errors.New("peer exploded") + } + files, ok := p.records[addr] + if !ok { + return ErrNotFound + } + for name, content := range files { + full := filepath.Join(dst, name) + if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { + return err + } + if err := os.WriteFile(full, []byte(content), 0o600); err != nil { + return err + } + } + return nil +} + +func record() map[string]string { + return map[string]string{ + store.MetaFile: `{"id":"` + testID + `"}`, + filepath.Join(store.ExportDir, "mem"): "guest-pages", + } +} + +// TestPeerBackendContract: wrapping must not change local behavior. Every +// operation but a Fetch/ReadMeta miss is the local backend's, so the wrapper +// has to satisfy the same store contract the dir backend does. +func TestPeerBackendContract(t *testing.T) { + storetest.RunContract(t, New(localStore(t), nil, nil)) +} + +// TestInertWithoutOwnersOrPuller: a node with no mesh must degrade to its local +// backend, not fail. Healing is an addition, never a dependency. +func TestInertWithoutOwnersOrPuller(t *testing.T) { + s := New(localStore(t), nil, nil) + if _, _, _, err := s.Fetch(t.Context(), testID); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("Fetch error = %v, want store.ErrNotFound (a miss stays a miss)", err) + } +} + +// TestHealPublishesLocally is L3's whole point: after healing once, the record +// is local, so the next branch is served at L1 speed and this node becomes an +// owner that can serve its peers. The transfer must be paid once, not per use. +func TestHealPublishesLocally(t *testing.T) { + local := localStore(t) + puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} + s := New(local, func(string) []string { return []string{"peer-a:7777"} }, puller) + + dir1, _, release, err := s.Fetch(t.Context(), testID) + if err != nil { + t.Fatalf("Fetch after heal: %v", err) + } + release() + if got, err := os.ReadFile(filepath.Join(dir1, "mem")); err != nil || string(got) != "guest-pages" { + t.Fatalf("healed export = %q, %v; want the peer's bytes", got, err) + } + + // The record is now in the LOCAL backend: reading it directly, with no + // wrapper and no puller, must succeed. + if _, err := local.ReadMeta(t.Context(), testID); err != nil { + t.Fatalf("record not published locally after heal: %v", err) + } + // And a second Fetch must not touch the network again. + if _, _, release2, err := s.Fetch(t.Context(), testID); err != nil { + t.Fatalf("second Fetch: %v", err) + } else { + release2() + } + if len(puller.asked) != 1 { + t.Errorf("puller called %d times (%v); the transfer must be paid once", len(puller.asked), puller.asked) + } +} + +// TestHealTriesNextOwner: a gossiped view can be stale or a peer can be broken. +// One bad owner must not fail the heal while another can still serve it. +func TestHealTriesNextOwner(t *testing.T) { + puller := &fakePuller{ + records: map[string]map[string]string{"peer-b:7777": record()}, + failAddr: "peer-a:7777", + } + s := New(localStore(t), func(string) []string { return []string{"peer-a:7777", "peer-b:7777"} }, puller) + + _, _, release, err := s.Fetch(t.Context(), testID) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + release() + if len(puller.asked) != 2 { + t.Errorf("asked = %v, want both owners tried in order", puller.asked) + } +} + +// TestHealAllOwnersMissStaysNotFound: when every owner answers "not found" the +// gossiped view was simply stale. That must surface as store.ErrNotFound so the +// caller's existing not-found handling (a 404, not a 500) is unchanged. +func TestHealAllOwnersMissStaysNotFound(t *testing.T) { + puller := &fakePuller{records: map[string]map[string]string{}} + s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) + + _, _, _, err := s.Fetch(t.Context(), testID) + if !errors.Is(err, store.ErrNotFound) { + t.Fatalf("Fetch error = %v, want store.ErrNotFound", err) + } +} + +// TestHealNoOwners: nothing gossiped the record, so there is nobody to ask. +func TestHealNoOwners(t *testing.T) { + puller := &fakePuller{} + s := New(localStore(t), func(string) []string { return nil }, puller) + + if _, err := s.ReadMeta(t.Context(), testID); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("ReadMeta error = %v, want store.ErrNotFound", err) + } + if len(puller.asked) != 0 { + t.Errorf("puller was called with no owners: %v", puller.asked) + } +} + +// TestHealErrorIsReported: a peer that fails for a real reason (not a miss) +// must not be flattened into "not found" — that would hide a broken transfer +// behind a 404 and send the operator looking for a deleted checkpoint. +func TestHealErrorIsReported(t *testing.T) { + puller := &fakePuller{failAddr: "peer-a:7777"} + s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) + + _, _, _, err := s.Fetch(t.Context(), testID) + if err == nil || errors.Is(err, store.ErrNotFound) { + t.Fatalf("Fetch error = %v, want the peer failure surfaced", err) + } + if !bytes.Contains([]byte(err.Error()), []byte("peer exploded")) { + t.Errorf("error = %v, want the underlying peer failure named", err) + } +} + +// TestMetasStaysLocal: a cluster-wide listing is the control plane's job (it +// already scatter-gathers every node). If each node answered for its peers the +// same record would come back N times. +func TestMetasStaysLocal(t *testing.T) { + puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} + s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) + + metas, err := s.Metas(t.Context()) + if err != nil { + t.Fatalf("Metas: %v", err) + } + if len(metas) != 0 { + t.Errorf("Metas returned %d record(s); a peer's records must not appear in a local listing", len(metas)) + } +} diff --git a/sandboxd/store/peer/transport.go b/sandboxd/store/peer/transport.go new file mode 100644 index 0000000..de18d06 --- /dev/null +++ b/sandboxd/store/peer/transport.go @@ -0,0 +1,274 @@ +// Package peer adds cross-node reach to a node-local record store: when a +// record is missing locally, it is pulled from a node that gossiped it, cached +// locally, and served from there. +// +// The transport is sandboxd's own — deliberately NOT cocoon's image/snapshot +// transfer. cocoon's mover is bound to its VM store layout and its own +// addressing, and reusing it would tie a checkpoint's mobility to the engine's +// release cycle. This one moves exactly one thing (a store record: an export +// directory plus its meta.json) between two sandboxd nodes that already +// authenticate to each other with the fleet api_token, over the control-plane +// port they already share. It is a tar stream, so a pull costs one round trip +// and never materializes the record twice. +package peer + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +// pullTimeout bounds a single record transfer. A checkpoint carries a guest's +// memory image, so this is generous — but it is never unbounded: a wedged peer +// must fail the pull and let the next owner be tried, not hang the branch. +const pullTimeout = 30 * time.Minute + +// maxRecordBytes caps a pulled record. It is a decompression-bomb guard, not a +// business limit: a peer is authenticated, but a compromised or buggy one must +// not be able to fill this node's disk. +const maxRecordBytes = 1 << 40 // 1 TiB + +// The record layout a pull must reproduce, mirroring the store's own names. +const ( + metaFile = "meta.json" + exportPrefix = "export/" +) + +// Puller fetches a record from a peer into a local directory. +type Puller interface { + // Pull writes the record's contents (export/ and meta.json) into dst, + // which the caller has created. It returns ErrNotFound when the peer does + // not hold the record, so the caller can try the next owner. + Pull(ctx context.Context, addr, id, dst string) error +} + +// ErrNotFound reports that a peer does not hold the requested record. It +// mirrors store.ErrNotFound but is declared here so the transport does not +// import the store package it is a backend for. +var ErrNotFound = errors.New("peer does not hold record") + +// HTTPPuller pulls records over sandboxd's control-plane HTTP port. +type HTTPPuller struct { + // Client is the HTTP client. A nil Client uses a default with no overall + // timeout — the per-pull deadline comes from the context instead, so a + // large record is not cut off mid-stream. + Client *http.Client + // Token is the fleet api_token presented to the serving peer. + Token string +} + +// Pull implements Puller. +func (p *HTTPPuller) Pull(ctx context.Context, addr, id, dst string) error { + ctx, cancel := context.WithTimeout(ctx, pullTimeout) + defer cancel() + + base := addr + if !strings.Contains(base, "://") { + base = "http://" + base + } + u := base + "/v1/checkpoints/" + url.PathEscape(id) + "/blob" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return err + } + if p.Token != "" { + req.Header.Set("Authorization", "Bearer "+p.Token) + } + + client := p.Client + if client == nil { + client = http.DefaultClient + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("pull %s from %s: %w", id, addr, err) + } + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return ErrNotFound + default: + return fmt.Errorf("pull %s from %s: unexpected status %d", id, addr, resp.StatusCode) + } + return Untar(resp.Body, dst) +} + +// TarRecord streams a whole store record: the export directory under +// "export/" plus its "meta.json". A record is only branchable with both — the +// meta carries the pool key and lineage the claim path reads before it ever +// touches the export, so shipping the export alone produces a directory that +// publishes cleanly and then fails every read as "unknown checkpoint". +func TarRecord(exportDir string, meta []byte, w io.Writer) error { + tw := tar.NewWriter(w) + defer func() { _ = tw.Close() }() + + if err := tw.WriteHeader(&tar.Header{ + Name: metaFile, Mode: 0o600, Size: int64(len(meta)), Typeflag: tar.TypeReg, + }); err != nil { + return fmt.Errorf("tar record meta: %w", err) + } + if _, err := tw.Write(meta); err != nil { + return fmt.Errorf("tar record meta: %w", err) + } + if err := tw.WriteHeader(&tar.Header{ + Name: exportPrefix, Mode: 0o700, Typeflag: tar.TypeDir, + }); err != nil { + return fmt.Errorf("tar record export dir: %w", err) + } + if err := tarInto(exportDir, exportPrefix, tw); err != nil { + return err + } + return tw.Close() +} + +// Tar streams src's contents as a tar archive. Only regular files and +// directories are emitted: a record is data, and a symlink or device node in +// the stream would be a way to write outside the reader's destination. +func Tar(src string, w io.Writer) error { + tw := tar.NewWriter(w) + defer func() { _ = tw.Close() }() + if err := tarInto(src, "", tw); err != nil { + return err + } + return tw.Close() +} + +// tarInto walks src into tw, prefixing every entry name with prefix. +func tarInto(src, prefix string, tw *tar.Writer) error { + err := filepath.Walk(src, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + name := prefix + rel + switch { + case fi.IsDir(): + return tw.WriteHeader(&tar.Header{ + Name: name + "/", + Mode: int64(fi.Mode().Perm()), + Typeflag: tar.TypeDir, + }) + case fi.Mode().IsRegular(): + if err := tw.WriteHeader(&tar.Header{ + Name: name, + Mode: int64(fi.Mode().Perm()), + Size: fi.Size(), + Typeflag: tar.TypeReg, + }); err != nil { + return err + } + f, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + _, err = io.Copy(tw, f) + return err + default: + // Skip anything that is not data. + return nil + } + }) + if err != nil { + return fmt.Errorf("tar %s: %w", src, err) + } + return nil +} + +// Untar writes a tar stream into dst. Entry names are validated against path +// traversal — a peer is authenticated, but an authenticated peer is still not +// allowed to write outside the destination this node chose. +func Untar(r io.Reader, dst string) error { + tr := tar.NewReader(r) + var written int64 + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("untar: %w", err) + } + target, err := safeJoin(dst, hdr.Name) + if err != nil { + return err + } + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o700); err != nil { + return fmt.Errorf("untar mkdir %s: %w", hdr.Name, err) + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return fmt.Errorf("untar mkdir %s: %w", filepath.Dir(hdr.Name), err) + } + written += hdr.Size + if written > maxRecordBytes { + return fmt.Errorf("untar: record exceeds %d bytes", int64(maxRecordBytes)) + } + if err := writeFile(target, tr, os.FileMode(hdr.Mode).Perm()); err != nil { + return err + } + default: + // Ignore non-data entries rather than trusting them. + continue + } + } +} + +// writeFile copies one entry to disk. +func writeFile(target string, r io.Reader, mode os.FileMode) error { + if mode == 0 { + mode = 0o600 + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("untar create %s: %w", target, err) + } + defer func() { _ = f.Close() }() + if _, err := io.Copy(f, io.LimitReader(r, maxRecordBytes)); err != nil { + return fmt.Errorf("untar write %s: %w", target, err) + } + return f.Close() +} + +// safeJoin resolves name under root, rejecting absolute paths and any name +// that would escape root. +func safeJoin(root, name string) (string, error) { + if name == "" || filepath.IsAbs(name) || strings.Contains(name, `\`) { + return "", fmt.Errorf("untar: refusing entry %q", name) + } + // Clean the name as-is, NOT as "/"+name: a leading slash would absorb any + // leading "..", silently rewriting ../escape into /escape and landing the + // entry inside root under a surprising name. A record this node produced + // never contains "..", so its presence means corruption or an attack — + // reject it loudly instead of neutralizing it. + clean := filepath.Clean(name) + if clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("untar: entry %q escapes destination", name) + } + target := filepath.Join(root, clean) + if target != root && !strings.HasPrefix(target, root+string(os.PathSeparator)) { + return "", fmt.Errorf("untar: entry %q escapes destination", name) + } + return target, nil +} diff --git a/sandboxd/store/peer/transport_test.go b/sandboxd/store/peer/transport_test.go new file mode 100644 index 0000000..6d15cd0 --- /dev/null +++ b/sandboxd/store/peer/transport_test.go @@ -0,0 +1,260 @@ +package peer + +import ( + "archive/tar" + "bytes" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestTarUntarRoundTrip is the transport's core promise: a record that goes out +// as a tar comes back byte-identical on the other node, directory structure +// included. A checkpoint is a guest memory image plus its meta — a transport +// that silently drops a file would produce a clone of the wrong state. +func TestTarUntarRoundTrip(t *testing.T) { + src := t.TempDir() + files := map[string]string{ + "meta.json": `{"id":"ck_0000000000000001"}`, + "export/memory-range-0": "guest-memory-page-contents", + "export/disk.qcow2": "disk-bytes", + "export/nested/deeper/file": "deep", + } + for name, content := range files { + p := filepath.Join(src, name) + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatalf("mkdir %s: %v", name, err) + } + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + + var buf bytes.Buffer + if err := Tar(src, &buf); err != nil { + t.Fatalf("Tar: %v", err) + } + dst := t.TempDir() + if err := Untar(&buf, dst); err != nil { + t.Fatalf("Untar: %v", err) + } + + for name, want := range files { + got, err := os.ReadFile(filepath.Join(dst, name)) + if err != nil { + t.Errorf("read back %s: %v", name, err) + continue + } + if string(got) != want { + t.Errorf("%s = %q, want %q", name, got, want) + } + } +} + +// TestUntarRejectsPathTraversal is the security invariant. A peer authenticates +// with the fleet token, but an authenticated peer is still not allowed to write +// outside the destination this node chose — a compromised or buggy one must not +// be able to overwrite the host's files. +func TestUntarRejectsPathTraversal(t *testing.T) { + for _, name := range []string{ + "../escape", + "../../etc/passwd", + "export/../../escape", + "/absolute/path", + `..\windows-style`, + } { + t.Run(name, func(t *testing.T) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + body := []byte("pwned") + if err := tw.WriteHeader(&tar.Header{ + Name: name, Mode: 0o600, Size: int64(len(body)), Typeflag: tar.TypeReg, + }); err != nil { + t.Fatalf("WriteHeader: %v", err) + } + if _, err := tw.Write(body); err != nil { + t.Fatalf("Write: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + dst := t.TempDir() + err := Untar(&buf, dst) + if err == nil { + t.Fatalf("Untar accepted %q; it must refuse to write outside the destination", name) + } + if !strings.Contains(err.Error(), "untar") { + t.Errorf("error = %v, want an untar rejection", err) + } + }) + } +} + +// TestUntarSkipsNonDataEntries keeps symlinks and device nodes out of a record. +// A record is data; honoring a symlink entry would be an indirect way to make a +// later write land outside the destination. +func TestUntarSkipsNonDataEntries(t *testing.T) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + if err := tw.WriteHeader(&tar.Header{ + Name: "evil-link", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd", Mode: 0o777, + }); err != nil { + t.Fatalf("WriteHeader: %v", err) + } + if err := tw.WriteHeader(&tar.Header{ + Name: "meta.json", Mode: 0o600, Size: 2, Typeflag: tar.TypeReg, + }); err != nil { + t.Fatalf("WriteHeader: %v", err) + } + if _, err := tw.Write([]byte("{}")); err != nil { + t.Fatalf("Write: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + dst := t.TempDir() + if err := Untar(&buf, dst); err != nil { + t.Fatalf("Untar: %v", err) + } + if _, err := os.Lstat(filepath.Join(dst, "evil-link")); err == nil { + t.Error("symlink entry was materialized; non-data entries must be skipped") + } + // The legitimate entry alongside it still lands. + if _, err := os.Stat(filepath.Join(dst, "meta.json")); err != nil { + t.Errorf("meta.json missing after skipping the symlink: %v", err) + } +} + +// TestTarSkipsSymlinksAtSource is the same rule on the sending side: a record +// that somehow contains a symlink must not put one on the wire. +func TestTarSkipsSymlinksAtSource(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "real"), []byte("x"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + if err := os.Symlink("/etc/passwd", filepath.Join(src, "link")); err != nil { + t.Skipf("symlink unsupported here: %v", err) + } + + var buf bytes.Buffer + if err := Tar(src, &buf); err != nil { + t.Fatalf("Tar: %v", err) + } + tr := tar.NewReader(&buf) + for { + hdr, err := tr.Next() + if err != nil { + break + } + if hdr.Typeflag == tar.TypeSymlink { + t.Errorf("Tar emitted a symlink entry %q", hdr.Name) + } + } +} + +// TestHTTPPullerNotFound maps a peer's 404 to ErrNotFound so the caller keeps +// trying the next owner instead of failing the whole heal. A gossiped view can +// be stale — one owner not having the record is normal, not an error. +func TestHTTPPullerNotFound(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + p := &HTTPPuller{Client: srv.Client()} + err := p.Pull(t.Context(), srv.URL, "ck_0000000000000001", t.TempDir()) + if !errors.Is(err, ErrNotFound) { + t.Fatalf("Pull error = %v, want ErrNotFound so the next owner is tried", err) + } +} + +// TestHTTPPullerPresentsToken proves the fleet token rides on the pull: the +// serving peer authorizes this exactly like any other control-plane read. +func TestHTTPPullerPresentsToken(t *testing.T) { + var gotAuth, gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth, gotPath = r.Header.Get("Authorization"), r.URL.Path + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + _ = tw.WriteHeader(&tar.Header{Name: "meta.json", Mode: 0o600, Size: 2, Typeflag: tar.TypeReg}) + _, _ = tw.Write([]byte("{}")) + _ = tw.Close() + _, _ = w.Write(buf.Bytes()) + })) + defer srv.Close() + + p := &HTTPPuller{Client: srv.Client(), Token: "fleet-token"} + dst := t.TempDir() + if err := p.Pull(t.Context(), srv.URL, "ck_0000000000000001", dst); err != nil { + t.Fatalf("Pull: %v", err) + } + if gotAuth != "Bearer fleet-token" { + t.Errorf("Authorization = %q, want the fleet token", gotAuth) + } + if want := "/v1/checkpoints/ck_0000000000000001/blob"; gotPath != want { + t.Errorf("path = %q, want %q", gotPath, want) + } + if _, err := os.Stat(filepath.Join(dst, "meta.json")); err != nil { + t.Errorf("pulled record not materialized: %v", err) + } +} + +// TestTarRecordRoundTripsAWholeRecord reproduces a bug found in the cluster: +// the blob endpoint tarred only the export directory Fetch() returns, so a +// pulled record published cleanly and then failed every read as "unknown +// checkpoint" — the meta carrying the pool key and lineage was never shipped. +// A record on the wire must be exactly what the local store lays out. +func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { + exportDir := t.TempDir() + for name, content := range map[string]string{ + "memory-ranges": "guest-pages", + "cow.raw": "disk", + "snapshot.json": "{}", + } { + if err := os.WriteFile(filepath.Join(exportDir, name), []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + meta := []byte(`{"id":"ck_0000000000000001","key":{"template":"t"}}`) + + var buf bytes.Buffer + if err := TarRecord(exportDir, meta, &buf); err != nil { + t.Fatalf("TarRecord: %v", err) + } + dst := t.TempDir() + if err := Untar(&buf, dst); err != nil { + t.Fatalf("Untar: %v", err) + } + + // meta.json at the record root — without it the record is unreadable. + got, err := os.ReadFile(filepath.Join(dst, "meta.json")) + if err != nil { + t.Fatalf("meta.json missing from the shipped record: %v", err) + } + if string(got) != string(meta) { + t.Errorf("meta.json = %q, want %q", got, meta) + } + // export/ contents under the export prefix, not at the root. + for name, want := range map[string]string{ + "memory-ranges": "guest-pages", + "cow.raw": "disk", + } { + got, err := os.ReadFile(filepath.Join(dst, "export", name)) + if err != nil { + t.Errorf("export/%s missing: %v", name, err) + continue + } + if string(got) != want { + t.Errorf("export/%s = %q, want %q", name, got, want) + } + } + if _, err := os.Stat(filepath.Join(dst, "memory-ranges")); err == nil { + t.Error("export contents landed at the record root; they must be under export/") + } +} From 1904cf9279d0da283b0e7716e1390be86266bbb0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 12:49:08 +0800 Subject: [PATCH 04/31] simplify: drop the dead tar export and the tier-memory parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tar had no production caller — TarRecord is the only path the blob handler uses — so the round-trip tests now drive tarInto directly. parseMemSize existed to parse SizeSpec.Memory, whose whole domain is four literals from one table; the table carries MemoryBytes instead. Untar already caps a record by accumulating hdr.Size, and archive/tar bounds each read to the entry size, so writeFile's LimitReader guarded nothing. --- sandboxd/pool/stats.go | 59 ++++----------- sandboxd/store/peer/transport.go | 100 ++++++++------------------ sandboxd/store/peer/transport_test.go | 20 ++++-- sandboxd/types/types.go | 16 +++-- 4 files changed, 70 insertions(+), 125 deletions(-) diff --git a/sandboxd/pool/stats.go b/sandboxd/pool/stats.go index add8b9f..8698c7d 100644 --- a/sandboxd/pool/stats.go +++ b/sandboxd/pool/stats.go @@ -8,24 +8,19 @@ import ( "time" ) -// SandboxStats is one sandbox's resource usage. The declared fields (CPUCount, -// MemTotalBytes) come from the size tier, which is what the VM was actually -// booted with, so they are authoritative. The measured field (MemUsedBytes) is -// the host VMM process's resident set — for a microVM that is the guest's -// backing memory plus a small VMM overhead, and it is the only usage signal -// available without a guest agent that reports its own /proc. -// -// MemUsedBytes is zero when the VMM process cannot be found (a hibernated -// sandbox has no process at all); callers must treat zero as "not measured" -// rather than "idle". +// SandboxStats is one sandbox's resource usage. CPUCount and MemTotalBytes come +// from the size tier the VM was booted with. MemUsedBytes is the host VMM +// process's resident set, the only usage signal available without a guest agent; +// MemUsedMeasured is false when no VMM process was found (a hibernated sandbox +// has none), so a zero is never read as idle. type SandboxStats struct { ID string `json:"id"` CPUCount int `json:"cpu_count"` MemTotalBytes int64 `json:"mem_total_bytes"` MemUsedBytes int64 `json:"mem_used_bytes"` + MemUsedMeasured bool `json:"mem_used_measured"` Hibernated bool `json:"hibernated"` MeasuredAt time.Time `json:"measured_at"` - MemUsedMeasured bool `json:"mem_used_measured"` } // Stats reports one live claim's resource usage. @@ -38,7 +33,7 @@ func (m *Manager) Stats(id string) (SandboxStats, bool) { st := SandboxStats{ ID: sb.ID, CPUCount: spec.CPU, - MemTotalBytes: parseMemSize(spec.Memory), + MemTotalBytes: spec.MemoryBytes, Hibernated: sb.HibernateSnap != "", MeasuredAt: time.Now().UTC(), } @@ -50,31 +45,9 @@ func (m *Manager) Stats(id string) (SandboxStats, bool) { return st, true } -// parseMemSize converts a size tier's memory string ("512M", "8G") to bytes. -func parseMemSize(s string) int64 { - if s == "" { - return 0 - } - mult := int64(1) - switch s[len(s)-1] { - case 'K', 'k': - mult, s = 1<<10, s[:len(s)-1] - case 'M', 'm': - mult, s = 1<<20, s[:len(s)-1] - case 'G', 'g': - mult, s = 1<<30, s[:len(s)-1] - } - n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64) - if err != nil { - return 0 - } - return n * mult -} - -// vmmResidentBytes finds the hypervisor process serving vmName and reports its -// resident set. The VM name appears in the VMM's argv (its run directory), so -// the scan matches on that; it is O(processes) and therefore meant for a -// single-sandbox read, never a whole-fleet sweep. +// vmmResidentBytes reports the resident set of the hypervisor process serving +// vmName, matched on the run directory in the VMM's argv. The scan is +// O(processes), so this is a single-sandbox read, never a fleet sweep. func vmmResidentBytes(vmName string) (int64, bool) { if vmName == "" { return 0, false @@ -84,14 +57,11 @@ func vmmResidentBytes(vmName string) (int64, bool) { return 0, false } for _, e := range entries { - if !e.IsDir() { - continue - } pid := e.Name() - if pid[0] < '0' || pid[0] > '9' { + if !e.IsDir() || pid[0] < '0' || pid[0] > '9' { continue } - cmdline, err := os.ReadFile(filepath.Join("/proc", pid, "cmdline")) + cmdline, err := os.ReadFile(filepath.Join("/proc", pid, "cmdline")) //nolint:gosec // pid enumerated from /proc if err != nil || !strings.Contains(string(cmdline), vmName) { continue } @@ -102,10 +72,9 @@ func vmmResidentBytes(vmName string) (int64, bool) { return 0, false } -// residentBytes reads a process's resident set from /proc//statm, whose -// second field is the resident page count. +// residentBytes reads a process's resident page count from statm's second field. func residentBytes(pid string) (int64, bool) { - b, err := os.ReadFile(filepath.Join("/proc", pid, "statm")) + b, err := os.ReadFile(filepath.Join("/proc", pid, "statm")) //nolint:gosec // pid enumerated from /proc if err != nil { return 0, false } diff --git a/sandboxd/store/peer/transport.go b/sandboxd/store/peer/transport.go index de18d06..eba98be 100644 --- a/sandboxd/store/peer/transport.go +++ b/sandboxd/store/peer/transport.go @@ -1,15 +1,6 @@ -// Package peer adds cross-node reach to a node-local record store: when a -// record is missing locally, it is pulled from a node that gossiped it, cached -// locally, and served from there. -// -// The transport is sandboxd's own — deliberately NOT cocoon's image/snapshot -// transfer. cocoon's mover is bound to its VM store layout and its own -// addressing, and reusing it would tie a checkpoint's mobility to the engine's -// release cycle. This one moves exactly one thing (a store record: an export -// directory plus its meta.json) between two sandboxd nodes that already -// authenticate to each other with the fleet api_token, over the control-plane -// port they already share. It is a tar stream, so a pull costs one round trip -// and never materializes the record twice. +// Package peer adds cross-node reach to a node-local record store: a record +// missing locally is pulled from a node that gossiped it, published locally, +// and served from there. package peer import ( @@ -26,43 +17,32 @@ import ( "time" ) -// pullTimeout bounds a single record transfer. A checkpoint carries a guest's -// memory image, so this is generous — but it is never unbounded: a wedged peer -// must fail the pull and let the next owner be tried, not hang the branch. -const pullTimeout = 30 * time.Minute +const ( + // A checkpoint carries a guest memory image, so this is generous — but a + // wedged peer must fail the pull so the next owner is tried, not hang it. + pullTimeout = 30 * time.Minute -// maxRecordBytes caps a pulled record. It is a decompression-bomb guard, not a -// business limit: a peer is authenticated, but a compromised or buggy one must -// not be able to fill this node's disk. -const maxRecordBytes = 1 << 40 // 1 TiB + maxRecordBytes = 1 << 40 // 1 TiB -// The record layout a pull must reproduce, mirroring the store's own names. -const ( metaFile = "meta.json" exportPrefix = "export/" ) +// ErrNotFound reports that a peer does not hold the requested record. Declared +// here so the transport does not import the store package it is a backend for. +var ErrNotFound = errors.New("peer does not hold record") + // Puller fetches a record from a peer into a local directory. type Puller interface { - // Pull writes the record's contents (export/ and meta.json) into dst, - // which the caller has created. It returns ErrNotFound when the peer does - // not hold the record, so the caller can try the next owner. Pull(ctx context.Context, addr, id, dst string) error } -// ErrNotFound reports that a peer does not hold the requested record. It -// mirrors store.ErrNotFound but is declared here so the transport does not -// import the store package it is a backend for. -var ErrNotFound = errors.New("peer does not hold record") - // HTTPPuller pulls records over sandboxd's control-plane HTTP port. type HTTPPuller struct { - // Client is the HTTP client. A nil Client uses a default with no overall - // timeout — the per-pull deadline comes from the context instead, so a - // large record is not cut off mid-stream. + // A nil Client uses a default with no overall timeout: the per-pull + // deadline comes from the context, so a large record is not cut off. Client *http.Client - // Token is the fleet api_token presented to the serving peer. - Token string + Token string } // Pull implements Puller. @@ -87,7 +67,7 @@ func (p *HTTPPuller) Pull(ctx context.Context, addr, id, dst string) error { if client == nil { client = http.DefaultClient } - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:gosec // addr comes from the mesh's own member view if err != nil { return fmt.Errorf("pull %s from %s: %w", id, addr, err) } @@ -106,11 +86,9 @@ func (p *HTTPPuller) Pull(ctx context.Context, addr, id, dst string) error { return Untar(resp.Body, dst) } -// TarRecord streams a whole store record: the export directory under -// "export/" plus its "meta.json". A record is only branchable with both — the -// meta carries the pool key and lineage the claim path reads before it ever -// touches the export, so shipping the export alone produces a directory that -// publishes cleanly and then fails every read as "unknown checkpoint". +// TarRecord streams a whole store record: export/ plus meta.json. Both are +// required — the claim path reads the meta's pool key and lineage before it +// ever touches the export, so an export-only copy fails every read. func TarRecord(exportDir string, meta []byte, w io.Writer) error { tw := tar.NewWriter(w) defer func() { _ = tw.Close() }() @@ -134,19 +112,9 @@ func TarRecord(exportDir string, meta []byte, w io.Writer) error { return tw.Close() } -// Tar streams src's contents as a tar archive. Only regular files and -// directories are emitted: a record is data, and a symlink or device node in -// the stream would be a way to write outside the reader's destination. -func Tar(src string, w io.Writer) error { - tw := tar.NewWriter(w) - defer func() { _ = tw.Close() }() - if err := tarInto(src, "", tw); err != nil { - return err - } - return tw.Close() -} - -// tarInto walks src into tw, prefixing every entry name with prefix. +// tarInto walks src into tw under prefix, emitting only regular files and +// directories: a symlink or device node would let the reader be steered outside +// its destination. func tarInto(src, prefix string, tw *tar.Writer) error { err := filepath.Walk(src, func(path string, fi os.FileInfo, err error) error { if err != nil { @@ -176,7 +144,7 @@ func tarInto(src, prefix string, tw *tar.Writer) error { }); err != nil { return err } - f, err := os.Open(path) + f, err := os.Open(path) //nolint:gosec // path comes from Walk over src if err != nil { return err } @@ -184,7 +152,6 @@ func tarInto(src, prefix string, tw *tar.Writer) error { _, err = io.Copy(tw, f) return err default: - // Skip anything that is not data. return nil } }) @@ -194,9 +161,9 @@ func tarInto(src, prefix string, tw *tar.Writer) error { return nil } -// Untar writes a tar stream into dst. Entry names are validated against path -// traversal — a peer is authenticated, but an authenticated peer is still not -// allowed to write outside the destination this node chose. +// Untar writes a tar stream into dst. An authenticated peer is still not +// allowed to write outside the destination this node chose, so every entry +// name is validated against traversal. func Untar(r io.Reader, dst string) error { tr := tar.NewReader(r) var written int64 @@ -225,27 +192,25 @@ func Untar(r io.Reader, dst string) error { if written > maxRecordBytes { return fmt.Errorf("untar: record exceeds %d bytes", int64(maxRecordBytes)) } - if err := writeFile(target, tr, os.FileMode(hdr.Mode).Perm()); err != nil { + if err := writeFile(target, tr, os.FileMode(hdr.Mode).Perm()); err != nil { //nolint:gosec // Perm masks to 0777 return err } default: - // Ignore non-data entries rather than trusting them. continue } } } -// writeFile copies one entry to disk. func writeFile(target string, r io.Reader, mode os.FileMode) error { if mode == 0 { mode = 0o600 } - f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) //nolint:gosec // target resolved by safeJoin if err != nil { return fmt.Errorf("untar create %s: %w", target, err) } defer func() { _ = f.Close() }() - if _, err := io.Copy(f, io.LimitReader(r, maxRecordBytes)); err != nil { + if _, err := io.Copy(f, r); err != nil { return fmt.Errorf("untar write %s: %w", target, err) } return f.Close() @@ -257,11 +222,8 @@ func safeJoin(root, name string) (string, error) { if name == "" || filepath.IsAbs(name) || strings.Contains(name, `\`) { return "", fmt.Errorf("untar: refusing entry %q", name) } - // Clean the name as-is, NOT as "/"+name: a leading slash would absorb any - // leading "..", silently rewriting ../escape into /escape and landing the - // entry inside root under a surprising name. A record this node produced - // never contains "..", so its presence means corruption or an attack — - // reject it loudly instead of neutralizing it. + // Clean as-is, never as "/"+name: a leading slash absorbs a leading "..", + // rewriting ../escape to /escape instead of rejecting it. clean := filepath.Clean(name) if clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) { return "", fmt.Errorf("untar: entry %q escapes destination", name) diff --git a/sandboxd/store/peer/transport_test.go b/sandboxd/store/peer/transport_test.go index 6d15cd0..ea6325b 100644 --- a/sandboxd/store/peer/transport_test.go +++ b/sandboxd/store/peer/transport_test.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "errors" + "io" "net/http" "net/http/httptest" "os" @@ -35,8 +36,8 @@ func TestTarUntarRoundTrip(t *testing.T) { } var buf bytes.Buffer - if err := Tar(src, &buf); err != nil { - t.Fatalf("Tar: %v", err) + if err := tarDir(src, &buf); err != nil { + t.Fatalf("tarDir: %v", err) } dst := t.TempDir() if err := Untar(&buf, dst); err != nil { @@ -143,8 +144,8 @@ func TestTarSkipsSymlinksAtSource(t *testing.T) { } var buf bytes.Buffer - if err := Tar(src, &buf); err != nil { - t.Fatalf("Tar: %v", err) + if err := tarDir(src, &buf); err != nil { + t.Fatalf("tarDir: %v", err) } tr := tar.NewReader(&buf) for { @@ -258,3 +259,14 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { t.Error("export contents landed at the record root; they must be under export/") } } + +// tarDir streams src's contents with no record layout, exercising tarInto and +// Untar on their own. +func tarDir(src string, w io.Writer) error { + tw := tar.NewWriter(w) + defer func() { _ = tw.Close() }() + if err := tarInto(src, "", tw); err != nil { + return err + } + return tw.Close() +} diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 27812cf..66e9167 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -36,10 +36,10 @@ var ( NameRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,62}$`) sizeSpecs = map[Size]SizeSpec{ - SizeSmall: {CPU: 1, Memory: "512M"}, - SizeMedium: {CPU: 2, Memory: "1G"}, - SizeLarge: {CPU: 4, Memory: "4G"}, - SizeXLarge: {CPU: 4, Memory: "8G"}, + SizeSmall: {CPU: 1, Memory: "512M", MemoryBytes: 512 << 20}, + SizeMedium: {CPU: 2, Memory: "1G", MemoryBytes: 1 << 30}, + SizeLarge: {CPU: 4, Memory: "4G", MemoryBytes: 4 << 30}, + SizeXLarge: {CPU: 4, Memory: "8G", MemoryBytes: 8 << 30}, } ) @@ -79,10 +79,12 @@ func (e Engine) Validate() error { // warm pools, so only tiers are accepted. type Size string -// SizeSpec is the concrete allocation behind a tier, in cocoon flag units. +// SizeSpec is the concrete allocation behind a tier. Memory is in cocoon flag +// units; MemoryBytes is the same figure for callers that report bytes. type SizeSpec struct { - CPU int - Memory string + CPU int + Memory string + MemoryBytes int64 } // Spec resolves a tier to its allocation; ok is false for unknown tiers. From 838220c9bff121b59090b0f16b66619c1b97d6ad Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 12:49:18 +0800 Subject: [PATCH 05/31] review: comment budget, declaration layout and the lint/asl gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both gates were clean on main and this branch broke them: golangci-lint reported 8 issues and asl 57. Restores zero on both, dual-GOOS. Layout rules applied: one top-level const block per file and the package var block above the types (peer/transport.go had three const blocks and declared ErrNotFound below an interface); the compile-time interface check sits immediately above its type (peer/peer.go); tests precede helpers, which the new tests broke in server_test.go, mesh_test.go and peer_test.go; exported above unexported (pool/operator.go). isRootToken's godoc had been orphaned — rootRequest was inserted between the comment and the function, so go doc rendered it against the wrong symbol. handleSandboxVerb's two inline func types are named (tokenVerb, operatorVerb) so the signature reads on one line. gosec's file-inclusion and SSRF findings are annotated with why the input is already pinned, matching how the dir and pool stores annotate theirs. Comments: 208 added lines against 603 of code was 33%, twice this repo's own 16%. Now 19%, cutting design narrative that duplicated the commit messages and per-field comments that restated the field name. --- sandboxd/config/config.go | 9 +- sandboxd/main.go | 3 +- sandboxd/mesh/mesh.go | 24 ++-- sandboxd/mesh/mesh_test.go | 71 +++++----- sandboxd/pool/checkpoint.go | 14 +- sandboxd/pool/fork.go | 5 +- sandboxd/pool/operator.go | 39 ++---- sandboxd/pool/pool.go | 7 +- sandboxd/pool/template.go | 4 +- sandboxd/server/server.go | 50 ++++--- sandboxd/server/server_test.go | 234 +++++++++++++++---------------- sandboxd/store/peer/peer.go | 53 +++---- sandboxd/store/peer/peer_test.go | 109 +++++++------- sandboxd/types/api.go | 6 +- 14 files changed, 292 insertions(+), 336 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index fcb59ff..2f5d678 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -226,13 +226,8 @@ type Config struct { CheckpointStore *StoreConfig `json:"checkpoint_store,omitempty"` // CheckpointPeerHeal lets a node pull a checkpoint it does not hold from a - // node that gossiped it, instead of failing the branch. It is the last - // tier of the snapshot placement design: placement normally follows the - // data (a branch is redirected to the owning node and clones on its local - // reflink fast path), and this moves the data only when no owner can serve - // — the owner is gone, draining, or full. Requires a mesh; ignored without - // one. Off by default: it trades a transfer for availability, and that is - // the operator's call. + // node that gossiped it instead of failing the branch. Requires a mesh; off + // by default because it trades a transfer for availability. CheckpointPeerHeal bool `json:"checkpoint_peer_heal,omitempty"` // CheckpointTTLHours ages out checkpoints (0 = keep forever); the diff --git a/sandboxd/main.go b/sandboxd/main.go index 960f0ee..54d2c2a 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -102,8 +102,7 @@ func main() { defer func() { _ = msh.Shutdown() }() placer = msh mgr.SetTemplateNotifier(func() { msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs()) }) - // Peer healing needs the mesh's gossiped view to resolve who holds a - // record, so it is wired here rather than at store construction. + // Wired after the mesh: the owners resolver is the mesh's own view. mgr.WithPeerHeal(cfg.CheckpointPeerHeal, msh.CheckpointOwners, cfg.APIToken) go gossipNodeState(ctx, msh, mgr) logger.Infof(ctx, "mesh %s joined (%d seeds)", cmp.Or(cfg.Mesh.NodeID, cfg.Mesh.Bind), len(cfg.Mesh.Join)) diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index a9d694a..4641974 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -28,18 +28,13 @@ const leaveTimeout = time.Second // NodeState is one node's gossiped placement view. Epoch resolves merges: the // higher epoch for a given node wins. type NodeState struct { - NodeID string `json:"node_id"` - Addr string `json:"addr"` // data-plane advertise address - Epoch uint64 `json:"epoch"` - Pools map[string]int `json:"pools"` // PoolKey hash → warm count - Templates []string `json:"templates,omitempty"` // promoted-template key hashes on disk - // Checkpoints lists the checkpoint ids this node holds. Checkpoints are - // node-local like templates, so a resume or branch of one this node does - // not hold is answered with a redirect to a node that does — the placement - // moves to the data instead of the data moving to the placement, which is - // what keeps the clone on its local reflink fast path. - Checkpoints []string `json:"checkpoints,omitempty"` - Digest string `json:"digest,omitempty"` // cluster-invariant config digest + NodeID string `json:"node_id"` + Addr string `json:"addr"` // data-plane advertise address + Epoch uint64 `json:"epoch"` + Pools map[string]int `json:"pools"` // PoolKey hash → warm count + Templates []string `json:"templates,omitempty"` // promoted-template key hashes on disk + Checkpoints []string `json:"checkpoints,omitempty"` // checkpoint ids on disk + Digest string `json:"digest,omitempty"` // cluster-invariant config digest } // Mesh is the node's view of the cluster and its own gossiped state. @@ -224,9 +219,8 @@ func (m *Mesh) TemplateOwners(keyHash string) []string { } // CheckpointOwners returns up to two peer addresses whose gossiped checkpoint -// set contains ckptID — the redirect targets for a branch of a checkpoint this -// node does not hold. Self is excluded: the caller has already checked its own -// store. Mirrors TemplateOwners; both answer "who has this immutable record". +// set contains ckptID, the redirect targets for a branch this node cannot +// serve. Self is excluded: the caller already checked its own store. func (m *Mesh) CheckpointOwners(ckptID string) []string { m.mu.Lock() var owners []string diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index 11968ba..99bb011 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -138,41 +138,6 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { t.Fatalf("node-b never learned node-a's state: view=%+v", b.mesh.Members()) } -func newTestMesh(t *testing.T, id string) *Mesh { - t.Helper() - return &Mesh{ - epochPath: filepath.Join(t.TempDir(), "mesh-epoch"), - self: NodeState{NodeID: id, Addr: id + ":7777", Pools: map[string]int{}}, - view: map[string]NodeState{id: {NodeID: id, Addr: id + ":7777"}}, - } -} - -func discardLogger() *log.Logger { return log.New(io.Discard, "", 0) } - -type node struct { - mesh *Mesh - addr string -} - -func startNode(t *testing.T, host string, port int, id string) *node { - t.Helper() - cfg := memberlist.DefaultLocalConfig() - cfg.BindAddr = host - cfg.BindPort = port - cfg.AdvertiseAddr = host - cfg.PushPullInterval = 200 * time.Millisecond - cfg.Logger = discardLogger() - m, err := New(cfg, id, id+":7777", nil, t.TempDir()) - if err != nil { - t.Fatalf("new mesh %s: %v", id, err) - } - t.Cleanup(func() { _ = m.Shutdown() }) - return &node{mesh: m, addr: fmt.Sprintf("%s:%d", host, m.ml.LocalNode().Port)} -} - -// TestCheckpointOwnersExcludeSelfAndUnknown mirrors the template case: the -// caller has already checked its own store, so self is never an answer — a -// redirect to ourselves would bounce the request in place. func TestCheckpointOwnersExcludeSelfAndUnknown(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(nil, nil, []string{"ck_00000000000000aa"}) // self holds it, still not an owner candidate @@ -222,3 +187,39 @@ func TestUpdateSelfGossipsCheckpoints(t *testing.T) { t.Fatalf("self.Checkpoints = %v, want the record to be gossiped", self.Checkpoints) } } + +func newTestMesh(t *testing.T, id string) *Mesh { + t.Helper() + return &Mesh{ + epochPath: filepath.Join(t.TempDir(), "mesh-epoch"), + self: NodeState{NodeID: id, Addr: id + ":7777", Pools: map[string]int{}}, + view: map[string]NodeState{id: {NodeID: id, Addr: id + ":7777"}}, + } +} + +func discardLogger() *log.Logger { return log.New(io.Discard, "", 0) } + +type node struct { + mesh *Mesh + addr string +} + +func startNode(t *testing.T, host string, port int, id string) *node { + t.Helper() + cfg := memberlist.DefaultLocalConfig() + cfg.BindAddr = host + cfg.BindPort = port + cfg.AdvertiseAddr = host + cfg.PushPullInterval = 200 * time.Millisecond + cfg.Logger = discardLogger() + m, err := New(cfg, id, id+":7777", nil, t.TempDir()) + if err != nil { + t.Fatalf("new mesh %s: %v", id, err) + } + t.Cleanup(func() { _ = m.Shutdown() }) + return &node{mesh: m, addr: fmt.Sprintf("%s:%d", host, m.ml.LocalNode().Port)} +} + +// TestCheckpointOwnersExcludeSelfAndUnknown mirrors the template case: the +// caller has already checked its own store, so self is never an answer — a +// redirect to ourselves would bounce the request in place. diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 55b7232..c5d2bd4 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -34,8 +34,6 @@ func (m *Manager) Checkpoint(ctx context.Context, id, token, name, tenant string } // CheckpointOperator checkpoints a sandbox by id without a per-sandbox token. -// It is the operator (root) path, authorized by the node's root api_token -// before the call — mirroring ReleaseOperator. func (m *Manager) CheckpointOperator(ctx context.Context, id, name, tenant string) (types.Checkpoint, error) { sb, ok := m.byID(id) if !ok { @@ -276,11 +274,9 @@ func parseCheckpoint(raw []byte) (types.Checkpoint, error) { return ckpt, nil } -// CheckpointIDs lists this node's checkpoint ids for the mesh to gossip, so a -// peer can resolve a branch of a checkpoint it does not hold to the node that -// does. Archive records are excluded: they are lifecycle-internal wake images, -// never a branch target. Errors are swallowed to nil — gossip is best-effort -// and a transient store read must not take the tick down. +// CheckpointIDs lists this node's checkpoint ids for the mesh to gossip. +// Archive records are excluded: a wake image is never a branch target. A store +// read error yields nil — gossip is best-effort and must not take the tick down. func (m *Manager) CheckpointIDs() []string { ckpts, err := m.Checkpoints(context.Background(), "") if err != nil { @@ -297,9 +293,7 @@ func (m *Manager) CheckpointIDs() []string { } // FetchCheckpoint materializes a checkpoint's export for a peer transfer, -// returning the local directory, its meta, and the release to call when the -// copy is done. It is the read half of the peer-heal path; the id is validated -// against the store's namespace before any disk is touched. +// returning the local directory, its meta, and the release to call when done. func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, []byte, func(), error) { if _, err := m.loadCheckpoint(ctx, ckptID); err != nil { return "", nil, nil, err diff --git a/sandboxd/pool/fork.go b/sandboxd/pool/fork.go index 20531e1..7fe5618 100644 --- a/sandboxd/pool/fork.go +++ b/sandboxd/pool/fork.go @@ -25,10 +25,7 @@ func (m *Manager) Fork(ctx context.Context, id, token string, count int, ttl tim return m.forkResolved(ctx, sb, count, ttl) } -// ForkOperator forks a sandbox by id without a per-sandbox token. It is the -// operator (root) path: the server authorizes it by the node's root api_token -// before calling, so no token check happens here — mirroring ReleaseOperator. -// A tenant token never reaches this method (see server.go). +// ForkOperator forks a sandbox by id without a per-sandbox token. func (m *Manager) ForkOperator(ctx context.Context, id string, count int, ttl time.Duration) ([]*types.Sandbox, error) { sb, ok := m.byID(id) if !ok { diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index 9dbefb5..2becaba 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -6,22 +6,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// The operator paths let a control plane holding the node's root api_token -// drive a sandbox's lifecycle without its per-sandbox token. They exist -// because a stateless aggregated control plane cannot keep one secret per -// sandbox without turning O(nodes) storage into O(sandboxes) — the property -// the whole design rests on. ReleaseOperator established the pattern; these -// follow it exactly: the server authorizes by root api_token before calling, -// so no token check happens here, and a tenant token never reaches them. - -// byID resolves a live claim by id alone, with no ownership proof. -func (m *Manager) byID(id string) (*types.Sandbox, bool) { - m.mu.Lock() - defer m.mu.Unlock() - sb := m.claimed[id] - return sb, sb != nil -} - // Sandbox reports one live claim's summary, the single-sandbox read the // whole-node listing otherwise forces a caller to scan for. func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { @@ -47,10 +31,9 @@ func (m *Manager) HibernateOperator(ctx context.Context, id string) error { return m.hibernateLocked(ctx, sb) } -// Wake restores a hibernated sandbox and leaves it running, the explicit -// counterpart to Hibernate. Waking is otherwise only a side effect of opening -// an agent connection, which gives a control plane no way to resume a sandbox -// it is not about to talk to. Idempotent on an already-running sandbox. +// Wake restores a hibernated sandbox and leaves it running, so a control plane +// can resume one it is not about to talk to — waking is otherwise only a side +// effect of opening an agent connection. Idempotent on a running sandbox. func (m *Manager) Wake(ctx context.Context, id, token string) error { sb, ok := m.claim(id, token) if !ok { @@ -68,10 +51,18 @@ func (m *Manager) WakeOperator(ctx context.Context, id string) error { return m.wake(ctx, sb) } -// wake is the body shared by both wake entry points. It reuses the relay's -// resolve path, which restores through cocoon's mmap fast path (~55 ms) and -// queues concurrent wakes on the transition lock, then discards the resolved -// socket: the caller wants the VM running, not a connection to it. +// byID resolves a live claim by id alone, with no ownership proof. The server +// authorizes the operator paths by root api_token before the call, so a tenant +// token never reaches one. +func (m *Manager) byID(id string) (*types.Sandbox, bool) { + m.mu.Lock() + defer m.mu.Unlock() + sb := m.claimed[id] + return sb, sb != nil +} + +// wake reuses the relay's resolve path, then discards the resolved socket: the +// caller wants the VM running, not a connection to it. func (m *Manager) wake(ctx context.Context, sb *types.Sandbox) error { sb.Touch() _, err := m.wakeResolved(ctx, sb) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 4b2807a..2569b81 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -555,10 +555,9 @@ func newStoreView(ctx context.Context, cfg *config.Config, staging string, idRe } // WithPeerHeal wraps the manager's checkpoint store so a record this node does -// not hold is pulled from a node that gossiped it. It is wired after the mesh -// exists (the owners resolver is the mesh's view), and is a no-op unless -// checkpoint_peer_heal is set — a shared backend (s3, a FUSE mount) already -// resolves every record from every node and needs no healing. +// not hold is pulled from a node that gossiped it. A no-op unless +// checkpoint_peer_heal is set: a shared backend already resolves every record +// from every node. func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { if !enabled || owners == nil { return diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index d86a1ad..a19a8d9 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -36,9 +36,7 @@ func (m *Manager) Promote(ctx context.Context, id, token, template, tenant strin return m.promoteResolved(ctx, sb, template, tenant) } -// PromoteOperator promotes a sandbox by id without a per-sandbox token. It is -// the operator (root) path, authorized by the node's root api_token before the -// call — mirroring ReleaseOperator. +// PromoteOperator promotes a sandbox by id without a per-sandbox token. func (m *Manager) PromoteOperator(ctx context.Context, id, template, tenant string) (types.PoolKey, error) { sb, ok := m.byID(id) if !ok { diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index c569b63..2241337 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -98,6 +98,13 @@ type Dialer interface { DialSilkd(ctx context.Context, vsockSocket string) (net.Conn, error) } +// The two shapes a sandbox-scoped verb takes: proof of ownership by the +// per-sandbox token, or by id alone once the root api_token authorized it. +type ( + tokenVerb func(ctx context.Context, id, token string) error + operatorVerb func(ctx context.Context, id string) error +) + // Placer names peers for redirect placement and lists the mesh for Lookup; // nil on a single-node deployment (no mesh). type Placer interface { @@ -302,11 +309,7 @@ func (s *Server) handleSandboxStats(w http.ResponseWriter, r *http.Request) { // HTTP: per-sandbox bearer auth, 404 on unknown, 204 on success. The node's // root api_token takes the operator path instead, exactly as release does, so // a control plane holding only the fleet token can drive the lifecycle. -func (s *Server) handleSandboxVerb( - verb string, - do func(ctx context.Context, id, token string) error, - operator func(ctx context.Context, id string) error, -) http.HandlerFunc { +func (s *Server) handleSandboxVerb(verb string, do tokenVerb, operator operatorVerb) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) if !ok { @@ -333,9 +336,9 @@ func (s *Server) handleFork(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - // An operator (root) caller proves authority with the node api_token in the - // header and carries no per-sandbox token; a tenant caller must still - // present the sandbox's own token as ownership proof. + // An operator caller proves authority with the node api_token in the header + // and carries no per-sandbox token; a tenant must still present the + // sandbox's own token as ownership proof. var children []*types.Sandbox var err error if req.Token == "" && s.rootRequest(r) { @@ -398,11 +401,9 @@ func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { } ckptID := r.PathValue("id") sb, err := s.mgr.ClaimCheckpoint(r.Context(), ckptID, req.TTL(), tenantFrom(r.Context())) - // Checkpoints are node-local. When this node does not hold the record, a - // peer that gossiped it does: redirect the branch there rather than failing, - // so the clone still runs on the node whose disk already has the data (its - // local reflink fast path). A no_redirect retry must resolve locally, or two - // nodes would bounce the request between them. + // Checkpoints are node-local: redirect to a peer that gossiped the record + // so the clone runs on the disk that already holds it. A no_redirect retry + // must resolve locally or two nodes would bounce the request. if errors.Is(err, pool.ErrUnknownCheckpoint) && s.placer != nil && !req.NoRedirect && writeRedirect(w, s.placer.CheckpointOwners(ckptID)) { return @@ -412,10 +413,8 @@ func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { }) } -// handleCheckpointBlob streams a checkpoint record (its export directory and -// meta.json) as a tar, so a peer that must serve a branch locally can pull it. -// It is a read of an immutable record: the same api/tenant token class that -// may branch a checkpoint may also copy one. +// handleCheckpointBlob streams a checkpoint record as a tar, so a peer that +// must serve a branch locally can pull it. func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { ckptID := r.PathValue("id") dir, meta, release, err := s.mgr.FetchCheckpoint(r.Context(), ckptID) @@ -431,9 +430,8 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { defer release() w.Header().Set("Content-Type", "application/x-tar") - // The status is committed before the walk begins, so a mid-stream failure - // can only truncate the tar — the reader detects that as a short archive - // and treats the pull as failed, which is the honest outcome. + // Status committed before the walk, so a mid-stream failure truncates the + // tar and the reader fails the pull on the short archive. w.WriteHeader(http.StatusOK) if err := peer.TarRecord(dir, meta, w); err != nil { log.WithFunc("server.handleCheckpointBlob").Error(r.Context(), err, "stream checkpoint") @@ -575,17 +573,17 @@ func (s *Server) requireRoot(next http.HandlerFunc) http.HandlerFunc { }) } -// isRootToken reports whether token is the configured root api_token (the -// operator credential). It mirrors resolveScope's root branch: an unset api -// token matches nothing, so an open node grants no operator elevation on the -// per-sandbox release path. Tenant tokens never match — they live in s.tenants. -// rootRequest reports whether this request presented the node's root -// api_token, the credential that authorizes the operator lifecycle paths. +// rootRequest reports whether this request presented the node's root api_token, +// the credential that authorizes the operator lifecycle paths. func (s *Server) rootRequest(r *http.Request) bool { token, ok := bearerToken(r) return ok && s.isRootToken(token) } +// isRootToken reports whether token is the configured root api_token (the +// operator credential). It mirrors resolveScope's root branch: an unset api +// token matches nothing, so an open node grants no operator elevation on the +// per-sandbox release path. Tenant tokens never match — they live in s.tenants. func (s *Server) isRootToken(token string) bool { return s.apiToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.apiToken)) == 1 } diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index d02ffd4..c7aa2f5 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -936,6 +936,123 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { } } +// TestCheckpointClaimRedirectsToOwner is L2: checkpoints are node-local, so a +// branch that lands on a node without the record must be pointed at one that +// has it — the clone then runs on the node whose disk already holds the data, +// on its local reflink fast path. Failing here instead would make "snapshot on +// A, branch from B" simply not work. +func TestCheckpointClaimRedirectsToOwner(t *testing.T) { + mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", mgr, placer) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 carrying a redirect (the mesh redirect is a 200, not a 3xx)", resp.StatusCode) + } + var got types.ClaimResponse + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if len(got.Redirect) != 1 || got.Redirect[0] != "owner-a:7777" { + t.Errorf("redirect = %v, want [owner-a:7777]", got.Redirect) + } + if got.ID != "" { + t.Errorf("id = %q, want empty: a redirect and a delivered sandbox are mutually exclusive", got.ID) + } +} + +// TestCheckpointClaimNoRedirectResolvesLocally: the retry at a redirect target +// must resolve there or two nodes would bounce the request between them. +func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { + placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", &fakeManager{}, placer) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{"no_redirect":true}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404: a no_redirect retry must not bounce again", resp.StatusCode) + } +} + +// TestCheckpointClaimNoOwnersIs404: nothing gossiped the record, so a miss is +// an honest miss rather than a redirect to nowhere. +func TestCheckpointClaimNoOwnersIs404(t *testing.T) { + ts := newPlacerTestServer(t, "sekret", &fakeManager{}, &fakePlacer{}) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404", resp.StatusCode) + } +} + +// TestCheckpointBlobUnknownIs404 lets a puller move on to the next owner +// instead of treating a stale gossip entry as a transfer failure. +func TestCheckpointBlobUnknownIs404(t *testing.T) { + ts := newTestServer(t, "sekret", &fakeManager{}, nil) + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, + ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", nil) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Authorization", "Bearer sekret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404 so the puller tries the next owner", resp.StatusCode) + } +} + +// TestCheckpointBlobStreamsRecord is the serving half of L3: the record leaves +// as a tar the peer can unpack, with its export contents intact. +func TestCheckpointBlobStreamsRecord(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "mem"), []byte("guest-pages"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + mgr := &fakeManager{ckptDir: src} + ts := newTestServer(t, "sekret", mgr, nil) + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, + ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", nil) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Authorization", "Bearer sekret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + dst := t.TempDir() + if err := peer.Untar(resp.Body, dst); err != nil { + t.Fatalf("Untar the streamed record: %v", err) + } + // The blob is a whole record: meta.json at the root plus the export under + // export/. Streaming only the export produced a directory that published + // cleanly and then failed every read as "unknown checkpoint". + if got, err := os.ReadFile(filepath.Join(dst, "export", "mem")); err != nil || string(got) != "guest-pages" { + t.Fatalf("streamed export/mem = %q, %v; want the record's bytes", got, err) + } + if _, err := os.Stat(filepath.Join(dst, "meta.json")); err != nil { + t.Fatalf("meta.json missing from the streamed record: %v", err) + } +} + func newTestServer(t *testing.T, apiToken string, mgr Manager, dialer Dialer) *httptest.Server { t.Helper() return newTenantTestServer(t, apiToken, nil, mgr, dialer) @@ -1228,120 +1345,3 @@ func postJSON(t *testing.T, url, token, body string) *http.Response { } return resp } - -// TestCheckpointClaimRedirectsToOwner is L2: checkpoints are node-local, so a -// branch that lands on a node without the record must be pointed at one that -// has it — the clone then runs on the node whose disk already holds the data, -// on its local reflink fast path. Failing here instead would make "snapshot on -// A, branch from B" simply not work. -func TestCheckpointClaimRedirectsToOwner(t *testing.T) { - mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint - placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", mgr, placer) - - resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want 200 carrying a redirect (the mesh redirect is a 200, not a 3xx)", resp.StatusCode) - } - var got types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { - t.Fatalf("decode: %v", err) - } - if len(got.Redirect) != 1 || got.Redirect[0] != "owner-a:7777" { - t.Errorf("redirect = %v, want [owner-a:7777]", got.Redirect) - } - if got.ID != "" { - t.Errorf("id = %q, want empty: a redirect and a delivered sandbox are mutually exclusive", got.ID) - } -} - -// TestCheckpointClaimNoRedirectResolvesLocally: the retry at a redirect target -// must resolve there or two nodes would bounce the request between them. -func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { - placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", &fakeManager{}, placer) - - resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{"no_redirect":true}`) - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("status = %d, want 404: a no_redirect retry must not bounce again", resp.StatusCode) - } -} - -// TestCheckpointClaimNoOwnersIs404: nothing gossiped the record, so a miss is -// an honest miss rather than a redirect to nowhere. -func TestCheckpointClaimNoOwnersIs404(t *testing.T) { - ts := newPlacerTestServer(t, "sekret", &fakeManager{}, &fakePlacer{}) - - resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("status = %d, want 404", resp.StatusCode) - } -} - -// TestCheckpointBlobUnknownIs404 lets a puller move on to the next owner -// instead of treating a stale gossip entry as a transfer failure. -func TestCheckpointBlobUnknownIs404(t *testing.T) { - ts := newTestServer(t, "sekret", &fakeManager{}, nil) - - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, - ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer sekret") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("do: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("status = %d, want 404 so the puller tries the next owner", resp.StatusCode) - } -} - -// TestCheckpointBlobStreamsRecord is the serving half of L3: the record leaves -// as a tar the peer can unpack, with its export contents intact. -func TestCheckpointBlobStreamsRecord(t *testing.T) { - src := t.TempDir() - if err := os.WriteFile(filepath.Join(src, "mem"), []byte("guest-pages"), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - mgr := &fakeManager{ckptDir: src} - ts := newTestServer(t, "sekret", mgr, nil) - - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, - ts.URL+"/v1/checkpoints/ck_00000000000000aa/blob", nil) - if err != nil { - t.Fatalf("request: %v", err) - } - req.Header.Set("Authorization", "Bearer sekret") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("do: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want 200", resp.StatusCode) - } - dst := t.TempDir() - if err := peer.Untar(resp.Body, dst); err != nil { - t.Fatalf("Untar the streamed record: %v", err) - } - // The blob is a whole record: meta.json at the root plus the export under - // export/. Streaming only the export produced a directory that published - // cleanly and then failed every read as "unknown checkpoint". - if got, err := os.ReadFile(filepath.Join(dst, "export", "mem")); err != nil || string(got) != "guest-pages" { - t.Fatalf("streamed export/mem = %q, %v; want the record's bytes", got, err) - } - if _, err := os.Stat(filepath.Join(dst, "meta.json")); err != nil { - t.Fatalf("meta.json missing from the streamed record: %v", err) - } -} diff --git a/sandboxd/store/peer/peer.go b/sandboxd/store/peer/peer.go index 6f10082..d962e7c 100644 --- a/sandboxd/store/peer/peer.go +++ b/sandboxd/store/peer/peer.go @@ -9,39 +9,24 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/store" ) -// Owners resolves a record id to the peer addresses that hold it. It is the -// mesh's gossiped view (Mesh.CheckpointOwners), injected so the store stays -// testable without a live cluster. +// Owners resolves a record id to the peer addresses that hold it — the mesh's +// gossiped view, injected so the store stays testable without a live cluster. type Owners func(id string) []string -// Store is a node-local record store that can heal a miss from a peer. -// -// It is the third tier of the snapshot placement design, and deliberately the -// last resort. The first two tiers never move data: a branch normally runs on -// the node that already holds the record, on its local reflink fast path, and -// a request that lands elsewhere is redirected there. This tier exists for the -// case redirect cannot answer — the owning node is gone, draining, or full — -// where the choice is between paying one transfer and failing the request. -// -// A pull publishes the record into the local store, so the cost is paid once: -// afterwards this node is itself an owner, gossips the record, and serves -// branches of it locally like any other. +var _ store.Store = (*Store)(nil) + +// Store is a node-local record store that heals a Fetch/ReadMeta miss by +// pulling the record from a peer and publishing it locally, so the transfer is +// paid once: this node then owns the record and serves later reads locally. type Store struct { - // local is the node's own store; every operation but a Fetch/ReadMeta miss - // is served entirely by it. - local store.Store - // owners resolves who to pull from. Nil disables healing (the store then - // behaves exactly like its local backend). + local store.Store owners Owners - // puller moves the bytes. puller Puller } -var _ store.Store = (*Store)(nil) - // New wraps local with peer healing. A nil owners or puller leaves the wrapper -// inert — it degrades to the local backend rather than failing, so a node with -// no mesh keeps working. +// inert, so a node with no mesh degrades to the local backend rather than +// failing. func New(local store.Store, owners Owners, puller Puller) *Store { return &Store{local: local, owners: owners, puller: puller} } @@ -56,9 +41,8 @@ func (s *Store) Delete(ctx context.Context, id string) error { return s.local.De func (s *Store) SweepStaging() error { return s.local.SweepStaging() } -// Metas lists local records only. A cluster-wide listing is the control -// plane's job (it already scatter-gathers every node); having each node -// answer for its peers would return the same record N times. +// Metas lists local records only: answering for peers would return the same +// record once per node to a control plane that already scatter-gathers. func (s *Store) Metas(ctx context.Context) ([][]byte, error) { return s.local.Metas(ctx) } // Fetch serves the record locally, healing from a peer on a miss. @@ -85,9 +69,8 @@ func (s *Store) ReadMeta(ctx context.Context, id string) ([]byte, error) { return s.local.ReadMeta(ctx, id) } -// heal pulls id from the first peer that serves it and publishes it locally. -// Returns store.ErrNotFound when no peer has it, so a miss stays a miss and -// the caller's existing not-found handling is unchanged. +// heal pulls id from the first peer that serves it and publishes it locally, +// returning store.ErrNotFound when no peer has it so a miss stays a miss. func (s *Store) heal(ctx context.Context, id string) error { if s.owners == nil || s.puller == nil { return store.ErrNotFound @@ -96,9 +79,8 @@ func (s *Store) heal(ctx context.Context, id string) error { if len(addrs) == 0 { return store.ErrNotFound } - // A started pull must finish even if the requesting client hangs up: - // abandoning it halfway would leave staging to the sweeper and make the - // next branch pay the whole transfer again. + // A client hanging up must not abandon a started pull: the next branch + // would pay the whole transfer again. ctx = context.WithoutCancel(ctx) var errs []error @@ -114,8 +96,7 @@ func (s *Store) heal(ctx context.Context, id string) error { if len(errs) > 0 { return fmt.Errorf("heal %s from %d peer(s): %w", id, len(addrs), errors.Join(errs...)) } - // Every owner answered "not found": the gossiped view was stale. - return store.ErrNotFound + return store.ErrNotFound // every owner answered not-found: stale gossip } // pullFrom stages a peer's copy and publishes it, so the record becomes local diff --git a/sandboxd/store/peer/peer_test.go b/sandboxd/store/peer/peer_test.go index 6aa216b..2d5cb11 100644 --- a/sandboxd/store/peer/peer_test.go +++ b/sandboxd/store/peer/peer_test.go @@ -15,53 +15,6 @@ import ( const testID = "ck_00000000000000aa" -func localStore(t *testing.T) *dir.Store { - t.Helper() - st, err := dir.New(t.TempDir(), store.CheckpointIDRe) - if err != nil { - t.Fatalf("dir.New: %v", err) - } - return st -} - -// fakePuller serves records from an in-memory table, recording who was asked. -type fakePuller struct { - records map[string]map[string]string // addr → file → content - asked []string - failAddr string -} - -func (p *fakePuller) Pull(_ context.Context, addr, _ string, dst string) error { - p.asked = append(p.asked, addr) - if addr == p.failAddr { - return errors.New("peer exploded") - } - files, ok := p.records[addr] - if !ok { - return ErrNotFound - } - for name, content := range files { - full := filepath.Join(dst, name) - if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { - return err - } - if err := os.WriteFile(full, []byte(content), 0o600); err != nil { - return err - } - } - return nil -} - -func record() map[string]string { - return map[string]string{ - store.MetaFile: `{"id":"` + testID + `"}`, - filepath.Join(store.ExportDir, "mem"): "guest-pages", - } -} - -// TestPeerBackendContract: wrapping must not change local behavior. Every -// operation but a Fetch/ReadMeta miss is the local backend's, so the wrapper -// has to satisfy the same store contract the dir backend does. func TestPeerBackendContract(t *testing.T) { storetest.RunContract(t, New(localStore(t), nil, nil)) } @@ -134,7 +87,7 @@ func TestHealAllOwnersMissStaysNotFound(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{}} s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) - _, _, _, err := s.Fetch(t.Context(), testID) + err := fetchErr(t, s) if !errors.Is(err, store.ErrNotFound) { t.Fatalf("Fetch error = %v, want store.ErrNotFound", err) } @@ -160,7 +113,7 @@ func TestHealErrorIsReported(t *testing.T) { puller := &fakePuller{failAddr: "peer-a:7777"} s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) - _, _, _, err := s.Fetch(t.Context(), testID) + err := fetchErr(t, s) if err == nil || errors.Is(err, store.ErrNotFound) { t.Fatalf("Fetch error = %v, want the peer failure surfaced", err) } @@ -184,3 +137,61 @@ func TestMetasStaysLocal(t *testing.T) { t.Errorf("Metas returned %d record(s); a peer's records must not appear in a local listing", len(metas)) } } + +func localStore(t *testing.T) *dir.Store { + t.Helper() + st, err := dir.New(t.TempDir(), store.CheckpointIDRe) + if err != nil { + t.Fatalf("dir.New: %v", err) + } + return st +} + +// fakePuller serves records from an in-memory table, recording who was asked. +type fakePuller struct { + records map[string]map[string]string // addr → file → content + asked []string + failAddr string +} + +func (p *fakePuller) Pull(_ context.Context, addr, _, dst string) error { + p.asked = append(p.asked, addr) + if addr == p.failAddr { + return errors.New("peer exploded") + } + files, ok := p.records[addr] + if !ok { + return ErrNotFound + } + for name, content := range files { + full := filepath.Join(dst, name) + if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { + return err + } + if err := os.WriteFile(full, []byte(content), 0o600); err != nil { + return err + } + } + return nil +} + +func record() map[string]string { + return map[string]string{ + store.MetaFile: `{"id":"` + testID + `"}`, + filepath.Join(store.ExportDir, "mem"): "guest-pages", + } +} + +// TestPeerBackendContract: wrapping must not change local behavior. Every +// operation but a Fetch/ReadMeta miss is the local backend's, so the wrapper +// has to satisfy the same store contract the dir backend does. + +// fetchErr keeps the three unused Fetch results out of the assertions. +func fetchErr(t *testing.T, s *Store) error { + t.Helper() + _, _, release, err := s.Fetch(t.Context(), testID) //nolint:dogsled // only err is asserted + if release != nil { + release() + } + return err +} diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 42be43b..e0c2e1d 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -84,10 +84,8 @@ type CheckpointResponse struct { // TTLSeconds semantics match a claim's; zero means the server default. type CheckpointClaimRequest struct { TTLSeconds int `json:"ttl_seconds,omitempty"` - // NoRedirect is set by a client retrying at a redirect target: checkpoints - // are node-local, so a branch of a record this node lacks is answered with - // the owning peer's address. The retry must resolve locally or two nodes - // would bounce it between them. + // NoRedirect is set by a client retrying at a redirect target, so the retry + // resolves locally instead of bouncing between two nodes. NoRedirect bool `json:"no_redirect,omitempty"` } From a2a5a562ce43b417898044b355e7e39120c0d27b Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 12:53:41 +0800 Subject: [PATCH 06/31] simplify: drop a dead archive filter and three copy-forward duplications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checkpoints already skips archive wake images, so CheckpointIDs' own !c.Archive test could never be false — and the godoc claimed that dead condition was what excluded them. forkResolved/promoteResolved opened with `id := sb.ID` only to keep two pre-existing error sites compiling after the claim/resolved split; checkpointResolved, split the same way, reads sb.ID directly. Sandbox rebuilt the SandboxSummary literal Sandboxes already had. --- sandboxd/pool/checkpoint.go | 9 ++++----- sandboxd/pool/fork.go | 5 ++--- sandboxd/pool/operator.go | 6 +----- sandboxd/pool/pool.go | 14 +++++++++----- sandboxd/pool/template.go | 5 ++--- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index c5d2bd4..a568b95 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -275,8 +275,9 @@ func parseCheckpoint(raw []byte) (types.Checkpoint, error) { } // CheckpointIDs lists this node's checkpoint ids for the mesh to gossip. -// Archive records are excluded: a wake image is never a branch target. A store -// read error yields nil — gossip is best-effort and must not take the tick down. +// Checkpoints already drops archive wake images, which are never a branch +// target. A store read error yields nil: gossip is best-effort and must not +// take the tick down. func (m *Manager) CheckpointIDs() []string { ckpts, err := m.Checkpoints(context.Background(), "") if err != nil { @@ -284,9 +285,7 @@ func (m *Manager) CheckpointIDs() []string { } ids := make([]string, 0, len(ckpts)) for _, c := range ckpts { - if !c.Archive { - ids = append(ids, c.ID) - } + ids = append(ids, c.ID) } slices.Sort(ids) return ids diff --git a/sandboxd/pool/fork.go b/sandboxd/pool/fork.go index 7fe5618..d7c210c 100644 --- a/sandboxd/pool/fork.go +++ b/sandboxd/pool/fork.go @@ -36,7 +36,6 @@ func (m *Manager) ForkOperator(ctx context.Context, id string, count int, ttl ti // forkResolved is Fork's body once the source claim is resolved. func (m *Manager) forkResolved(ctx context.Context, sb *types.Sandbox, count int, ttl time.Duration) ([]*types.Sandbox, error) { - id := sb.ID if count < 1 || count > m.maxFork { return nil, fmt.Errorf("%w: %d not in 1..%d", ErrBadCount, count, m.maxFork) } @@ -51,13 +50,13 @@ func (m *Manager) forkResolved(ctx context.Context, sb *types.Sandbox, count int children, err := m.forkClones(ctx, sb, count) if err != nil { - return nil, fmt.Errorf("fork %s: %w", id, err) + return nil, fmt.Errorf("fork %s: %w", sb.ID, err) } for _, c := range children { c.Tenant = sb.Tenant } if err := m.finalizeBatch(ctx, children, ttl); err != nil { - return nil, fmt.Errorf("fork %s: %w", id, err) + return nil, fmt.Errorf("fork %s: %w", sb.ID, err) } m.counters.forks.Add(1) m.counters.claimsClone.Add(uint64(len(children))) //nolint:gosec // count is bounded by maxFork diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index 2becaba..f65bb1f 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -13,11 +13,7 @@ func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { if !ok { return SandboxSummary{}, false } - return SandboxSummary{ - ID: sb.ID, Key: sb.Key, Deadline: sb.Deadline, - Hibernated: sb.HibernateSnap != "", Archived: sb.ArchiveCk != "", - FromCheckpoint: sb.FromCheckpoint, ClaimRef: sb.ClaimRef, - }, true + return summarize(sb), true } // HibernateOperator hibernates a sandbox by id without a per-sandbox token. diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 2569b81..59a10bf 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -485,16 +485,20 @@ func (m *Manager) Sandboxes() []SandboxSummary { defer m.mu.Unlock() out := make([]SandboxSummary, 0, len(m.claimed)) for _, sb := range m.claimed { - out = append(out, SandboxSummary{ - ID: sb.ID, Key: sb.Key, Deadline: sb.Deadline, - Hibernated: sb.HibernateSnap != "", Archived: sb.ArchiveCk != "", FromCheckpoint: sb.FromCheckpoint, - ClaimRef: sb.ClaimRef, - }) + out = append(out, summarize(sb)) } slices.SortFunc(out, func(a, b SandboxSummary) int { return strings.Compare(a.ID, b.ID) }) return out } +func summarize(sb *types.Sandbox) SandboxSummary { + return SandboxSummary{ + ID: sb.ID, Key: sb.Key, Deadline: sb.Deadline, + Hibernated: sb.HibernateSnap != "", Archived: sb.ArchiveCk != "", + FromCheckpoint: sb.FromCheckpoint, ClaimRef: sb.ClaimRef, + } +} + // WarmCounts is the per-pool-key-hash warm count, for gossiping placement. func (m *Manager) WarmCounts() map[string]int { m.mu.Lock() diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index a19a8d9..c2d15e0 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -47,7 +47,6 @@ func (m *Manager) PromoteOperator(ctx context.Context, id, template, tenant stri // promoteResolved is Promote's body once the source claim is resolved. func (m *Manager) promoteResolved(ctx context.Context, sb *types.Sandbox, template, tenant string) (types.PoolKey, error) { - id := sb.ID if !types.NameRe.MatchString(template) { return types.PoolKey{}, fmt.Errorf("%w: template %q must match %s", ErrBadKey, template, types.NameRe) } @@ -73,11 +72,11 @@ func (m *Manager) promoteResolved(ctx context.Context, sb *types.Sandbox, templa snap, cleanup, err := m.sourceSnap(ctx, sb) if err != nil { - return types.PoolKey{}, fmt.Errorf("promote %s: %w", id, err) + return types.PoolKey{}, fmt.Errorf("promote %s: %w", sb.ID, err) } defer cleanup() if err := m.publishTemplate(ctx, snap, key, tenant); err != nil { - return types.PoolKey{}, fmt.Errorf("promote %s: %w", id, err) + return types.PoolKey{}, fmt.Errorf("promote %s: %w", sb.ID, err) } if m.notifyTemplates != nil { m.notifyTemplates() From 2383e9d740ab59a63c39daa4d9a53850c5d1a9a1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 13:04:14 +0800 Subject: [PATCH 07/31] review: propagate ctx into CheckpointIDs, dedup the owners scans CheckpointIDs reads the store but built its own context.Background while both call sites (the gossip tick and the template notifier) hold main's ctx. TemplateOwners and CheckpointOwners were the same lock/scan/truncate body with only the NodeState field swapped; both now wrap recordOwners. UpdateSelf's godoc still enumerated only pools and templates. --- sandboxd/main.go | 4 +-- sandboxd/mesh/mesh.go | 50 ++++++++++++++++--------------------- sandboxd/pool/checkpoint.go | 4 +-- 3 files changed, 26 insertions(+), 32 deletions(-) diff --git a/sandboxd/main.go b/sandboxd/main.go index 54d2c2a..2b6303a 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -101,7 +101,7 @@ func main() { } defer func() { _ = msh.Shutdown() }() placer = msh - mgr.SetTemplateNotifier(func() { msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs()) }) + mgr.SetTemplateNotifier(func() { msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs(ctx)) }) // Wired after the mesh: the owners resolver is the mesh's own view. mgr.WithPeerHeal(cfg.CheckpointPeerHeal, msh.CheckpointOwners, cfg.APIToken) go gossipNodeState(ctx, msh, mgr) @@ -196,7 +196,7 @@ func gossipNodeState(ctx context.Context, msh *mesh.Mesh, mgr *pool.Manager) { case <-ctx.Done(): return case <-t.C: - msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs()) + msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs(ctx)) } } } diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 4641974..6334446 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -100,8 +100,8 @@ func (m *Mesh) Join(seeds []string) error { return nil } -// UpdateSelf republishes this node's warm-pool counts and promoted-template -// set, bumping the epoch so peers adopt the new view. An unchanged view is +// UpdateSelf republishes this node's warm-pool counts and template and +// checkpoint sets, bumping the epoch so peers adopt the new view. An unchanged view is // not republished — the periodic tick would otherwise gossip a fresh epoch // every second for nothing. func (m *Mesh) UpdateSelf(pools map[string]int, templates, checkpoints []string) { @@ -202,38 +202,14 @@ func (m *Mesh) Candidates(keyHash string) []string { // delete of a template this node does not hold. Self is excluded: the caller // has already checked its own disk. func (m *Mesh) TemplateOwners(keyHash string) []string { - m.mu.Lock() - var owners []string - for id, st := range m.view { - if id != m.self.NodeID && slices.Contains(st.Templates, keyHash) { - owners = append(owners, st.Addr) - } - } - m.mu.Unlock() - // Which two survive truncation is already jittered by map iteration - // order; unlike Candidates there is no warmth to rank by. - if len(owners) > 2 { - owners = owners[:2] - } - return owners + return m.recordOwners(keyHash, func(st NodeState) []string { return st.Templates }) } // CheckpointOwners returns up to two peer addresses whose gossiped checkpoint // set contains ckptID, the redirect targets for a branch this node cannot // serve. Self is excluded: the caller already checked its own store. func (m *Mesh) CheckpointOwners(ckptID string) []string { - m.mu.Lock() - var owners []string - for id, st := range m.view { - if id != m.self.NodeID && slices.Contains(st.Checkpoints, ckptID) { - owners = append(owners, st.Addr) - } - } - m.mu.Unlock() - if len(owners) > 2 { - owners = owners[:2] - } - return owners + return m.recordOwners(ckptID, func(st NodeState) []string { return st.Checkpoints }) } // Members returns the current cluster view (self included). @@ -264,6 +240,24 @@ func (m *Mesh) Shutdown() error { } // persistEpoch durably records the epoch. +// recordOwners scans the view for peers whose listed record set contains id. +func (m *Mesh) recordOwners(id string, list func(NodeState) []string) []string { + m.mu.Lock() + var owners []string + for nid, st := range m.view { + if nid != m.self.NodeID && slices.Contains(list(st), id) { + owners = append(owners, st.Addr) + } + } + m.mu.Unlock() + // Which two survive truncation is already jittered by map iteration + // order; unlike Candidates there is no warmth to rank by. + if len(owners) > 2 { + owners = owners[:2] + } + return owners +} + func (m *Mesh) persistEpoch(epoch uint64) error { return storeEpoch(m.epochPath, epoch) } diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index a568b95..6b37697 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -278,8 +278,8 @@ func parseCheckpoint(raw []byte) (types.Checkpoint, error) { // Checkpoints already drops archive wake images, which are never a branch // target. A store read error yields nil: gossip is best-effort and must not // take the tick down. -func (m *Manager) CheckpointIDs() []string { - ckpts, err := m.Checkpoints(context.Background(), "") +func (m *Manager) CheckpointIDs(ctx context.Context) []string { + ckpts, err := m.Checkpoints(ctx, "") if err != nil { return nil } From 48d9aaf9efdde607455ec3fecd62be1936673edc Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 13:07:39 +0800 Subject: [PATCH 08/31] review: propagate ctx through UpdateSelf Its two callers hold main's ctx; only merge, behind memberlist's ctx-less delegate interface, still builds its own. --- sandboxd/main.go | 4 ++-- sandboxd/mesh/mesh.go | 4 ++-- sandboxd/mesh/mesh_test.go | 14 +++++++------- sandboxd/mesh/state_test.go | 8 ++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sandboxd/main.go b/sandboxd/main.go index 2b6303a..3837fc9 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -101,7 +101,7 @@ func main() { } defer func() { _ = msh.Shutdown() }() placer = msh - mgr.SetTemplateNotifier(func() { msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs(ctx)) }) + mgr.SetTemplateNotifier(func() { msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs(ctx)) }) // Wired after the mesh: the owners resolver is the mesh's own view. mgr.WithPeerHeal(cfg.CheckpointPeerHeal, msh.CheckpointOwners, cfg.APIToken) go gossipNodeState(ctx, msh, mgr) @@ -196,7 +196,7 @@ func gossipNodeState(ctx context.Context, msh *mesh.Mesh, mgr *pool.Manager) { case <-ctx.Done(): return case <-t.C: - msh.UpdateSelf(mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs(ctx)) + msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs(ctx)) } } } diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 6334446..428f725 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -104,7 +104,7 @@ func (m *Mesh) Join(seeds []string) error { // checkpoint sets, bumping the epoch so peers adopt the new view. An unchanged view is // not republished — the periodic tick would otherwise gossip a fresh epoch // every second for nothing. -func (m *Mesh) UpdateSelf(pools map[string]int, templates, checkpoints []string) { +func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates, checkpoints []string) { m.updateMu.Lock() defer m.updateMu.Unlock() m.mu.Lock() @@ -119,7 +119,7 @@ func (m *Mesh) UpdateSelf(pools map[string]int, templates, checkpoints []string) // instant it enters the view, so a crash before the write would strand peers // on an epoch a backwards-clock restart can't beat. Hold old state on failure. if err := m.persistEpoch(epoch); err != nil { - log.WithFunc("mesh.UpdateSelf").Warnf(context.Background(), "persist epoch: %v", err) + log.WithFunc("mesh.UpdateSelf").Warnf(ctx, "persist epoch: %v", err) return } m.mu.Lock() diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index 99bb011..bae9170 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -31,7 +31,7 @@ func TestMergeKeepsHigherEpoch(t *testing.T) { func TestMergeNeverOverwritesSelf(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(map[string]int{"k": 3}, nil, nil) + m.UpdateSelf(t.Context(), map[string]int{"k": 3}, nil, nil) // A peer claiming to be "a" must not clobber our authoritative self entry. m.merge([]NodeState{{NodeID: "a", Addr: "evil:9999", Epoch: 999, Pools: map[string]int{"k": 0}}}) @@ -44,7 +44,7 @@ func TestMergeNeverOverwritesSelf(t *testing.T) { func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(map[string]int{"k": 5}, nil, nil) // self has warm, but is never a candidate + m.UpdateSelf(t.Context(), map[string]int{"k": 5}, nil, nil) // self has warm, but is never a candidate m.merge([]NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 0}}, // no warm @@ -62,7 +62,7 @@ func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { func TestTemplateOwnersExcludeSelfAndUnknown(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(nil, []string{"tpl"}, nil) // self holds it, but is never an owner candidate + m.UpdateSelf(t.Context(), nil, []string{"tpl"}, nil) // self holds it, but is never an owner candidate m.merge([]NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Templates: []string{"tpl", "other"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Templates: []string{"other"}}, @@ -87,7 +87,7 @@ func TestForgetPrunesDeadNode(t *testing.T) { t.Errorf("candidates after forget %v, want nil (b pruned)", got) } // forgetting self is a no-op. - m.UpdateSelf(map[string]int{"k": 1}, nil, nil) + m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) m.forget("a") if len(m.Members()) != 1 { t.Error("forget removed self") @@ -121,7 +121,7 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { if err := b.mesh.Join([]string{a.addr}); err != nil { t.Fatalf("join: %v", err) } - a.mesh.UpdateSelf(map[string]int{"kk": 4}, []string{"tpl-hash"}, nil) + a.mesh.UpdateSelf(t.Context(), map[string]int{"kk": 4}, []string{"tpl-hash"}, nil) // Push/pull sync propagates a's warm counts and template set to b within // a few intervals. @@ -140,7 +140,7 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { func TestCheckpointOwnersExcludeSelfAndUnknown(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(nil, nil, []string{"ck_00000000000000aa"}) // self holds it, still not an owner candidate + m.UpdateSelf(t.Context(), nil, nil, []string{"ck_00000000000000aa"}) // self holds it, still not an owner candidate m.merge([]NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Checkpoints: []string{"ck_00000000000000aa", "ck_00000000000000bb"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Checkpoints: []string{"ck_00000000000000bb"}}, @@ -175,7 +175,7 @@ func TestCheckpointOwnersTruncates(t *testing.T) { // into "branch fails on the wrong node" without any visible error. func TestUpdateSelfGossipsCheckpoints(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(nil, nil, []string{"ck_00000000000000aa"}) + m.UpdateSelf(t.Context(), nil, nil, []string{"ck_00000000000000aa"}) var self NodeState for _, st := range m.Members() { diff --git a/sandboxd/mesh/state_test.go b/sandboxd/mesh/state_test.go index a37921c..a9ed899 100644 --- a/sandboxd/mesh/state_test.go +++ b/sandboxd/mesh/state_test.go @@ -52,7 +52,7 @@ func TestUpdateSelfPersistFailKeepsOldState(t *testing.T) { before := m.self.Epoch // A non-existent dir makes the durable write fail: the epoch must not publish. m.epochPath = filepath.Join(dir, "gone", "mesh-epoch") - m.UpdateSelf(map[string]int{"k": 1}, nil, nil) + m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) if m.self.Epoch != before { t.Errorf("self epoch advanced to %d on a failed persist, want %d held", m.self.Epoch, before) } @@ -65,7 +65,7 @@ func TestUpdateSelfPersistsEpoch(t *testing.T) { dir := t.TempDir() m := newBoundMesh(t, dir) before := m.self.Epoch - m.UpdateSelf(map[string]int{"k": 1}, nil, nil) + m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) if got := loadEpoch(filepath.Join(dir, "mesh-epoch")); got <= before { t.Errorf("persisted epoch %d did not advance past the seed %d", got, before) } @@ -78,8 +78,8 @@ func TestUpdateSelfConcurrentDropsNothing(t *testing.T) { a := map[string]int{"a": i + 1} b := map[string]int{"b": i + 1} var wg sync.WaitGroup - wg.Go(func() { m.UpdateSelf(a, nil, nil) }) - wg.Go(func() { m.UpdateSelf(b, nil, nil) }) + wg.Go(func() { m.UpdateSelf(t.Context(), a, nil, nil) }) + wg.Go(func() { m.UpdateSelf(t.Context(), b, nil, nil) }) wg.Wait() // Serialized updates with distinct payloads must both land: the loser // of the old TOCTOU race silently dropped its payload and one bump. From 0feb179d26e0cec7f6410bbb29b5e5edab35cbf2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 13:18:20 +0800 Subject: [PATCH 09/31] review: carry the daemon ctx into mesh for callback logging memberlist's delegate interfaces have no context parameter, so merge logged with a fresh Background; New now stores the daemon's ctx for those callbacks. --- sandboxd/main.go | 6 +++--- sandboxd/mesh/mesh.go | 8 ++++++-- sandboxd/mesh/mesh_test.go | 2 +- sandboxd/mesh/state_test.go | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/sandboxd/main.go b/sandboxd/main.go index 3837fc9..5559aee 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -95,7 +95,7 @@ func main() { var placer server.Placer if cfg.Mesh != nil { - msh, err := startMesh(cfg, mgr) + msh, err := startMesh(ctx, cfg, mgr) if err != nil { logger.Fatalf(ctx, err, "start mesh") } @@ -156,7 +156,7 @@ func main() { logger.Info(ctx, "sandboxd stopped; VMs stay alive for the next reconcile") } -func startMesh(cfg *config.Config, mgr *pool.Manager) (*mesh.Mesh, error) { +func startMesh(ctx context.Context, cfg *config.Config, mgr *pool.Manager) (*mesh.Mesh, error) { mc := cfg.Mesh mlCfg := memberlist.DefaultLANConfig() host, port, err := mc.ParsedBind() @@ -173,7 +173,7 @@ func startMesh(cfg *config.Config, mgr *pool.Manager) (*mesh.Mesh, error) { if err != nil { return nil, err } - msh, err := mesh.New(mlCfg, nodeID, cfg.AdvertiseAddr, key, cfg.DataDir) + msh, err := mesh.New(ctx, mlCfg, nodeID, cfg.AdvertiseAddr, key, cfg.DataDir) if err != nil { return nil, err } diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 428f725..a1d825e 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -41,6 +41,9 @@ type NodeState struct { type Mesh struct { ml *memberlist.Memberlist epochPath string + // ctx is the daemon's, for logging inside memberlist callbacks — the + // delegate interfaces carry no context of their own. + ctx context.Context // updateMu serializes UpdateSelf end-to-end: two updates reading one epoch // would both bump to E+1 and the loser's payload would be dropped. @@ -54,13 +57,14 @@ type Mesh struct { // New starts a mesh member listening per cfg. selfAddr is the data-plane // address peers should dial for a redirect; secretKey (16/24/32 bytes) enables // gossip encryption when non-empty; dataDir holds the persisted epoch. -func New(cfg *memberlist.Config, nodeID, selfAddr string, secretKey []byte, dataDir string) (*Mesh, error) { +func New(ctx context.Context, cfg *memberlist.Config, nodeID, selfAddr string, secretKey []byte, dataDir string) (*Mesh, error) { epochPath := filepath.Join(dataDir, "mesh-epoch") // Seed strictly above the persisted floor (the last epoch peers saw): // seeding at it ties their stale copy and merge's `>` rejects the restart's // fresh state. loadEpoch caps the floor so the +1 cannot wrap. epoch := max(uint64(time.Now().UnixNano()), loadEpoch(epochPath)+1) //nolint:gosec // UnixNano is positive for current times m := &Mesh{ + ctx: ctx, epochPath: epochPath, self: NodeState{ NodeID: nodeID, @@ -290,7 +294,7 @@ func (m *Mesh) merge(states []NodeState) { // redirects and fails interception. Warn-only — refusing would partition // a rolling credential rotation. if m.self.Digest != "" && st.Digest != "" && st.Digest != m.self.Digest && (!ok || cur.Digest != st.Digest) { - log.WithFunc("mesh.merge").Warnf(context.Background(), + log.WithFunc("mesh.merge").Warnf(m.ctx, "peer %s config digest %s differs from this node's %s: cluster-invariant config diverges (redirects may 401, interception may fail)", st.NodeID, short(st.Digest), short(m.self.Digest)) } diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index bae9170..2ecabe7 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -212,7 +212,7 @@ func startNode(t *testing.T, host string, port int, id string) *node { cfg.AdvertiseAddr = host cfg.PushPullInterval = 200 * time.Millisecond cfg.Logger = discardLogger() - m, err := New(cfg, id, id+":7777", nil, t.TempDir()) + m, err := New(t.Context(), cfg, id, id+":7777", nil, t.TempDir()) if err != nil { t.Fatalf("new mesh %s: %v", id, err) } diff --git a/sandboxd/mesh/state_test.go b/sandboxd/mesh/state_test.go index a9ed899..31902d1 100644 --- a/sandboxd/mesh/state_test.go +++ b/sandboxd/mesh/state_test.go @@ -105,7 +105,7 @@ func newBoundMesh(t *testing.T, dataDir string) *Mesh { cfg := memberlist.DefaultLocalConfig() cfg.BindPort = 0 cfg.Logger = discardLogger() - m, err := New(cfg, "self", "self:7777", nil, dataDir) + m, err := New(t.Context(), cfg, "self", "self:7777", nil, dataDir) if err != nil { t.Fatalf("new mesh: %v", err) } From c72dae8f2792ffaf02413887c0a6c58296097988 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 13:40:38 +0800 Subject: [PATCH 10/31] sandboxd: heal only after redirect cannot answer, gossip from memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The peer-heal wrapper was installed as the claim path's own store, so a branch of a record a live owner could serve pulled the whole multi-GB image before the handler's zero-byte redirect branch ever ran — the redirect was dead code exactly when it should win. The healer now lives on its own Manager field and the server sequences the tiers explicitly: local claim, then redirect while a gossiped owner exists (and the request is not already the redirect retry), and only then ClaimCheckpointHeal. The blob endpoint reads the local store alone, so a peer's pull can never recurse into another pull. Gossip no longer lists the store every tick: an in-memory id set mirrors tplSet — seeded from one scan at boot, kept current by publish/delete/heal — so the 1s tick costs a map walk, not N meta reads (measured 15.4ms/tick at 1000 records before). A shared (s3) checkpoint store gossips nothing and installs no healer: every node already resolves every record. Concurrent misses of one id share a single transfer via singleflight, the idiom the s3 backend already uses. peer.Store embeds the local backend instead of forwarding five methods by hand. checkpoint_peer_heal now requires mesh.cluster_key: the heal path presents the fleet api_token to gossip-learned addresses, so gossip must be authenticated before that credential rides on it. --- sandboxd/config/config.go | 8 +- sandboxd/config/config_test.go | 2 + sandboxd/main.go | 4 +- sandboxd/pool/archive.go | 3 + sandboxd/pool/checkpoint.go | 85 +++++++++-- sandboxd/pool/checkpoint_heal_test.go | 208 ++++++++++++++++++++++++++ sandboxd/pool/pool.go | 50 ++++++- sandboxd/server/server.go | 15 +- sandboxd/server/server_test.go | 108 ++++++++++++- sandboxd/store/peer/peer.go | 46 +++--- sandboxd/store/peer/peer_test.go | 28 ++++ 11 files changed, 501 insertions(+), 56 deletions(-) create mode 100644 sandboxd/pool/checkpoint_heal_test.go diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 2f5d678..93becc7 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -226,8 +226,9 @@ type Config struct { CheckpointStore *StoreConfig `json:"checkpoint_store,omitempty"` // CheckpointPeerHeal lets a node pull a checkpoint it does not hold from a - // node that gossiped it instead of failing the branch. Requires a mesh; off - // by default because it trades a transfer for availability. + // node that gossiped it instead of failing the branch. Requires an encrypted + // mesh (cluster_key): the heal path presents the fleet token to gossip-learned + // addresses. Off by default because it trades a transfer for availability. CheckpointPeerHeal bool `json:"checkpoint_peer_heal,omitempty"` // CheckpointTTLHours ages out checkpoints (0 = keep forever); the @@ -369,6 +370,9 @@ func (c *Config) validate() error { if c.CheckpointTTLHours < 0 { return fmt.Errorf("checkpoint_ttl_hours must not be negative") } + if c.CheckpointPeerHeal && (c.Mesh == nil || c.Mesh.ClusterKey == "") { + return fmt.Errorf("checkpoint_peer_heal requires an encrypted mesh (set mesh.cluster_key)") + } if err := c.validateTenants(); err != nil { return err } diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 8f8744a..5097e5e 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -121,6 +121,8 @@ func TestLoadRejectsInvalid(t *testing.T) { {"mesh bind wildcard host", `{"pools":[],"mesh":{"bind":":7946"}}`, "explicit host"}, {"mesh cluster key not base64", `{"pools":[],"mesh":{"bind":"node1:7946","cluster_key":"not!base64"}}`, "not valid base64"}, {"mesh cluster key wrong length", `{"pools":[],"mesh":{"bind":"node1:7946","cluster_key":"YWJj"}}`, "want 16, 24, or 32"}, + {"checkpoint peer heal without mesh", `{"checkpoint_peer_heal":true,"pools":[]}`, "requires an encrypted mesh"}, + {"checkpoint peer heal without cluster key", `{"checkpoint_peer_heal":true,"pools":[],"mesh":{"bind":"node1:7946"}}`, "requires an encrypted mesh"}, } { t.Run(tt.name, func(t *testing.T) { _, err := Load(writeConfig(t, tt.body)) diff --git a/sandboxd/main.go b/sandboxd/main.go index 5559aee..f7a21a5 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -101,7 +101,7 @@ func main() { } defer func() { _ = msh.Shutdown() }() placer = msh - mgr.SetTemplateNotifier(func() { msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs(ctx)) }) + mgr.SetTemplateNotifier(func() { msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs()) }) // Wired after the mesh: the owners resolver is the mesh's own view. mgr.WithPeerHeal(cfg.CheckpointPeerHeal, msh.CheckpointOwners, cfg.APIToken) go gossipNodeState(ctx, msh, mgr) @@ -196,7 +196,7 @@ func gossipNodeState(ctx context.Context, msh *mesh.Mesh, mgr *pool.Manager) { case <-ctx.Done(): return case <-t.C: - msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs(ctx)) + msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs()) } } } diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 7dd1841..57e3153 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -188,6 +188,7 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, log.WithFunc("pool.wakeArchived").Warnf(ctx, "delete consumed archive ck %s: %v", ck, delErr) } else { m.dropRecLock(ck) + m.dropCkpt(ck) } // Only the none lane reaches here (the egress guard above fails closed), so // this rebinds the none-lane proxy; there is no NIC to re-lock. @@ -234,5 +235,7 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc func (m *Manager) deleteOrphanArchiveCk(ctx context.Context, ckID string) { if err := m.ckpts.Delete(ctx, ckID); err != nil { log.WithFunc("pool.deleteOrphanArchiveCk").Warnf(ctx, "delete orphaned archive ck %s: %v", ckID, err) + return } + m.dropCkpt(ckID) } diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 6b37697..f474c3f 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -94,6 +94,9 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID if err := m.ckpts.Publish(ctx, staging, ckpt.ID); err != nil { return types.Checkpoint{}, "", fmt.Errorf("commit checkpoint: %w", err) } + if !archive { + m.noteCkpt(ckpt) + } return ckpt, srcSnap, nil } @@ -113,10 +116,35 @@ func (m *Manager) ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.D if err != nil { return nil, err } - l := m.recLock(ckptID) + return m.claimLoaded(ctx, ckpt, ttl, tenant) +} + +// ClaimCheckpointHeal claims a checkpoint this node does not hold locally, +// pulling it from a peer first. The server calls it only after the local +// claim missed and a redirect could not answer. +func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) { + if m.healer == nil { + return nil, ErrUnknownCheckpoint + } + if err := m.overQuota(1, tenant); err != nil { + return nil, err + } + ckpt, err := m.loadCheckpointFrom(ctx, m.healer, ckptID) + if err != nil { + return nil, err + } + m.noteCkpt(ckpt) // record is now local: gossip it + return m.claimLoaded(ctx, ckpt, ttl, tenant) +} + +// claimLoaded is ClaimCheckpoint's and ClaimCheckpointHeal's shared body once +// ckpt's meta is resolved: it re-fetches under the record lock (immune to a +// delete racing the pre-check), provisions the branch, and finalizes the claim. +func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl time.Duration, tenant string) (*types.Sandbox, error) { + l := m.recLock(ckpt.ID) l.RLock() defer l.RUnlock() - dir, _, release, err := m.ckpts.Fetch(ctx, ckptID) + dir, _, release, err := m.ckpts.Fetch(ctx, ckpt.ID) if errors.Is(err, store.ErrNotFound) { return nil, ErrUnknownCheckpoint // deleted between the pre-check and the lock } @@ -249,14 +277,22 @@ func (m *Manager) deleteCkLocked(ctx context.Context, ckID string) error { return err } m.dropRecLock(ckID) + m.dropCkpt(ckID) return nil } +// loadCheckpoint reads and parses a checkpoint's meta from the local store. func (m *Manager) loadCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { + return m.loadCheckpointFrom(ctx, m.ckpts, ckptID) +} + +// loadCheckpointFrom reads and parses a checkpoint's meta from st — the local +// store for an ordinary claim, or the healer for ClaimCheckpointHeal. +func (m *Manager) loadCheckpointFrom(ctx context.Context, st store.Store, ckptID string) (types.Checkpoint, error) { if !store.CheckpointIDRe.MatchString(ckptID) { return types.Checkpoint{}, ErrUnknownCheckpoint } - raw, err := m.ckpts.ReadMeta(ctx, ckptID) + raw, err := st.ReadMeta(ctx, ckptID) if errors.Is(err, store.ErrNotFound) { return types.Checkpoint{}, ErrUnknownCheckpoint } @@ -274,23 +310,46 @@ func parseCheckpoint(raw []byte) (types.Checkpoint, error) { return ckpt, nil } -// CheckpointIDs lists this node's checkpoint ids for the mesh to gossip. -// Checkpoints already drops archive wake images, which are never a branch -// target. A store read error yields nil: gossip is best-effort and must not -// take the tick down. -func (m *Manager) CheckpointIDs(ctx context.Context) []string { - ckpts, err := m.Checkpoints(ctx, "") - if err != nil { +// CheckpointIDs lists this node's checkpoint ids for the mesh to gossip, from +// the in-memory set — mirrors TemplateHashes so the 1s gossip tick never +// touches the store backend. Archive wake images never enter the set. Nil on +// a shared checkpoint store (every node already resolves every record). +func (m *Manager) CheckpointIDs() []string { + m.ckptMu.Lock() + defer m.ckptMu.Unlock() + if m.ckptSet == nil { return nil } - ids := make([]string, 0, len(ckpts)) - for _, c := range ckpts { - ids = append(ids, c.ID) + ids := make([]string, 0, len(m.ckptSet)) + for id := range m.ckptSet { + ids = append(ids, id) } slices.Sort(ids) return ids } +// noteCkpt adds a fresh non-archive record to the gossip set: called after +// publishCheckpoint's Publish commits, and after ClaimCheckpointHeal pulls a +// peer's record into this node's local store. +func (m *Manager) noteCkpt(ckpt types.Checkpoint) { + if m.ckptSet == nil || ckpt.Archive { + return + } + m.ckptMu.Lock() + m.ckptSet[ckpt.ID] = struct{}{} + m.ckptMu.Unlock() +} + +// dropCkpt removes id from the gossip set once its store record is deleted. +func (m *Manager) dropCkpt(id string) { + if m.ckptSet == nil { + return + } + m.ckptMu.Lock() + delete(m.ckptSet, id) + m.ckptMu.Unlock() +} + // FetchCheckpoint materializes a checkpoint's export for a peer transfer, // returning the local directory, its meta, and the release to call when done. func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, []byte, func(), error) { diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go new file mode 100644 index 0000000..a8d35d5 --- /dev/null +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -0,0 +1,208 @@ +package pool + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/store/peer" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +// TestClaimCheckpointNeverPullsOnMiss: ClaimCheckpoint answers from the local +// store alone — a redirect exists precisely so this path never pays a peer +// transfer; only ClaimCheckpointHeal may. +func TestClaimCheckpointNeverPullsOnMiss(t *testing.T) { + ckpt := types.Checkpoint{ID: "ck_00000000000000bb", Key: testKey, CreatedAt: time.Now()} + m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) + + if _, err := m.ClaimCheckpoint(t.Context(), "ck_00000000000000cc", time.Hour, ""); !errors.Is(err, ErrUnknownCheckpoint) { + t.Errorf("claim: %v, want ErrUnknownCheckpoint", err) + } + if got := puller.count(); got != 0 { + t.Errorf("puller called %d times, want 0: ClaimCheckpoint must never pull", got) + } +} + +// TestClaimCheckpointHealPullsAndGossips: a heal pays the transfer once and +// makes the record locally listed, so the mesh advertises it going forward. +func TestClaimCheckpointHealPullsAndGossips(t *testing.T) { + id := "ck_00000000000000bb" + ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} + m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) + + sb, err := m.ClaimCheckpointHeal(t.Context(), id, time.Hour, "") + if err != nil { + t.Fatalf("ClaimCheckpointHeal: %v", err) + } + if sb.FromCheckpoint != id { + t.Errorf("FromCheckpoint = %q, want %q", sb.FromCheckpoint, id) + } + if got := puller.count(); got != 1 { + t.Errorf("puller called %d times, want 1", got) + } + if ids := m.CheckpointIDs(); !slices.Contains(ids, id) { + t.Errorf("CheckpointIDs() = %v, want %s gossiped after heal", ids, id) + } +} + +// TestClaimCheckpointAfterHealStaysLocal: once healed, the record is served +// from the local store — a later branch must not pull again. +func TestClaimCheckpointAfterHealStaysLocal(t *testing.T) { + id := "ck_00000000000000bb" + ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} + m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) + + if _, err := m.ClaimCheckpointHeal(t.Context(), id, time.Hour, ""); err != nil { + t.Fatalf("ClaimCheckpointHeal: %v", err) + } + if _, err := m.ClaimCheckpoint(t.Context(), id, time.Hour, ""); err != nil { + t.Fatalf("ClaimCheckpoint after heal: %v", err) + } + if got := puller.count(); got != 1 { + t.Errorf("puller called %d times after a local claim, want still 1 (pay once)", got) + } +} + +// TestClaimCheckpointHealDedupsConcurrentPulls: two branches racing to heal +// the same missing checkpoint must share one transfer, not each pay for it. +func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { + id := "ck_00000000000000bb" + ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} + m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) + puller.release = make(chan struct{}) + + var wg sync.WaitGroup + errs := make([]error, 2) + for i := range 2 { + wg.Go(func() { + _, err := m.ClaimCheckpointHeal(t.Context(), id, time.Hour, "") + errs[i] = err + }) + } + waitFor(t, func() bool { return puller.count() >= 1 }) + time.Sleep(50 * time.Millisecond) // let the second goroutine join the same flight + close(puller.release) + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Errorf("goroutine %d: %v", i, err) + } + } + if got := puller.count(); got != 1 { + t.Errorf("puller called %d times for two concurrent heals of one id, want 1", got) + } +} + +// TestFetchCheckpointNeverPulls: the peer-transfer blob endpoint must never +// trigger a recursive pull, even with a healer installed that could serve it. +func TestFetchCheckpointNeverPulls(t *testing.T) { + id := "ck_00000000000000bb" + ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} + m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) + + if _, _, _, err := m.FetchCheckpoint(t.Context(), id); !errors.Is(err, ErrUnknownCheckpoint) { + t.Errorf("FetchCheckpoint: %v, want ErrUnknownCheckpoint", err) + } + if got := puller.count(); got != 0 { + t.Errorf("puller called %d times, want 0: the blob endpoint must never pull", got) + } +} + +// TestCheckpointIDsReflectsPublishAndDelete: the gossip set tracks a +// checkpoint's whole lifecycle, and an archive wake image never enters it. +func TestCheckpointIDsReflectsPublishAndDelete(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, archivePool(3600)) + src := mustClaim(t, m, testKey) + + ckpt, err := m.Checkpoint(t.Context(), src.ID, src.Token, "", "") + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + if ids := m.CheckpointIDs(); !slices.Contains(ids, ckpt.ID) { + t.Errorf("CheckpointIDs() = %v, want %s listed after publish", ids, ckpt.ID) + } + + if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, ""); err != nil { + t.Fatalf("delete: %v", err) + } + if ids := m.CheckpointIDs(); slices.Contains(ids, ckpt.ID) { + t.Errorf("CheckpointIDs() = %v, want %s gone after delete", ids, ckpt.ID) + } + + sb := mustClaim(t, m, testKey) + mustArchive(t, m, sb) + if ids := m.CheckpointIDs(); slices.Contains(ids, sb.ArchiveCk) { + t.Errorf("CheckpointIDs() = %v, want the archive wake image %s never listed", ids, sb.ArchiveCk) + } +} + +// TestCheckpointIDsReseedsOnBoot: a fresh Manager over the same dataDir must +// rebuild the gossip set from disk, not start empty until the next publish. +func TestCheckpointIDsReseedsOnBoot(t *testing.T) { + dataDir := t.TempDir() + m1 := newTestManagerAt(t, newFakeEngine(), dataDir) + src := mustClaim(t, m1, testKey) + ckpt, err := m1.Checkpoint(t.Context(), src.ID, src.Token, "", "") + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + + m2 := newTestManagerAt(t, newFakeEngine(), dataDir) + if ids := m2.CheckpointIDs(); !slices.Contains(ids, ckpt.ID) { + t.Errorf("fresh manager CheckpointIDs() = %v, want %s re-seeded from disk", ids, ckpt.ID) + } +} + +// newHealManager builds a Manager with a healer wired to a puller serving +// ckpt at addrs, mirroring WithPeerHeal's construction directly (tests are in +// the same package). +func newHealManager(t *testing.T, ckpt types.Checkpoint, addrs []string) (*Manager, *healPuller) { + t.Helper() + m := newTestManager(t, newFakeEngine()) + puller := &healPuller{ckpt: ckpt} + m.healer = peer.New(m.ckpts, func(string) []string { return addrs }, puller) + return m, puller +} + +// healPuller fakes peer.Puller: it writes a valid record (meta.json + an +// empty export/ dir the fakeEngine "clones" from) and can block until +// release, so a test can prove concurrent misses dedup to one pull. +type healPuller struct { + mu sync.Mutex + calls int + ckpt types.Checkpoint + release chan struct{} // non-nil: Pull blocks until closed +} + +func (p *healPuller) Pull(_ context.Context, _, _, dst string) error { + p.mu.Lock() + p.calls++ + p.mu.Unlock() + if p.release != nil { + <-p.release + } + if err := os.MkdirAll(filepath.Join(dst, store.ExportDir), 0o750); err != nil { + return err + } + meta, err := json.Marshal(p.ckpt) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dst, store.MetaFile), meta, 0o600) +} + +func (p *healPuller) count() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.calls +} diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 59a10bf..179fdd2 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -222,6 +222,14 @@ type Manager struct { audit *journal counters counters ckpts store.Store + // ckptsShared marks an s3 (or other cluster-wide) checkpoint backend: + // every node already resolves every record, so peer heal and its gossip + // set are no-ops. + ckptsShared bool + // healer is the checkpoint-only read path ClaimCheckpointHeal pulls + // through; nil unless checkpoint_peer_heal wired it. m.ckpts itself is + // never wrapped, so the server's redirect branch stays reachable. + healer store.Store tpls store.Store ckptTTL time.Duration ckptSweeping atomic.Bool @@ -232,6 +240,13 @@ type Manager struct { tplMu sync.Mutex tplSet map[string]struct{} + // ckptSet mirrors tplSet for checkpoints: publish/delete/heal keep it + // current, startup seeds it from one Metas scan. Nil on a shared + // checkpoint store — gossiping it there would have every node advertise + // every record. + ckptMu sync.Mutex + ckptSet map[string]struct{} + // recLocks serializes same-id store record mutations and holds off a // re-publish swap while a clone reads the old generation (per id, RW). recLocks sync.Map @@ -308,6 +323,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg if m.ckpts, err = newStoreView(ctx, cfg, "checkpoint-staging", store.CheckpointIDRe); err != nil { return nil, err } + m.ckptsShared = cfg.CheckpointStore != nil && cfg.CheckpointStore.Kind == "s3" if m.tpls, err = newStoreView(ctx, cfg, "template-staging", store.TemplateIDRe); err != nil { return nil, err } @@ -321,6 +337,9 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg } } } + if !m.ckptsShared { + m.seedCkptSet(ctx) + } usage, err := newJournal(filepath.Join(cfg.DataDir, "usage.jsonl")) if err != nil { return nil, fmt.Errorf("open usage journal: %w", err) @@ -558,15 +577,34 @@ func newStoreView(ctx context.Context, cfg *config.Config, staging string, idRe return dir.New(cmp.Or(cfg.CheckpointDir, filepath.Join(cfg.DataDir, "checkpoints")), idRe) } -// WithPeerHeal wraps the manager's checkpoint store so a record this node does -// not hold is pulled from a node that gossiped it. A no-op unless -// checkpoint_peer_heal is set: a shared backend already resolves every record -// from every node. +// WithPeerHeal installs the checkpoint healer: a read path ClaimCheckpointHeal +// pulls through only after a local claim missed and a redirect could not +// answer, so a peer transfer is paid only when nothing cheaper resolves the +// claim. A no-op unless checkpoint_peer_heal is set, or the checkpoint store +// is already shared cluster-wide (every node resolves every record already). func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { - if !enabled || owners == nil { + if !enabled || owners == nil || m.ckptsShared { return } - m.ckpts = peer.New(m.ckpts, owners, &peer.HTTPPuller{Token: token}) + m.healer = peer.New(m.ckpts, owners, &peer.HTTPPuller{Token: token}) +} + +// seedCkptSet rebuilds the gossip set from one store scan at boot, so records +// published by a prior life are advertised again without a re-publish. +func (m *Manager) seedCkptSet(ctx context.Context) { + m.ckptSet = map[string]struct{}{} + metas, err := m.ckpts.Metas(ctx) + if err != nil { + // Existing records stay unadvertised until re-published; say so. + log.WithFunc("pool.seedCkptSet").Warnf(ctx, "seed checkpoint gossip set: %v", err) + return + } + for _, raw := range metas { + var ckpt types.Checkpoint + if json.Unmarshal(raw, &ckpt) == nil && ckpt.ID != "" && !ckpt.Archive { + m.ckptSet[ckpt.ID] = struct{}{} + } + } } func dirExists(path string) bool { diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 2241337..841ba9f 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -80,6 +80,7 @@ type Manager interface { Audit(ctx context.Context, id string, line []byte) AuditEnabled() bool ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) + ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) Checkpoints(ctx context.Context, tenant string) ([]types.Checkpoint, error) FetchCheckpoint(ctx context.Context, ckptID string) (dir string, meta []byte, release func(), err error) DeleteCheckpoint(ctx context.Context, ckptID, tenant string) error @@ -394,6 +395,9 @@ func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { } // handleClaimCheckpoint claims a fresh sandbox branched from a checkpoint. +// Tier order: local claim first; then a redirect (zero bytes moved) when a +// gossiped owner exists and this is not already the redirect retry; heal (one +// peer transfer) only when neither could answer. func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.CheckpointClaimRequest](w, r) if !ok { @@ -401,12 +405,11 @@ func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { } ckptID := r.PathValue("id") sb, err := s.mgr.ClaimCheckpoint(r.Context(), ckptID, req.TTL(), tenantFrom(r.Context())) - // Checkpoints are node-local: redirect to a peer that gossiped the record - // so the clone runs on the disk that already holds it. A no_redirect retry - // must resolve locally or two nodes would bounce the request. - if errors.Is(err, pool.ErrUnknownCheckpoint) && s.placer != nil && !req.NoRedirect && - writeRedirect(w, s.placer.CheckpointOwners(ckptID)) { - return + if errors.Is(err, pool.ErrUnknownCheckpoint) { + if !req.NoRedirect && s.placer != nil && writeRedirect(w, s.placer.CheckpointOwners(ckptID)) { + return + } + sb, err = s.mgr.ClaimCheckpointHeal(r.Context(), ckptID, req.TTL(), tenantFrom(r.Context())) } writeResult(w, r, "claim checkpoint", ckptID, "provisioning failed", err, func() { writeJSON(w, http.StatusOK, s.claimResponse(sb)) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index c7aa2f5..e99a84f 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -978,10 +978,12 @@ func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { } } -// TestCheckpointClaimNoOwnersIs404: nothing gossiped the record, so a miss is -// an honest miss rather than a redirect to nowhere. +// TestCheckpointClaimNoOwnersIs404: nothing gossiped the record, so a miss +// falls through to heal, and a heal miss (item 4) is an honest 404 rather +// than a redirect to nowhere. func TestCheckpointClaimNoOwnersIs404(t *testing.T) { - ts := newPlacerTestServer(t, "sekret", &fakeManager{}, &fakePlacer{}) + mgr := &fakeManager{} // ClaimCheckpointHeal defaults to ErrUnknownCheckpoint + ts := newPlacerTestServer(t, "sekret", mgr, &fakePlacer{}) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -989,6 +991,95 @@ func TestCheckpointClaimNoOwnersIs404(t *testing.T) { if resp.StatusCode != http.StatusNotFound { t.Fatalf("status = %d, want 404", resp.StatusCode) } + if mgr.healCalls != 1 { + t.Errorf("heal called %d times, want 1: a miss with no owner must still try heal", mgr.healCalls) + } +} + +// TestCheckpointClaimRedirectBeatsHeal is the tier-order regression test: a +// gossiped owner must redirect (zero bytes moved) and never fall through to +// the heal pull, even though the local ClaimCheckpoint missed. +func TestCheckpointClaimRedirectBeatsHeal(t *testing.T) { + mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", mgr, placer) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 carrying a redirect", resp.StatusCode) + } + var got types.ClaimResponse + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if len(got.Redirect) != 1 || got.Redirect[0] != "owner-a:7777" { + t.Errorf("redirect = %v, want [owner-a:7777]", got.Redirect) + } + if mgr.healCalls != 0 { + t.Errorf("heal called %d times, want 0: a known owner must redirect, not heal", mgr.healCalls) + } +} + +// TestCheckpointClaimFallsBackToHealWhenNoOwner: no gossiped owner to redirect +// to, so the server pulls the record itself via heal. +func TestCheckpointClaimFallsBackToHealWhenNoOwner(t *testing.T) { + mgr := &fakeManager{ + healCheckpoint: func(ckptID string) (*types.Sandbox, error) { + return &types.Sandbox{ID: "sb_healed", Token: "htok", FromCheckpoint: ckptID}, nil + }, + } + ts := newPlacerTestServer(t, "sekret", mgr, &fakePlacer{}) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var got types.ClaimResponse + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.ID != "sb_healed" { + t.Errorf("id = %q, want the healed claim", got.ID) + } + if mgr.healCalls != 1 { + t.Errorf("heal called %d times, want 1", mgr.healCalls) + } +} + +// TestCheckpointClaimNoRedirectGoesStraightToHeal: a no_redirect retry must +// not bounce again, but it must still try heal rather than answer 404 outright. +func TestCheckpointClaimNoRedirectGoesStraightToHeal(t *testing.T) { + mgr := &fakeManager{ + healCheckpoint: func(ckptID string) (*types.Sandbox, error) { + return &types.Sandbox{ID: "sb_healed", Token: "htok", FromCheckpoint: ckptID}, nil + }, + } + placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", mgr, placer) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{"no_redirect":true}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var got types.ClaimResponse + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if len(got.Redirect) != 0 { + t.Errorf("redirect = %v, want none: no_redirect must not bounce", got.Redirect) + } + if got.ID != "sb_healed" { + t.Errorf("id = %q, want the healed claim", got.ID) + } + if mgr.healCalls != 1 { + t.Errorf("heal called %d times, want 1", mgr.healCalls) + } } // TestCheckpointBlobUnknownIs404 lets a puller move on to the next owner @@ -1093,6 +1184,8 @@ type fakeManager struct { audited func(id string, line []byte) checkpoint func(id, token, name string) (types.Checkpoint, error) claimCheckpoint func(ckptID string) (*types.Sandbox, error) + healCheckpoint func(ckptID string) (*types.Sandbox, error) + healCalls int checkpoints []types.Checkpoint deleteCheckpoint func(ckptID string) error setPools func(pools []config.PoolSpec) error @@ -1260,6 +1353,15 @@ func (f *fakeManager) ClaimCheckpoint(_ context.Context, ckptID string, _ time.D return f.claimCheckpoint(ckptID) } +func (f *fakeManager) ClaimCheckpointHeal(_ context.Context, ckptID string, _ time.Duration, tenant string) (*types.Sandbox, error) { + f.gotTenant = tenant + f.healCalls++ + if f.healCheckpoint == nil { + return nil, pool.ErrUnknownCheckpoint + } + return f.healCheckpoint(ckptID) +} + func (f *fakeManager) Checkpoints(_ context.Context, tenant string) ([]types.Checkpoint, error) { f.gotTenant = tenant return f.checkpoints, nil diff --git a/sandboxd/store/peer/peer.go b/sandboxd/store/peer/peer.go index d962e7c..87ecd1c 100644 --- a/sandboxd/store/peer/peer.go +++ b/sandboxd/store/peer/peer.go @@ -6,6 +6,8 @@ import ( "fmt" "os" + "golang.org/x/sync/singleflight" + "github.com/cocoonstack/sandbox/sandboxd/store" ) @@ -19,58 +21,46 @@ var _ store.Store = (*Store)(nil) // pulling the record from a peer and publishing it locally, so the transfer is // paid once: this node then owns the record and serves later reads locally. type Store struct { - local store.Store - owners Owners - puller Puller + store.Store + owners Owners + puller Puller + flights singleflight.Group } // New wraps local with peer healing. A nil owners or puller leaves the wrapper // inert, so a node with no mesh degrades to the local backend rather than // failing. func New(local store.Store, owners Owners, puller Puller) *Store { - return &Store{local: local, owners: owners, puller: puller} -} - -func (s *Store) Stage(id string) (string, error) { return s.local.Stage(id) } - -func (s *Store) Publish(ctx context.Context, staging, id string) error { - return s.local.Publish(ctx, staging, id) + return &Store{Store: local, owners: owners, puller: puller} } -func (s *Store) Delete(ctx context.Context, id string) error { return s.local.Delete(ctx, id) } - -func (s *Store) SweepStaging() error { return s.local.SweepStaging() } - -// Metas lists local records only: answering for peers would return the same -// record once per node to a control plane that already scatter-gathers. -func (s *Store) Metas(ctx context.Context) ([][]byte, error) { return s.local.Metas(ctx) } - // Fetch serves the record locally, healing from a peer on a miss. func (s *Store) Fetch(ctx context.Context, id string) (string, []byte, func(), error) { - dir, meta, release, err := s.local.Fetch(ctx, id) + dir, meta, release, err := s.Store.Fetch(ctx, id) if !errors.Is(err, store.ErrNotFound) { return dir, meta, release, err } if err := s.heal(ctx, id); err != nil { return "", nil, nil, err } - return s.local.Fetch(ctx, id) + return s.Store.Fetch(ctx, id) } // ReadMeta reads the record's metadata, healing from a peer on a miss. func (s *Store) ReadMeta(ctx context.Context, id string) ([]byte, error) { - meta, err := s.local.ReadMeta(ctx, id) + meta, err := s.Store.ReadMeta(ctx, id) if !errors.Is(err, store.ErrNotFound) { return meta, err } if err := s.heal(ctx, id); err != nil { return nil, err } - return s.local.ReadMeta(ctx, id) + return s.Store.ReadMeta(ctx, id) } // heal pulls id from the first peer that serves it and publishes it locally, // returning store.ErrNotFound when no peer has it so a miss stays a miss. +// Concurrent misses of the same id share one pull. func (s *Store) heal(ctx context.Context, id string) error { if s.owners == nil || s.puller == nil { return store.ErrNotFound @@ -82,7 +72,15 @@ func (s *Store) heal(ctx context.Context, id string) error { // A client hanging up must not abandon a started pull: the next branch // would pay the whole transfer again. ctx = context.WithoutCancel(ctx) + _, err, _ := s.flights.Do(id, func() (any, error) { + return nil, s.healFrom(ctx, id, addrs) + }) + return err +} +// healFrom tries each owner in order until one serves id; callers hold the +// id's singleflight slot. +func (s *Store) healFrom(ctx context.Context, id string, addrs []string) error { var errs []error for _, addr := range addrs { if err := s.pullFrom(ctx, addr, id); err != nil { @@ -102,7 +100,7 @@ func (s *Store) heal(ctx context.Context, id string) error { // pullFrom stages a peer's copy and publishes it, so the record becomes local // through the same atomic path a locally-created one takes. func (s *Store) pullFrom(ctx context.Context, addr, id string) error { - staging, err := s.local.Stage(id) + staging, err := s.Stage(id) if err != nil { return fmt.Errorf("stage: %w", err) } @@ -111,7 +109,7 @@ func (s *Store) pullFrom(ctx context.Context, addr, id string) error { if err := s.puller.Pull(ctx, addr, id, staging); err != nil { return err } - if err := s.local.Publish(ctx, staging, id); err != nil { + if err := s.Publish(ctx, staging, id); err != nil { return fmt.Errorf("publish pulled record: %w", err) } return nil diff --git a/sandboxd/store/peer/peer_test.go b/sandboxd/store/peer/peer_test.go index 2d5cb11..73172ef 100644 --- a/sandboxd/store/peer/peer_test.go +++ b/sandboxd/store/peer/peer_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "sync" "testing" "github.com/cocoonstack/sandbox/sandboxd/store" @@ -122,6 +123,30 @@ func TestHealErrorIsReported(t *testing.T) { } } +// TestHealDedupsConcurrentMisses: N concurrent Fetch misses of the same id +// must trigger exactly one pull — a stampede of branches racing to heal the +// same checkpoint must not each pay for the whole transfer. +func TestHealDedupsConcurrentMisses(t *testing.T) { + puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} + s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) + + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + _, _, release, err := s.Fetch(t.Context(), testID) + if err != nil { + t.Errorf("Fetch: %v", err) + return + } + release() + }) + } + wg.Wait() + if len(puller.asked) != 1 { + t.Errorf("puller called %d times (%v); concurrent misses must dedup to one pull", len(puller.asked), puller.asked) + } +} + // TestMetasStaysLocal: a cluster-wide listing is the control plane's job (it // already scatter-gathers every node). If each node answered for its peers the // same record would come back N times. @@ -149,13 +174,16 @@ func localStore(t *testing.T) *dir.Store { // fakePuller serves records from an in-memory table, recording who was asked. type fakePuller struct { + mu sync.Mutex records map[string]map[string]string // addr → file → content asked []string failAddr string } func (p *fakePuller) Pull(_ context.Context, addr, _, dst string) error { + p.mu.Lock() p.asked = append(p.asked, addr) + p.mu.Unlock() if addr == p.failAddr { return errors.New("peer exploded") } From 6850744007a1885de9f5e60f415e490393cd3c01 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 13:56:12 +0800 Subject: [PATCH 11/31] sdk: follow a checkpoint redirect instead of claiming nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A branch of a checkpoint the node no longer holds is answered with 200 and the owning peers' addresses — the same contract the generic claim path has followed since the mesh landed. The checkpoint path passed that reply straight to handleFrom, so Go built a sandbox with an empty id and token and Python raised on the missing key. Both now retry each candidate with no_redirect set and bind the sandbox to the address that served it, so the redirect tier is reachable from a client at all. --- sdk/go/checkpoint.go | 36 +++++++- sdk/go/checkpoint_test.go | 122 +++++++++++++++++++++++++ sdk/python/cocoonsandbox/checkpoint.py | 27 +++++- sdk/python/tests/test_checkpoint.py | 90 ++++++++++++++++++ 4 files changed, 268 insertions(+), 7 deletions(-) create mode 100644 sdk/go/checkpoint_test.go create mode 100644 sdk/python/tests/test_checkpoint.py diff --git a/sdk/go/checkpoint.go b/sdk/go/checkpoint.go index c94d883..8e53845 100644 --- a/sdk/go/checkpoint.go +++ b/sdk/go/checkpoint.go @@ -3,6 +3,7 @@ package sandbox import ( "bytes" "context" + "fmt" "net/http" "time" ) @@ -22,7 +23,9 @@ type Checkpoint struct { } // New claims a sandbox cloned from the checkpoint, on the checkpoint's -// node. The snapshot pins the key axes; WithTimeout may set the TTL. +// node. The snapshot pins the key axes; WithTimeout may set the TTL. A node +// that no longer holds the checkpoint redirects, which New follows +// transparently, mirroring Client.New. func (ck *Checkpoint) New(ctx context.Context, opts ...Option) (*Sandbox, error) { var claim claimRequest for _, opt := range opts { @@ -35,10 +38,32 @@ func (ck *Checkpoint) New(ctx context.Context, opts ...Option) (*Sandbox, error) if err != nil { return nil, err } - cr, err := doJSON[claimResponse](ctx, ck.c, http.MethodPost, ck.addr, "/v1/checkpoints/"+ck.ID+"/claim", bytes.NewReader(body), ck.c.apiToken, "claim checkpoint") + cr, err := ck.claimAt(ctx, ck.addr, body) if err != nil { return nil, err } + if len(cr.Redirect) > 0 { + body, err = encodeBody("checkpoint claim", checkpointClaimRequest{TTLSeconds: claim.TTLSeconds, NoRedirect: true}) + if err != nil { + return nil, err + } + var sb *Sandbox + retryAny := func(error) bool { return true } + if tryErr := tryEach(cr.Redirect, func(addr string) error { + target, claimErr := ck.claimAt(ctx, addr, body) + if claimErr != nil { + return claimErr + } + if len(target.Redirect) > 0 { + return fmt.Errorf("%s redirected again despite no_redirect", addr) + } + sb = ck.c.handleFrom(addr, target) + return nil + }, retryAny); tryErr != nil { + return nil, fmt.Errorf("claim checkpoint: all redirect targets failed: %w", tryErr) + } + return sb, nil + } return ck.c.handleFrom(ck.addr, cr), nil } @@ -47,6 +72,10 @@ func (ck *Checkpoint) Delete(ctx context.Context) error { return doNoContent(ctx, ck.c, http.MethodDelete, ck.addr, "/v1/checkpoints/"+ck.ID, nil, ck.c.apiToken, "delete checkpoint") } +func (ck *Checkpoint) claimAt(ctx context.Context, addr string, body []byte) (claimResponse, error) { + return doJSON[claimResponse](ctx, ck.c, http.MethodPost, addr, "/v1/checkpoints/"+ck.ID+"/claim", bytes.NewReader(body), ck.c.apiToken, "claim checkpoint") +} + // Checkpoint captures the sandbox's full state — memory, disk, running // processes — without stopping it, and returns a handle that branches new // sandboxes from that exact moment. name is an optional label. @@ -100,7 +129,8 @@ type checkpointResponse struct { } type checkpointClaimRequest struct { - TTLSeconds int `json:"ttl_seconds,omitempty"` + TTLSeconds int `json:"ttl_seconds,omitempty"` + NoRedirect bool `json:"no_redirect,omitempty"` } type checkpointListResponse struct { diff --git a/sdk/go/checkpoint_test.go b/sdk/go/checkpoint_test.go new file mode 100644 index 0000000..65cd74f --- /dev/null +++ b/sdk/go/checkpoint_test.go @@ -0,0 +1,122 @@ +package sandbox + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestCheckpointNewFollowsRedirect(t *testing.T) { + // The checkpoint's node no longer holds it and redirects; the retry + // must land on node B, bound there, exactly like Client.New's redirect. + var nodeACalls int + nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req checkpointClaimRequest + _ = json.NewDecoder(r.Body).Decode(&req) + if !req.NoRedirect { + t.Error("retry did not carry no_redirect") + } + _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_ck1", Token: "tok"}) + })) + t.Cleanup(nodeB.Close) + + nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + nodeACalls++ + _ = json.NewEncoder(w).Encode(claimResponse{ + Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, + }) + })) + t.Cleanup(nodeA.Close) + + c := testClient(t, nodeA) + ck := checkpointHandle(c, c.addr, checkpointRecord{ID: "ck_1"}) + sb, err := ck.New(t.Context()) + if err != nil { + t.Fatalf("New: %v", err) + } + if sb.ID != "sb_ck1" { + t.Errorf("id %q, want sb_ck1 (claimed at the redirect target)", sb.ID) + } + if sb.owner != strings.TrimPrefix(nodeB.URL, "http://") { + t.Errorf("owner %q, want node B", sb.owner) + } + if nodeACalls != 1 { + t.Errorf("node A called %d times, want 1", nodeACalls) + } +} + +func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { + broken := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(broken.Close) + + entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(claimResponse{ + Redirect: []string{strings.TrimPrefix(broken.URL, "http://")}, + }) + })) + t.Cleanup(entry.Close) + + c := testClient(t, entry) + ck := checkpointHandle(c, c.addr, checkpointRecord{ID: "ck_2"}) + sb, err := ck.New(t.Context()) + if err == nil { + t.Fatal("New: want error, got nil") + } + if !strings.Contains(err.Error(), "http 500") { + t.Errorf("err %v, want the 500 surfaced", err) + } + if sb != nil { + t.Errorf("sb %+v, want nil on failure", sb) + } +} + +// TestCheckpointNewRedirectNeverYieldsEmptyID is a regression test: New used +// to hand a bare redirect reply straight to handleFrom, producing a Sandbox +// with no id/token. It must now follow the redirect and fail loudly when no +// candidate answers, never return a Sandbox with an empty ID. +func TestCheckpointNewRedirectNeverYieldsEmptyID(t *testing.T) { + entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{"127.0.0.1:1"}}) + })) + t.Cleanup(entry.Close) + + c := testClient(t, entry) + ck := checkpointHandle(c, c.addr, checkpointRecord{ID: "ck_3"}) + sb, err := ck.New(t.Context()) + if err == nil { + t.Fatal("New: want error (dead redirect target), got nil") + } + if sb != nil { + t.Errorf("sb %+v, want nil", sb) + } +} + +func TestCheckpointNewSecondLevelRedirectFails(t *testing.T) { + // A compliant server never redirects a no_redirect retry; if one does + // anyway, it must be treated as a failed candidate rather than followed. + nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{"127.0.0.1:1"}}) + })) + t.Cleanup(nodeB.Close) + + nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(claimResponse{ + Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, + }) + })) + t.Cleanup(nodeA.Close) + + c := testClient(t, nodeA) + ck := checkpointHandle(c, c.addr, checkpointRecord{ID: "ck_4"}) + sb, err := ck.New(t.Context()) + if err == nil { + t.Fatal("New: want error, got nil") + } + if sb != nil { + t.Errorf("sb %+v, want nil", sb) + } +} diff --git a/sdk/python/cocoonsandbox/checkpoint.py b/sdk/python/cocoonsandbox/checkpoint.py index 0bb09a1..2b1cc9a 100644 --- a/sdk/python/cocoonsandbox/checkpoint.py +++ b/sdk/python/cocoonsandbox/checkpoint.py @@ -6,6 +6,8 @@ from typing import TYPE_CHECKING +from .errors import APIError + if TYPE_CHECKING: from .sandbox import Sandbox @@ -22,10 +24,27 @@ def __init__(self, client, addr: str, rec: dict): self.created_at = rec.get("created_at", "") def new(self, ttl_seconds: int = 0) -> Sandbox: - """Claims a fresh sandbox branched from the checkpoint.""" - body = {"ttl_seconds": ttl_seconds} if ttl_seconds else {} - reply = self._client._post_json(self._addr, f"/v1/checkpoints/{self.id}/claim", body, "claim checkpoint") - return self._client._handle_from(self._addr, reply) + """Claims a fresh sandbox branched from the checkpoint, following a + redirect to the node that actually holds it.""" + # Local import: a top-level one would close the client -> sandbox -> + # checkpoint cycle. + from .client import _try_each + + claim = {"ttl_seconds": ttl_seconds} if ttl_seconds else {} + path = f"/v1/checkpoints/{self.id}/claim" + reply = self._client._post_json(self._addr, path, claim, "claim checkpoint") + redirect = reply.get("redirect") or [] + if not redirect: + return self._client._handle_from(self._addr, reply) + claim["no_redirect"] = True + + def claim_at(peer: str) -> Sandbox: + r = self._client._post_json(peer, path, claim, "claim checkpoint") + if r.get("redirect"): + raise APIError("claim checkpoint", 0, f"{peer} redirected again despite no_redirect") + return self._client._handle_from(peer, r) + + return _try_each(redirect, claim_at, retry=lambda _: True) def delete(self) -> None: """Removes the checkpoint from its node.""" diff --git a/sdk/python/tests/test_checkpoint.py b/sdk/python/tests/test_checkpoint.py new file mode 100644 index 0000000..56f3a04 --- /dev/null +++ b/sdk/python/tests/test_checkpoint.py @@ -0,0 +1,90 @@ +"""Checkpoint claim redirect: mirrors the cluster redirect follow tested for +Client.new in test_fault_matrix.py, applied to Checkpoint.new.""" + +import socket +import threading +from http.server import HTTPServer + +import pytest +from test_client import FakeNode + +from cocoonsandbox import APIError, Client + + +@pytest.fixture +def spawn_node(): + servers = [] + + def spawn(routes): + handler = type("Node", (FakeNode,), {"routes": routes}) + server = HTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + servers.append(server) + return f"127.0.0.1:{server.server_port}" + + yield spawn + for server in servers: + server.shutdown() + + +@pytest.fixture +def dead_addr(): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + return f"127.0.0.1:{port}" + + +def test_checkpoint_new_follows_redirect(spawn_node): + seen = [] + + def claim_at_b(body, path): + seen.append(body) + return 200, {"id": "sb_ck1", "token": "tok"} + + node_b = spawn_node({("POST", "/v1/checkpoints/ck_1/claim"): claim_at_b}) + + entry_hits = [] + + def redirect(body, path): + entry_hits.append(body) + return 200, {"redirect": [node_b]} + + entry = spawn_node({("POST", "/v1/checkpoints/ck_1/claim"): redirect}) + + sb = Client(entry).checkpoint("ck_1").new() + assert sb.id == "sb_ck1" + assert sb.owner == node_b + assert len(entry_hits) == 1 + assert seen[0]["no_redirect"] is True + + +def test_checkpoint_new_all_candidates_fail(spawn_node): + broken = spawn_node({("POST", "/v1/checkpoints/ck_2/claim"): lambda body, path: (500, {"error": "boom"})}) + entry = spawn_node({("POST", "/v1/checkpoints/ck_2/claim"): lambda body, path: (200, {"redirect": [broken]})}) + + with pytest.raises(APIError) as exc: + Client(entry).checkpoint("ck_2").new() + assert exc.value.status == 500 and "boom" in exc.value.message + + +def test_checkpoint_new_redirect_never_yields_empty_id(spawn_node, dead_addr): + # Regression: new() used to hand a bare redirect reply straight to + # _handle_from, producing a Sandbox with no id/token. It must now follow + # the redirect and raise when no candidate answers. + entry = spawn_node({("POST", "/v1/checkpoints/ck_3/claim"): lambda body, path: (200, {"redirect": [dead_addr]})}) + + with pytest.raises(APIError): + Client(entry).checkpoint("ck_3").new() + + +def test_checkpoint_new_second_level_redirect_fails(spawn_node): + # A compliant server never redirects a no_redirect retry; if one does + # anyway, it must be treated as a failed candidate rather than followed. + path = "/v1/checkpoints/ck_4/claim" + node_b = spawn_node({("POST", path): lambda body, p: (200, {"redirect": ["127.0.0.1:1"]})}) + entry = spawn_node({("POST", path): lambda body, p: (200, {"redirect": [node_b]})}) + + with pytest.raises(APIError): + Client(entry).checkpoint("ck_4").new() From eeaed376930522f5b07905a7d3781e2f7419b98d Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 14:00:03 +0800 Subject: [PATCH 12/31] sandboxd: one credential type instead of paired operator verbs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six verbs each carried a second Operator method that differed only in how it found the sandbox, and the server decided "root api_token or per-sandbox token" in two different shapes across five handlers. A Cred carries the resolved authority instead: the server builds it once per request, resolve looks the claim up by token or by id, and the Manager interface loses six methods. resolve authorizes off the manager mutex, unlike the old Release which held it across the token check and the delete. releaseResolved therefore re-checks under the lock that the claim it was handed is still the live one, so a second release racing on the same id is a miss rather than a second teardown — TestReleaseAfterResolveTearsDownOnce fails without that check. An empty token is now rejected explicitly rather than compared, so a caller presenting nothing can never match a claim. --- sandboxd/pool/archive_test.go | 18 ++--- sandboxd/pool/checkpoint.go | 18 +---- sandboxd/pool/checkpoint_heal_test.go | 4 +- sandboxd/pool/checkpoint_test.go | 14 ++-- sandboxd/pool/claim.go | 32 +++------ sandboxd/pool/egress_test.go | 6 +- sandboxd/pool/fork.go | 18 +---- sandboxd/pool/fork_test.go | 22 +++--- sandboxd/pool/hibernate.go | 4 +- sandboxd/pool/hibernate_test.go | 52 +++++++------- sandboxd/pool/intercept_test.go | 2 +- sandboxd/pool/operator.go | 44 ++++++------ sandboxd/pool/pool_test.go | 60 ++++++++++++++--- sandboxd/pool/promote_test.go | 28 ++++---- sandboxd/pool/telemetry_test.go | 8 +-- sandboxd/pool/template.go | 18 +---- sandboxd/server/server.go | 97 ++++++++++----------------- sandboxd/server/server_test.go | 88 +++++++++++------------- 18 files changed, 243 insertions(+), 290 deletions(-) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 3be795d..536b645 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -22,7 +22,7 @@ func TestArchivePublishWindowPinsCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } stall := &stallingStore{Store: m.ckpts, published: make(chan string, 1), release: make(chan struct{})} @@ -56,7 +56,7 @@ func TestArchiveWakeRehibernateWindowAborts(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } stall := &stallingStore{Store: m.ckpts, published: make(chan string, 1), release: make(chan struct{})} @@ -69,7 +69,7 @@ func TestArchiveWakeRehibernateWindowAborts(t *testing.T) { if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake mid-publish: %v", err) } - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("re-hibernate mid-publish: %v", err) } m.mu.Lock() @@ -342,7 +342,7 @@ func TestReleaseArchivedDeletesCheckpoint(t *testing.T) { ck := sb.ArchiveCk removesBefore := len(eng.removedNames()) - if err := m.Release(t.Context(), id, token); err != nil { + if err := m.Release(t.Context(), id, Cred{Token: token}); err != nil { t.Fatalf("release archived: %v", err) } if ckExists(t, m, ck) { @@ -410,7 +410,7 @@ func TestArchivePersistFailureRollsBack(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } snap, vm := sb.HibernateSnap, sb.VMName @@ -446,7 +446,7 @@ func TestReleaseArchivedRollsBackOnPersistFailure(t *testing.T) { ck := sb.ArchiveCk breakStore(t, m) - if err := m.Release(t.Context(), id, token); err == nil { + if err := m.Release(t.Context(), id, Cred{Token: token}); err == nil { t.Fatal("release succeeded despite a persist failure") } if _, ok := m.claim(id, token); !ok { @@ -518,7 +518,7 @@ func TestArchiveOnceSkipsInFlight(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } backdate(m, sb, time.Hour) @@ -546,7 +546,7 @@ func TestArchiveWakeEvictsRecLock(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } countLocks := func() int { @@ -622,7 +622,7 @@ func archivePool(delete int) config.PoolSpec { // can start from a known archived record. func mustArchive(t *testing.T, m *Manager, sb *types.Sandbox) { t.Helper() - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } if err := m.archive(t.Context(), sb); err != nil { diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index f474c3f..7f9f804 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -25,25 +25,11 @@ var ( // keeps running (a hibernated one is captured from its wake image). Branches // clone that exact state, and a source can be checkpointed again — a tree. // tenant attributes the record; empty means the operator (root). -func (m *Manager) Checkpoint(ctx context.Context, id, token, name, tenant string) (types.Checkpoint, error) { - sb, ok := m.claim(id, token) +func (m *Manager) Checkpoint(ctx context.Context, id string, cred Cred, name, tenant string) (types.Checkpoint, error) { + sb, ok := m.resolve(id, cred) if !ok { return types.Checkpoint{}, ErrUnknownSandbox } - return m.checkpointResolved(ctx, sb, name, tenant) -} - -// CheckpointOperator checkpoints a sandbox by id without a per-sandbox token. -func (m *Manager) CheckpointOperator(ctx context.Context, id, name, tenant string) (types.Checkpoint, error) { - sb, ok := m.byID(id) - if !ok { - return types.Checkpoint{}, ErrUnknownSandbox - } - return m.checkpointResolved(ctx, sb, name, tenant) -} - -// checkpointResolved is Checkpoint's body once the source claim is resolved. -func (m *Manager) checkpointResolved(ctx context.Context, sb *types.Sandbox, name, tenant string) (types.Checkpoint, error) { if name != "" && !types.NameRe.MatchString(name) { return types.Checkpoint{}, fmt.Errorf("%w: %q must match %s", ErrBadName, name, types.NameRe) } diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index a8d35d5..16c38a8 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -124,7 +124,7 @@ func TestCheckpointIDsReflectsPublishAndDelete(t *testing.T) { m := newTestManager(t, eng, archivePool(3600)) src := mustClaim(t, m, testKey) - ckpt, err := m.Checkpoint(t.Context(), src.ID, src.Token, "", "") + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") if err != nil { t.Fatalf("checkpoint: %v", err) } @@ -152,7 +152,7 @@ func TestCheckpointIDsReseedsOnBoot(t *testing.T) { dataDir := t.TempDir() m1 := newTestManagerAt(t, newFakeEngine(), dataDir) src := mustClaim(t, m1, testKey) - ckpt, err := m1.Checkpoint(t.Context(), src.ID, src.Token, "", "") + ckpt, err := m1.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") if err != nil { t.Fatalf("checkpoint: %v", err) } diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index 197f550..96f6388 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -17,7 +17,7 @@ func TestCheckpointThenBranch(t *testing.T) { m := newTestManager(t, eng) src := mustClaim(t, m, testKey) - ckpt, err := m.Checkpoint(t.Context(), src.ID, src.Token, "step-1", "") + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "step-1", "") if err != nil { t.Fatalf("Checkpoint: %v", err) } @@ -57,10 +57,10 @@ func TestCheckpointValidation(t *testing.T) { m := newTestManager(t, eng) src := mustClaim(t, m, testKey) - if _, err := m.Checkpoint(t.Context(), src.ID, "wrong-token", "", ""); !errors.Is(err, ErrUnknownSandbox) { + if _, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: "wrong-token"}, "", ""); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("bad token: %v, want ErrUnknownSandbox", err) } - if _, err := m.Checkpoint(t.Context(), src.ID, src.Token, "bad name", ""); !errors.Is(err, ErrBadName) { + if _, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "bad name", ""); !errors.Is(err, ErrBadName) { t.Errorf("bad name: %v, want ErrBadName", err) } for _, id := range []string{"../../etc", "ck_zz", "", "ck_0011223344556677x"} { @@ -128,11 +128,11 @@ func TestCheckpointTenantIsolation(t *testing.T) { if err != nil { t.Fatalf("beta claim: %v", err) } - ckA, err := m.Checkpoint(t.Context(), srcA.ID, srcA.Token, "a-step", "acme") + ckA, err := m.Checkpoint(t.Context(), srcA.ID, Cred{Token: srcA.Token}, "a-step", "acme") if err != nil { t.Fatalf("acme checkpoint: %v", err) } - ckB, err := m.Checkpoint(t.Context(), srcB.ID, srcB.Token, "b-step", "beta") + ckB, err := m.Checkpoint(t.Context(), srcB.ID, Cred{Token: srcB.Token}, "b-step", "beta") if err != nil { t.Fatalf("beta checkpoint: %v", err) } @@ -176,7 +176,7 @@ func TestCheckpointDeleteEvictsRecLock(t *testing.T) { var ids []string for range 5 { - ckpt, err := m.Checkpoint(t.Context(), src.ID, src.Token, "", "") + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") if err != nil { t.Fatalf("checkpoint: %v", err) } @@ -221,7 +221,7 @@ func TestRejectedCheckpointOpsLeakNoRecLock(t *testing.T) { // A wrong-tenant delete of a real checkpoint must also leak nothing. src := mustClaim(t, m, testKey) - ckpt, err := m.Checkpoint(t.Context(), src.ID, src.Token, "", "acme") + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "acme") if err != nil { t.Fatalf("checkpoint: %v", err) } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 1366dd7..766888a 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -90,35 +90,25 @@ func (m *Manager) ClaimProvision(ctx context.Context, key types.PoolKey, ttl tim return out, err } -// Release destroys a claimed sandbox after validating its per-sandbox token. -func (m *Manager) Release(ctx context.Context, id, token string) error { - m.mu.Lock() - sb, ok := m.authed(id, token) +// Release destroys a claimed sandbox after authorizing cred. +func (m *Manager) Release(ctx context.Context, id string, cred Cred) error { + sb, ok := m.resolve(id, cred) if !ok { - m.mu.Unlock() return ErrUnknownSandbox } - return m.releaseLocked(ctx, id, sb) + return m.releaseResolved(ctx, id, sb) } -// ReleaseOperator destroys a claimed sandbox by id without a per-sandbox token. -// It is the operator (root) path: the server authorizes it by the node's root -// api_token before calling, so no token check happens here — the claim is looked -// up by id directly. A tenant token never reaches this method (see server.go). -func (m *Manager) ReleaseOperator(ctx context.Context, id string) error { +// releaseResolved drops the resolved claim (id, sb) and tears down its VM. +// resolve authorizes and unlocks before this runs, so it re-validates under +// m.mu that sb is still the live claim — a second release racing in on the +// same id must not double-tear-down. +func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sandbox) error { m.mu.Lock() - sb, ok := m.claimed[id] - if !ok { + if m.claimed[id] != sb { m.mu.Unlock() return ErrUnknownSandbox } - return m.releaseLocked(ctx, id, sb) -} - -// releaseLocked drops the already-resolved claim (id, sb) and tears down its VM. -// The caller holds m.mu and has resolved sb; releaseLocked owns the unlock. It -// is the shared body of Release (per-sandbox token) and ReleaseOperator (root). -func (m *Manager) releaseLocked(ctx context.Context, id string, sb *types.Sandbox) error { delete(m.claimed, id) m.tenantDelta(sb.Tenant, -1) snap, ck, vmName := sb.HibernateSnap, sb.ArchiveCk, sb.VMName @@ -131,7 +121,7 @@ func (m *Manager) releaseLocked(ctx context.Context, id string, sb *types.Sandbo rb := m.store.snapshot(m.claimed) m.mu.Unlock() m.recommit(ctx, rb) - log.WithFunc("pool.releaseLocked").Errorf(ctx, saveErr, "persist release of %s", id) + log.WithFunc("pool.releaseResolved").Errorf(ctx, saveErr, "persist release of %s", id) return fmt.Errorf("release %s: %w", id, saveErr) } // Cleanup must survive the caller hanging up; the claim is already dropped. diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index 2455ce2..ec80d64 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -439,13 +439,13 @@ func TestEgressLaneCannotForkOrCheckpoint(t *testing.T) { m.mu.Lock() m.claimed[sb.ID] = sb m.mu.Unlock() - if _, err := m.Fork(t.Context(), sb.ID, "tok", 1, time.Minute); !errors.Is(err, ErrNoEgressFork) { + if _, err := m.Fork(t.Context(), sb.ID, Cred{Token: "tok"}, 1, time.Minute); !errors.Is(err, ErrNoEgressFork) { t.Errorf("Fork on egress lane: got %v, want ErrNoEgressFork", err) } - if _, err := m.Checkpoint(t.Context(), sb.ID, "tok", "", ""); !errors.Is(err, ErrNoEgressFork) { + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: "tok"}, "", ""); !errors.Is(err, ErrNoEgressFork) { t.Errorf("Checkpoint on egress lane: got %v, want ErrNoEgressFork", err) } - if _, err := m.Promote(t.Context(), sb.ID, "tok", "tpl", ""); !errors.Is(err, ErrNoEgressFork) { + if _, err := m.Promote(t.Context(), sb.ID, Cred{Token: "tok"}, "tpl", ""); !errors.Is(err, ErrNoEgressFork) { t.Errorf("Promote on egress lane: got %v, want ErrNoEgressFork", err) } } diff --git a/sandboxd/pool/fork.go b/sandboxd/pool/fork.go index d7c210c..8326270 100644 --- a/sandboxd/pool/fork.go +++ b/sandboxd/pool/fork.go @@ -17,25 +17,11 @@ import ( // child a distinct machine identity. Children inherit the parent's tenant // and count against its quota. All-or-nothing: any child failing destroys // the ones already built, so an error means no child survived. -func (m *Manager) Fork(ctx context.Context, id, token string, count int, ttl time.Duration) ([]*types.Sandbox, error) { - sb, ok := m.claim(id, token) +func (m *Manager) Fork(ctx context.Context, id string, cred Cred, count int, ttl time.Duration) ([]*types.Sandbox, error) { + sb, ok := m.resolve(id, cred) if !ok { return nil, ErrUnknownSandbox } - return m.forkResolved(ctx, sb, count, ttl) -} - -// ForkOperator forks a sandbox by id without a per-sandbox token. -func (m *Manager) ForkOperator(ctx context.Context, id string, count int, ttl time.Duration) ([]*types.Sandbox, error) { - sb, ok := m.byID(id) - if !ok { - return nil, ErrUnknownSandbox - } - return m.forkResolved(ctx, sb, count, ttl) -} - -// forkResolved is Fork's body once the source claim is resolved. -func (m *Manager) forkResolved(ctx context.Context, sb *types.Sandbox, count int, ttl time.Duration) ([]*types.Sandbox, error) { if count < 1 || count > m.maxFork { return nil, fmt.Errorf("%w: %d not in 1..%d", ErrBadCount, count, m.maxFork) } diff --git a/sandboxd/pool/fork_test.go b/sandboxd/pool/fork_test.go index fb7ac16..552f779 100644 --- a/sandboxd/pool/fork_test.go +++ b/sandboxd/pool/fork_test.go @@ -21,7 +21,7 @@ func TestForkRacingHibernateUsesPrivateSource(t *testing.T) { eng.hibernateStall = make(chan struct{}) var wg sync.WaitGroup wg.Go(func() { - if err := m.Hibernate(t.Context(), parent.ID, parent.Token); err != nil { + if err := m.Hibernate(t.Context(), parent.ID, Cred{Token: parent.Token}); err != nil { t.Errorf("Hibernate: %v", err) } }) @@ -32,7 +32,7 @@ func TestForkRacingHibernateUsesPrivateSource(t *testing.T) { }) var children []string wg.Go(func() { - sbs, err := m.Fork(t.Context(), parent.ID, parent.Token, 2, time.Hour) + sbs, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 2, time.Hour) if err != nil { t.Errorf("Fork: %v", err) return @@ -63,7 +63,7 @@ func TestForkFromRunning(t *testing.T) { m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - children, err := m.Fork(t.Context(), parent.ID, parent.Token, 3, time.Hour) + children, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 3, time.Hour) if err != nil { t.Fatalf("Fork: %v", err) } @@ -109,11 +109,11 @@ func TestForkFromHibernatedUsesWakeImage(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), parent.ID, parent.Token); err != nil { + if err := m.Hibernate(t.Context(), parent.ID, Cred{Token: parent.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } - children, err := m.Fork(t.Context(), parent.ID, parent.Token, 2, 0) + children, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 2, 0) if err != nil { t.Fatalf("Fork: %v", err) } @@ -141,7 +141,7 @@ func TestForkCountValidation(t *testing.T) { m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) for _, count := range []int{0, -1, m.maxFork + 1} { - if _, err := m.Fork(t.Context(), parent.ID, parent.Token, count, 0); !errors.Is(err, ErrBadCount) { + if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, count, 0); !errors.Is(err, ErrBadCount) { t.Errorf("count %d: err %v, want ErrBadCount", count, err) } } @@ -154,7 +154,7 @@ func TestForkUnknownSandbox(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - if _, err := m.Fork(t.Context(), parent.ID, "wrong-token", 1, 0); !errors.Is(err, ErrUnknownSandbox) { + if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: "wrong-token"}, 1, 0); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("err %v, want ErrUnknownSandbox", err) } } @@ -168,7 +168,7 @@ func TestForkAllOrNothing(t *testing.T) { } eng.cloneFailNth = 2 // fail one of the three fork clones - if _, err := m.Fork(t.Context(), parent.ID, parent.Token, 3, 0); err == nil { + if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 3, 0); err == nil { t.Fatal("Fork succeeded despite a failed clone") } // Every successfully built child is destroyed; only the parent's claim @@ -191,7 +191,7 @@ func TestReconcileSweepsOrphanSnapshots(t *testing.T) { dataDir := t.TempDir() m := newTestManagerAt(t, eng, dataDir) parent := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), parent.ID, parent.Token); err != nil { + if err := m.Hibernate(t.Context(), parent.ID, Cred{Token: parent.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } referenced := eng.hibernates[0] @@ -235,7 +235,7 @@ func TestForkChildrenInheritTenantAndQuota(t *testing.T) { t.Fatalf("claim: %v", err) } - children, err := m.Fork(t.Context(), parent.ID, parent.Token, 2, 0) + children, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 2, 0) if err != nil { t.Fatalf("Fork: %v", err) } @@ -245,7 +245,7 @@ func TestForkChildrenInheritTenantAndQuota(t *testing.T) { } } // Parent plus two children fill acme's cap of 3; one more child is over. - if _, err := m.Fork(t.Context(), parent.ID, parent.Token, 1, 0); !errors.Is(err, ErrQuota) { + if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 1, 0); !errors.Is(err, ErrQuota) { t.Errorf("fork past the tenant cap: %v, want ErrQuota", err) } } diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index f97b466..a9228e1 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -17,8 +17,8 @@ import ( // memory; the next agent access wakes it. Idempotent on an already-hibernated // sandbox. When to hibernate is the caller's policy — the node only provides // the transition. -func (m *Manager) Hibernate(ctx context.Context, id, token string) error { - sb, ok := m.claim(id, token) +func (m *Manager) Hibernate(ctx context.Context, id string, cred Cred) error { + sb, ok := m.resolve(id, cred) if !ok { return ErrUnknownSandbox } diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index 6a03cc2..63b2b13 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -24,10 +24,10 @@ func TestHibernatePersistFailureSurfacesAndConverges(t *testing.T) { broken := filepath.Join(t.TempDir(), "gone", "claims.json") m.store.path = broken - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err == nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success despite a persist failure") } - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err == nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate retry reported success while the journal is unwritable") } if len(eng.hibernates) != 0 { @@ -36,7 +36,7 @@ func TestHibernatePersistFailureSurfacesAndConverges(t *testing.T) { if err := os.MkdirAll(filepath.Dir(broken), 0o750); err != nil { t.Fatalf("heal store dir: %v", err) } - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate retry after heal: %v", err) } got, err := (&claimStore{path: broken}).load() @@ -53,7 +53,7 @@ func TestWakePersistFailureSurfaces(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } snap := eng.hibernates[0] @@ -100,7 +100,7 @@ func TestHibernateAmbiguousErrorTreatedAsDone(t *testing.T) { eng.hibernateErr = errors.New("cli timeout") eng.hibernateErrCompletes = true // the snapshot landed before the timeout - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v, want nil (the snapshot landed despite the error)", err) } if _, g := m.Info(); g.Hibernated != 1 { @@ -121,7 +121,7 @@ func TestHibernateVerifiedFailureClearsIntent(t *testing.T) { sb := mustClaim(t, m, testKey) eng.hibernateErr = errors.New("cli failed") // completes=false: no snapshot - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err == nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success despite a verified failure") } if _, g := m.Info(); g.Hibernated != 0 { @@ -132,7 +132,7 @@ func TestHibernateVerifiedFailureClearsIntent(t *testing.T) { t.Fatalf("journal %+v, want running (no snaps)", got[sb.ID]) } eng.hibernateErr = nil - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate retry: %v", err) } } @@ -148,7 +148,7 @@ func TestHibernateUnverifiableErrorKeepsIntent(t *testing.T) { eng.hibernateErr = errors.New("cli timeout") eng.snapListErr = errors.New("cocoon unreachable") - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err == nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success on an unconfirmed transition") } if _, g := m.Info(); g.Hibernated != 0 { @@ -168,7 +168,7 @@ func TestWakeRestoreErrorDestroysAndRetries(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } snap := eng.hibernates[0] @@ -205,7 +205,7 @@ func TestRetryResolvesDanglingIntent(t *testing.T) { eng.hibernateErr = errors.New("cli timeout") eng.hibernateErrCompletes = true eng.snapListErr = errors.New("cocoon unreachable") - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err == nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success on an unconfirmed transition") } realSnap := eng.hibernates[0] @@ -216,7 +216,7 @@ func TestRetryResolvesDanglingIntent(t *testing.T) { // cocoon recovers; the retry must adopt the real snapshot, not re-hibernate. eng.hibernateErr = nil eng.snapListErr = nil - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate retry: %v", err) } if len(eng.hibernates) != 1 { @@ -247,7 +247,7 @@ func TestWakeResolvesDanglingIntent(t *testing.T) { eng.hibernateErr = errors.New("cli timeout") eng.hibernateErrCompletes = true eng.snapListErr = errors.New("cocoon unreachable") - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err == nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success on an unconfirmed transition") } @@ -280,7 +280,7 @@ func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) { eng.hibernateErr = errors.New("cli timeout") eng.hibernateErrCompletes = true eng.snapListErr = errors.New("cocoon unreachable") - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err == nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success on an unconfirmed transition") } realSnap := eng.hibernates[0] @@ -295,7 +295,7 @@ func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) { wakeErr <- err }() waitFor(t, func() bool { return eng.snapListCalls() > callsBefore }) - if err := m.Release(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("release: %v", err) } close(eng.snapListStall) @@ -316,14 +316,14 @@ func TestResolveAdoptBillsWhenPersistFails(t *testing.T) { eng.hibernateErr = errors.New("cli timeout") eng.hibernateErrCompletes = true eng.snapListErr = errors.New("cocoon unreachable") - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err == nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success on an unconfirmed transition") } // The list recovers, but the claims journal is now unwritable. eng.snapListErr = nil broken := filepath.Join(t.TempDir(), "gone", "claims.json") m.store.path = broken - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err == nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success despite a persist failure") } if n := m.Counters().Hibernates; n != 1 { @@ -346,7 +346,7 @@ func TestWakeProbeFailureDestroysAndRetries(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } snap := eng.hibernates[0] @@ -377,7 +377,7 @@ func TestFlushClaimsClosesShutdownWindow(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } broken := filepath.Join(t.TempDir(), "gone", "claims.json") @@ -403,10 +403,10 @@ func TestHibernateWakeCycle(t *testing.T) { m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("repeat Hibernate: %v", err) } if len(eng.hibernates) != 1 { @@ -453,7 +453,7 @@ func TestWakeDropsSnapshotOffReturnPath(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } eng.snapRemoveStall = make(chan struct{}) @@ -476,7 +476,7 @@ func TestWakeFailureKeepsHibernated(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } @@ -496,11 +496,11 @@ func TestReleaseHibernatedDropsSnapshot(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } - if err := m.Release(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Release: %v", err) } if !slices.Contains(eng.removes, sb.VMName) { @@ -518,7 +518,7 @@ func TestReleaseMidHibernateDropsOrphanSnapshot(t *testing.T) { sb := mustClaim(t, m, testKey) hibErr := make(chan error, 1) - go func() { hibErr <- m.Hibernate(t.Context(), sb.ID, sb.Token) }() + go func() { hibErr <- m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}) }() waitFor(t, func() bool { eng.mu.Lock() defer eng.mu.Unlock() @@ -527,7 +527,7 @@ func TestReleaseMidHibernateDropsOrphanSnapshot(t *testing.T) { // Release lands while the engine transition is in flight: the claim is // gone before Hibernate can commit, so its snapshot must not leak. - if err := m.Release(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Release: %v", err) } close(eng.hibernateStall) diff --git a/sandboxd/pool/intercept_test.go b/sandboxd/pool/intercept_test.go index ecb646b..f9899ac 100644 --- a/sandboxd/pool/intercept_test.go +++ b/sandboxd/pool/intercept_test.go @@ -124,7 +124,7 @@ func TestInterceptPoolAllowsPromote(t *testing.T) { m.mu.Unlock() // The cluster root is shared, so an interception sandbox's disk carries no // node-private material: promote/checkpoint are unrestricted. - if _, err := m.Promote(t.Context(), sb.ID, "tok", "tpl:x", ""); err != nil { + if _, err := m.Promote(t.Context(), sb.ID, Cred{Token: "tok"}, "tpl:x", ""); err != nil { t.Errorf("Promote of an interception-pool sandbox: %v, want success", err) } } diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index f65bb1f..55bf0d3 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -6,6 +6,14 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +// Cred is a caller's resolved authority over one sandbox: the per-sandbox +// token, or Operator for the node's root api_token (verified by the server +// before the call). +type Cred struct { + Token string + Operator bool +} + // Sandbox reports one live claim's summary, the single-sandbox read the // whole-node listing otherwise forces a caller to scan for. func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { @@ -16,40 +24,32 @@ func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { return summarize(sb), true } -// HibernateOperator hibernates a sandbox by id without a per-sandbox token. -func (m *Manager) HibernateOperator(ctx context.Context, id string) error { - sb, ok := m.byID(id) - if !ok { - return ErrUnknownSandbox - } - sb.Transition.Lock() - defer sb.Transition.Unlock() - return m.hibernateLocked(ctx, sb) -} - // Wake restores a hibernated sandbox and leaves it running, so a control plane // can resume one it is not about to talk to — waking is otherwise only a side // effect of opening an agent connection. Idempotent on a running sandbox. -func (m *Manager) Wake(ctx context.Context, id, token string) error { - sb, ok := m.claim(id, token) +func (m *Manager) Wake(ctx context.Context, id string, cred Cred) error { + sb, ok := m.resolve(id, cred) if !ok { return ErrUnknownSandbox } return m.wake(ctx, sb) } -// WakeOperator wakes a sandbox by id without a per-sandbox token. -func (m *Manager) WakeOperator(ctx context.Context, id string) error { - sb, ok := m.byID(id) - if !ok { - return ErrUnknownSandbox +// resolve authorizes id under cred: Operator resolves by id alone (the +// server has already verified the root api_token), otherwise the token must +// match the claim — an unclaimed slot must never match an empty token. +func (m *Manager) resolve(id string, cred Cred) (*types.Sandbox, bool) { + if cred.Operator { + return m.byID(id) } - return m.wake(ctx, sb) + if cred.Token == "" { + return nil, false + } + return m.claim(id, cred.Token) } -// byID resolves a live claim by id alone, with no ownership proof. The server -// authorizes the operator paths by root api_token before the call, so a tenant -// token never reaches one. +// byID resolves a live claim by id alone, with no ownership proof. Callers +// must have authorized the Operator credential themselves before reaching it. func (m *Manager) byID(id string) (*types.Sandbox, bool) { m.mu.Lock() defer m.mu.Unlock() diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 0309191..9e0d553 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -169,13 +169,13 @@ func TestReleaseValidatesToken(t *testing.T) { m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Release(t.Context(), sb.ID, "wrong"); !errors.Is(err, ErrUnknownSandbox) { + if err := m.Release(t.Context(), sb.ID, Cred{Token: "wrong"}); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("got %v, want ErrUnknownSandbox", err) } - if err := m.Release(t.Context(), "sb_nope", sb.Token); !errors.Is(err, ErrUnknownSandbox) { + if err := m.Release(t.Context(), "sb_nope", Cred{Token: sb.Token}); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("got %v, want ErrUnknownSandbox", err) } - if err := m.Release(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Release: %v", err) } if !slices.Contains(eng.removes, sb.VMName) { @@ -186,18 +186,48 @@ func TestReleaseValidatesToken(t *testing.T) { } } -func TestReleaseOperator(t *testing.T) { +// TestReleaseAfterResolveTearsDownOnce: resolve authorizes off the manager +// mutex, so two callers can both hold the same resolved claim before either +// drops it — the state a live double-release reaches. Exactly one may tear the +// VM down; the loser must read as unknown rather than remove it twice. +func TestReleaseAfterResolveTearsDownOnce(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) + cred := Cred{Token: sb.Token} - // Unknown id is 404-equivalent, exactly like Release. - if err := m.ReleaseOperator(t.Context(), "sb_nope"); !errors.Is(err, ErrUnknownSandbox) { + first, ok := m.resolve(sb.ID, cred) + if !ok { + t.Fatal("resolve: claim not found") + } + second, ok := m.resolve(sb.ID, cred) + if !ok { + t.Fatal("resolve: claim not found on the second caller") + } + + if err := m.releaseResolved(t.Context(), sb.ID, first); err != nil { + t.Fatalf("first release: %v", err) + } + if err := m.releaseResolved(t.Context(), sb.ID, second); !errors.Is(err, ErrUnknownSandbox) { + t.Errorf("second release: %v, want ErrUnknownSandbox", err) + } + if got := slices.Compact(slices.Clone(eng.removes)); len(got) != 1 || got[0] != sb.VMName { + t.Errorf("removes=%v, want %s torn down exactly once", eng.removes, sb.VMName) + } +} + +func TestReleaseByOperatorCred(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + sb := mustClaim(t, m, testKey) + + // Unknown id is 404-equivalent, exactly like a token release. + if err := m.Release(t.Context(), "sb_nope", Cred{Operator: true}); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("got %v, want ErrUnknownSandbox", err) } // The operator releases by id alone — no per-sandbox token needed. - if err := m.ReleaseOperator(t.Context(), sb.ID); err != nil { - t.Fatalf("ReleaseOperator: %v", err) + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatalf("Release operator: %v", err) } if !slices.Contains(eng.removes, sb.VMName) { t.Errorf("removes=%v, want %s", eng.removes, sb.VMName) @@ -207,6 +237,20 @@ func TestReleaseOperator(t *testing.T) { } } +// TestResolveRejectsEmptyToken documents resolve's explicit guard: an +// unclaimed slot must never match an empty token. +func TestResolveRejectsEmptyToken(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + sb := mustClaim(t, m, testKey) + + if _, ok := m.resolve(sb.ID, Cred{Token: ""}); ok { + t.Error("empty token resolved a claimed sandbox") + } + if _, ok := m.resolve(sb.ID, Cred{Operator: true}); !ok { + t.Error("operator credential did not resolve the claim") + } +} + func TestAgentSocketValidatesToken(t *testing.T) { m := newTestManager(t, newFakeEngine()) sb := mustClaim(t, m, testKey) diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index 67c688e..8f581b4 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -18,7 +18,7 @@ func TestPromoteThenClaimClonesFromTemplate(t *testing.T) { m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - gotKey, err := m.Promote(t.Context(), parent.ID, parent.Token, "tpl:x", "") + gotKey, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") if err != nil { t.Fatalf("Promote: %v", err) } @@ -50,11 +50,11 @@ func TestPromoteHibernatedUsesWakeImage(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), parent.ID, parent.Token); err != nil { + if err := m.Hibernate(t.Context(), parent.ID, Cred{Token: parent.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } - if _, err := m.Promote(t.Context(), parent.ID, parent.Token, "tpl:hib", ""); err != nil { + if _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:hib", ""); err != nil { t.Fatalf("Promote: %v", err) } if len(eng.snapSaves) != 0 { @@ -73,15 +73,15 @@ func TestPromoteValidations(t *testing.T) { m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 0}) parent := mustClaim(t, m, testKey) - if _, err := m.Promote(t.Context(), parent.ID, parent.Token, "_bad", ""); !errors.Is(err, ErrBadKey) { + if _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "_bad", ""); !errors.Is(err, ErrBadKey) { t.Errorf("bad name: %v, want ErrBadKey", err) } - if _, err := m.Promote(t.Context(), parent.ID, "wrong", "tpl:x", ""); !errors.Is(err, ErrUnknownSandbox) { + if _, err := m.Promote(t.Context(), parent.ID, Cred{Token: "wrong"}, "tpl:x", ""); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("bad token: %v, want ErrUnknownSandbox", err) } // Same template/net/size as the configured pool: the golden path would // collide with the pool's own. - if _, err := m.Promote(t.Context(), parent.ID, parent.Token, testKey.Template, ""); !errors.Is(err, ErrPooledTemplate) { + if _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, testKey.Template, ""); !errors.Is(err, ErrPooledTemplate) { t.Errorf("pooled key: %v, want ErrPooledTemplate", err) } if len(eng.snapSaves) != 0 { @@ -93,7 +93,7 @@ func TestDeleteTemplate(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 0}) parent := mustClaim(t, m, testKey) - if _, err := m.Promote(t.Context(), parent.ID, parent.Token, "tpl:del", ""); err != nil { + if _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:del", ""); err != nil { t.Fatalf("Promote: %v", err) } key := types.PoolKey{Template: "tpl:del", Net: testKey.Net, Size: testKey.Size, Engine: testKey.Engine} @@ -167,7 +167,7 @@ func TestTemplateTenantScopedDelete(t *testing.T) { if err != nil { t.Fatalf("claim: %v", err) } - key, err := m.Promote(t.Context(), parent.ID, parent.Token, "tpl:tenant", "acme") + key, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:tenant", "acme") if err != nil { t.Fatalf("Promote: %v", err) } @@ -178,7 +178,7 @@ func TestTemplateTenantScopedDelete(t *testing.T) { if err := m.DeleteTemplate(t.Context(), key, "acme"); err != nil { t.Errorf("own delete: %v", err) } - if _, err := m.Promote(t.Context(), parent.ID, parent.Token, "tpl:tenant", "acme"); err != nil { + if _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:tenant", "acme"); err != nil { t.Fatalf("re-promote: %v", err) } if err := m.DeleteTemplate(t.Context(), key, ""); err != nil { @@ -223,7 +223,7 @@ func TestPromoteFailsClosedOnMetaError(t *testing.T) { if err != nil { t.Fatalf("claim: %v", err) } - if _, err := m.Promote(t.Context(), a.ID, a.Token, "shared:v1", "acme"); err != nil { + if _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "shared:v1", "acme"); err != nil { t.Fatalf("promote: %v", err) } key := types.PoolKey{Template: "shared:v1", Net: testKey.Net, Size: testKey.Size, Engine: testKey.Engine} @@ -232,7 +232,7 @@ func TestPromoteFailsClosedOnMetaError(t *testing.T) { t.Fatalf("chmod: %v", err) } t.Cleanup(func() { _ = os.Chmod(meta, 0o600) }) - if _, err := m.Promote(t.Context(), a.ID, a.Token, "shared:v1", "beta"); err == nil { + if _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "shared:v1", "beta"); err == nil { t.Fatal("promote succeeded despite an unreadable owner record") } } @@ -249,15 +249,15 @@ func TestPromoteRefusesCrossTenantOverwrite(t *testing.T) { return sb } a := claim("acme") - if _, err := m.Promote(t.Context(), a.ID, a.Token, "shared:v1", "acme"); err != nil { + if _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "shared:v1", "acme"); err != nil { t.Fatalf("acme promote: %v", err) } b := claim("beta") - if _, err := m.Promote(t.Context(), b.ID, b.Token, "shared:v1", "beta"); !errors.Is(err, ErrTemplateOwned) { + if _, err := m.Promote(t.Context(), b.ID, Cred{Token: b.Token}, "shared:v1", "beta"); !errors.Is(err, ErrTemplateOwned) { t.Errorf("beta overwrite: %v, want ErrTemplateOwned", err) } r := claim("") // root may replace anything - if _, err := m.Promote(t.Context(), r.ID, r.Token, "shared:v1", ""); err != nil { + if _, err := m.Promote(t.Context(), r.ID, Cred{Token: r.Token}, "shared:v1", ""); err != nil { t.Errorf("root replace: %v, want ok", err) } } diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index f43a80e..b328711 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -17,13 +17,13 @@ func TestUsageJournalRecordsLifecycle(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) - if err := m.Hibernate(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } if _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("Wake: %v", err) } - if err := m.Release(t.Context(), sb.ID, sb.Token); err != nil { + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Release: %v", err) } @@ -86,7 +86,7 @@ func TestQuotaRefusesClaimsPastCap(t *testing.T) { t.Fatalf("claim past cap: %v, want ErrQuota", err) } // The refused VM must not leak, and the pool state stays sane. - if err := m.Release(t.Context(), first.ID, first.Token); err != nil { + if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { t.Fatalf("Release: %v", err) } if _, err := claimAny(t.Context(), m, testKey, time.Hour); err != nil { @@ -121,7 +121,7 @@ func TestTenantQuotaBindsPerTenant(t *testing.T) { if counts["acme"] != 1 || counts["beta"] != 1 { t.Errorf("TenantClaims %v, want acme=1 beta=1", counts) } - if err := m.Release(t.Context(), first.ID, first.Token); err != nil { + if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { t.Fatalf("Release: %v", err) } if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", ""); err != nil { diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index c2d15e0..3ff92d9 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -28,25 +28,11 @@ type templateRecord struct { // net, parent size); later claims for that key clone from it. Re-promoting the // same name replaces it, and the caller owns its lifecycle (DeleteTemplate). // tenant attributes the record; empty means the operator (root). -func (m *Manager) Promote(ctx context.Context, id, token, template, tenant string) (types.PoolKey, error) { - sb, ok := m.claim(id, token) +func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, tenant string) (types.PoolKey, error) { + sb, ok := m.resolve(id, cred) if !ok { return types.PoolKey{}, ErrUnknownSandbox } - return m.promoteResolved(ctx, sb, template, tenant) -} - -// PromoteOperator promotes a sandbox by id without a per-sandbox token. -func (m *Manager) PromoteOperator(ctx context.Context, id, template, tenant string) (types.PoolKey, error) { - sb, ok := m.byID(id) - if !ok { - return types.PoolKey{}, ErrUnknownSandbox - } - return m.promoteResolved(ctx, sb, template, tenant) -} - -// promoteResolved is Promote's body once the source claim is resolved. -func (m *Manager) promoteResolved(ctx context.Context, sb *types.Sandbox, template, tenant string) (types.PoolKey, error) { if !types.NameRe.MatchString(template) { return types.PoolKey{}, fmt.Errorf("%w: template %q must match %s", ErrBadKey, template, types.NameRe) } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 841ba9f..1eb2764 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -59,19 +59,13 @@ var poolErrHTTP = []struct { type Manager interface { ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string) (*types.Sandbox, error) ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string) (*types.Sandbox, error) - Release(ctx context.Context, id, token string) error - ReleaseOperator(ctx context.Context, id string) error - Hibernate(ctx context.Context, id, token string) error - HibernateOperator(ctx context.Context, id string) error - Wake(ctx context.Context, id, token string) error - WakeOperator(ctx context.Context, id string) error - Fork(ctx context.Context, id, token string, count int, ttl time.Duration) ([]*types.Sandbox, error) - ForkOperator(ctx context.Context, id string, count int, ttl time.Duration) ([]*types.Sandbox, error) - Promote(ctx context.Context, id, token, template, tenant string) (types.PoolKey, error) - PromoteOperator(ctx context.Context, id, template, tenant string) (types.PoolKey, error) + Release(ctx context.Context, id string, cred pool.Cred) error + Hibernate(ctx context.Context, id string, cred pool.Cred) error + Wake(ctx context.Context, id string, cred pool.Cred) error + Fork(ctx context.Context, id string, cred pool.Cred, count int, ttl time.Duration) ([]*types.Sandbox, error) + Promote(ctx context.Context, id string, cred pool.Cred, template, tenant string) (types.PoolKey, error) DeleteTemplate(ctx context.Context, key types.PoolKey, tenant string) error - Checkpoint(ctx context.Context, id, token, name, tenant string) (types.Checkpoint, error) - CheckpointOperator(ctx context.Context, id, name, tenant string) (types.Checkpoint, error) + Checkpoint(ctx context.Context, id string, cred pool.Cred, name, tenant string) (types.Checkpoint, error) Counters() pool.Counters TenantClaims() map[string]int Sandboxes() []pool.SandboxSummary @@ -99,13 +93,6 @@ type Dialer interface { DialSilkd(ctx context.Context, vsockSocket string) (net.Conn, error) } -// The two shapes a sandbox-scoped verb takes: proof of ownership by the -// per-sandbox token, or by id alone once the root api_token authorized it. -type ( - tokenVerb func(ctx context.Context, id, token string) error - operatorVerb func(ctx context.Context, id string) error -) - // Placer names peers for redirect placement and lists the mesh for Lookup; // nil on a single-node deployment (no mesh). type Placer interface { @@ -176,8 +163,8 @@ func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/claim", s.requireToken(s.handleClaim)) mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleRelease) - mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate, s.mgr.HibernateOperator)) - mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake, s.mgr.WakeOperator)) + mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) + mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) mux.HandleFunc("GET /v1/sandboxes/{id}", s.requireRoot(s.handleSandbox)) mux.HandleFunc("GET /v1/sandboxes/{id}/stats", s.requireRoot(s.handleSandboxStats)) // Fork and promote create node resources, so they take the same token @@ -258,20 +245,15 @@ func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req t // node's root api_token (the operator) may release any sandbox by id, so // aggregated/control-plane teardown works without holding the per-sandbox token; // a per-sandbox token releases only its own claim, unchanged. A tenant token is -// neither — it is not the root api_token, so it takes the per-sandbox path and -// 404s (it matches no sandbox token). Tenants never get operator release. +// neither — it is not the root api_token, so it resolves as a (non-matching) +// sandbox token and 404s. Tenants never get operator release. func (s *Server) handleRelease(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) if !ok { return } id := r.PathValue("id") - var err error - if s.isRootToken(token) { - err = s.mgr.ReleaseOperator(r.Context(), id) - } else { - err = s.mgr.Release(r.Context(), id, token) - } + err := s.mgr.Release(r.Context(), id, s.sandboxCred(token)) switch { case writePoolErr(w, err): case err != nil: @@ -308,21 +290,16 @@ func (s *Server) handleSandboxStats(w http.ResponseWriter, r *http.Request) { // handleSandboxVerb adapts a sandbox-scoped manager call (hibernate, wake) to // HTTP: per-sandbox bearer auth, 404 on unknown, 204 on success. The node's -// root api_token takes the operator path instead, exactly as release does, so -// a control plane holding only the fleet token can drive the lifecycle. -func (s *Server) handleSandboxVerb(verb string, do tokenVerb, operator operatorVerb) http.HandlerFunc { +// root api_token resolves as an Operator credential instead, so a control +// plane holding only the fleet token can drive the lifecycle. +func (s *Server) handleSandboxVerb(verb string, do func(ctx context.Context, id string, cred pool.Cred) error) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) if !ok { return } id := r.PathValue("id") - var err error - if s.isRootToken(token) { - err = operator(r.Context(), id) - } else { - err = do(r.Context(), id, token) - } + err := do(r.Context(), id, s.sandboxCred(token)) writeResult(w, r, verb, id, verb+" failed", err, func() { w.WriteHeader(http.StatusNoContent) }) @@ -337,16 +314,7 @@ func (s *Server) handleFork(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - // An operator caller proves authority with the node api_token in the header - // and carries no per-sandbox token; a tenant must still present the - // sandbox's own token as ownership proof. - var children []*types.Sandbox - var err error - if req.Token == "" && s.rootRequest(r) { - children, err = s.mgr.ForkOperator(r.Context(), id, req.Count, req.TTL()) - } else { - children, err = s.mgr.Fork(r.Context(), id, req.Token, req.Count, req.TTL()) - } + children, err := s.mgr.Fork(r.Context(), id, s.bodyCred(r, req.Token), req.Count, req.TTL()) writeResult(w, r, "fork", id, "fork failed", err, func() { resp := types.ForkResponse{Children: make([]types.ClaimResponse, len(children))} for i, c := range children { @@ -363,13 +331,7 @@ func (s *Server) handlePromote(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - var key types.PoolKey - var err error - if req.Token == "" && s.rootRequest(r) { - key, err = s.mgr.PromoteOperator(r.Context(), id, req.Template, tenantFrom(r.Context())) - } else { - key, err = s.mgr.Promote(r.Context(), id, req.Token, req.Template, tenantFrom(r.Context())) - } + key, err := s.mgr.Promote(r.Context(), id, s.bodyCred(r, req.Token), req.Template, tenantFrom(r.Context())) writeResult(w, r, "promote", id, "promote failed", err, func() { writeJSON(w, http.StatusOK, types.PromoteResponse{Key: key}) }) @@ -382,13 +344,7 @@ func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - var ckpt types.Checkpoint - var err error - if req.Token == "" && s.rootRequest(r) { - ckpt, err = s.mgr.CheckpointOperator(r.Context(), id, req.Name, tenantFrom(r.Context())) - } else { - ckpt, err = s.mgr.Checkpoint(r.Context(), id, req.Token, req.Name, tenantFrom(r.Context())) - } + ckpt, err := s.mgr.Checkpoint(r.Context(), id, s.bodyCred(r, req.Token), req.Name, tenantFrom(r.Context())) writeResult(w, r, "checkpoint", id, "checkpoint failed", err, func() { writeJSON(w, http.StatusOK, types.CheckpointResponse{Checkpoint: ckpt}) }) @@ -591,6 +547,23 @@ func (s *Server) isRootToken(token string) bool { return s.apiToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.apiToken)) == 1 } +// sandboxCred resolves a header-borne bearer token to a Cred: the root +// api_token elevates to Operator (the manager must never see it as a +// sandbox token), anything else is carried as the per-sandbox token as-is. +func (s *Server) sandboxCred(token string) pool.Cred { + if s.isRootToken(token) { + return pool.Cred{Operator: true} + } + return pool.Cred{Token: token} +} + +// bodyCred resolves a body-carried sandbox token to a Cred: an empty token +// from a root-authenticated request (header api_token) is Operator; a tenant +// must still present the sandbox's own token as ownership proof. +func (s *Server) bodyCred(r *http.Request, bodyToken string) pool.Cred { + return pool.Cred{Token: bodyToken, Operator: bodyToken == "" && s.rootRequest(r)} +} + // resolveScope matches the bearer token to root ("") or a tenant name. With // no api token and no tenants the node-level endpoints stay open. func (s *Server) resolveScope(r *http.Request) (string, bool) { diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index e99a84f..265e29c 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -373,20 +373,20 @@ func TestSandboxVerbFlows(t *testing.T) { } // TestReleaseOperatorToken proves the release route's authorization split: the -// node root api_token releases any sandbox by id via ReleaseOperator (no +// node root api_token releases any sandbox by id via an Operator credential (no // per-sandbox token), while a per-sandbox token still takes the token-checked -// Release path and a tenant/wrong token gets no operator elevation (it falls to -// Release and 404s). Exactly one manager method runs per request. +// path and a tenant/wrong token gets no operator elevation (it resolves as a +// non-matching sandbox token and 404s). Exactly one hook runs per request. func TestReleaseOperatorToken(t *testing.T) { const rootTok, sbTok = "sekret", "sb-secret" tenants := []config.TenantSpec{{Name: "acme", Token: "acme-tok"}} tests := []struct { name string auth string - wantOp bool // ReleaseOperator expected - wantRelease bool // per-sandbox Release expected - wantToken string // token Release should receive (per-sandbox path) - releaseErr error // error the per-sandbox Release returns + wantOp bool // operator (Cred.Operator) release expected + wantRelease bool // per-sandbox release expected + wantToken string // token the per-sandbox release should receive + releaseErr error // error the per-sandbox release returns want int }{ {"root token releases by id", "Bearer " + rootTok, true, false, "", nil, http.StatusNoContent}, @@ -426,13 +426,13 @@ func TestReleaseOperatorToken(t *testing.T) { t.Errorf("status %d, want %d", resp.StatusCode, tt.want) } if opCalled != tt.wantOp { - t.Errorf("ReleaseOperator called=%v, want %v", opCalled, tt.wantOp) + t.Errorf("operator release called=%v, want %v", opCalled, tt.wantOp) } if relCalled != tt.wantRelease { t.Errorf("Release called=%v, want %v", relCalled, tt.wantRelease) } if tt.wantOp && opID != "sb_1" { - t.Errorf("ReleaseOperator id=%q, want sb_1", opID) + t.Errorf("operator release id=%q, want sb_1", opID) } if tt.wantRelease && (relID != "sb_1" || relToken != tt.wantToken) { t.Errorf("Release(%q, %q), want (sb_1, %q)", relID, relToken, tt.wantToken) @@ -1163,6 +1163,15 @@ func newTenantTestServer(t *testing.T, apiToken string, tenants []config.TenantS return ts } +// credToken recovers the token a fake hook expects from cred: empty for an +// Operator credential, mirroring the collapsed manager's own translation. +func credToken(cred pool.Cred) string { + if cred.Operator { + return "" + } + return cred.Token +} + // fakeManager implements Manager with overridable behavior. ClaimWarm always // misses, so the server's warm-miss → redirect → provision path is exercised; // the claim hook stands in for the provision result. Tenant-scoped methods @@ -1213,18 +1222,19 @@ func (f *fakeManager) ClaimProvision(ctx context.Context, key types.PoolKey, ttl return f.claim(ctx, key, ttl) } -func (f *fakeManager) Release(_ context.Context, id, token string) error { - if f.release == nil { - return nil +// Release dispatches to the operator or per-sandbox hook, mirroring the +// collapsed manager's own Cred split. +func (f *fakeManager) Release(_ context.Context, id string, cred pool.Cred) error { + if cred.Operator { + if f.releaseOp == nil { + return nil + } + return f.releaseOp(id) } - return f.release(id, token) -} - -func (f *fakeManager) ReleaseOperator(_ context.Context, id string) error { - if f.releaseOp == nil { + if f.release == nil { return nil } - return f.releaseOp(id) + return f.release(id, cred.Token) } func (f *fakeManager) AgentSocket(id, token string) (string, error) { @@ -1238,26 +1248,26 @@ func (f *fakeManager) WakeAgentSocket(_ context.Context, id, token string) (stri return f.AgentSocket(id, token) } -func (f *fakeManager) Hibernate(_ context.Context, id, token string) error { +func (f *fakeManager) Hibernate(_ context.Context, id string, cred pool.Cred) error { if f.hibernate == nil { return nil } - return f.hibernate(id, token) + return f.hibernate(id, credToken(cred)) } -func (f *fakeManager) Fork(_ context.Context, id, token string, count int, ttl time.Duration) ([]*types.Sandbox, error) { +func (f *fakeManager) Fork(_ context.Context, id string, cred pool.Cred, count int, ttl time.Duration) ([]*types.Sandbox, error) { if f.fork == nil { return nil, pool.ErrUnknownSandbox } - return f.fork(id, token, count, ttl) + return f.fork(id, credToken(cred), count, ttl) } -func (f *fakeManager) Promote(_ context.Context, id, token, template, tenant string) (types.PoolKey, error) { +func (f *fakeManager) Promote(_ context.Context, id string, cred pool.Cred, template, tenant string) (types.PoolKey, error) { f.gotTenant = tenant if f.promote == nil { return types.PoolKey{}, pool.ErrUnknownSandbox } - if err := f.promote(id, token, template); err != nil { + if err := f.promote(id, credToken(cred), template); err != nil { return types.PoolKey{}, err } return types.PoolKey{Template: template, Net: types.NetNone, Size: types.SizeSmall}, nil @@ -1300,33 +1310,11 @@ func (f *fakeManager) Sandbox(string) (pool.SandboxSummary, bool) { func (f *fakeManager) Stats(string) (pool.SandboxStats, bool) { return pool.SandboxStats{}, false } -// The operator variants take no per-sandbox token; the fake records them -// through the same hooks with an empty token, mirroring releaseOp. -func (f *fakeManager) HibernateOperator(ctx context.Context, id string) error { - return f.Hibernate(ctx, id, "") -} - -func (f *fakeManager) Wake(_ context.Context, id, token string) error { +func (f *fakeManager) Wake(_ context.Context, id string, cred pool.Cred) error { if f.wake == nil { return nil } - return f.wake(id, token) -} - -func (f *fakeManager) WakeOperator(ctx context.Context, id string) error { - return f.Wake(ctx, id, "") -} - -func (f *fakeManager) ForkOperator(ctx context.Context, id string, count int, ttl time.Duration) ([]*types.Sandbox, error) { - return f.Fork(ctx, id, "", count, ttl) -} - -func (f *fakeManager) PromoteOperator(ctx context.Context, id, template, tenant string) (types.PoolKey, error) { - return f.Promote(ctx, id, "", template, tenant) -} - -func (f *fakeManager) CheckpointOperator(ctx context.Context, id, name, tenant string) (types.Checkpoint, error) { - return f.Checkpoint(ctx, id, "", name, tenant) + return f.wake(id, credToken(cred)) } func (f *fakeManager) Audit(_ context.Context, id string, line []byte) { @@ -1337,12 +1325,12 @@ func (f *fakeManager) Audit(_ context.Context, id string, line []byte) { func (f *fakeManager) AuditEnabled() bool { return f.audited != nil } -func (f *fakeManager) Checkpoint(_ context.Context, id, token, name, tenant string) (types.Checkpoint, error) { +func (f *fakeManager) Checkpoint(_ context.Context, id string, cred pool.Cred, name, tenant string) (types.Checkpoint, error) { f.gotTenant = tenant if f.checkpoint == nil { return types.Checkpoint{}, pool.ErrUnknownSandbox } - return f.checkpoint(id, token, name) + return f.checkpoint(id, credToken(cred), name) } func (f *fakeManager) ClaimCheckpoint(_ context.Context, ckptID string, _ time.Duration, tenant string) (*types.Sandbox, error) { From b961e3b8fb4f8c3561214449d5775b1fe9fb60b4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 16:04:19 +0800 Subject: [PATCH 13/31] sandboxd: probe peers for a checkpoint instead of gossiping every id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Warm counts are volatile, so they belong in a 1s gossip tick. Checkpoint ownership is stable but tenant-created and unbounded, and it rode the same tick: measured on bare metal, a 20-node view at 2000 records per node cost 4.94ms of serialize plus deserialize and ~5MB of payload every second, per node, growing linearly with a number no operator controls. Ownership is now asked for on the rare cross-node miss instead: a HEAD against each peer's blob endpoint, in parallel, capped at three answers. The answer is live, so a redirect can no longer point at a node that lost the record between two gossip rounds — which also retires the in-memory id set, its boot seeding, and the truncation that could hide a real owner behind two stale ones. The probe carries no credential. The checkpoint id is itself the unguessable capability to branch, so an unauthenticated HEAD tells a caller only what it already knows; the GET that streams the record keeps its token. --- e2e/e2e_test.go | 2 +- sandboxd/main.go | 19 +++- sandboxd/mesh/mesh.go | 63 ++++------- sandboxd/mesh/mesh_test.go | 64 +---------- sandboxd/mesh/state_test.go | 8 +- sandboxd/pool/archive.go | 3 - sandboxd/pool/checkpoint.go | 48 +------- sandboxd/pool/checkpoint_heal_test.go | 50 +++------ sandboxd/pool/pool.go | 28 ----- sandboxd/server/relay_test.go | 2 +- sandboxd/server/server.go | 38 ++++++- sandboxd/server/server_test.go | 151 +++++++++++++++++++------- sandboxd/store/peer/probe.go | 96 ++++++++++++++++ sandboxd/store/peer/probe_test.go | 104 ++++++++++++++++++ 14 files changed, 414 insertions(+), 262 deletions(-) create mode 100644 sandboxd/store/peer/probe.go create mode 100644 sandboxd/store/peer/probe_test.go diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 7ccc4be..52efbdc 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -304,7 +304,7 @@ func startTenantStack(t *testing.T, apiToken string, tenants []config.TenantSpec } go mgr.Run(t.Context()) - ts := httptest.NewServer(server.New(apiToken, tenants, "", mgr, eng.real, nil, nil).Handler()) + ts := httptest.NewServer(server.New(apiToken, tenants, "", mgr, eng.real, nil, nil, nil).Handler()) t.Cleanup(ts.Close) addr := strings.TrimPrefix(ts.URL, "http://") client, err := sandbox.Connect(addr, sandbox.WithAPIToken(apiToken)) diff --git a/sandboxd/main.go b/sandboxd/main.go index f7a21a5..4ede365 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -28,11 +28,15 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/mesh" "github.com/cocoonstack/sandbox/sandboxd/pool" "github.com/cocoonstack/sandbox/sandboxd/server" + "github.com/cocoonstack/sandbox/sandboxd/store/peer" ) const ( shutdownGrace = 5 * time.Second gossipInterval = time.Second + // probeBudget bounds a checkpoint-owner probe fan-out for WithPeerHeal's + // resolver; the HTTPProber itself times out each individual peer sooner. + probeBudget = 5 * time.Second // Slowloris protection; ReadTimeout/WriteTimeout must stay zero — cold // claims block up to the cold probe timeout and relays stream forever. readHeaderTimeout = 5 * time.Second @@ -94,6 +98,7 @@ func main() { go mgr.Run(ctx) var placer server.Placer + var prober server.CheckpointProber if cfg.Mesh != nil { msh, err := startMesh(ctx, cfg, mgr) if err != nil { @@ -101,9 +106,13 @@ func main() { } defer func() { _ = msh.Shutdown() }() placer = msh - mgr.SetTemplateNotifier(func() { msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs()) }) - // Wired after the mesh: the owners resolver is the mesh's own view. - mgr.WithPeerHeal(cfg.CheckpointPeerHeal, msh.CheckpointOwners, cfg.APIToken) + mgr.SetTemplateNotifier(func() { msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes()) }) + prober = &peer.HTTPProber{Peers: msh.PeerAddrs} + mgr.WithPeerHeal(cfg.CheckpointPeerHeal, func(id string) []string { + probeCtx, cancel := context.WithTimeout(ctx, probeBudget) + defer cancel() + return prober.Owners(probeCtx, id) + }, cfg.APIToken) go gossipNodeState(ctx, msh, mgr) logger.Infof(ctx, "mesh %s joined (%d seeds)", cmp.Or(cfg.Mesh.NodeID, cfg.Mesh.Bind), len(cfg.Mesh.Join)) } @@ -112,7 +121,7 @@ func main() { if cfg.PreviewListen != "" { preview = server.NewPreviewServer(cfg.PreviewSecret, cfg.PreviewAdvertise, mgr) } - srv := server.New(cfg.APIToken, cfg.Tenants, cfg.AdvertiseAddr, mgr, eng, placer, preview) + srv := server.New(cfg.APIToken, cfg.Tenants, cfg.AdvertiseAddr, mgr, eng, placer, prober, preview) httpSrv := &http.Server{ Addr: cfg.Listen, Handler: srv.Handler(), @@ -196,7 +205,7 @@ func gossipNodeState(ctx context.Context, msh *mesh.Mesh, mgr *pool.Manager) { case <-ctx.Done(): return case <-t.C: - msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes(), mgr.CheckpointIDs()) + msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes()) } } } diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index a1d825e..34a37b8 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -28,13 +28,12 @@ const leaveTimeout = time.Second // NodeState is one node's gossiped placement view. Epoch resolves merges: the // higher epoch for a given node wins. type NodeState struct { - NodeID string `json:"node_id"` - Addr string `json:"addr"` // data-plane advertise address - Epoch uint64 `json:"epoch"` - Pools map[string]int `json:"pools"` // PoolKey hash → warm count - Templates []string `json:"templates,omitempty"` // promoted-template key hashes on disk - Checkpoints []string `json:"checkpoints,omitempty"` // checkpoint ids on disk - Digest string `json:"digest,omitempty"` // cluster-invariant config digest + NodeID string `json:"node_id"` + Addr string `json:"addr"` // data-plane advertise address + Epoch uint64 `json:"epoch"` + Pools map[string]int `json:"pools"` // PoolKey hash → warm count + Templates []string `json:"templates,omitempty"` // promoted-template key hashes on disk + Digest string `json:"digest,omitempty"` // cluster-invariant config digest } // Mesh is the node's view of the cluster and its own gossiped state. @@ -104,16 +103,15 @@ func (m *Mesh) Join(seeds []string) error { return nil } -// UpdateSelf republishes this node's warm-pool counts and template and -// checkpoint sets, bumping the epoch so peers adopt the new view. An unchanged view is +// UpdateSelf republishes this node's warm-pool counts and promoted-template +// set, bumping the epoch so peers adopt the new view. An unchanged view is // not republished — the periodic tick would otherwise gossip a fresh epoch // every second for nothing. -func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates, checkpoints []string) { +func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates []string) { m.updateMu.Lock() defer m.updateMu.Unlock() m.mu.Lock() - if maps.Equal(m.self.Pools, pools) && slices.Equal(m.self.Templates, templates) && - slices.Equal(m.self.Checkpoints, checkpoints) { + if maps.Equal(m.self.Pools, pools) && slices.Equal(m.self.Templates, templates) { m.mu.Unlock() return } @@ -130,7 +128,6 @@ func (m *Mesh) UpdateSelf(ctx context.Context, pools map[string]int, templates, m.self.Epoch = epoch m.self.Pools = pools m.self.Templates = templates - m.self.Checkpoints = checkpoints m.view[m.self.NodeID] = m.self m.mu.Unlock() } @@ -206,14 +203,20 @@ func (m *Mesh) Candidates(keyHash string) []string { // delete of a template this node does not hold. Self is excluded: the caller // has already checked its own disk. func (m *Mesh) TemplateOwners(keyHash string) []string { - return m.recordOwners(keyHash, func(st NodeState) []string { return st.Templates }) -} - -// CheckpointOwners returns up to two peer addresses whose gossiped checkpoint -// set contains ckptID, the redirect targets for a branch this node cannot -// serve. Self is excluded: the caller already checked its own store. -func (m *Mesh) CheckpointOwners(ckptID string) []string { - return m.recordOwners(ckptID, func(st NodeState) []string { return st.Checkpoints }) + m.mu.Lock() + var owners []string + for id, st := range m.view { + if id != m.self.NodeID && slices.Contains(st.Templates, keyHash) { + owners = append(owners, st.Addr) + } + } + m.mu.Unlock() + // Which two survive truncation is already jittered by map iteration + // order; unlike Candidates there is no warmth to rank by. + if len(owners) > 2 { + owners = owners[:2] + } + return owners } // Members returns the current cluster view (self included). @@ -244,24 +247,6 @@ func (m *Mesh) Shutdown() error { } // persistEpoch durably records the epoch. -// recordOwners scans the view for peers whose listed record set contains id. -func (m *Mesh) recordOwners(id string, list func(NodeState) []string) []string { - m.mu.Lock() - var owners []string - for nid, st := range m.view { - if nid != m.self.NodeID && slices.Contains(list(st), id) { - owners = append(owners, st.Addr) - } - } - m.mu.Unlock() - // Which two survive truncation is already jittered by map iteration - // order; unlike Candidates there is no warmth to rank by. - if len(owners) > 2 { - owners = owners[:2] - } - return owners -} - func (m *Mesh) persistEpoch(epoch uint64) error { return storeEpoch(m.epochPath, epoch) } diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index 2ecabe7..17e327d 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -31,7 +31,7 @@ func TestMergeKeepsHigherEpoch(t *testing.T) { func TestMergeNeverOverwritesSelf(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(t.Context(), map[string]int{"k": 3}, nil, nil) + m.UpdateSelf(t.Context(), map[string]int{"k": 3}, nil) // A peer claiming to be "a" must not clobber our authoritative self entry. m.merge([]NodeState{{NodeID: "a", Addr: "evil:9999", Epoch: 999, Pools: map[string]int{"k": 0}}}) @@ -44,7 +44,7 @@ func TestMergeNeverOverwritesSelf(t *testing.T) { func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(t.Context(), map[string]int{"k": 5}, nil, nil) // self has warm, but is never a candidate + m.UpdateSelf(t.Context(), map[string]int{"k": 5}, nil) // self has warm, but is never a candidate m.merge([]NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 0}}, // no warm @@ -62,7 +62,7 @@ func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { func TestTemplateOwnersExcludeSelfAndUnknown(t *testing.T) { m := newTestMesh(t, "a") - m.UpdateSelf(t.Context(), nil, []string{"tpl"}, nil) // self holds it, but is never an owner candidate + m.UpdateSelf(t.Context(), nil, []string{"tpl"}) // self holds it, but is never an owner candidate m.merge([]NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Templates: []string{"tpl", "other"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Templates: []string{"other"}}, @@ -87,7 +87,7 @@ func TestForgetPrunesDeadNode(t *testing.T) { t.Errorf("candidates after forget %v, want nil (b pruned)", got) } // forgetting self is a no-op. - m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) + m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil) m.forget("a") if len(m.Members()) != 1 { t.Error("forget removed self") @@ -121,7 +121,7 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { if err := b.mesh.Join([]string{a.addr}); err != nil { t.Fatalf("join: %v", err) } - a.mesh.UpdateSelf(t.Context(), map[string]int{"kk": 4}, []string{"tpl-hash"}, nil) + a.mesh.UpdateSelf(t.Context(), map[string]int{"kk": 4}, []string{"tpl-hash"}) // Push/pull sync propagates a's warm counts and template set to b within // a few intervals. @@ -138,56 +138,6 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { t.Fatalf("node-b never learned node-a's state: view=%+v", b.mesh.Members()) } -func TestCheckpointOwnersExcludeSelfAndUnknown(t *testing.T) { - m := newTestMesh(t, "a") - m.UpdateSelf(t.Context(), nil, nil, []string{"ck_00000000000000aa"}) // self holds it, still not an owner candidate - m.merge([]NodeState{ - {NodeID: "b", Addr: "b:7777", Epoch: 1, Checkpoints: []string{"ck_00000000000000aa", "ck_00000000000000bb"}}, - {NodeID: "c", Addr: "c:7777", Epoch: 1, Checkpoints: []string{"ck_00000000000000bb"}}, - }) - - if owners := m.CheckpointOwners("ck_00000000000000aa"); len(owners) != 1 || owners[0] != "b:7777" { - t.Errorf("owners %v, want [b:7777]", owners) - } - if owners := m.CheckpointOwners("ck_deadbeefdeadbeef"); owners != nil { - t.Errorf("owners for absent checkpoint %v, want nil", owners) - } -} - -// TestCheckpointOwnersTruncates keeps a redirect answer small: the client only -// needs somewhere to retry, not the whole fleet. -func TestCheckpointOwnersTruncates(t *testing.T) { - m := newTestMesh(t, "a") - id := "ck_00000000000000aa" - m.merge([]NodeState{ - {NodeID: "b", Addr: "b:7777", Epoch: 1, Checkpoints: []string{id}}, - {NodeID: "c", Addr: "c:7777", Epoch: 1, Checkpoints: []string{id}}, - {NodeID: "d", Addr: "d:7777", Epoch: 1, Checkpoints: []string{id}}, - }) - - if owners := m.CheckpointOwners(id); len(owners) != 2 { - t.Errorf("owners = %v, want at most 2", owners) - } -} - -// TestUpdateSelfGossipsCheckpoints guards the wiring: a checkpoint set that -// never reaches NodeState is a redirect that never happens, which degrades L2 -// into "branch fails on the wrong node" without any visible error. -func TestUpdateSelfGossipsCheckpoints(t *testing.T) { - m := newTestMesh(t, "a") - m.UpdateSelf(t.Context(), nil, nil, []string{"ck_00000000000000aa"}) - - var self NodeState - for _, st := range m.Members() { - if st.NodeID == "a" { - self = st - } - } - if len(self.Checkpoints) != 1 || self.Checkpoints[0] != "ck_00000000000000aa" { - t.Fatalf("self.Checkpoints = %v, want the record to be gossiped", self.Checkpoints) - } -} - func newTestMesh(t *testing.T, id string) *Mesh { t.Helper() return &Mesh{ @@ -219,7 +169,3 @@ func startNode(t *testing.T, host string, port int, id string) *node { t.Cleanup(func() { _ = m.Shutdown() }) return &node{mesh: m, addr: fmt.Sprintf("%s:%d", host, m.ml.LocalNode().Port)} } - -// TestCheckpointOwnersExcludeSelfAndUnknown mirrors the template case: the -// caller has already checked its own store, so self is never an answer — a -// redirect to ourselves would bounce the request in place. diff --git a/sandboxd/mesh/state_test.go b/sandboxd/mesh/state_test.go index 31902d1..699dc39 100644 --- a/sandboxd/mesh/state_test.go +++ b/sandboxd/mesh/state_test.go @@ -52,7 +52,7 @@ func TestUpdateSelfPersistFailKeepsOldState(t *testing.T) { before := m.self.Epoch // A non-existent dir makes the durable write fail: the epoch must not publish. m.epochPath = filepath.Join(dir, "gone", "mesh-epoch") - m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) + m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil) if m.self.Epoch != before { t.Errorf("self epoch advanced to %d on a failed persist, want %d held", m.self.Epoch, before) } @@ -65,7 +65,7 @@ func TestUpdateSelfPersistsEpoch(t *testing.T) { dir := t.TempDir() m := newBoundMesh(t, dir) before := m.self.Epoch - m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil, nil) + m.UpdateSelf(t.Context(), map[string]int{"k": 1}, nil) if got := loadEpoch(filepath.Join(dir, "mesh-epoch")); got <= before { t.Errorf("persisted epoch %d did not advance past the seed %d", got, before) } @@ -78,8 +78,8 @@ func TestUpdateSelfConcurrentDropsNothing(t *testing.T) { a := map[string]int{"a": i + 1} b := map[string]int{"b": i + 1} var wg sync.WaitGroup - wg.Go(func() { m.UpdateSelf(t.Context(), a, nil, nil) }) - wg.Go(func() { m.UpdateSelf(t.Context(), b, nil, nil) }) + wg.Go(func() { m.UpdateSelf(t.Context(), a, nil) }) + wg.Go(func() { m.UpdateSelf(t.Context(), b, nil) }) wg.Wait() // Serialized updates with distinct payloads must both land: the loser // of the old TOCTOU race silently dropped its payload and one bump. diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 57e3153..7dd1841 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -188,7 +188,6 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, log.WithFunc("pool.wakeArchived").Warnf(ctx, "delete consumed archive ck %s: %v", ck, delErr) } else { m.dropRecLock(ck) - m.dropCkpt(ck) } // Only the none lane reaches here (the egress guard above fails closed), so // this rebinds the none-lane proxy; there is no NIC to re-lock. @@ -235,7 +234,5 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc func (m *Manager) deleteOrphanArchiveCk(ctx context.Context, ckID string) { if err := m.ckpts.Delete(ctx, ckID); err != nil { log.WithFunc("pool.deleteOrphanArchiveCk").Warnf(ctx, "delete orphaned archive ck %s: %v", ckID, err) - return } - m.dropCkpt(ckID) } diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 7f9f804..3bee649 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -80,9 +80,6 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID if err := m.ckpts.Publish(ctx, staging, ckpt.ID); err != nil { return types.Checkpoint{}, "", fmt.Errorf("commit checkpoint: %w", err) } - if !archive { - m.noteCkpt(ckpt) - } return ckpt, srcSnap, nil } @@ -119,7 +116,6 @@ func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl ti if err != nil { return nil, err } - m.noteCkpt(ckpt) // record is now local: gossip it return m.claimLoaded(ctx, ckpt, ttl, tenant) } @@ -263,7 +259,6 @@ func (m *Manager) deleteCkLocked(ctx context.Context, ckID string) error { return err } m.dropRecLock(ckID) - m.dropCkpt(ckID) return nil } @@ -296,44 +291,11 @@ func parseCheckpoint(raw []byte) (types.Checkpoint, error) { return ckpt, nil } -// CheckpointIDs lists this node's checkpoint ids for the mesh to gossip, from -// the in-memory set — mirrors TemplateHashes so the 1s gossip tick never -// touches the store backend. Archive wake images never enter the set. Nil on -// a shared checkpoint store (every node already resolves every record). -func (m *Manager) CheckpointIDs() []string { - m.ckptMu.Lock() - defer m.ckptMu.Unlock() - if m.ckptSet == nil { - return nil - } - ids := make([]string, 0, len(m.ckptSet)) - for id := range m.ckptSet { - ids = append(ids, id) - } - slices.Sort(ids) - return ids -} - -// noteCkpt adds a fresh non-archive record to the gossip set: called after -// publishCheckpoint's Publish commits, and after ClaimCheckpointHeal pulls a -// peer's record into this node's local store. -func (m *Manager) noteCkpt(ckpt types.Checkpoint) { - if m.ckptSet == nil || ckpt.Archive { - return - } - m.ckptMu.Lock() - m.ckptSet[ckpt.ID] = struct{}{} - m.ckptMu.Unlock() -} - -// dropCkpt removes id from the gossip set once its store record is deleted. -func (m *Manager) dropCkpt(id string) { - if m.ckptSet == nil { - return - } - m.ckptMu.Lock() - delete(m.ckptSet, id) - m.ckptMu.Unlock() +// HasCheckpoint reports whether this node's local store holds a branchable +// (non-archive) record for id — the probe answer, never a fetch. +func (m *Manager) HasCheckpoint(ctx context.Context, ckptID string) bool { + ckpt, err := m.loadCheckpoint(ctx, ckptID) + return err == nil && !ckpt.Archive } // FetchCheckpoint materializes a checkpoint's export for a peer transfer, diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index 16c38a8..6a52048 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -6,7 +6,6 @@ import ( "errors" "os" "path/filepath" - "slices" "sync" "testing" "time" @@ -31,9 +30,9 @@ func TestClaimCheckpointNeverPullsOnMiss(t *testing.T) { } } -// TestClaimCheckpointHealPullsAndGossips: a heal pays the transfer once and -// makes the record locally listed, so the mesh advertises it going forward. -func TestClaimCheckpointHealPullsAndGossips(t *testing.T) { +// TestClaimCheckpointHealPullsOnce: a heal pays the transfer once and leaves +// the record served from the local store from then on. +func TestClaimCheckpointHealPullsOnce(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) @@ -48,8 +47,8 @@ func TestClaimCheckpointHealPullsAndGossips(t *testing.T) { if got := puller.count(); got != 1 { t.Errorf("puller called %d times, want 1", got) } - if ids := m.CheckpointIDs(); !slices.Contains(ids, id) { - t.Errorf("CheckpointIDs() = %v, want %s gossiped after heal", ids, id) + if !m.HasCheckpoint(t.Context(), id) { + t.Errorf("HasCheckpoint(%s) = false, want true after heal", id) } } @@ -117,9 +116,9 @@ func TestFetchCheckpointNeverPulls(t *testing.T) { } } -// TestCheckpointIDsReflectsPublishAndDelete: the gossip set tracks a -// checkpoint's whole lifecycle, and an archive wake image never enters it. -func TestCheckpointIDsReflectsPublishAndDelete(t *testing.T) { +// TestHasCheckpoint covers the probe answer across a present, missing, and +// archive record: only a live branchable checkpoint reports true. +func TestHasCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) src := mustClaim(t, m, testKey) @@ -128,38 +127,17 @@ func TestCheckpointIDsReflectsPublishAndDelete(t *testing.T) { if err != nil { t.Fatalf("checkpoint: %v", err) } - if ids := m.CheckpointIDs(); !slices.Contains(ids, ckpt.ID) { - t.Errorf("CheckpointIDs() = %v, want %s listed after publish", ids, ckpt.ID) + if !m.HasCheckpoint(t.Context(), ckpt.ID) { + t.Errorf("HasCheckpoint(%s) = false, want true for a published record", ckpt.ID) } - - if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, ""); err != nil { - t.Fatalf("delete: %v", err) - } - if ids := m.CheckpointIDs(); slices.Contains(ids, ckpt.ID) { - t.Errorf("CheckpointIDs() = %v, want %s gone after delete", ids, ckpt.ID) + if m.HasCheckpoint(t.Context(), "ck_00000000000000ff") { + t.Error("HasCheckpoint = true for a missing id, want false") } sb := mustClaim(t, m, testKey) mustArchive(t, m, sb) - if ids := m.CheckpointIDs(); slices.Contains(ids, sb.ArchiveCk) { - t.Errorf("CheckpointIDs() = %v, want the archive wake image %s never listed", ids, sb.ArchiveCk) - } -} - -// TestCheckpointIDsReseedsOnBoot: a fresh Manager over the same dataDir must -// rebuild the gossip set from disk, not start empty until the next publish. -func TestCheckpointIDsReseedsOnBoot(t *testing.T) { - dataDir := t.TempDir() - m1 := newTestManagerAt(t, newFakeEngine(), dataDir) - src := mustClaim(t, m1, testKey) - ckpt, err := m1.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") - if err != nil { - t.Fatalf("checkpoint: %v", err) - } - - m2 := newTestManagerAt(t, newFakeEngine(), dataDir) - if ids := m2.CheckpointIDs(); !slices.Contains(ids, ckpt.ID) { - t.Errorf("fresh manager CheckpointIDs() = %v, want %s re-seeded from disk", ids, ckpt.ID) + if m.HasCheckpoint(t.Context(), sb.ArchiveCk) { + t.Errorf("HasCheckpoint(%s) = true for an archive wake image, want false", sb.ArchiveCk) } } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 179fdd2..395af49 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -240,13 +240,6 @@ type Manager struct { tplMu sync.Mutex tplSet map[string]struct{} - // ckptSet mirrors tplSet for checkpoints: publish/delete/heal keep it - // current, startup seeds it from one Metas scan. Nil on a shared - // checkpoint store — gossiping it there would have every node advertise - // every record. - ckptMu sync.Mutex - ckptSet map[string]struct{} - // recLocks serializes same-id store record mutations and holds off a // re-publish swap while a clone reads the old generation (per id, RW). recLocks sync.Map @@ -337,9 +330,6 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg } } } - if !m.ckptsShared { - m.seedCkptSet(ctx) - } usage, err := newJournal(filepath.Join(cfg.DataDir, "usage.jsonl")) if err != nil { return nil, fmt.Errorf("open usage journal: %w", err) @@ -589,24 +579,6 @@ func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { m.healer = peer.New(m.ckpts, owners, &peer.HTTPPuller{Token: token}) } -// seedCkptSet rebuilds the gossip set from one store scan at boot, so records -// published by a prior life are advertised again without a re-publish. -func (m *Manager) seedCkptSet(ctx context.Context) { - m.ckptSet = map[string]struct{}{} - metas, err := m.ckpts.Metas(ctx) - if err != nil { - // Existing records stay unadvertised until re-published; say so. - log.WithFunc("pool.seedCkptSet").Warnf(ctx, "seed checkpoint gossip set: %v", err) - return - } - for _, raw := range metas { - var ckpt types.Checkpoint - if json.Unmarshal(raw, &ckpt) == nil && ckpt.ID != "" && !ckpt.Archive { - m.ckptSet[ckpt.ID] = struct{}{} - } - } -} - func dirExists(path string) bool { fi, err := os.Stat(path) return err == nil && fi.IsDir() diff --git a/sandboxd/server/relay_test.go b/sandboxd/server/relay_test.go index 7ae9651..e770f7c 100644 --- a/sandboxd/server/relay_test.go +++ b/sandboxd/server/relay_test.go @@ -156,7 +156,7 @@ func newRelayServer(t *testing.T, guest func(net.Conn)) (*httptest.Server, *Serv go guest(guestEnd) return relayEnd, nil }} - srv := New("", nil, "node:7777", &fakeManager{}, dialer, nil, nil) + srv := New("", nil, "node:7777", &fakeManager{}, dialer, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { srv.CloseRelays() diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 1eb2764..ed06d1d 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -76,6 +76,7 @@ type Manager interface { ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) Checkpoints(ctx context.Context, tenant string) ([]types.Checkpoint, error) + HasCheckpoint(ctx context.Context, ckptID string) bool FetchCheckpoint(ctx context.Context, ckptID string) (dir string, meta []byte, release func(), err error) DeleteCheckpoint(ctx context.Context, ckptID, tenant string) error ClaimDeadline(id, token string) (time.Time, error) @@ -98,11 +99,17 @@ type Dialer interface { type Placer interface { Candidates(keyHash string) []string TemplateOwners(keyHash string) []string - CheckpointOwners(ckptID string) []string PeerAddrs() []string ConfigMismatches() int } +// CheckpointProber answers which peers currently hold a checkpoint, queried +// live on a redirect decision rather than gossiped; nil disables probing +// (single-node). +type CheckpointProber interface { + Owners(ctx context.Context, id string) []string +} + // InfoResponse is the wire reply of GET /v1/info. Peers lists the other nodes' // data-plane addresses so a client can scatter a Lookup across the cluster. type InfoResponse struct { @@ -127,6 +134,7 @@ type Server struct { mgr Manager dialer Dialer placer Placer + prober CheckpointProber apiToken string tenants []config.TenantSpec advertise string @@ -142,12 +150,14 @@ type Server struct { // node-level endpoints open (per-sandbox tokens still guard sandbox-scoped // calls). tenants adds per-tenant tokens next to the root apiToken. // advertise is this node's data-plane address, returned as a claim's owner -// address. A nil placer disables mesh redirects (single node). -func New(apiToken string, tenants []config.TenantSpec, advertise string, mgr Manager, dialer Dialer, placer Placer, preview *PreviewServer) *Server { +// address. A nil placer disables mesh redirects and a nil prober disables +// checkpoint-owner probing (both single-node). +func New(apiToken string, tenants []config.TenantSpec, advertise string, mgr Manager, dialer Dialer, placer Placer, prober CheckpointProber, preview *PreviewServer) *Server { return &Server{ mgr: mgr, dialer: dialer, placer: placer, + prober: prober, apiToken: apiToken, tenants: tenants, advertise: advertise, @@ -179,6 +189,9 @@ func (s *Server) Handler() http.Handler { // The peer-transfer half of the snapshot placement design: a node that // cannot redirect a branch pulls the record from an owner through this. mux.HandleFunc("GET /v1/checkpoints/{id}/blob", s.requireToken(s.handleCheckpointBlob)) + // HEAD is the probe endpoint a redirect decision fans out to: it must stay + // unauthenticated, so it is registered outside requireToken. + mux.HandleFunc("HEAD /v1/checkpoints/{id}/blob", s.handleCheckpointProbe) mux.HandleFunc("DELETE /v1/checkpoints/{id}", s.requireToken(s.handleDeleteCheckpoint)) mux.HandleFunc("DELETE /v1/templates", s.requireToken(s.handleDeleteTemplate)) mux.HandleFunc("PUT /v1/pools", s.requireRoot(s.handlePutPools)) @@ -352,8 +365,10 @@ func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { // handleClaimCheckpoint claims a fresh sandbox branched from a checkpoint. // Tier order: local claim first; then a redirect (zero bytes moved) when a -// gossiped owner exists and this is not already the redirect retry; heal (one -// peer transfer) only when neither could answer. +// live probe finds an owner and this is not already the redirect retry — +// redirect targets come from a live probe, not gossip, so a redirect never +// points at a node that already lost the record; heal (one peer transfer) +// only when neither could answer. func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.CheckpointClaimRequest](w, r) if !ok { @@ -362,7 +377,7 @@ func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { ckptID := r.PathValue("id") sb, err := s.mgr.ClaimCheckpoint(r.Context(), ckptID, req.TTL(), tenantFrom(r.Context())) if errors.Is(err, pool.ErrUnknownCheckpoint) { - if !req.NoRedirect && s.placer != nil && writeRedirect(w, s.placer.CheckpointOwners(ckptID)) { + if !req.NoRedirect && s.prober != nil && writeRedirect(w, s.prober.Owners(r.Context(), ckptID)) { return } sb, err = s.mgr.ClaimCheckpointHeal(r.Context(), ckptID, req.TTL(), tenantFrom(r.Context())) @@ -397,6 +412,17 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { } } +// handleCheckpointProbe answers a peer's HEAD probe for a checkpoint. +// Unauthenticated on purpose: the id is the unguessable capability, so this +// leaks only existence to a caller who already holds it. +func (s *Server) handleCheckpointProbe(w http.ResponseWriter, r *http.Request) { + if s.mgr.HasCheckpoint(r.Context(), r.PathValue("id")) { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) +} + // handleListCheckpoints lists this node's checkpoints, newest first — a // tenant caller sees only its own records. func (s *Server) handleListCheckpoints(w http.ResponseWriter, r *http.Request) { diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 265e29c..fe61762 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -715,7 +715,7 @@ func TestClaimRedirectsOnWarmMiss(t *testing.T) { provisioned = true return &types.Sandbox{ID: "sb_local"}, nil }} - srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: []string{"node-b:7777", "node-c:7777"}}, nil) + srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: []string{"node-b:7777", "node-c:7777"}}, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -760,7 +760,7 @@ func TestClaimRedirectsToTemplateOwner(t *testing.T) { return &types.Sandbox{ID: "sb_local", Token: "tok"}, nil }, } - srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil) + srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -788,7 +788,7 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { // Unknown locally + gossip names an owner → the claim redirect shape; // unknown everywhere stays 404. mgr := &fakeManager{} // DeleteTemplate → ErrUnknownTemplate - srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil) + srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -820,7 +820,7 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { t.Errorf("status %d, want 404 with no_redirect despite known owners", resp2.StatusCode) } - srvNoOwner := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{}, nil) + srvNoOwner := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{}, nil, nil) ts2 := httptest.NewServer(srvNoOwner.Handler()) t.Cleanup(func() { ts2.Close(); srvNoOwner.CloseRelays() }) req3, _ := http.NewRequest(http.MethodDelete, ts2.URL+"/v1/templates?template=tpl", nil) @@ -839,7 +839,7 @@ func TestClaimProvisionsWhenNoCandidate(t *testing.T) { mgr := &fakeManager{claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { return &types.Sandbox{ID: "sb_local", Token: "tok"}, nil }} - srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: nil}, nil) + srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: nil}, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -867,7 +867,7 @@ func TestOwnerEndpoint(t *testing.T) { } return "", pool.ErrUnknownSandbox }} - srv := New("", nil, "node-b:7777", mgr, &fakeDialer{}, nil, nil) + srv := New("", nil, "node-b:7777", mgr, &fakeDialer{}, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -909,7 +909,7 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { return time.Time{}, nil }} ps := NewPreviewServer("secret", "node:7777", &fakePreviewMgr{}) - srv := New("", nil, "node:7777", mgr, &fakeDialer{}, nil, ps) + srv := New("", nil, "node:7777", mgr, &fakeDialer{}, nil, nil, ps) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -943,8 +943,8 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { // A, branch from B" simply not work. func TestCheckpointClaimRedirectsToOwner(t *testing.T) { mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint - placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", mgr, placer) + prober := &fakeProber{owners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", mgr, prober) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -967,8 +967,8 @@ func TestCheckpointClaimRedirectsToOwner(t *testing.T) { // TestCheckpointClaimNoRedirectResolvesLocally: the retry at a redirect target // must resolve there or two nodes would bounce the request between them. func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { - placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", &fakeManager{}, placer) + prober := &fakeProber{owners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", &fakeManager{}, prober) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{"no_redirect":true}`) defer resp.Body.Close() @@ -983,7 +983,7 @@ func TestCheckpointClaimNoRedirectResolvesLocally(t *testing.T) { // than a redirect to nowhere. func TestCheckpointClaimNoOwnersIs404(t *testing.T) { mgr := &fakeManager{} // ClaimCheckpointHeal defaults to ErrUnknownCheckpoint - ts := newPlacerTestServer(t, "sekret", mgr, &fakePlacer{}) + ts := newPlacerTestServer(t, "sekret", mgr, nil) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -1001,8 +1001,8 @@ func TestCheckpointClaimNoOwnersIs404(t *testing.T) { // the heal pull, even though the local ClaimCheckpoint missed. func TestCheckpointClaimRedirectBeatsHeal(t *testing.T) { mgr := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint - placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", mgr, placer) + prober := &fakeProber{owners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", mgr, prober) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -1030,7 +1030,7 @@ func TestCheckpointClaimFallsBackToHealWhenNoOwner(t *testing.T) { return &types.Sandbox{ID: "sb_healed", Token: "htok", FromCheckpoint: ckptID}, nil }, } - ts := newPlacerTestServer(t, "sekret", mgr, &fakePlacer{}) + ts := newPlacerTestServer(t, "sekret", mgr, nil) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) defer resp.Body.Close() @@ -1058,8 +1058,8 @@ func TestCheckpointClaimNoRedirectGoesStraightToHeal(t *testing.T) { return &types.Sandbox{ID: "sb_healed", Token: "htok", FromCheckpoint: ckptID}, nil }, } - placer := &fakePlacer{ckptOwners: []string{"owner-a:7777"}} - ts := newPlacerTestServer(t, "sekret", mgr, placer) + prober := &fakeProber{owners: []string{"owner-a:7777"}} + ts := newPlacerTestServer(t, "sekret", mgr, prober) resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{"no_redirect":true}`) defer resp.Body.Close() @@ -1144,6 +1144,71 @@ func TestCheckpointBlobStreamsRecord(t *testing.T) { } } +// TestCheckpointBlobHeadUnauthenticated proves the probe route bypasses +// requireToken while GET still enforces it: the id is the unguessable +// capability, so an unauthenticated HEAD only confirms existence. +func TestCheckpointBlobHeadUnauthenticated(t *testing.T) { + const ckptID = "ck_00000000000000aa" + mgr := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} + ts := newTestServer(t, "sekret", mgr, nil) + + head := func(id string) int { + resp, err := http.Head(ts.URL + "/v1/checkpoints/" + id + "/blob") + if err != nil { + t.Fatalf("head: %v", err) + } + defer resp.Body.Close() + return resp.StatusCode + } + if got := head(ckptID); got != http.StatusOK { + t.Errorf("HEAD present, no token: status = %d, want 200", got) + } + if got := head("ck_00000000000000ff"); got != http.StatusNotFound { + t.Errorf("HEAD missing, no token: status = %d, want 404", got) + } + + resp, err := http.Get(ts.URL + "/v1/checkpoints/" + ckptID + "/blob") + if err != nil { + t.Fatalf("get: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("GET no token: status = %d, want 401", resp.StatusCode) + } +} + +// TestCheckpointClaimProbesRealPeerAndRedirects is the end-to-end probe: a +// live HTTPProber HEADs an actual peer server rather than a fake, proving the +// probe wire protocol (unauthenticated HEAD, addr scheme defaulting) against +// the real handler, not just an interface stub. +func TestCheckpointClaimProbesRealPeerAndRedirects(t *testing.T) { + const ckptID = "ck_00000000000000aa" + mgrB := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} + srvB := New("sekret", nil, "node-b:7777", mgrB, &fakeDialer{}, nil, nil, nil) + tsB := httptest.NewServer(srvB.Handler()) + t.Cleanup(tsB.Close) + + prober := &peer.HTTPProber{Peers: func() []string { return []string{tsB.URL} }} + mgrA := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint + srvA := New("sekret", nil, "node-a:7777", mgrA, &fakeDialer{}, nil, prober, nil) + tsA := httptest.NewServer(srvA.Handler()) + t.Cleanup(tsA.Close) + + resp := postJSON(t, tsA.URL+"/v1/checkpoints/"+ckptID+"/claim", "sekret", `{}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 carrying a redirect", resp.StatusCode) + } + var got types.ClaimResponse + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if len(got.Redirect) != 1 || got.Redirect[0] != tsB.URL { + t.Errorf("redirect = %v, want [%s]", got.Redirect, tsB.URL) + } +} + func newTestServer(t *testing.T, apiToken string, mgr Manager, dialer Dialer) *httptest.Server { t.Helper() return newTenantTestServer(t, apiToken, nil, mgr, dialer) @@ -1154,7 +1219,7 @@ func newTenantTestServer(t *testing.T, apiToken string, tenants []config.TenantS if dialer == nil { dialer = &fakeDialer{} } - srv := New(apiToken, tenants, "node:7777", mgr, dialer, nil, nil) + srv := New(apiToken, tenants, "node:7777", mgr, dialer, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close() @@ -1177,15 +1242,16 @@ func credToken(cred pool.Cred) string { // the claim hook stands in for the provision result. Tenant-scoped methods // record the tenant they were handed in gotTenant. type fakeManager struct { - ckptDir string - claim func(ctx context.Context, key types.PoolKey, ttl time.Duration) (*types.Sandbox, error) - release func(id, token string) error - releaseOp func(id string) error - socket func(id, token string) (string, error) - hibernate func(id, token string) error - wake func(id, token string) error - fork func(id, token string, count int, ttl time.Duration) ([]*types.Sandbox, error) - promote func(id, token, template string) error + ckptDir string + hasCheckpoint map[string]bool + claim func(ctx context.Context, key types.PoolKey, ttl time.Duration) (*types.Sandbox, error) + release func(id, token string) error + releaseOp func(id string) error + socket func(id, token string) (string, error) + hibernate func(id, token string) error + wake func(id, token string) error + fork func(id, token string, count int, ttl time.Duration) ([]*types.Sandbox, error) + promote func(id, token, template string) error deleteGolden func(key types.PoolKey) error hasGolden bool @@ -1391,17 +1457,20 @@ func (f *fakeDialer) DialSilkd(ctx context.Context, sock string) (net.Conn, erro } type fakePlacer struct { - ckptOwners []string - addrs []string - owners []string + addrs []string + owners []string } func (f *fakePlacer) Candidates(string) []string { return f.addrs } func (f *fakePlacer) TemplateOwners(string) []string { return f.owners } +func (f *fakePlacer) PeerAddrs() []string { return f.addrs } +func (f *fakePlacer) ConfigMismatches() int { return 0 } -func (f *fakePlacer) CheckpointOwners(string) []string { return f.ckptOwners } -func (f *fakePlacer) PeerAddrs() []string { return f.addrs } -func (f *fakePlacer) ConfigMismatches() int { return 0 } +// fakeProber stubs CheckpointProber with a fixed answer, standing in for a +// live HEAD fan-out. +type fakeProber struct{ owners []string } + +func (f *fakeProber) Owners(context.Context, string) []string { return f.owners } // FetchCheckpoint serves the peer-transfer read; the fake reports every record // missing unless a test supplies a directory. @@ -1412,11 +1481,19 @@ func (f *fakeManager) FetchCheckpoint(_ context.Context, _ string) (string, []by return f.ckptDir, []byte(`{"id":"ck_00000000000000aa"}`), func() {}, nil } -// newPlacerTestServer builds a server with a mesh placer, which the shared -// helper deliberately leaves nil (most tests are single-node). -func newPlacerTestServer(t *testing.T, apiToken string, mgr Manager, placer Placer) *httptest.Server { +// HasCheckpoint answers the probe endpoint straight from the fake's map; +// unset ids report false. +func (f *fakeManager) HasCheckpoint(_ context.Context, ckptID string) bool { + return f.hasCheckpoint[ckptID] +} + +// newPlacerTestServer builds a server with a checkpoint prober, which the +// shared helper deliberately leaves nil (most tests are single-node); the +// mesh placer stays nil throughout — these tests exercise checkpoint claims +// only. +func newPlacerTestServer(t *testing.T, apiToken string, mgr Manager, prober CheckpointProber) *httptest.Server { t.Helper() - srv := New(apiToken, nil, "node:7777", mgr, &fakeDialer{}, placer, nil) + srv := New(apiToken, nil, "node:7777", mgr, &fakeDialer{}, nil, prober, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(ts.Close) return ts diff --git a/sandboxd/store/peer/probe.go b/sandboxd/store/peer/probe.go new file mode 100644 index 0000000..a051509 --- /dev/null +++ b/sandboxd/store/peer/probe.go @@ -0,0 +1,96 @@ +package peer + +import ( + "context" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +const ( + probeTimeout = time.Second + defaultProbeClientTimeout = 2 * time.Second + maxProbeOwners = 3 +) + +// HTTPProber finds which peers hold a checkpoint by probing them directly: +// ownership is stable but unbounded, so it is queried on the rare cross-node +// miss instead of gossiped every second by every node. +type HTTPProber struct { + // A nil Client uses a default with a 2-second timeout: a probe must never + // hang on a wedged peer. + Client *http.Client + Peers func() []string +} + +// Owners fans a HEAD out to every peer in parallel and returns up to 3 +// addresses that answered 200, in no particular order. +func (p *HTTPProber) Owners(ctx context.Context, id string) []string { + addrs := dedupAddrs(p.Peers()) + if len(addrs) == 0 { + return nil + } + client := p.Client + if client == nil { + client = &http.Client{Timeout: defaultProbeClientTimeout} + } + + var mu sync.Mutex + var owners []string + var wg sync.WaitGroup + for _, addr := range addrs { + wg.Go(func() { + if probeOwner(ctx, client, addr, id) { + mu.Lock() + owners = append(owners, addr) + mu.Unlock() + } + }) + } + wg.Wait() + if len(owners) > maxProbeOwners { + owners = owners[:maxProbeOwners] + } + return owners +} + +// probeOwner sends no Authorization header: the id itself is the +// unguessable capability, so an unauthenticated HEAD leaks only existence +// to a caller who already holds it. +func probeOwner(ctx context.Context, client *http.Client, addr, id string) bool { + ctx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + + base := addr + if !strings.Contains(base, "://") { + base = "http://" + base + } + u := base + "/v1/checkpoints/" + url.PathEscape(id) + "/blob" + req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil) + if err != nil { + return false + } + resp, err := client.Do(req) //nolint:gosec // addr comes from the mesh's own member view + if err != nil { + return false + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode == http.StatusOK +} + +// dedupAddrs drops repeated addresses so a stale or duplicated peer list +// never probes the same node twice. +func dedupAddrs(addrs []string) []string { + seen := make(map[string]struct{}, len(addrs)) + out := make([]string, 0, len(addrs)) + for _, a := range addrs { + if _, ok := seen[a]; ok { + continue + } + seen[a] = struct{}{} + out = append(out, a) + } + return out +} diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go new file mode 100644 index 0000000..ba42933 --- /dev/null +++ b/sandboxd/store/peer/probe_test.go @@ -0,0 +1,104 @@ +package peer + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +// TestOwnersReturnsOnly200: a mixed 200/404 answer set redirects only to the +// peer that actually holds the record. +func TestOwnersReturnsOnly200(t *testing.T) { + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer ok.Close() + miss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer miss.Close() + + p := &HTTPProber{Peers: func() []string { return []string{ok.URL, miss.URL} }} + owners := p.Owners(t.Context(), testID) + if len(owners) != 1 || owners[0] != ok.URL { + t.Errorf("owners = %v, want [%s]", owners, ok.URL) + } +} + +// TestOwnersAllMissIsEmpty: nobody answering 200 means nobody to redirect to. +func TestOwnersAllMissIsEmpty(t *testing.T) { + miss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer miss.Close() + + p := &HTTPProber{Peers: func() []string { return []string{miss.URL} }} + if owners := p.Owners(t.Context(), testID); len(owners) != 0 { + t.Errorf("owners = %v, want none", owners) + } +} + +// TestOwnersHungPeerExcludedWithoutBlockingOthers: a wedged peer must not +// stall the whole probe past its own timeout, nor keep a healthy peer's +// answer from coming back. +func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { + block := make(chan struct{}) + hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + <-block + })) + defer hung.Close() + defer close(block) + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer ok.Close() + + start := time.Now() + p := &HTTPProber{Peers: func() []string { return []string{hung.URL, ok.URL} }} + owners := p.Owners(t.Context(), testID) + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Errorf("Owners took %v, want bounded by the per-probe timeout", elapsed) + } + if len(owners) != 1 || owners[0] != ok.URL { + t.Errorf("owners = %v, want [%s]", owners, ok.URL) + } +} + +// TestOwnersDedupsAddrs: a duplicated peer list must probe each address once. +func TestOwnersDedupsAddrs(t *testing.T) { + var hits atomic.Int32 + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer ok.Close() + + p := &HTTPProber{Peers: func() []string { return []string{ok.URL, ok.URL, ok.URL} }} + owners := p.Owners(t.Context(), testID) + if len(owners) != 1 || owners[0] != ok.URL { + t.Errorf("owners = %v, want a single deduped [%s]", owners, ok.URL) + } + if got := hits.Load(); got != 1 { + t.Errorf("server probed %d times, want 1 (dupes deduped before the fan-out)", got) + } +} + +// TestOwnersCapsAtThree: a redirect answer must stay small even with a large +// fleet behind the record. +func TestOwnersCapsAtThree(t *testing.T) { + var addrs []string + for range 4 { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + addrs = append(addrs, srv.URL) + } + + p := &HTTPProber{Peers: func() []string { return addrs }} + if owners := p.Owners(t.Context(), testID); len(owners) != 3 { + t.Errorf("owners = %v, want capped at 3", owners) + } +} From 19f6a8620a5dbc47d4e1e534f46f0e79d48f85fc Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 16:36:10 +0800 Subject: [PATCH 14/31] sandboxd: bound the heal, gate the blob, and let a delete reach a replica MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six defects in the heal path, found once the tiers were sequenced properly. Quota was checked before the local record lookup, so a full or draining node answered 429 instead of "not here" — the handler never reached the redirect or heal tiers, which is exactly the case they exist for. The lookup is a cheap local read, so it goes first now; the heal path keeps quota ahead of the pull, since resolving a record there means moving a guest memory image. GET on the blob endpoint took any tenant token, streaming a raw checkpoint — guest memory and disk — outside the tenant scoping every other checkpoint verb enforces. Its only caller is the peer puller, which holds the fleet token, so it is operator-only now. The HEAD probe stays unauthenticated: the id is the capability, and it answers nothing a caller does not already know. Healing published outside the record lock, against the store's own contract that same-id publish and delete are the caller's to serialize; a delete or TTL sweep racing a heal could resurrect a record or swap a directory under a reader. The heal now runs entirely under the id's write lock, and the peer package stops pretending to be a store: it pulls into a staging dir the manager owns and publishes nothing itself. A pulled record was published unread. A buggy or hostile peer could install an unreadable record — which then suppressed every later heal — or one whose meta named a different checkpoint. The staged copy is now checked for shape and identity before it is trusted, and a rejected copy moves on to the next owner rather than failing the branch. Transfers were bounded per owner, not overall, so two slow owners cost an hour, and nothing capped concurrent heals node-wide. One budget now covers the whole attempt, and a node runs at most four at once. Deleting a checkpoint only removed the local copy, so a healed replica kept serving branches of a record its owner had deleted. A fleet-scoped delete broadcasts to peers; a forwarded one does not, which is what stops the loop. A healed copy carries the source's CreatedAt, so every node's TTL sweep ages it out at the same moment without coordinating. --- sandboxd/main.go | 2 + sandboxd/pool/archive_test.go | 6 +- sandboxd/pool/checkpoint.go | 124 ++++++++++++-- sandboxd/pool/checkpoint_heal_test.go | 235 +++++++++++++++++++++++++- sandboxd/pool/checkpoint_test.go | 16 +- sandboxd/pool/pool.go | 35 +++- sandboxd/pool/pool_test.go | 7 +- sandboxd/pool/stats.go | 39 ++--- sandboxd/pool/stats_test.go | 85 ++++++++++ sandboxd/server/server.go | 22 ++- sandboxd/server/server_test.go | 42 ++++- sandboxd/store/peer/broadcast.go | 84 +++++++++ sandboxd/store/peer/broadcast_test.go | 95 +++++++++++ sandboxd/store/peer/peer.go | 132 ++++++++------- sandboxd/store/peer/peer_test.go | 201 +++++++++++----------- sandboxd/types/types.go | 1 + 16 files changed, 896 insertions(+), 230 deletions(-) create mode 100644 sandboxd/pool/stats_test.go create mode 100644 sandboxd/store/peer/broadcast.go create mode 100644 sandboxd/store/peer/broadcast_test.go diff --git a/sandboxd/main.go b/sandboxd/main.go index 4ede365..d1c0beb 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -113,6 +113,8 @@ func main() { defer cancel() return prober.Owners(probeCtx, id) }, cfg.APIToken) + broadcaster := &peer.Broadcaster{Peers: msh.PeerAddrs, Token: cfg.APIToken} + mgr.WithPeerDelete(broadcaster.Delete) go gossipNodeState(ctx, msh, mgr) logger.Infof(ctx, "mesh %s joined (%d seeds)", cmp.Or(cfg.Mesh.NodeID, cfg.Mesh.Bind), len(cfg.Mesh.Join)) } diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 536b645..6744a8f 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -35,7 +35,7 @@ func TestArchivePublishWindowPinsCheckpoint(t *testing.T) { if ckpts, err := m.Checkpoints(t.Context(), ""); err != nil || len(ckpts) != 0 { t.Errorf("mid-publish checkpoint visible: %v, %v", ckpts, err) } - if err := m.DeleteCheckpoint(t.Context(), ck, ""); !errors.Is(err, ErrUnknownCheckpoint) { + if err := m.DeleteCheckpoint(t.Context(), ck, "", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("mid-publish delete: %v, want ErrUnknownCheckpoint", err) } close(stall.release) @@ -123,7 +123,7 @@ func TestArchiveCkHiddenAcrossNodes(t *testing.T) { if ckpts, err := mB.Checkpoints(t.Context(), ""); err != nil || len(ckpts) != 0 { t.Errorf("peer lists the archive wake image: %v, %v", ckpts, err) } - if err := mB.DeleteCheckpoint(t.Context(), ck, ""); !errors.Is(err, ErrUnknownCheckpoint) { + if err := mB.DeleteCheckpoint(t.Context(), ck, "", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("peer delete: %v, want ErrUnknownCheckpoint", err) } mB.ckptTTL = time.Nanosecond @@ -392,7 +392,7 @@ func TestDeleteCheckpointCannotBrickArchive(t *testing.T) { if ckpts, err := m.Checkpoints(t.Context(), ""); err != nil || len(ckpts) != 0 { t.Fatalf("Checkpoints listed %d records, want the archive image hidden (%v)", len(ckpts), err) } - if err := m.DeleteCheckpoint(t.Context(), ck, ""); !errors.Is(err, ErrUnknownCheckpoint) { + if err := m.DeleteCheckpoint(t.Context(), ck, "", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Fatalf("DeleteCheckpoint(archive ck) = %v, want ErrUnknownCheckpoint", err) } if !ckExists(t, m, ck) { diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 3bee649..3b513a3 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -16,9 +16,20 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +// DeleteScope selects how far a checkpoint delete reaches. The fleet-wide +// value is the zero one: a caller that does not think about scope gets the +// safe behavior, and the narrower local-only delete must be asked for. +type DeleteScope int + +const ( + DeleteFleet DeleteScope = iota + DeleteLocal +) + var ( ErrBadName = errors.New("invalid checkpoint name") ErrUnknownCheckpoint = errors.New("unknown checkpoint") + ErrHealBusy = errors.New("too many checkpoint heals in flight") ) // Checkpoint captures a claimed sandbox's state under a fresh id; the source @@ -89,16 +100,18 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID // attributed to tenant, not the checkpoint's recorder — the unguessable id // is the capability to branch. func (m *Manager) ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) { - if err := m.overQuota(1, tenant); err != nil { - return nil, err - } // Reject a bad or unknown id before recLock: a rejected id must not leave // a lock-map entry (only a delete evicts one). Checkpoints are immutable, - // so this parse stands in for the fetched meta below. + // so this parse stands in for the fetched meta below. The local read is + // cheap, so it goes before quota: a full node must still answer "not + // here" on a miss, or the handler's redirect/heal tiers never run. ckpt, err := m.loadCheckpoint(ctx, ckptID) if err != nil { return nil, err } + if err := m.overQuota(1, tenant); err != nil { + return nil, err + } return m.claimLoaded(ctx, ckpt, ttl, tenant) } @@ -106,13 +119,16 @@ func (m *Manager) ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.D // pulling it from a peer first. The server calls it only after the local // claim missed and a redirect could not answer. func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) { - if m.healer == nil { + if m.healer == nil || !store.CheckpointIDRe.MatchString(ckptID) { return nil, ErrUnknownCheckpoint } + // Unlike ClaimCheckpoint, resolving the record here means a peer + // transfer of a whole guest memory image: quota must reject a full node + // before paying that cost, not after. if err := m.overQuota(1, tenant); err != nil { return nil, err } - ckpt, err := m.loadCheckpointFrom(ctx, m.healer, ckptID) + ckpt, err := m.healCheckpoint(ctx, ckptID) if err != nil { return nil, err } @@ -153,6 +169,78 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl ti return out, err } +// healCheckpoint pulls ckptID from a peer under its record lock, so the +// transfer, validation, and publish race no concurrent heal, delete, or TTL +// sweep of the same id. A record already local when the lock is acquired +// (another heal won the race) is used as-is. claimLoaded takes its own +// RLock on the same recLock, so this releases before returning instead of +// holding it into the claim; a delete racing in between is handled there +// (Fetch answers store.ErrNotFound). +func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { + l := m.recLock(ckptID) + l.Lock() + defer l.Unlock() + if ckpt, err := m.loadCheckpoint(ctx, ckptID); err == nil { + return ckpt, nil + } + select { + case m.healSem <- struct{}{}: + defer func() { <-m.healSem }() + default: + return types.Checkpoint{}, ErrHealBusy + } + staging, err := m.ckpts.Stage(ckptID) + if err != nil { + return types.Checkpoint{}, fmt.Errorf("stage healed checkpoint: %w", err) + } + defer func() { _ = os.RemoveAll(staging) }() + validate := func(dir string) error { return validateHealedCheckpoint(dir, ckptID) } + if err := m.healer.Pull(ctx, ckptID, staging, validate); err != nil { + if errors.Is(err, store.ErrNotFound) { + return types.Checkpoint{}, ErrUnknownCheckpoint + } + return types.Checkpoint{}, fmt.Errorf("heal checkpoint: %w", err) + } + // A client hanging up must not abort a transfer that already landed. + if err := m.ckpts.Publish(context.WithoutCancel(ctx), staging, ckptID); err != nil { + return types.Checkpoint{}, fmt.Errorf("publish healed checkpoint: %w", err) + } + return m.loadCheckpoint(ctx, ckptID) +} + +// validateHealedCheckpoint checks a staged pull's shape before it is trusted +// enough to publish: a hostile or buggy peer must not be able to install an +// unreadable or misattributed record that then suppresses future healing. +func validateHealedCheckpoint(staging, wantID string) error { + raw, err := os.ReadFile(filepath.Join(staging, store.MetaFile)) //nolint:gosec // staging dir is this manager's own + if err != nil { + return fmt.Errorf("read healed meta: %w", err) + } + ckpt, err := parseCheckpoint(raw) + if err != nil { + return fmt.Errorf("parse healed meta: %w", err) + } + if ckpt.ID != wantID { + return fmt.Errorf("healed record id %q does not match requested %q", ckpt.ID, wantID) + } + if ckpt.Archive { + return fmt.Errorf("healed record %s is a wake image, not a checkpoint", wantID) + } + if !dirExists(filepath.Join(staging, store.ExportDir)) { + return fmt.Errorf("healed record %s missing export dir", wantID) + } + entries, err := os.ReadDir(staging) + if err != nil { + return fmt.Errorf("read staging: %w", err) + } + for _, e := range entries { + if e.Name() != store.MetaFile && e.Name() != store.ExportDir { + return fmt.Errorf("healed record %s has unexpected entry %q", wantID, e.Name()) + } + } + return nil +} + // Checkpoints lists the store's checkpoints, newest first — on a shared // backend (a FUSE mount, a bucket), that is the cluster's set, not one // node's. A non-empty tenant filters to that tenant's records; empty (root) @@ -197,10 +285,13 @@ func (m *Manager) pinnedArchiveCks() map[string]struct{} { return pinned } -// DeleteCheckpoint removes a checkpoint's snapshot and record. A tenant may -// delete only its own records — anything else answers ErrUnknownCheckpoint, -// never a hint that the id exists; root (empty tenant) deletes anything. -func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string) error { +// DeleteCheckpoint removes a checkpoint's snapshot and record. A fleet-scoped +// delete then best-effort broadcasts to peers, so a copy healed onto one does +// not outlive it; a forwarded delete is DeleteLocal, since re-forwarding would +// loop the fleet forever. A tenant may delete only its own records — anything +// else answers ErrUnknownCheckpoint, never a hint that the id exists; root +// (empty tenant) deletes anything. +func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope DeleteScope) error { // Validate off the lock — a rejected id must not leave a lock-map entry; // loadCheckpoint is safe unlocked (checkpoints are single-publish, immutable). ckpt, err := m.loadCheckpoint(ctx, ckptID) @@ -218,6 +309,11 @@ func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string) e if err := m.deleteCkLocked(ctx, ckptID); err != nil { return fmt.Errorf("delete checkpoint: %w", err) } + // A shared backend (s3) has no per-node replicas to chase: the delete + // above already removed the one copy every node sees. + if scope == DeleteFleet && m.peerDelete != nil && !m.ckptsShared { + m.peerDelete(context.WithoutCancel(ctx), ckptID) + } return nil } @@ -264,16 +360,10 @@ func (m *Manager) deleteCkLocked(ctx context.Context, ckID string) error { // loadCheckpoint reads and parses a checkpoint's meta from the local store. func (m *Manager) loadCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { - return m.loadCheckpointFrom(ctx, m.ckpts, ckptID) -} - -// loadCheckpointFrom reads and parses a checkpoint's meta from st — the local -// store for an ordinary claim, or the healer for ClaimCheckpointHeal. -func (m *Manager) loadCheckpointFrom(ctx context.Context, st store.Store, ckptID string) (types.Checkpoint, error) { if !store.CheckpointIDRe.MatchString(ckptID) { return types.Checkpoint{}, ErrUnknownCheckpoint } - raw, err := st.ReadMeta(ctx, ckptID) + raw, err := m.ckpts.ReadMeta(ctx, ckptID) if errors.Is(err, store.ErrNotFound) { return types.Checkpoint{}, ErrUnknownCheckpoint } diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index 6a52048..8bf7784 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -71,7 +71,9 @@ func TestClaimCheckpointAfterHealStaysLocal(t *testing.T) { } // TestClaimCheckpointHealDedupsConcurrentPulls: two branches racing to heal -// the same missing checkpoint must share one transfer, not each pay for it. +// the same missing checkpoint must share one transfer, not each pay for it — +// the id's recLock now serializes them, so the second waits for the first's +// publish and never reaches the puller at all. func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -87,7 +89,7 @@ func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { }) } waitFor(t, func() bool { return puller.count() >= 1 }) - time.Sleep(50 * time.Millisecond) // let the second goroutine join the same flight + time.Sleep(50 * time.Millisecond) // let the second goroutine block on the recLock close(puller.release) wg.Wait() @@ -141,6 +143,196 @@ func TestHasCheckpoint(t *testing.T) { } } +// TestClaimCheckpointQuotaDoesNotMaskMiss: a full node must still answer +// "not here" for a checkpoint it does not hold, or the handler's +// redirect/heal tiers behind ErrUnknownCheckpoint never run; a full node +// that DOES hold the record still answers the quota error. +func TestClaimCheckpointQuotaDoesNotMaskMiss(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + src := mustClaim(t, m, testKey) + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + m.draining = true + + if _, err := m.ClaimCheckpoint(t.Context(), "ck_00000000000000ff", time.Hour, ""); !errors.Is(err, ErrUnknownCheckpoint) { + t.Errorf("claim missing id while draining: %v, want ErrUnknownCheckpoint (so the handler can redirect)", err) + } + if _, err := m.ClaimCheckpoint(t.Context(), ckpt.ID, time.Hour, ""); !errors.Is(err, ErrQuota) { + t.Errorf("claim present id while draining: %v, want ErrQuota", err) + } +} + +// TestClaimCheckpointHealQuotaBeforePull: unlike ClaimCheckpoint's cheap +// local read, resolving a heal means a peer transfer — quota must reject a +// full node before that cost, so the puller must never be called. +func TestClaimCheckpointHealQuotaBeforePull(t *testing.T) { + id := "ck_00000000000000bb" + ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} + m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) + m.draining = true + + if _, err := m.ClaimCheckpointHeal(t.Context(), id, time.Hour, ""); !errors.Is(err, ErrQuota) { + t.Errorf("heal while draining: %v, want ErrQuota", err) + } + if got := puller.count(); got != 0 { + t.Errorf("puller called %d times, want 0: quota must reject before the pull", got) + } +} + +// TestClaimCheckpointHealConcurrencyCapRejectsExtra: heals of DIFFERENT ids +// (so the recLock does not serialize them) must still be capped node-wide — +// each pulls a whole guest memory image, so distinct ids are not free just +// because they don't contend on the same lock. +func TestClaimCheckpointHealConcurrencyCapRejectsExtra(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + m.healSem = make(chan struct{}, 1) + puller := &blockingPuller{started: make(chan struct{}), release: make(chan struct{})} + m.healer = peer.NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) + + done := make(chan error, 1) + go func() { + _, err := m.ClaimCheckpointHeal(t.Context(), "ck_00000000000000aa", time.Hour, "") + done <- err + }() + <-puller.started + + if _, err := m.ClaimCheckpointHeal(t.Context(), "ck_00000000000000bb", time.Hour, ""); !errors.Is(err, ErrHealBusy) { + t.Errorf("second heal while one is in flight: %v, want ErrHealBusy", err) + } + close(puller.release) + if err := <-done; err != nil { + t.Errorf("first heal: %v", err) + } +} + +// TestValidateHealedCheckpointAcceptsValid: a well-formed staged record must +// pass every check. +func TestValidateHealedCheckpointAcceptsValid(t *testing.T) { + dir := t.TempDir() + plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa"}) + if err := validateHealedCheckpoint(dir, "ck_00000000000000aa"); err != nil { + t.Errorf("validate: %v, want a valid record accepted", err) + } +} + +// TestValidateHealedCheckpointRejectsMismatchedID: a peer's meta.json must +// name the id that was actually requested, or a hostile peer could serve a +// crafted record under someone else's id. +func TestValidateHealedCheckpointRejectsMismatchedID(t *testing.T) { + dir := t.TempDir() + plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000bb"}) + if err := validateHealedCheckpoint(dir, "ck_00000000000000aa"); err == nil { + t.Error("validate accepted a mismatched id") + } +} + +// TestValidateHealedCheckpointRejectsMissingMeta: no meta.json means nothing +// to trust. +func TestValidateHealedCheckpointRejectsMissingMeta(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { + t.Fatalf("setup: %v", err) + } + if err := validateHealedCheckpoint(dir, "ck_00000000000000aa"); err == nil { + t.Error("validate accepted a record with no meta.json") + } +} + +// TestValidateHealedCheckpointRejectsMissingExport: a meta with no export +// dir cannot back a branch. +func TestValidateHealedCheckpointRejectsMissingExport(t *testing.T) { + dir := t.TempDir() + meta, err := json.Marshal(types.Checkpoint{ID: "ck_00000000000000aa"}) + if err != nil { + t.Fatalf("setup: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, store.MetaFile), meta, 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + if err := validateHealedCheckpoint(dir, "ck_00000000000000aa"); err == nil { + t.Error("validate accepted a record with no export dir") + } +} + +// TestValidateHealedCheckpointRejectsArchive: a wake image must never be +// healed in as a branchable checkpoint. +func TestValidateHealedCheckpointRejectsArchive(t *testing.T) { + dir := t.TempDir() + plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa", Archive: true}) + if err := validateHealedCheckpoint(dir, "ck_00000000000000aa"); err == nil { + t.Error("validate accepted an archive-flagged record") + } +} + +// TestDeleteCheckpointBroadcasts: a local delete calls the peer-delete hook +// once so a copy healed onto another node is removed too. +func TestDeleteCheckpointBroadcasts(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + src := mustClaim(t, m, testKey) + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + var calls []string + m.peerDelete = func(_ context.Context, id string) { calls = append(calls, id) } + + if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteFleet); err != nil { + t.Fatalf("DeleteCheckpoint: %v", err) + } + if len(calls) != 1 || calls[0] != ckpt.ID { + t.Errorf("peerDelete calls = %v, want one call for %s", calls, ckpt.ID) + } +} + +// TestDeleteCheckpointNoForwardSkipsBroadcast: a delete arriving as a +// broadcast (no_forward) must not re-broadcast, or the fleet would loop +// forever. +func TestDeleteCheckpointNoForwardSkipsBroadcast(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + src := mustClaim(t, m, testKey) + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + called := false + m.peerDelete = func(context.Context, string) { called = true } + + if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteLocal); err != nil { + t.Fatalf("DeleteCheckpoint: %v", err) + } + if called { + t.Error("peerDelete called despite no_forward") + } +} + +// TestDeleteCheckpointSharedStoreSkipsBroadcast: an s3 (or other shared) +// backend has one copy every node already sees, so broadcasting would be a +// no-op fan-out; DeleteCheckpoint must skip it. +func TestDeleteCheckpointSharedStoreSkipsBroadcast(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + src := mustClaim(t, m, testKey) + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + m.ckptsShared = true + called := false + m.peerDelete = func(context.Context, string) { called = true } + + if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteFleet); err != nil { + t.Fatalf("DeleteCheckpoint: %v", err) + } + if called { + t.Error("peerDelete called on a shared-store node") + } +} + // newHealManager builds a Manager with a healer wired to a puller serving // ckpt at addrs, mirroring WithPeerHeal's construction directly (tests are in // the same package). @@ -148,7 +340,7 @@ func newHealManager(t *testing.T, ckpt types.Checkpoint, addrs []string) (*Manag t.Helper() m := newTestManager(t, newFakeEngine()) puller := &healPuller{ckpt: ckpt} - m.healer = peer.New(m.ckpts, func(string) []string { return addrs }, puller) + m.healer = peer.NewHealer(func(string) []string { return addrs }, puller) return m, puller } @@ -184,3 +376,40 @@ func (p *healPuller) count() int { defer p.mu.Unlock() return p.calls } + +// blockingPuller writes a valid record stamped with the requested id, after +// signaling started and then blocking until release — for proving the +// node-wide heal concurrency cap against distinct, non-contending ids. +type blockingPuller struct { + started chan struct{} + release chan struct{} + once sync.Once +} + +func (p *blockingPuller) Pull(_ context.Context, _, id, dst string) error { + p.once.Do(func() { close(p.started) }) + <-p.release + if err := os.MkdirAll(filepath.Join(dst, store.ExportDir), 0o750); err != nil { + return err + } + meta, err := json.Marshal(types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()}) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dst, store.MetaFile), meta, 0o600) +} + +// plantHealedRecord writes a valid-shaped staged record for validate tests. +func plantHealedRecord(t *testing.T, dir string, ckpt types.Checkpoint) { + t.Helper() + if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { + t.Fatalf("setup: %v", err) + } + meta, err := json.Marshal(ckpt) + if err != nil { + t.Fatalf("setup: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, store.MetaFile), meta, 0o600); err != nil { + t.Fatalf("setup: %v", err) + } +} diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index 96f6388..4c93bd4 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -44,7 +44,7 @@ func TestCheckpointThenBranch(t *testing.T) { t.Errorf("Checkpoints() = %+v, %v; want the one record", ckpts, err) } - if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, ""); err != nil { + if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteFleet); err != nil { t.Fatalf("DeleteCheckpoint: %v", err) } if _, err := m.ClaimCheckpoint(t.Context(), ckpt.ID, time.Hour, ""); !errors.Is(err, ErrUnknownCheckpoint) { @@ -67,7 +67,7 @@ func TestCheckpointValidation(t *testing.T) { if _, err := m.ClaimCheckpoint(t.Context(), id, time.Hour, ""); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("claim %q: %v, want ErrUnknownCheckpoint", id, err) } - if err := m.DeleteCheckpoint(t.Context(), id, ""); !errors.Is(err, ErrUnknownCheckpoint) { + if err := m.DeleteCheckpoint(t.Context(), id, "", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("delete %q: %v, want ErrUnknownCheckpoint", id, err) } } @@ -148,13 +148,13 @@ func TestCheckpointTenantIsolation(t *testing.T) { } // A tenant cannot delete another's record — and learns nothing beyond 404. - if err := m.DeleteCheckpoint(t.Context(), ckA.ID, "beta"); !errors.Is(err, ErrUnknownCheckpoint) { + if err := m.DeleteCheckpoint(t.Context(), ckA.ID, "beta", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("cross-tenant delete: %v, want ErrUnknownCheckpoint", err) } - if err := m.DeleteCheckpoint(t.Context(), ckA.ID, "acme"); err != nil { + if err := m.DeleteCheckpoint(t.Context(), ckA.ID, "acme", DeleteFleet); err != nil { t.Errorf("own delete: %v", err) } - if err := m.DeleteCheckpoint(t.Context(), ckB.ID, ""); err != nil { + if err := m.DeleteCheckpoint(t.Context(), ckB.ID, "", DeleteFleet); err != nil { t.Errorf("root delete: %v", err) } } @@ -183,7 +183,7 @@ func TestCheckpointDeleteEvictsRecLock(t *testing.T) { ids = append(ids, ckpt.ID) } for _, id := range ids { - if err := m.DeleteCheckpoint(t.Context(), id, ""); err != nil { + if err := m.DeleteCheckpoint(t.Context(), id, "", DeleteFleet); err != nil { t.Fatalf("delete %s: %v", id, err) } if _, ok := m.recLocks.Load(id); ok { @@ -211,7 +211,7 @@ func TestRejectedCheckpointOpsLeakNoRecLock(t *testing.T) { if _, err := m.ClaimCheckpoint(t.Context(), id, time.Hour, ""); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("claim %q: %v, want ErrUnknownCheckpoint", id, err) } - if err := m.DeleteCheckpoint(t.Context(), id, ""); !errors.Is(err, ErrUnknownCheckpoint) { + if err := m.DeleteCheckpoint(t.Context(), id, "", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("delete %q: %v, want ErrUnknownCheckpoint", id, err) } } @@ -226,7 +226,7 @@ func TestRejectedCheckpointOpsLeakNoRecLock(t *testing.T) { t.Fatalf("checkpoint: %v", err) } before := countLocks() - if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "other"); !errors.Is(err, ErrUnknownCheckpoint) { + if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "other", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("wrong-tenant delete: %v, want ErrUnknownCheckpoint", err) } if got := countLocks(); got != before { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 395af49..cec5b98 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -50,6 +50,12 @@ const ( defaultMaxFork = 16 defaultRefill = 4 + // maxConcurrentHeals bounds node-wide in-flight checkpoint heals: each + // pulls a full guest memory image over the network, so a handful in + // flight already saturates a node's disk and NIC; further ones queue + // for nothing. + maxConcurrentHeals = 4 + vmPrefix = "sbx-" goldenPrefix = "sbx-golden-" hibernatePrefix = "sbx-hib-" @@ -169,6 +175,10 @@ func (p *pool) trimWarm(target int) []string { return trim } +// PeerDeleteFunc broadcasts a checkpoint delete to every peer; wired via +// WithPeerDelete, nil means single node. +type PeerDeleteFunc func(ctx context.Context, id string) + // Manager owns the node's pools, claims, and their persistence. type Manager struct { eng Engine @@ -226,10 +236,17 @@ type Manager struct { // every node already resolves every record, so peer heal and its gossip // set are no-ops. ckptsShared bool - // healer is the checkpoint-only read path ClaimCheckpointHeal pulls - // through; nil unless checkpoint_peer_heal wired it. m.ckpts itself is - // never wrapped, so the server's redirect branch stays reachable. - healer store.Store + // healer pulls a checkpoint this node does not hold from a peer; + // ClaimCheckpointHeal is its only caller. nil unless checkpoint_peer_heal + // wired it. m.ckpts itself is never wrapped, so the server's redirect + // branch stays reachable. + healer *peer.Healer + // healSem bounds node-wide in-flight heals to maxConcurrentHeals. + healSem chan struct{} + // peerDelete broadcasts a checkpoint delete to every peer after a + // successful local delete, so a healed replica does not outlive the + // source record; nil means single node (no mesh). + peerDelete PeerDeleteFunc tpls store.Store ckptTTL time.Duration ckptSweeping atomic.Bool @@ -305,6 +322,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg refillSem: make(chan struct{}, refill), probeSem: make(chan struct{}, refill), refillKick: make(chan struct{}, 1), + healSem: make(chan struct{}, maxConcurrentHeals), } if err := os.MkdirAll(m.goldensDir(), 0o750); err != nil { return nil, fmt.Errorf("create goldens dir: %w", err) @@ -576,7 +594,14 @@ func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { if !enabled || owners == nil || m.ckptsShared { return } - m.healer = peer.New(m.ckpts, owners, &peer.HTTPPuller{Token: token}) + m.healer = peer.NewHealer(owners, &peer.HTTPPuller{Token: token}) +} + +// WithPeerDelete wires the fleet-wide delete broadcast: DeleteCheckpoint calls +// fn after a successful local delete so a checkpoint healed onto a peer does +// not outlive the source record. Nil (the default) keeps delete single-node. +func (m *Manager) WithPeerDelete(fn PeerDeleteFunc) { + m.peerDelete = fn } func dirExists(path string) bool { diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 9e0d553..83ee48b 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -767,7 +767,8 @@ type fakeEngine struct { caInstalls []string // vsock sockets InstallCACert was called on installCAErr error stopped map[string]bool - vsockLateN int // List calls that report no socket yet + pids map[string]int // VM name → PID, for Stats' resident-set lookup + vsockLateN int // List calls that report no socket yet cloneErr, runColdErr, probeErr, hibernateErr, restoreErr, snapSaveErr, snapListErr error removeErrFor string // VM name whose Remove fails; "" = never @@ -786,7 +787,7 @@ type fakeEngine struct { } func newFakeEngine() *fakeEngine { - return &fakeEngine{vms: map[string]string{}, stopped: map[string]bool{}} + return &fakeEngine{vms: map[string]string{}, stopped: map[string]bool{}, pids: map[string]int{}} } func (f *fakeEngine) Clone(_ context.Context, fromDir, name string, _ types.PoolKey) (types.VMRecord, error) { @@ -981,7 +982,7 @@ func (f *fakeEngine) List(_ context.Context, filters ...string) ([]types.VMRecor if f.stopped[name] { state = "stopped" } - rec := types.VMRecord{State: state, VsockSocket: sock, Config: types.VMConfig{Name: name}} + rec := types.VMRecord{State: state, PID: f.pids[name], VsockSocket: sock, Config: types.VMConfig{Name: name}} if f.tap != "" { rec.NetworkConfigs = []types.VMNetConfig{{TAP: f.tap}} } diff --git a/sandboxd/pool/stats.go b/sandboxd/pool/stats.go index 8698c7d..1e2b633 100644 --- a/sandboxd/pool/stats.go +++ b/sandboxd/pool/stats.go @@ -1,11 +1,15 @@ package pool import ( + "context" "os" "path/filepath" + "slices" "strconv" "strings" "time" + + "github.com/cocoonstack/sandbox/sandboxd/types" ) // SandboxStats is one sandbox's resource usage. CPUCount and MemTotalBytes come @@ -24,7 +28,7 @@ type SandboxStats struct { } // Stats reports one live claim's resource usage. -func (m *Manager) Stats(id string) (SandboxStats, bool) { +func (m *Manager) Stats(ctx context.Context, id string) (SandboxStats, bool) { sb, ok := m.byID(id) if !ok { return SandboxStats{}, false @@ -38,43 +42,34 @@ func (m *Manager) Stats(id string) (SandboxStats, bool) { MeasuredAt: time.Now().UTC(), } if !st.Hibernated { - if rss, ok := vmmResidentBytes(sb.VMName); ok { + if rss, ok := m.vmResidentBytes(ctx, sb.VMName); ok { st.MemUsedBytes, st.MemUsedMeasured = rss, true } } return st, true } -// vmmResidentBytes reports the resident set of the hypervisor process serving -// vmName, matched on the run directory in the VMM's argv. The scan is -// O(processes), so this is a single-sandbox read, never a fleet sweep. -func vmmResidentBytes(vmName string) (int64, bool) { +// vmResidentBytes reports the resident set of the hypervisor process serving +// vmName, read straight from engine.List's VMRecord.PID rather than scanning +// /proc for a matching argv. +func (m *Manager) vmResidentBytes(ctx context.Context, vmName string) (int64, bool) { if vmName == "" { return 0, false } - entries, err := os.ReadDir("/proc") + vms, err := m.eng.List(ctx, vmName) if err != nil { return 0, false } - for _, e := range entries { - pid := e.Name() - if !e.IsDir() || pid[0] < '0' || pid[0] > '9' { - continue - } - cmdline, err := os.ReadFile(filepath.Join("/proc", pid, "cmdline")) //nolint:gosec // pid enumerated from /proc - if err != nil || !strings.Contains(string(cmdline), vmName) { - continue - } - if rss, ok := residentBytes(pid); ok { - return rss, true - } + i := slices.IndexFunc(vms, func(vm types.VMRecord) bool { return vm.Config.Name == vmName }) + if i < 0 || vms[i].PID <= 0 { + return 0, false } - return 0, false + return residentBytes(vms[i].PID) } // residentBytes reads a process's resident page count from statm's second field. -func residentBytes(pid string) (int64, bool) { - b, err := os.ReadFile(filepath.Join("/proc", pid, "statm")) //nolint:gosec // pid enumerated from /proc +func residentBytes(pid int) (int64, bool) { + b, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "statm")) //nolint:gosec // pid comes from cocoon's own VM record if err != nil { return 0, false } diff --git a/sandboxd/pool/stats_test.go b/sandboxd/pool/stats_test.go new file mode 100644 index 0000000..f8fb112 --- /dev/null +++ b/sandboxd/pool/stats_test.go @@ -0,0 +1,85 @@ +package pool + +import ( + "os" + "runtime" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" +) + +// TestStatsUnknownSandbox: an unclaimed or bad id must not surface any usage. +func TestStatsUnknownSandbox(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + if _, ok := m.Stats(t.Context(), "sb_missing"); ok { + t.Error("Stats on an unknown id, want ok=false") + } +} + +// TestStatsHibernatedIsUnmeasured: a hibernated claim has no VMM process, so +// MemUsedMeasured must stay false rather than reading a stale or zero RSS — +// and Stats must not even ask the engine, which a hibernated claim can't answer. +func TestStatsHibernatedIsUnmeasured(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) + sb := mustClaim(t, m, testKey) + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("hibernate: %v", err) + } + before := eng.listCalls() + + st, ok := m.Stats(t.Context(), sb.ID) + if !ok { + t.Fatal("Stats: want ok=true for a claimed sandbox") + } + if !st.Hibernated { + t.Error("Hibernated = false, want true") + } + if st.MemUsedMeasured { + t.Error("MemUsedMeasured = true for a hibernated claim, want false") + } + if got := eng.listCalls(); got != before { + t.Errorf("engine.List called %d more time(s), want 0: a hibernated claim must not consult the engine", got-before) + } +} + +// TestStatsNoPIDIsUnmeasured: engine.List answering with no PID (VM not found, +// or not yet running) must not be read as a zero-byte RSS. +func TestStatsNoPIDIsUnmeasured(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) + sb := mustClaim(t, m, testKey) + + st, ok := m.Stats(t.Context(), sb.ID) + if !ok { + t.Fatal("Stats: want ok=true for a claimed sandbox") + } + if st.MemUsedMeasured { + t.Error("MemUsedMeasured = true with no PID in the engine's record, want false") + } +} + +// TestStatsReadsResidentSetFromPID proves the PID engine.List reports is the +// one actually read: pointing it at this test process's own PID, the +// resident-set read must succeed and land on a real value. /proc is +// Linux-only, so this is skipped elsewhere. +func TestStatsReadsResidentSetFromPID(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("/proc/PID/statm is Linux-only") + } + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) + sb := mustClaim(t, m, testKey) + eng.pids[sb.VMName] = os.Getpid() + + st, ok := m.Stats(t.Context(), sb.ID) + if !ok { + t.Fatal("Stats: want ok=true for a claimed sandbox") + } + if !st.MemUsedMeasured { + t.Fatal("MemUsedMeasured = false, want true with a live PID") + } + if st.MemUsedBytes <= 0 { + t.Errorf("MemUsedBytes = %d, want > 0", st.MemUsedBytes) + } +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index ed06d1d..26bb784 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -46,6 +46,7 @@ var poolErrHTTP = []struct { {pool.ErrNoEgressHibernate, http.StatusConflict, ""}, {pool.ErrNoEgressFork, http.StatusConflict, ""}, {pool.ErrQuota, http.StatusTooManyRequests, ""}, + {pool.ErrHealBusy, http.StatusServiceUnavailable, ""}, {pool.ErrPooledTemplate, http.StatusConflict, ""}, {pool.ErrTemplateOwned, http.StatusConflict, ""}, {pool.ErrUnknownSandbox, http.StatusNotFound, "unknown sandbox"}, @@ -70,7 +71,7 @@ type Manager interface { TenantClaims() map[string]int Sandboxes() []pool.SandboxSummary Sandbox(id string) (pool.SandboxSummary, bool) - Stats(id string) (pool.SandboxStats, bool) + Stats(ctx context.Context, id string) (pool.SandboxStats, bool) Audit(ctx context.Context, id string, line []byte) AuditEnabled() bool ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) @@ -78,7 +79,7 @@ type Manager interface { Checkpoints(ctx context.Context, tenant string) ([]types.Checkpoint, error) HasCheckpoint(ctx context.Context, ckptID string) bool FetchCheckpoint(ctx context.Context, ckptID string) (dir string, meta []byte, release func(), err error) - DeleteCheckpoint(ctx context.Context, ckptID, tenant string) error + DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope pool.DeleteScope) error ClaimDeadline(id, token string) (time.Time, error) HasGolden(ctx context.Context, key types.PoolKey) bool AgentSocket(id, token string) (string, error) @@ -188,7 +189,10 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /v1/checkpoints", s.requireToken(s.handleListCheckpoints)) // The peer-transfer half of the snapshot placement design: a node that // cannot redirect a branch pulls the record from an owner through this. - mux.HandleFunc("GET /v1/checkpoints/{id}/blob", s.requireToken(s.handleCheckpointBlob)) + // It streams a full checkpoint (guest memory + disk) with no tenant + // scoping, so only the fleet's root api_token may call it — its one + // intended caller is the peer puller, authenticated as the operator. + mux.HandleFunc("GET /v1/checkpoints/{id}/blob", s.requireRoot(s.handleCheckpointBlob)) // HEAD is the probe endpoint a redirect decision fans out to: it must stay // unauthenticated, so it is registered outside requireToken. mux.HandleFunc("HEAD /v1/checkpoints/{id}/blob", s.handleCheckpointProbe) @@ -293,7 +297,7 @@ func (s *Server) handleSandbox(w http.ResponseWriter, r *http.Request) { // per-sandbox usage surface: /metrics is node- and pool-scoped by design. func (s *Server) handleSandboxStats(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") - st, ok := s.mgr.Stats(id) + st, ok := s.mgr.Stats(r.Context(), id) if !ok { writeErr(w, http.StatusNotFound, "unknown sandbox") return @@ -436,9 +440,15 @@ func (s *Server) handleListCheckpoints(w http.ResponseWriter, r *http.Request) { } // handleDeleteCheckpoint removes a checkpoint; a tenant caller may delete -// only its own records (anything else is 404). +// only its own records (anything else is 404). no_forward marks a delete +// arriving from another node's own broadcast, so this one does not +// re-broadcast and loop the fleet forever. func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) { - err := s.mgr.DeleteCheckpoint(r.Context(), r.PathValue("id"), tenantFrom(r.Context())) + scope := pool.DeleteFleet + if r.URL.Query().Get("no_forward") != "" { + scope = pool.DeleteLocal + } + err := s.mgr.DeleteCheckpoint(r.Context(), r.PathValue("id"), tenantFrom(r.Context()), scope) writeResult(w, r, "delete checkpoint", r.PathValue("id"), "delete checkpoint failed", err, func() { w.WriteHeader(http.StatusNoContent) }) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index fe61762..a0403aa 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -184,6 +184,7 @@ func TestTenantAuthMatrix(t *testing.T) { {"tenant forbidden on index", http.MethodGet, "/v1/sandboxes", "acme-tok", http.StatusForbidden, ""}, {"tenant forbidden on metrics", http.MethodGet, "/metrics", "acme-tok", http.StatusForbidden, ""}, {"tenant forbidden on pools", http.MethodPut, "/v1/pools", "acme-tok", http.StatusForbidden, ""}, + {"tenant forbidden on checkpoint blob", http.MethodGet, "/v1/checkpoints/ck_00000000000000aa/blob", "acme-tok", http.StatusForbidden, ""}, {"wrong token on info stays 401", http.MethodGet, "/v1/info", "nope", http.StatusUnauthorized, ""}, {"root reads info", http.MethodGet, "/v1/info", "sekret", http.StatusOK, ""}, } { @@ -707,6 +708,39 @@ func TestCheckpointFlow(t *testing.T) { } } +// TestDeleteCheckpointNoForwardQueryParam proves the no_forward query param +// reaches the manager, mirroring how handleDeleteTemplate already threads +// no_redirect: a delete arriving from another node's broadcast must not +// re-broadcast, or the fleet loops the delete forever. +func TestDeleteCheckpointNoForwardQueryParam(t *testing.T) { + mgr := &fakeManager{deleteCheckpoint: func(string) error { return nil }} + ts := newTestServer(t, "api", mgr, &fakeDialer{}) + + del := func(path string) int { + req, _ := http.NewRequest(http.MethodDelete, ts.URL+path, nil) + req.Header.Set("Authorization", "Bearer api") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("delete: %v", err) + } + defer resp.Body.Close() + return resp.StatusCode + } + + if got := del("/v1/checkpoints/ck_0011223344556677"); got != http.StatusNoContent { + t.Fatalf("delete: status %d, want 204", got) + } + if mgr.gotNoForward { + t.Error("gotNoForward = true without the query param") + } + if got := del("/v1/checkpoints/ck_0011223344556677?no_forward=1"); got != http.StatusNoContent { + t.Fatalf("delete with no_forward: status %d, want 204", got) + } + if !mgr.gotNoForward { + t.Error("gotNoForward = false with no_forward=1 set") + } +} + func TestClaimRedirectsOnWarmMiss(t *testing.T) { // Warm miss (fakeManager.ClaimWarm always misses) + a placer with // candidates → a redirect response, and the local manager never provisions. @@ -1269,6 +1303,7 @@ type fakeManager struct { gotTenant string gotClaimRef string + gotNoForward bool tenantClaims map[string]int draining bool } @@ -1374,7 +1409,9 @@ func (f *fakeManager) Sandbox(string) (pool.SandboxSummary, bool) { return pool.SandboxSummary{}, false } -func (f *fakeManager) Stats(string) (pool.SandboxStats, bool) { return pool.SandboxStats{}, false } +func (f *fakeManager) Stats(context.Context, string) (pool.SandboxStats, bool) { + return pool.SandboxStats{}, false +} func (f *fakeManager) Wake(_ context.Context, id string, cred pool.Cred) error { if f.wake == nil { @@ -1421,8 +1458,9 @@ func (f *fakeManager) Checkpoints(_ context.Context, tenant string) ([]types.Che return f.checkpoints, nil } -func (f *fakeManager) DeleteCheckpoint(_ context.Context, ckptID, tenant string) error { +func (f *fakeManager) DeleteCheckpoint(_ context.Context, ckptID, tenant string, scope pool.DeleteScope) error { f.gotTenant = tenant + f.gotNoForward = scope == pool.DeleteLocal if f.deleteCheckpoint == nil { return pool.ErrUnknownCheckpoint } diff --git a/sandboxd/store/peer/broadcast.go b/sandboxd/store/peer/broadcast.go new file mode 100644 index 0000000..38409ab --- /dev/null +++ b/sandboxd/store/peer/broadcast.go @@ -0,0 +1,84 @@ +package peer + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/projecteru2/core/log" +) + +const ( + deleteTimeout = 5 * time.Second + defaultDeleteClientTimeout = 10 * time.Second +) + +// Broadcaster fans a checkpoint delete out to every peer after a local +// delete, so a record healed onto another node does not outlive the source. +type Broadcaster struct { + // A nil Client uses a default with a short timeout: one wedged peer must + // not hold up the fan-out. + Client *http.Client + Peers func() []string + Token string +} + +// Delete sends DELETE ?no_forward=1 to every peer in parallel. Failures are +// logged and never returned: a broadcast is best-effort and must never fail +// the local delete it follows. +func (b *Broadcaster) Delete(ctx context.Context, id string) { + addrs := dedupAddrs(b.Peers()) + if len(addrs) == 0 { + return + } + client := b.Client + if client == nil { + client = &http.Client{Timeout: defaultDeleteClientTimeout} + } + logger := log.WithFunc("peer.Broadcaster.Delete") + var wg sync.WaitGroup + for _, addr := range addrs { + wg.Go(func() { + if err := deleteOn(ctx, client, addr, id, b.Token); err != nil { + logger.Warnf(ctx, "broadcast delete %s to %s: %v", id, addr, err) + } + }) + } + wg.Wait() +} + +func deleteOn(ctx context.Context, client *http.Client, addr, id, token string) error { + ctx, cancel := context.WithTimeout(ctx, deleteTimeout) + defer cancel() + + base := addr + if !strings.Contains(base, "://") { + base = "http://" + base + } + u := base + "/v1/checkpoints/" + url.PathEscape(id) + "?no_forward=1" + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, u, nil) + if err != nil { + return err + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := client.Do(req) //nolint:gosec // addr comes from the mesh's own member view + if err != nil { + return err + } + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + // 404 means the peer never healed a copy — not a failure worth a retry. + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound { + return fmt.Errorf("unexpected status %d", resp.StatusCode) + } + return nil +} diff --git a/sandboxd/store/peer/broadcast_test.go b/sandboxd/store/peer/broadcast_test.go new file mode 100644 index 0000000..0797f6b --- /dev/null +++ b/sandboxd/store/peer/broadcast_test.go @@ -0,0 +1,95 @@ +package peer + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +// TestBroadcastDeleteSendsNoForward: a broadcast delete must carry +// no_forward=1 so the receiving peer's own delete does not re-broadcast, +// which would loop forever across the fleet. +func TestBroadcastDeleteSendsNoForward(t *testing.T) { + var gotNoForward, gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotNoForward = r.URL.Query().Get("no_forward") + gotAuth = r.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + b := &Broadcaster{Peers: func() []string { return []string{srv.URL} }, Token: "fleet-token"} + b.Delete(t.Context(), testID) + + if gotNoForward != "1" { + t.Errorf("no_forward = %q, want 1", gotNoForward) + } + if gotAuth != "Bearer fleet-token" { + t.Errorf("Authorization = %q, want the fleet token", gotAuth) + } +} + +// TestBroadcastDeleteFansOutToEveryPeer: every distinct peer must be +// contacted, not just the first. +func TestBroadcastDeleteFansOutToEveryPeer(t *testing.T) { + var hits atomic.Int32 + newSrv := func() *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + return srv + } + addrs := []string{newSrv().URL, newSrv().URL, newSrv().URL} + + b := &Broadcaster{Peers: func() []string { return addrs }} + b.Delete(t.Context(), testID) + + if got := hits.Load(); got != 3 { + t.Errorf("peers hit %d times, want 3 (one per distinct peer)", got) + } +} + +// TestBroadcastDeleteDedupsRepeatedAddrs: a duplicated peer list must not +// double-broadcast to the same address. +func TestBroadcastDeleteDedupsRepeatedAddrs(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + b := &Broadcaster{Peers: func() []string { return []string{srv.URL, srv.URL, srv.URL} }} + b.Delete(t.Context(), testID) + + if got := hits.Load(); got != 1 { + t.Errorf("server hit %d times, want 1 (dupes deduped)", got) + } +} + +// TestBroadcastDeleteSwallowsFailures: a broadcast never fails the local +// delete it follows, so an unreachable or erroring peer must not panic or +// block the caller past the fan-out. +func TestBroadcastDeleteSwallowsFailures(t *testing.T) { + fail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer fail.Close() + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer ok.Close() + + b := &Broadcaster{Peers: func() []string { return []string{fail.URL, ok.URL, "127.0.0.1:1"} }} + b.Delete(t.Context(), testID) // must return without panicking regardless of peer outcomes +} + +// TestBroadcastDeleteNoPeersNoOp: a single-node deployment must not dial +// anything. +func TestBroadcastDeleteNoPeersNoOp(t *testing.T) { + b := &Broadcaster{Peers: func() []string { return nil }} + b.Delete(t.Context(), testID) +} diff --git a/sandboxd/store/peer/peer.go b/sandboxd/store/peer/peer.go index 87ecd1c..15dbf50 100644 --- a/sandboxd/store/peer/peer.go +++ b/sandboxd/store/peer/peer.go @@ -1,94 +1,102 @@ package peer import ( + "cmp" "context" "errors" "fmt" "os" + "path/filepath" + "time" "golang.org/x/sync/singleflight" "github.com/cocoonstack/sandbox/sandboxd/store" ) +// healBudget bounds one Pull across every owner tried, replacing the old +// per-owner-only bound: a heal that tried N owners at 30 minutes each could +// run for N*30 minutes with nothing else limiting it. +const healBudget = 30 * time.Minute + // Owners resolves a record id to the peer addresses that hold it — the mesh's -// gossiped view, injected so the store stays testable without a live cluster. +// gossiped view, injected so the healer stays testable without a live cluster. type Owners func(id string) []string -var _ store.Store = (*Store)(nil) +// Validate checks a staged pull before Pull trusts the owner that sent it; a +// non-nil error tries the next owner instead of failing the whole heal. The +// healer does not know the record's shape, so the caller supplies this. +type Validate func(staging string) error -// Store is a node-local record store that heals a Fetch/ReadMeta miss by -// pulling the record from a peer and publishing it locally, so the transfer is -// paid once: this node then owns the record and serves later reads locally. -type Store struct { - store.Store - owners Owners - puller Puller - flights singleflight.Group -} +// Healer pulls a record this node does not hold from whichever peer gossiped +// it, staging the transfer into a caller-provided directory; the caller +// validates and publishes it. A nil owners or puller leaves Pull always +// answering store.ErrNotFound, so a node with no mesh degrades to unable to +// heal rather than failing. +type Healer struct { + owners Owners + puller Puller -// New wraps local with peer healing. A nil owners or puller leaves the wrapper -// inert, so a node with no mesh degrades to the local backend rather than -// failing. -func New(local store.Store, owners Owners, puller Puller) *Store { - return &Store{Store: local, owners: owners, puller: puller} -} + // budget overrides healBudget when set; a test seam, since 30 minutes is + // not something a test should wait out. + budget time.Duration -// Fetch serves the record locally, healing from a peer on a miss. -func (s *Store) Fetch(ctx context.Context, id string) (string, []byte, func(), error) { - dir, meta, release, err := s.Store.Fetch(ctx, id) - if !errors.Is(err, store.ErrNotFound) { - return dir, meta, release, err - } - if err := s.heal(ctx, id); err != nil { - return "", nil, nil, err - } - return s.Store.Fetch(ctx, id) + flights singleflight.Group } -// ReadMeta reads the record's metadata, healing from a peer on a miss. -func (s *Store) ReadMeta(ctx context.Context, id string) ([]byte, error) { - meta, err := s.Store.ReadMeta(ctx, id) - if !errors.Is(err, store.ErrNotFound) { - return meta, err - } - if err := s.heal(ctx, id); err != nil { - return nil, err - } - return s.Store.ReadMeta(ctx, id) +// NewHealer builds a Healer wired to owners and puller. +func NewHealer(owners Owners, puller Puller) *Healer { + return &Healer{owners: owners, puller: puller} } -// heal pulls id from the first peer that serves it and publishes it locally, -// returning store.ErrNotFound when no peer has it so a miss stays a miss. -// Concurrent misses of the same id share one pull. -func (s *Store) heal(ctx context.Context, id string) error { - if s.owners == nil || s.puller == nil { +// Pull fetches id into staging from the first owner whose transfer validates, +// bounding the whole attempt (every owner) to one budget so a wedged or +// endlessly-invalid peer cannot starve the rest. Concurrent pulls of the same +// id share one flight. +func (h *Healer) Pull(ctx context.Context, id, staging string, validate Validate) error { + if h.owners == nil || h.puller == nil { return store.ErrNotFound } - addrs := s.owners(id) + addrs := h.owners(id) if len(addrs) == 0 { return store.ErrNotFound } - // A client hanging up must not abandon a started pull: the next branch - // would pay the whole transfer again. - ctx = context.WithoutCancel(ctx) - _, err, _ := s.flights.Do(id, func() (any, error) { - return nil, s.healFrom(ctx, id, addrs) + budget := cmp.Or(h.budget, healBudget) + // A client hanging up must not abandon a started pull: the budget below + // bounds it instead of the caller's ctx. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), budget) + defer cancel() + _, err, _ := h.flights.Do(id, func() (any, error) { + return nil, h.pullFrom(ctx, id, staging, addrs, budget, validate) }) return err } -// healFrom tries each owner in order until one serves id; callers hold the -// id's singleflight slot. -func (s *Store) healFrom(ctx context.Context, id string, addrs []string) error { +// pullFrom tries each owner in turn, giving each an even slice of budget; +// callers hold id's singleflight slot. +func (h *Healer) pullFrom(ctx context.Context, id, staging string, addrs []string, budget time.Duration, validate Validate) error { + perOwner := budget / time.Duration(len(addrs)) var errs []error for _, addr := range addrs { - if err := s.pullFrom(ctx, addr, id); err != nil { + if err := clearDir(staging); err != nil { + return fmt.Errorf("reset staging: %w", err) + } + attemptCtx, cancel := context.WithTimeout(ctx, perOwner) + err := h.puller.Pull(attemptCtx, addr, id, staging) + cancel() + if err != nil { if !errors.Is(err, ErrNotFound) { errs = append(errs, fmt.Errorf("peer %s: %w", addr, err)) } continue } + if validate == nil { + return nil + } + if err := validate(staging); err != nil { + errs = append(errs, fmt.Errorf("peer %s: %w", addr, err)) + continue + } return nil } if len(errs) > 0 { @@ -97,20 +105,18 @@ func (s *Store) healFrom(ctx context.Context, id string, addrs []string) error { return store.ErrNotFound // every owner answered not-found: stale gossip } -// pullFrom stages a peer's copy and publishes it, so the record becomes local -// through the same atomic path a locally-created one takes. -func (s *Store) pullFrom(ctx context.Context, addr, id string) error { - staging, err := s.Stage(id) +// clearDir empties dir's contents (keeping dir itself) between owner +// attempts, so a rejected or partial transfer from one peer cannot linger +// into the next's. +func clearDir(dir string) error { + entries, err := os.ReadDir(dir) if err != nil { - return fmt.Errorf("stage: %w", err) - } - defer func() { _ = os.RemoveAll(staging) }() - - if err := s.puller.Pull(ctx, addr, id, staging); err != nil { return err } - if err := s.Publish(ctx, staging, id); err != nil { - return fmt.Errorf("publish pulled record: %w", err) + for _, e := range entries { + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { + return err + } } return nil } diff --git a/sandboxd/store/peer/peer_test.go b/sandboxd/store/peer/peer_test.go index 73172ef..7b2e9ff 100644 --- a/sandboxd/store/peer/peer_test.go +++ b/sandboxd/store/peer/peer_test.go @@ -8,99 +8,74 @@ import ( "path/filepath" "sync" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/store" - "github.com/cocoonstack/sandbox/sandboxd/store/dir" - "github.com/cocoonstack/sandbox/sandboxd/store/storetest" ) const testID = "ck_00000000000000aa" -func TestPeerBackendContract(t *testing.T) { - storetest.RunContract(t, New(localStore(t), nil, nil)) -} - -// TestInertWithoutOwnersOrPuller: a node with no mesh must degrade to its local -// backend, not fail. Healing is an addition, never a dependency. +// TestInertWithoutOwnersOrPuller: a node with no mesh must degrade to unable +// to heal, not fail oddly. Healing is an addition, never a dependency. func TestInertWithoutOwnersOrPuller(t *testing.T) { - s := New(localStore(t), nil, nil) - if _, _, _, err := s.Fetch(t.Context(), testID); !errors.Is(err, store.ErrNotFound) { - t.Fatalf("Fetch error = %v, want store.ErrNotFound (a miss stays a miss)", err) + h := NewHealer(nil, nil) + if err := h.Pull(t.Context(), testID, t.TempDir(), nil); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("Pull error = %v, want store.ErrNotFound", err) } } -// TestHealPublishesLocally is L3's whole point: after healing once, the record -// is local, so the next branch is served at L1 speed and this node becomes an -// owner that can serve its peers. The transfer must be paid once, not per use. -func TestHealPublishesLocally(t *testing.T) { - local := localStore(t) +// TestPullWritesStaging is Pull's whole point: the record lands in the +// caller-provided staging directory for the caller to validate and publish. +func TestPullWritesStaging(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} - s := New(local, func(string) []string { return []string{"peer-a:7777"} }, puller) + h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) - dir1, _, release, err := s.Fetch(t.Context(), testID) - if err != nil { - t.Fatalf("Fetch after heal: %v", err) + dst := t.TempDir() + if err := h.Pull(t.Context(), testID, dst, nil); err != nil { + t.Fatalf("Pull: %v", err) } - release() - if got, err := os.ReadFile(filepath.Join(dir1, "mem")); err != nil || string(got) != "guest-pages" { - t.Fatalf("healed export = %q, %v; want the peer's bytes", got, err) - } - - // The record is now in the LOCAL backend: reading it directly, with no - // wrapper and no puller, must succeed. - if _, err := local.ReadMeta(t.Context(), testID); err != nil { - t.Fatalf("record not published locally after heal: %v", err) - } - // And a second Fetch must not touch the network again. - if _, _, release2, err := s.Fetch(t.Context(), testID); err != nil { - t.Fatalf("second Fetch: %v", err) - } else { - release2() - } - if len(puller.asked) != 1 { - t.Errorf("puller called %d times (%v); the transfer must be paid once", len(puller.asked), puller.asked) + if got, err := os.ReadFile(filepath.Join(dst, store.ExportDir, "mem")); err != nil || string(got) != "guest-pages" { + t.Fatalf("staged export = %q, %v; want the peer's bytes", got, err) } } -// TestHealTriesNextOwner: a gossiped view can be stale or a peer can be broken. -// One bad owner must not fail the heal while another can still serve it. +// TestHealTriesNextOwner: a gossiped view can be stale or a peer can be +// broken. One bad owner must not fail the pull while another can still +// serve it. func TestHealTriesNextOwner(t *testing.T) { puller := &fakePuller{ records: map[string]map[string]string{"peer-b:7777": record()}, failAddr: "peer-a:7777", } - s := New(localStore(t), func(string) []string { return []string{"peer-a:7777", "peer-b:7777"} }, puller) + h := NewHealer(func(string) []string { return []string{"peer-a:7777", "peer-b:7777"} }, puller) - _, _, release, err := s.Fetch(t.Context(), testID) - if err != nil { - t.Fatalf("Fetch: %v", err) + if err := h.Pull(t.Context(), testID, t.TempDir(), nil); err != nil { + t.Fatalf("Pull: %v", err) } - release() if len(puller.asked) != 2 { t.Errorf("asked = %v, want both owners tried in order", puller.asked) } } -// TestHealAllOwnersMissStaysNotFound: when every owner answers "not found" the -// gossiped view was simply stale. That must surface as store.ErrNotFound so the -// caller's existing not-found handling (a 404, not a 500) is unchanged. +// TestHealAllOwnersMissStaysNotFound: when every owner answers "not found" +// the gossiped view was simply stale. That must surface as store.ErrNotFound +// so the caller's existing not-found handling is unchanged. func TestHealAllOwnersMissStaysNotFound(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{}} - s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) + h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) - err := fetchErr(t, s) - if !errors.Is(err, store.ErrNotFound) { - t.Fatalf("Fetch error = %v, want store.ErrNotFound", err) + if err := h.Pull(t.Context(), testID, t.TempDir(), nil); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("Pull error = %v, want store.ErrNotFound", err) } } // TestHealNoOwners: nothing gossiped the record, so there is nobody to ask. func TestHealNoOwners(t *testing.T) { puller := &fakePuller{} - s := New(localStore(t), func(string) []string { return nil }, puller) + h := NewHealer(func(string) []string { return nil }, puller) - if _, err := s.ReadMeta(t.Context(), testID); !errors.Is(err, store.ErrNotFound) { - t.Fatalf("ReadMeta error = %v, want store.ErrNotFound", err) + if err := h.Pull(t.Context(), testID, t.TempDir(), nil); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("Pull error = %v, want store.ErrNotFound", err) } if len(puller.asked) != 0 { t.Errorf("puller was called with no owners: %v", puller.asked) @@ -112,33 +87,31 @@ func TestHealNoOwners(t *testing.T) { // behind a 404 and send the operator looking for a deleted checkpoint. func TestHealErrorIsReported(t *testing.T) { puller := &fakePuller{failAddr: "peer-a:7777"} - s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) + h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) - err := fetchErr(t, s) + err := h.Pull(t.Context(), testID, t.TempDir(), nil) if err == nil || errors.Is(err, store.ErrNotFound) { - t.Fatalf("Fetch error = %v, want the peer failure surfaced", err) + t.Fatalf("Pull error = %v, want the peer failure surfaced", err) } if !bytes.Contains([]byte(err.Error()), []byte("peer exploded")) { t.Errorf("error = %v, want the underlying peer failure named", err) } } -// TestHealDedupsConcurrentMisses: N concurrent Fetch misses of the same id -// must trigger exactly one pull — a stampede of branches racing to heal the -// same checkpoint must not each pay for the whole transfer. +// TestHealDedupsConcurrentMisses: N concurrent Pulls of the same id sharing +// one staging destination must trigger exactly one transfer — a stampede of +// branches racing to heal the same checkpoint must not each pay for it. func TestHealDedupsConcurrentMisses(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} - s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) + h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) + dst := t.TempDir() var wg sync.WaitGroup for range 8 { wg.Go(func() { - _, _, release, err := s.Fetch(t.Context(), testID) - if err != nil { - t.Errorf("Fetch: %v", err) - return + if err := h.Pull(t.Context(), testID, dst, nil); err != nil { + t.Errorf("Pull: %v", err) } - release() }) } wg.Wait() @@ -147,29 +120,60 @@ func TestHealDedupsConcurrentMisses(t *testing.T) { } } -// TestMetasStaysLocal: a cluster-wide listing is the control plane's job (it -// already scatter-gathers every node). If each node answered for its peers the -// same record would come back N times. -func TestMetasStaysLocal(t *testing.T) { +// TestPullValidateRejectionTriesNextOwner: an owner whose transferred content +// fails validation is treated like a failed transfer — the next owner gets a +// chance instead of the whole heal aborting on one bad copy. +func TestPullValidateRejectionTriesNextOwner(t *testing.T) { + puller := &fakePuller{records: map[string]map[string]string{ + "peer-a:7777": record(), + "peer-b:7777": record(), + }} + h := NewHealer(func(string) []string { return []string{"peer-a:7777", "peer-b:7777"} }, puller) + + var calls int + validate := func(string) error { + calls++ + if calls == 1 { + return errors.New("invalid record") + } + return nil + } + if err := h.Pull(t.Context(), testID, t.TempDir(), validate); err != nil { + t.Fatalf("Pull: %v", err) + } + if len(puller.asked) != 2 { + t.Errorf("asked = %v, want both owners tried after the first failed validation", puller.asked) + } +} + +// TestPullValidateRejectsEveryOwner: when every owner's copy fails +// validation, Pull must report the failures, not silently succeed. +func TestPullValidateRejectsEveryOwner(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} - s := New(localStore(t), func(string) []string { return []string{"peer-a:7777"} }, puller) + h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) - metas, err := s.Metas(t.Context()) - if err != nil { - t.Fatalf("Metas: %v", err) - } - if len(metas) != 0 { - t.Errorf("Metas returned %d record(s); a peer's records must not appear in a local listing", len(metas)) + err := h.Pull(t.Context(), testID, t.TempDir(), func(string) error { return errors.New("bad shape") }) + if err == nil || errors.Is(err, store.ErrNotFound) { + t.Fatalf("Pull error = %v, want the validation failure surfaced", err) } } -func localStore(t *testing.T) *dir.Store { - t.Helper() - st, err := dir.New(t.TempDir(), store.CheckpointIDRe) - if err != nil { - t.Fatalf("dir.New: %v", err) +// TestPullBudgetBoundsSlowOwners: two owners that each sleep past their +// per-owner slice must not blow past the overall budget — 30 minutes PER +// owner (the old design) would let N slow owners run N*30 minutes. +func TestPullBudgetBoundsSlowOwners(t *testing.T) { + puller := &sleepyPuller{sleep: 300 * time.Millisecond} + h := NewHealer(func(string) []string { return []string{"peer-a:7777", "peer-b:7777"} }, puller) + h.budget = 200 * time.Millisecond // each owner's slice (100ms) is shorter than the sleep + + start := time.Now() + err := h.Pull(t.Context(), testID, t.TempDir(), nil) + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { + t.Errorf("Pull took %v, want bounded near the %v budget", elapsed, h.budget) + } + if err == nil { + t.Error("Pull with two owners that never finish in time should fail, not silently succeed") } - return st } // fakePuller serves records from an in-memory table, recording who was asked. @@ -203,23 +207,24 @@ func (p *fakePuller) Pull(_ context.Context, addr, _, dst string) error { return nil } +// sleepyPuller blocks until ctx expires or sleep elapses, whichever is +// first — for proving a per-owner slice actually cuts a slow owner off. +type sleepyPuller struct { + sleep time.Duration +} + +func (p *sleepyPuller) Pull(ctx context.Context, _, _, _ string) error { + select { + case <-time.After(p.sleep): + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + func record() map[string]string { return map[string]string{ store.MetaFile: `{"id":"` + testID + `"}`, filepath.Join(store.ExportDir, "mem"): "guest-pages", } } - -// TestPeerBackendContract: wrapping must not change local behavior. Every -// operation but a Fetch/ReadMeta miss is the local backend's, so the wrapper -// has to satisfy the same store contract the dir backend does. - -// fetchErr keeps the three unused Fetch results out of the assertions. -func fetchErr(t *testing.T, s *Store) error { - t.Helper() - _, _, release, err := s.Fetch(t.Context(), testID) //nolint:dogsled // only err is asserted - if release != nil { - release() - } - return err -} diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 66e9167..4595af2 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -228,6 +228,7 @@ type Checkpoint struct { // commands' `--output json` result. type VMRecord struct { State string `json:"state"` + PID int `json:"pid"` VsockSocket string `json:"vsock_socket"` NetworkConfigs []VMNetConfig `json:"network_configs,omitempty"` Config VMConfig `json:"config"` From 14a4f8813137d9324b8a83b750810564d67e9100 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 17:36:52 +0800 Subject: [PATCH 15/31] review: cut the comment budget back to the repo's own density Added comments were 27% of added code against a repo-wide 16%. The excess was design narrative that repeated the commit messages and justification prose that outlived the decision it argued for; what stays is godoc on the exported API this branch adds and the one-line reasons the code cannot carry itself. --- sandboxd/config/config.go | 7 +++-- sandboxd/pool/checkpoint.go | 53 +++++++++++++++---------------------- sandboxd/pool/claim.go | 7 +++-- sandboxd/pool/operator.go | 24 +++++++---------- sandboxd/pool/pool.go | 39 ++++++++------------------- sandboxd/server/server.go | 51 +++++++++++++---------------------- sandboxd/store/peer/peer.go | 35 +++++++++--------------- 7 files changed, 78 insertions(+), 138 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 93becc7..ea886fc 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -225,10 +225,9 @@ type Config struct { // storage (credentials from the AWS chain, never this file). CheckpointStore *StoreConfig `json:"checkpoint_store,omitempty"` - // CheckpointPeerHeal lets a node pull a checkpoint it does not hold from a - // node that gossiped it instead of failing the branch. Requires an encrypted - // mesh (cluster_key): the heal path presents the fleet token to gossip-learned - // addresses. Off by default because it trades a transfer for availability. + // CheckpointPeerHeal lets a node pull a checkpoint it lacks from a peer + // rather than failing the branch. Requires cluster_key: the pull presents + // the fleet token to a probed address. Off by default. CheckpointPeerHeal bool `json:"checkpoint_peer_heal,omitempty"` // CheckpointTTLHours ages out checkpoints (0 = keep forever); the diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 3b513a3..65b3c75 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -16,9 +16,8 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// DeleteScope selects how far a checkpoint delete reaches. The fleet-wide -// value is the zero one: a caller that does not think about scope gets the -// safe behavior, and the narrower local-only delete must be asked for. +// DeleteScope selects how far a checkpoint delete reaches. Fleet-wide is the +// zero value, so the narrower local-only delete must be asked for. type DeleteScope int const ( @@ -102,9 +101,8 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID func (m *Manager) ClaimCheckpoint(ctx context.Context, ckptID string, ttl time.Duration, tenant string) (*types.Sandbox, error) { // Reject a bad or unknown id before recLock: a rejected id must not leave // a lock-map entry (only a delete evicts one). Checkpoints are immutable, - // so this parse stands in for the fetched meta below. The local read is - // cheap, so it goes before quota: a full node must still answer "not - // here" on a miss, or the handler's redirect/heal tiers never run. + // so this parse stands in for the fetched meta below. It precedes quota: + // a full node must still answer "not here", or the tiers never run. ckpt, err := m.loadCheckpoint(ctx, ckptID) if err != nil { return nil, err @@ -122,9 +120,8 @@ func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl ti if m.healer == nil || !store.CheckpointIDRe.MatchString(ckptID) { return nil, ErrUnknownCheckpoint } - // Unlike ClaimCheckpoint, resolving the record here means a peer - // transfer of a whole guest memory image: quota must reject a full node - // before paying that cost, not after. + // Resolving here means moving a guest memory image, so quota rejects a + // full node before that cost, not after. if err := m.overQuota(1, tenant); err != nil { return nil, err } @@ -135,9 +132,8 @@ func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl ti return m.claimLoaded(ctx, ckpt, ttl, tenant) } -// claimLoaded is ClaimCheckpoint's and ClaimCheckpointHeal's shared body once -// ckpt's meta is resolved: it re-fetches under the record lock (immune to a -// delete racing the pre-check), provisions the branch, and finalizes the claim. +// claimLoaded is the shared body once ckpt's meta is resolved: it re-fetches +// under the record lock, so a delete racing the pre-check cannot slip through. func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl time.Duration, tenant string) (*types.Sandbox, error) { l := m.recLock(ckpt.ID) l.RLock() @@ -169,13 +165,10 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl ti return out, err } -// healCheckpoint pulls ckptID from a peer under its record lock, so the -// transfer, validation, and publish race no concurrent heal, delete, or TTL -// sweep of the same id. A record already local when the lock is acquired -// (another heal won the race) is used as-is. claimLoaded takes its own -// RLock on the same recLock, so this releases before returning instead of -// holding it into the claim; a delete racing in between is handled there -// (Fetch answers store.ErrNotFound). +// healCheckpoint pulls ckptID under its record lock, so the transfer, +// validation, and publish race no concurrent heal, delete, or TTL sweep of the +// same id. It releases before returning: claimLoaded takes an RLock on the +// same lock, and handles a delete landing in between. func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { l := m.recLock(ckptID) l.Lock() @@ -208,9 +201,8 @@ func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Chec return m.loadCheckpoint(ctx, ckptID) } -// validateHealedCheckpoint checks a staged pull's shape before it is trusted -// enough to publish: a hostile or buggy peer must not be able to install an -// unreadable or misattributed record that then suppresses future healing. +// validateHealedCheckpoint checks a staged pull's shape before publishing it: +// an unreadable or misattributed record would suppress every later heal. func validateHealedCheckpoint(staging, wantID string) error { raw, err := os.ReadFile(filepath.Join(staging, store.MetaFile)) //nolint:gosec // staging dir is this manager's own if err != nil { @@ -285,12 +277,10 @@ func (m *Manager) pinnedArchiveCks() map[string]struct{} { return pinned } -// DeleteCheckpoint removes a checkpoint's snapshot and record. A fleet-scoped -// delete then best-effort broadcasts to peers, so a copy healed onto one does -// not outlive it; a forwarded delete is DeleteLocal, since re-forwarding would -// loop the fleet forever. A tenant may delete only its own records — anything -// else answers ErrUnknownCheckpoint, never a hint that the id exists; root -// (empty tenant) deletes anything. +// DeleteCheckpoint removes a checkpoint's snapshot and record, then broadcasts +// to peers when fleet-scoped so a healed copy does not outlive it. A tenant may +// delete only its own records — anything else answers ErrUnknownCheckpoint, +// never a hint that the id exists; root (empty tenant) deletes anything. func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope DeleteScope) error { // Validate off the lock — a rejected id must not leave a lock-map entry; // loadCheckpoint is safe unlocked (checkpoints are single-publish, immutable). @@ -309,8 +299,7 @@ func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, s if err := m.deleteCkLocked(ctx, ckptID); err != nil { return fmt.Errorf("delete checkpoint: %w", err) } - // A shared backend (s3) has no per-node replicas to chase: the delete - // above already removed the one copy every node sees. + // A shared backend has no per-node replicas to chase. if scope == DeleteFleet && m.peerDelete != nil && !m.ckptsShared { m.peerDelete(context.WithoutCancel(ctx), ckptID) } @@ -381,8 +370,8 @@ func parseCheckpoint(raw []byte) (types.Checkpoint, error) { return ckpt, nil } -// HasCheckpoint reports whether this node's local store holds a branchable -// (non-archive) record for id — the probe answer, never a fetch. +// HasCheckpoint answers the ownership probe: a branchable local record, never +// a fetch. func (m *Manager) HasCheckpoint(ctx context.Context, ckptID string) bool { ckpt, err := m.loadCheckpoint(ctx, ckptID) return err == nil && !ckpt.Archive diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 766888a..570f0e2 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -99,10 +99,9 @@ func (m *Manager) Release(ctx context.Context, id string, cred Cred) error { return m.releaseResolved(ctx, id, sb) } -// releaseResolved drops the resolved claim (id, sb) and tears down its VM. -// resolve authorizes and unlocks before this runs, so it re-validates under -// m.mu that sb is still the live claim — a second release racing in on the -// same id must not double-tear-down. +// releaseResolved drops the resolved claim and tears down its VM. resolve +// unlocks before this runs, so it re-checks under m.mu that sb is still the +// live claim: a second release racing in must not tear down twice. func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sandbox) error { m.mu.Lock() if m.claimed[id] != sb { diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index 55bf0d3..fce3012 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -7,15 +7,13 @@ import ( ) // Cred is a caller's resolved authority over one sandbox: the per-sandbox -// token, or Operator for the node's root api_token (verified by the server -// before the call). +// token, or Operator for the root api_token the server already verified. type Cred struct { Token string Operator bool } -// Sandbox reports one live claim's summary, the single-sandbox read the -// whole-node listing otherwise forces a caller to scan for. +// Sandbox reports one live claim's summary. func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { sb, ok := m.byID(id) if !ok { @@ -24,9 +22,8 @@ func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { return summarize(sb), true } -// Wake restores a hibernated sandbox and leaves it running, so a control plane -// can resume one it is not about to talk to — waking is otherwise only a side -// effect of opening an agent connection. Idempotent on a running sandbox. +// Wake restores a hibernated sandbox and leaves it running: waking is +// otherwise only a side effect of opening an agent connection. Idempotent. func (m *Manager) Wake(ctx context.Context, id string, cred Cred) error { sb, ok := m.resolve(id, cred) if !ok { @@ -35,9 +32,8 @@ func (m *Manager) Wake(ctx context.Context, id string, cred Cred) error { return m.wake(ctx, sb) } -// resolve authorizes id under cred: Operator resolves by id alone (the -// server has already verified the root api_token), otherwise the token must -// match the claim — an unclaimed slot must never match an empty token. +// resolve authorizes id under cred: Operator by id alone, otherwise the token +// must match — an unclaimed slot must never match an empty token. func (m *Manager) resolve(id string, cred Cred) (*types.Sandbox, bool) { if cred.Operator { return m.byID(id) @@ -48,8 +44,8 @@ func (m *Manager) resolve(id string, cred Cred) (*types.Sandbox, bool) { return m.claim(id, cred.Token) } -// byID resolves a live claim by id alone, with no ownership proof. Callers -// must have authorized the Operator credential themselves before reaching it. +// byID resolves a live claim by id alone; callers must already have authorized +// the Operator credential. func (m *Manager) byID(id string) (*types.Sandbox, bool) { m.mu.Lock() defer m.mu.Unlock() @@ -57,8 +53,8 @@ func (m *Manager) byID(id string) (*types.Sandbox, bool) { return sb, sb != nil } -// wake reuses the relay's resolve path, then discards the resolved socket: the -// caller wants the VM running, not a connection to it. +// wake reuses the relay's resolve path, then discards the socket: the caller +// wants the VM running, not a connection to it. func (m *Manager) wake(ctx context.Context, sb *types.Sandbox) error { sb.Touch() _, err := m.wakeResolved(ctx, sb) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index cec5b98..d5b9aef 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -50,10 +50,7 @@ const ( defaultMaxFork = 16 defaultRefill = 4 - // maxConcurrentHeals bounds node-wide in-flight checkpoint heals: each - // pulls a full guest memory image over the network, so a handful in - // flight already saturates a node's disk and NIC; further ones queue - // for nothing. + // A heal pulls a whole guest memory image, so a few saturate disk and NIC. maxConcurrentHeals = 4 vmPrefix = "sbx-" @@ -175,8 +172,7 @@ func (p *pool) trimWarm(target int) []string { return trim } -// PeerDeleteFunc broadcasts a checkpoint delete to every peer; wired via -// WithPeerDelete, nil means single node. +// PeerDeleteFunc broadcasts a checkpoint delete to every peer. type PeerDeleteFunc func(ctx context.Context, id string) // Manager owns the node's pools, claims, and their persistence. @@ -232,20 +228,11 @@ type Manager struct { audit *journal counters counters ckpts store.Store - // ckptsShared marks an s3 (or other cluster-wide) checkpoint backend: - // every node already resolves every record, so peer heal and its gossip - // set are no-ops. - ckptsShared bool - // healer pulls a checkpoint this node does not hold from a peer; - // ClaimCheckpointHeal is its only caller. nil unless checkpoint_peer_heal - // wired it. m.ckpts itself is never wrapped, so the server's redirect - // branch stays reachable. - healer *peer.Healer - // healSem bounds node-wide in-flight heals to maxConcurrentHeals. - healSem chan struct{} - // peerDelete broadcasts a checkpoint delete to every peer after a - // successful local delete, so a healed replica does not outlive the - // source record; nil means single node (no mesh). + // A cluster-wide backend resolves every record from every node, so heal + // and the delete broadcast are both no-ops against it. + ckptsShared bool + healer *peer.Healer + healSem chan struct{} peerDelete PeerDeleteFunc tpls store.Store ckptTTL time.Duration @@ -585,11 +572,8 @@ func newStoreView(ctx context.Context, cfg *config.Config, staging string, idRe return dir.New(cmp.Or(cfg.CheckpointDir, filepath.Join(cfg.DataDir, "checkpoints")), idRe) } -// WithPeerHeal installs the checkpoint healer: a read path ClaimCheckpointHeal -// pulls through only after a local claim missed and a redirect could not -// answer, so a peer transfer is paid only when nothing cheaper resolves the -// claim. A no-op unless checkpoint_peer_heal is set, or the checkpoint store -// is already shared cluster-wide (every node resolves every record already). +// WithPeerHeal installs the healer ClaimCheckpointHeal pulls through, so a +// peer transfer is paid only once nothing cheaper resolves the claim. func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { if !enabled || owners == nil || m.ckptsShared { return @@ -597,9 +581,8 @@ func (m *Manager) WithPeerHeal(enabled bool, owners peer.Owners, token string) { m.healer = peer.NewHealer(owners, &peer.HTTPPuller{Token: token}) } -// WithPeerDelete wires the fleet-wide delete broadcast: DeleteCheckpoint calls -// fn after a successful local delete so a checkpoint healed onto a peer does -// not outlive the source record. Nil (the default) keeps delete single-node. +// WithPeerDelete wires the broadcast DeleteCheckpoint makes after a local +// delete, so a healed replica does not outlive the source record. func (m *Manager) WithPeerDelete(fn PeerDeleteFunc) { m.peerDelete = fn } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 26bb784..4169119 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -104,9 +104,8 @@ type Placer interface { ConfigMismatches() int } -// CheckpointProber answers which peers currently hold a checkpoint, queried -// live on a redirect decision rather than gossiped; nil disables probing -// (single-node). +// CheckpointProber answers which peers hold a checkpoint, asked live on a +// redirect decision rather than gossiped. type CheckpointProber interface { Owners(ctx context.Context, id string) []string } @@ -187,14 +186,9 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /v1/sandboxes/{id}/checkpoint", s.requireToken(s.handleCheckpoint)) mux.HandleFunc("POST /v1/checkpoints/{id}/claim", s.requireToken(s.handleClaimCheckpoint)) mux.HandleFunc("GET /v1/checkpoints", s.requireToken(s.handleListCheckpoints)) - // The peer-transfer half of the snapshot placement design: a node that - // cannot redirect a branch pulls the record from an owner through this. - // It streams a full checkpoint (guest memory + disk) with no tenant - // scoping, so only the fleet's root api_token may call it — its one - // intended caller is the peer puller, authenticated as the operator. + // GET streams a whole checkpoint with no tenant scoping, so it is + // operator-only; HEAD is the ownership probe and stays unauthenticated. mux.HandleFunc("GET /v1/checkpoints/{id}/blob", s.requireRoot(s.handleCheckpointBlob)) - // HEAD is the probe endpoint a redirect decision fans out to: it must stay - // unauthenticated, so it is registered outside requireToken. mux.HandleFunc("HEAD /v1/checkpoints/{id}/blob", s.handleCheckpointProbe) mux.HandleFunc("DELETE /v1/checkpoints/{id}", s.requireToken(s.handleDeleteCheckpoint)) mux.HandleFunc("DELETE /v1/templates", s.requireToken(s.handleDeleteTemplate)) @@ -281,8 +275,8 @@ func (s *Server) handleRelease(w http.ResponseWriter, r *http.Request) { } } -// handleSandbox reports one live claim, so a control plane can read a single -// sandbox instead of scanning the whole-node listing on every reconcile. +// handleSandbox reports one live claim, so a reconcile need not scan the +// whole-node listing. func (s *Server) handleSandbox(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") sb, ok := s.mgr.Sandbox(id) @@ -306,9 +300,8 @@ func (s *Server) handleSandboxStats(w http.ResponseWriter, r *http.Request) { } // handleSandboxVerb adapts a sandbox-scoped manager call (hibernate, wake) to -// HTTP: per-sandbox bearer auth, 404 on unknown, 204 on success. The node's -// root api_token resolves as an Operator credential instead, so a control -// plane holding only the fleet token can drive the lifecycle. +// HTTP: per-sandbox bearer auth, 404 on unknown, 204 on success. The root +// api_token resolves as an Operator credential instead. func (s *Server) handleSandboxVerb(verb string, do func(ctx context.Context, id string, cred pool.Cred) error) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) @@ -368,11 +361,8 @@ func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { } // handleClaimCheckpoint claims a fresh sandbox branched from a checkpoint. -// Tier order: local claim first; then a redirect (zero bytes moved) when a -// live probe finds an owner and this is not already the redirect retry — -// redirect targets come from a live probe, not gossip, so a redirect never -// points at a node that already lost the record; heal (one peer transfer) -// only when neither could answer. +// Tier order: the local claim, then a redirect to a probed owner (zero bytes +// moved, never a stale one), then one peer transfer if neither answered. func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.CheckpointClaimRequest](w, r) if !ok { @@ -391,8 +381,7 @@ func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { }) } -// handleCheckpointBlob streams a checkpoint record as a tar, so a peer that -// must serve a branch locally can pull it. +// handleCheckpointBlob streams a checkpoint record as a tar for a peer's pull. func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { ckptID := r.PathValue("id") dir, meta, release, err := s.mgr.FetchCheckpoint(r.Context(), ckptID) @@ -416,9 +405,8 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { } } -// handleCheckpointProbe answers a peer's HEAD probe for a checkpoint. -// Unauthenticated on purpose: the id is the unguessable capability, so this -// leaks only existence to a caller who already holds it. +// handleCheckpointProbe answers a peer's HEAD probe. Unauthenticated: the id +// is the capability, so this tells a caller only what it already knows. func (s *Server) handleCheckpointProbe(w http.ResponseWriter, r *http.Request) { if s.mgr.HasCheckpoint(r.Context(), r.PathValue("id")) { w.WriteHeader(http.StatusOK) @@ -568,8 +556,7 @@ func (s *Server) requireRoot(next http.HandlerFunc) http.HandlerFunc { }) } -// rootRequest reports whether this request presented the node's root api_token, -// the credential that authorizes the operator lifecycle paths. +// rootRequest reports whether this request presented the root api_token. func (s *Server) rootRequest(r *http.Request) bool { token, ok := bearerToken(r) return ok && s.isRootToken(token) @@ -583,9 +570,8 @@ func (s *Server) isRootToken(token string) bool { return s.apiToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.apiToken)) == 1 } -// sandboxCred resolves a header-borne bearer token to a Cred: the root -// api_token elevates to Operator (the manager must never see it as a -// sandbox token), anything else is carried as the per-sandbox token as-is. +// sandboxCred resolves a header bearer token to a Cred. The root api_token +// elevates to Operator: the manager must never see it as a sandbox token. func (s *Server) sandboxCred(token string) pool.Cred { if s.isRootToken(token) { return pool.Cred{Operator: true} @@ -593,9 +579,8 @@ func (s *Server) sandboxCred(token string) pool.Cred { return pool.Cred{Token: token} } -// bodyCred resolves a body-carried sandbox token to a Cred: an empty token -// from a root-authenticated request (header api_token) is Operator; a tenant -// must still present the sandbox's own token as ownership proof. +// bodyCred resolves a body-carried sandbox token: absent, on a +// root-authenticated request, is Operator; a tenant must still prove ownership. func (s *Server) bodyCred(r *http.Request, bodyToken string) pool.Cred { return pool.Cred{Token: bodyToken, Operator: bodyToken == "" && s.rootRequest(r)} } diff --git a/sandboxd/store/peer/peer.go b/sandboxd/store/peer/peer.go index 15dbf50..70cd4e7 100644 --- a/sandboxd/store/peer/peer.go +++ b/sandboxd/store/peer/peer.go @@ -14,31 +14,24 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/store" ) -// healBudget bounds one Pull across every owner tried, replacing the old -// per-owner-only bound: a heal that tried N owners at 30 minutes each could -// run for N*30 minutes with nothing else limiting it. +// healBudget bounds one Pull across every owner tried, not each. const healBudget = 30 * time.Minute -// Owners resolves a record id to the peer addresses that hold it — the mesh's -// gossiped view, injected so the healer stays testable without a live cluster. +// Owners resolves a record id to the peer addresses that hold it. type Owners func(id string) []string -// Validate checks a staged pull before Pull trusts the owner that sent it; a -// non-nil error tries the next owner instead of failing the whole heal. The -// healer does not know the record's shape, so the caller supplies this. +// Validate checks a staged pull before Pull trusts the owner that sent it; an +// error tries the next owner. The healer does not know the record's shape. type Validate func(staging string) error -// Healer pulls a record this node does not hold from whichever peer gossiped -// it, staging the transfer into a caller-provided directory; the caller -// validates and publishes it. A nil owners or puller leaves Pull always -// answering store.ErrNotFound, so a node with no mesh degrades to unable to -// heal rather than failing. +// Healer pulls a record this node does not hold into a caller-provided staging +// directory; the caller validates and publishes it. A nil owners or puller +// makes every Pull a miss, so a node with no mesh simply cannot heal. type Healer struct { owners Owners puller Puller - // budget overrides healBudget when set; a test seam, since 30 minutes is - // not something a test should wait out. + // budget overrides healBudget; a test seam, 30 minutes is unwaitable. budget time.Duration flights singleflight.Group @@ -50,9 +43,7 @@ func NewHealer(owners Owners, puller Puller) *Healer { } // Pull fetches id into staging from the first owner whose transfer validates, -// bounding the whole attempt (every owner) to one budget so a wedged or -// endlessly-invalid peer cannot starve the rest. Concurrent pulls of the same -// id share one flight. +// bounding every owner tried to one budget. Concurrent pulls share a flight. func (h *Healer) Pull(ctx context.Context, id, staging string, validate Validate) error { if h.owners == nil || h.puller == nil { return store.ErrNotFound @@ -62,8 +53,7 @@ func (h *Healer) Pull(ctx context.Context, id, staging string, validate Validate return store.ErrNotFound } budget := cmp.Or(h.budget, healBudget) - // A client hanging up must not abandon a started pull: the budget below - // bounds it instead of the caller's ctx. + // A client hanging up must not abandon a started pull; the budget bounds it. ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), budget) defer cancel() _, err, _ := h.flights.Do(id, func() (any, error) { @@ -105,9 +95,8 @@ func (h *Healer) pullFrom(ctx context.Context, id, staging string, addrs []strin return store.ErrNotFound // every owner answered not-found: stale gossip } -// clearDir empties dir's contents (keeping dir itself) between owner -// attempts, so a rejected or partial transfer from one peer cannot linger -// into the next's. +// clearDir empties dir between owner attempts, so one peer's rejected or +// partial transfer cannot linger into the next's. func clearDir(dir string) error { entries, err := os.ReadDir(dir) if err != nil { From c483590d56226a8cd40c3a5c540fe9be40f87383 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 17:46:16 +0800 Subject: [PATCH 16/31] config: a healed replica needs a finite life, and a fleet-agreed one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Peer heal makes revocation two-legged: a delete broadcasts to peers, and the TTL sweep ages out any copy the broadcast missed — a healed record carries the source's CreatedAt, so every node expires it at the same moment. Both legs had silent failure modes. checkpoint_ttl_hours = 0 (the default) cuts the second leg entirely: a node offline during the broadcast would keep a deleted checkpoint branchable forever. Heal now requires a nonzero TTL, making the revocation window finite and operator-chosen. A record that should live indefinitely is what promote is for. The TTL also had to match across the fleet for "expires at the same moment" to hold, and nothing checked that: it now rides ClusterDigest, so a node sweeping on a different clock is called out at join time instead of quietly outliving its peers' deletes. --- sandboxd/config/config.go | 17 +++++++++++------ sandboxd/config/config_test.go | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index ea886fc..7e6ace7 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -226,8 +226,9 @@ type Config struct { CheckpointStore *StoreConfig `json:"checkpoint_store,omitempty"` // CheckpointPeerHeal lets a node pull a checkpoint it lacks from a peer - // rather than failing the branch. Requires cluster_key: the pull presents - // the fleet token to a probed address. Off by default. + // rather than failing the branch. Requires cluster_key (the pull presents + // the fleet token to a probed address) and a nonzero checkpoint TTL (the + // revocation bound for a replica a delete broadcast missed). Off by default. CheckpointPeerHeal bool `json:"checkpoint_peer_heal,omitempty"` // CheckpointTTLHours ages out checkpoints (0 = keep forever); the @@ -269,8 +270,9 @@ func (c *Config) HasEgress() bool { } // ClusterDigest fingerprints the must-match config so a divergent node is -// caught early, not at a later 401. With cluster_key it HMACs token material -// too; without one only tenant names + CA root ride the (maybe cleartext) wire. +// caught early, not at a later 401 or an outlived replica. With cluster_key it +// HMACs token material too; without one only tenant names, the CA root and the +// checkpoint TTL ride the (maybe cleartext) wire. func (c *Config) ClusterDigest(caFingerprint string) string { names := make([]string, len(c.Tenants)) for i, t := range c.Tenants { @@ -285,13 +287,13 @@ func (c *Config) ClusterDigest(caFingerprint string) string { tenants[i] = auth{t.Name, t.Token} } slices.SortFunc(tenants, func(a, b auth) int { return strings.Compare(a.Name, b.Name) }) - raw, _ := json.Marshal([]any{c.APIToken, c.PreviewSecret, caFingerprint, tenants}) + raw, _ := json.Marshal([]any{c.APIToken, c.PreviewSecret, caFingerprint, tenants, c.CheckpointTTLHours}) mac := hmac.New(sha256.New, key) mac.Write(raw) return hex.EncodeToString(mac.Sum(nil)) } } - raw, _ := json.Marshal([]any{caFingerprint, names}) + raw, _ := json.Marshal([]any{caFingerprint, names, c.CheckpointTTLHours}) sum := sha256.Sum256(raw) return hex.EncodeToString(sum[:]) } @@ -372,6 +374,9 @@ func (c *Config) validate() error { if c.CheckpointPeerHeal && (c.Mesh == nil || c.Mesh.ClusterKey == "") { return fmt.Errorf("checkpoint_peer_heal requires an encrypted mesh (set mesh.cluster_key)") } + if c.CheckpointPeerHeal && c.CheckpointTTLHours == 0 { + return fmt.Errorf("checkpoint_peer_heal requires checkpoint_ttl_hours > 0: the ttl is what ages out a healed replica a delete broadcast missed") + } if err := c.validateTenants(); err != nil { return err } diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 5097e5e..cbedef7 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -123,6 +123,7 @@ func TestLoadRejectsInvalid(t *testing.T) { {"mesh cluster key wrong length", `{"pools":[],"mesh":{"bind":"node1:7946","cluster_key":"YWJj"}}`, "want 16, 24, or 32"}, {"checkpoint peer heal without mesh", `{"checkpoint_peer_heal":true,"pools":[]}`, "requires an encrypted mesh"}, {"checkpoint peer heal without cluster key", `{"checkpoint_peer_heal":true,"pools":[],"mesh":{"bind":"node1:7946"}}`, "requires an encrypted mesh"}, + {"checkpoint peer heal without ttl", `{"checkpoint_peer_heal":true,"pools":[],"mesh":{"bind":"node1:7946","cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="}}`, "requires checkpoint_ttl_hours"}, } { t.Run(tt.name, func(t *testing.T) { _, err := Load(writeConfig(t, tt.body)) From a19d72e324b9a65cfca33cfd4aa09481251a1298 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 18:27:20 +0800 Subject: [PATCH 17/31] sandboxd: make a delete and a heal of one id wait on each other MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Heal is the first thing that can make a checkpoint id live again, and two places still assumed an id, once gone, stays gone. DeleteCheckpoint decided "not here" before taking the record lock, on the comment's own reasoning that a checkpoint is single-publish and immutable. A delete landing while a heal was mid-transfer therefore saw nothing, returned unknown-checkpoint without ever contending for the lock, and the heal published moments later — a client that deleted a checkpoint could find it back. Only the id's format is checked before the lock now; whether the record exists is decided under the same lock heal holds. The record locks were also evicted from the map while still held, so a waiter kept the old mutex while a newcomer was handed a fresh one and both proceeded to mutate one record. Acquisition now counts references, and a lock is dropped only once none remain — every call site releases the mutex before releasing its reference, which is the ordering that makes the eviction safe. Both fixes are pinned by tests that fail without them: restoring the unlocked pre-check makes the delete return early against an unpublished heal, and evicting without checking the count strands a live holder. The SDK's redirect fallback now reports both halves of a failure — the peers' errors say why the claim left the origin, the origin's says why coming back did not help — instead of dropping the candidates' on the floor. --- docs/cluster.md | 46 ++++++++ docs/deploy.md | 3 +- docs/sandboxd-api.md | 69 ++++++++++- docs/security.md | 10 +- sandboxd/pool/archive.go | 17 ++- sandboxd/pool/checkpoint.go | 41 +++++-- sandboxd/pool/checkpoint_heal_test.go | 76 +++++++++++++ sandboxd/pool/pool.go | 9 +- sandboxd/pool/template.go | 50 +++++--- sandboxd/server/server.go | 4 +- sandboxd/server/server_http.go | 11 ++ sandboxd/server/server_test.go | 22 ++++ sdk/go/checkpoint.go | 48 ++++---- sdk/go/checkpoint_test.go | 60 +++++++++- sdk/go/client.go | 114 +++++++++++++++---- sdk/go/client_test.go | 152 +++++++++++++++++++++++++ sdk/python/cocoonsandbox/checkpoint.py | 26 +++-- sdk/python/cocoonsandbox/client.py | 66 +++++++++-- sdk/python/tests/test_checkpoint.py | 38 ++++++- sdk/python/tests/test_fault_matrix.py | 82 ++++++++++++- 20 files changed, 821 insertions(+), 123 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 0e9926e..0cac5ab 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -123,6 +123,51 @@ handle `Sandbox.Promote` returns is still owner-bound — `template.New` and `template.Delete` dial the owner directly, no gossip involved — and is the race-free choice immediately after a promote. +## Checkpoints on a cluster + +A checkpoint lives on whichever node captured it (the default +[`checkpoint_dir`](deploy.md#configuration)) unless the store is shared (a +FUSE mount or the s3 backend), in which case every node resolves every +checkpoint directly and nothing below applies. + +On the default per-node store, a branch claim (`Checkpoint.New` / +[`POST /v1/checkpoints/{id}/claim`](sandboxd-api.md#post-v1checkpointsidclaim)) +at a node that does not hold the record runs a tier order: + +1. **Local claim** — the fast path when the claiming node already holds it. +2. **Probe + redirect** — a live, unauthenticated `HEAD` fan-out to up to 3 + mesh peers in parallel, redirecting to whoever answers — the same + claim-redirect contract a warm miss already uses. The cap means the + answer is a hint, not an exhaustive list of every owner. The probe and + the follow-up claim are not atomic: a peer can answer the probe, then + lose the record — a delete's broadcast lands, or its own TTL sweep runs + (below) — before the retry reaches it, so a redirect can go stale + between the two calls. +3. **Heal** — when `checkpoint_peer_heal` is on and neither of the above + answered, the node pulls the record from a probed peer, validates its + shape and id before trusting it, and publishes it locally so later + branches are local — paid once per node. The pull is bounded to one + overall time budget across however many peers it tries (`healBudget` in + `store/peer`), and a node accepts only so many heals at once + (`maxConcurrentHeals` in `pool`) — past that cap it answers `503` rather + than queuing indefinitely. + +### Delete is eventual, not a fleet-wide revocation + +`Checkpoint.Delete` (`DELETE /v1/checkpoints/{id}`) removes the local +record, then best-effort broadcasts the delete to every peer this node +currently sees, so a replica a heal pulled earlier is cleaned up too, not +just the original. A peer that is offline or partitioned during that +broadcast keeps its copy until the checkpoint TTL ages it out: the +worst-case window in which a deleted checkpoint remains branchable by an +id-holder is `checkpoint_ttl_hours`. A healed replica carries the source +checkpoint's original `CreatedAt`, so every node's hourly TTL sweep ages it +out at the same wall-clock moment regardless of whether the broadcast +reached it — which is why heal *requires* a nonzero, fleet-matching TTL +(see [cluster-invariant config](#cluster-invariant-config)). With TTL +disabled (the default), that window never closes on a peer the broadcast +never reaches. + ## State ownership Each kind of state has exactly one source of truth, and a restart rebuilds @@ -134,6 +179,7 @@ fully from it: | API-applied pool targets (`PUT /v1/pools`) | `/pools.json` (machine owned) | yes | | claims | the claims journal + `Reconcile` | yes | | placement hints (warm counts, template sets) | gossip | rebuilt | +| checkpoint ownership | a live per-request probe (no gossip); a healed replica is this node's own persisted copy, aged out by `checkpoint_ttl_hours` | yes | Pools are managed API-first. The first time a node takes `PUT /v1/pools`, it writes the applied set to `/pools.json` and from then on **that file diff --git a/docs/deploy.md b/docs/deploy.md index 71bf7fc..64ef414 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -79,7 +79,8 @@ sandboxd reads one JSON file (`-config`, default | `preview_advertise` | = `preview_listen` | the base URL a browser/proxy reaches this node's preview server at | | `checkpoint_dir` | `/checkpoints` | where checkpoints and promoted templates live. Point it at a shared FUSE mount (JuiceFS over object storage, NFS) and every node sharing the mount can branch every checkpoint — the path's filesystem is the operator's choice. One contract on any shared root (mount or bucket): a template key has a single writer — promotes go to the sandbox's owner node, and operators must not race promotes of one name from different nodes (checkpoint ids are node-generated and never collide). A checkpoint deleted on one node while another is mid-branch from it fails that branch visibly | | `checkpoint_store` | dir | checkpoint AND promoted-template backend (both live in one store root, id-namespaced ck_/tp_): `{"kind": "s3", "s3": {"bucket": "…", "prefix": "ck/", "endpoint": "…", "region": "…", "force_path_style": true}}` stores checkpoints in object storage (any node claims any checkpoint, no shared mount needed). Credentials come from the standard AWS chain (env/IAM role), never this file. A crash between upload and the meta.json commit marker leaves orphan objects invisible to listings — add an S3 lifecycle rule to reclaim them. Absent = the dir backend at `checkpoint_dir` | -| `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it | +| `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the bound on a healed replica a delete broadcast missed | +| `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Two requirements, both enforced at config load: `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (the bound on how long a replica can outlive a delete that failed to reach it). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, a claim is first redirected to a warm peer) | | `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`); preview accesses record as op `preview_dial`. A request frame whose first line exceeds 4 KiB is skipped, never truncated | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 9438918..8d7253e 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -205,12 +205,51 @@ token, 409 egress-lane sandbox (see [egress](egress.md)). ## POST /v1/checkpoints/{id}/claim -Auth: node API token; body `{"ttl_seconds": 0}`. Claims a fresh sandbox -branched from the checkpoint (a normal claim response, attributed to the -caller); the checkpoint's recorded key applies — the unguessable id is the -capability to branch. 404 for an unknown checkpoint, 409 for an egress-lane -checkpoint (see [egress](egress.md)), 429 node or calling tenant at -`max_claims`, or the node draining. +Auth: node API token; body `{"ttl_seconds": 0, "no_redirect": false}`. +Claims a fresh sandbox branched from the checkpoint (a normal claim +response, attributed to the caller); the checkpoint's recorded key applies +— the unguessable id is the capability to branch. + +Checkpoints are node-local (unless the store is shared — see +[Configuration](deploy.md#configuration)), so a miss here runs a tier order: + +1. This node checks its own store first. +2. On a miss, it probes up to 3 peers directly — a parallel, unauthenticated + `HEAD` to each (see below) — and answers exactly like a warm-miss + [`POST /v1/claim`](#post-v1claim): `200 {"redirect": ["10.0.0.6:7777", + "10.0.0.7:7777"]}`, retry the same body (+`no_redirect: true`) at each + candidate until one answers. The probe and the follow-up claim are not + atomic: a peer can answer the probe, then lose the record — a delete's + broadcast lands, or its own TTL sweep runs (below) — before the retry + reaches it, so a redirect can go stale between the two calls. +3. If nothing answers the probe (or `no_redirect` is set), and the node has + `checkpoint_peer_heal` enabled, it pulls the record from a probed peer + itself, validates it, publishes it locally, and serves the claim from + there — paid once per node. See the full + [placement lifecycle](cluster.md#checkpoints-on-a-cluster) for how the + three tiers fit together. + +404 for an unknown checkpoint (locally, and after redirect and heal both +miss), 409 for an egress-lane checkpoint (see [egress](egress.md)), 429 node +or calling tenant at `max_claims` or the node draining, 503 when the node's +concurrent-heal cap is already full — the request is retryable. + +## GET/HEAD /v1/checkpoints/{id}/blob + +The peer-transfer route behind the probe and heal above — internal, not +part of the public API; an SDK caller has no reason to call it directly. + +- `GET` streams the whole record — guest memory, disk, and meta — as a tar. + Operator-token only: the stream carries no tenant scoping, and its only + real caller is a peer's heal pull, which presents the fleet `api_token`. + 401 missing or unrecognized token, 403 a valid tenant token (authenticated + but not the operator), 404 unknown checkpoint. +- `HEAD` is the ownership probe and is currently **unauthenticated** — the id + itself is treated as the capability, so this tells a caller only what it + already knows: 200 when this node holds a branchable (non-archive) copy, + 404 otherwise. Hardening this boundary (requiring a token, bounding who can + fan a probe out) is tracked as follow-up work; do not treat it as secured + today. ## GET /v1/checkpoints @@ -223,6 +262,24 @@ Auth: node API token. A tenant may delete only its own records — anything else is 404, never a hint the id exists; root deletes anything. 204 on success, 404 unknown. +**Delete removes the local record and then best-effort broadcasts to peers +so a healed replica does not outlive it — this is eventual best-effort +cleanup, not a fleet-wide revocation.** A peer that is offline or +partitioned during the broadcast keeps its copy until the checkpoint TTL +ages it out: the worst-case window in which a deleted checkpoint remains +branchable by an id-holder is `checkpoint_ttl_hours`. A healed replica +carries the source's original `CreatedAt`, so every node's sweep expires it +at the same wall-clock moment; the TTL must also match fleet-wide, which the +[cluster-invariant config](cluster.md#cluster-invariant-config) digest +checks. A window always exists because `checkpoint_peer_heal` cannot be +enabled with `checkpoint_ttl_hours: 0` — a replica that can outlive a delete +must have a bound on how long. A shared +checkpoint store skips the broadcast: every node already resolves every +record directly, so there is no replica to chase. `?no_forward=1` marks a +delete already arriving from another node's own broadcast, so it is not +itself re-broadcast (loop prevention); it is an internal parameter, not one +an SDK caller should set. + ## GET /v1/sandboxes Auth: root only (tenant tokens get 403). The operator index: `{"sandboxes": diff --git a/docs/security.md b/docs/security.md index bdd2534..4a12c94 100644 --- a/docs/security.md +++ b/docs/security.md @@ -77,9 +77,13 @@ per-sandbox tokens (that sandbox only — holding a handle amplifies to nothing node-level). Two capability tokens ride on top: preview URLs are HMAC-signed, expire with the claim's lease, and die with the sandbox (no revocation list to leak); a checkpoint id is the unguessable capability to -branch it. Tenants are isolated at the API layer — listings filter, -deletes answer 404 rather than confirming existence, and operator -surfaces answer tenants 403. +branch it. On a cluster, deleting a checkpoint does not revoke that +capability fleet-wide the instant it runs: the delete is best-effort — +broadcast to every peer the node currently sees — so a peer that is offline +or partitioned at that moment keeps its own replica branchable until +`checkpoint_ttl_hours` ages it out ([placement lifecycle](cluster.md#checkpoints-on-a-cluster)). +Tenants are isolated at the API layer — listings filter, deletes answer 404 +rather than confirming existence, and operator surfaces answer tenants 403. ## Known limitations diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 7dd1841..e1cc1e7 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -159,10 +159,21 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, } ctx = context.WithoutCancel(ctx) ck := sb.ArchiveCk - // Exclusive, not shared: this path deletes the consumed ck below. + // Exclusive, not shared: this path deletes the consumed ck below. The + // lock releases before eviction is considered, below — evicting while + // still held would let a concurrent recLock for ck split onto a + // different mutex. l := m.recLock(ck) l.Lock() - defer l.Unlock() + consumed := false + defer func() { + l.Unlock() + if consumed { + m.recDoneEvict(ck) + } else { + m.recDone(ck) + } + }() dir, _, release, err := m.ckpts.Fetch(ctx, ck) if errors.Is(err, store.ErrNotFound) { return "", ErrUnknownSandbox // record disagrees with the store @@ -187,7 +198,7 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string, if delErr := m.ckpts.Delete(ctx, ck); delErr != nil { log.WithFunc("pool.wakeArchived").Warnf(ctx, "delete consumed archive ck %s: %v", ck, delErr) } else { - m.dropRecLock(ck) + consumed = true } // Only the none lane reaches here (the egress guard above fails closed), so // this rebinds the none-lane proxy; there is no NIC to re-lock. diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 65b3c75..4b249c7 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -137,7 +137,7 @@ func (m *Manager) ClaimCheckpointHeal(ctx context.Context, ckptID string, ttl ti func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl time.Duration, tenant string) (*types.Sandbox, error) { l := m.recLock(ckpt.ID) l.RLock() - defer l.RUnlock() + defer func() { l.RUnlock(); m.recDone(ckpt.ID) }() dir, _, release, err := m.ckpts.Fetch(ctx, ckpt.ID) if errors.Is(err, store.ErrNotFound) { return nil, ErrUnknownCheckpoint // deleted between the pre-check and the lock @@ -172,7 +172,7 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl ti func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { l := m.recLock(ckptID) l.Lock() - defer l.Unlock() + defer func() { l.Unlock(); m.recDone(ckptID) }() if ckpt, err := m.loadCheckpoint(ctx, ckptID); err == nil { return ckpt, nil } @@ -281,9 +281,24 @@ func (m *Manager) pinnedArchiveCks() map[string]struct{} { // to peers when fleet-scoped so a healed copy does not outlive it. A tenant may // delete only its own records — anything else answers ErrUnknownCheckpoint, // never a hint that the id exists; root (empty tenant) deletes anything. +// Existence is checked under the record lock, not before it: heal broke the +// old assumption that a local miss means the id is truly gone — a concurrent +// heal can be mid-transfer, unpublished, when an unlocked check would see +// 404, and it would then publish right after this call gave up thinking +// there was nothing to delete. Taking the same lock heal takes makes the two +// wait on each other instead of racing. Every exit evicts the lock entry +// (recDoneEvict, not recDone) — unlike a template id, a checkpoint id is +// effectively one-shot, so nothing is lost keeping the map from growing +// per rejected or successful call alike. func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope DeleteScope) error { - // Validate off the lock — a rejected id must not leave a lock-map entry; - // loadCheckpoint is safe unlocked (checkpoints are single-publish, immutable). + // Reject a bad id before recLock: a rejected id must not leave a + // lock-map entry. + if !store.CheckpointIDRe.MatchString(ckptID) { + return ErrUnknownCheckpoint + } + l := m.recLock(ckptID) + l.Lock() + defer func() { l.Unlock(); m.recDoneEvict(ckptID) }() ckpt, err := m.loadCheckpoint(ctx, ckptID) if err != nil { return err @@ -296,7 +311,7 @@ func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, s if _, pinned := m.pinnedArchiveCks()[ckptID]; pinned || ckpt.Archive { return ErrUnknownCheckpoint // backs an archived sandbox, not a deletable checkpoint } - if err := m.deleteCkLocked(ctx, ckptID); err != nil { + if err := m.ckpts.Delete(ctx, ckptID); err != nil { return fmt.Errorf("delete checkpoint: %w", err) } // A shared backend has no per-node replicas to chase. @@ -335,15 +350,20 @@ func (m *Manager) sweepExpiredCheckpoints(ctx context.Context) { } // deleteCkLocked removes a checkpoint under its record lock, so the delete -// never runs beneath an in-flight branch clone or archived wake. +// never runs beneath an in-flight branch clone, heal, or archived wake. The +// lock is unlocked before the entry is considered for eviction: evicting +// while still (logically) held is what would let a concurrent recLock for +// the same id hand out a different mutex and split the serialization. func (m *Manager) deleteCkLocked(ctx context.Context, ckID string) error { l := m.recLock(ckID) l.Lock() - defer l.Unlock() - if err := m.ckpts.Delete(ctx, ckID); err != nil { + err := m.ckpts.Delete(ctx, ckID) + l.Unlock() + if err != nil { + m.recDone(ckID) return err } - m.dropRecLock(ckID) + m.recDoneEvict(ckID) return nil } @@ -388,6 +408,7 @@ func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, [ dir, meta, release, err := m.ckpts.Fetch(ctx, ckptID) if err != nil { l.RUnlock() + m.recDone(ckptID) if errors.Is(err, store.ErrNotFound) { return "", nil, nil, ErrUnknownCheckpoint } @@ -395,5 +416,5 @@ func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, [ } // The read lock spans the transfer: a delete must not pull the export out // from under a stream already writing it to a peer. - return dir, meta, func() { release(); l.RUnlock() }, nil + return dir, meta, func() { release(); l.RUnlock(); m.recDone(ckptID) }, nil } diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index 8bf7784..aa0f396 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -103,6 +103,82 @@ func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { } } +// TestDeleteCheckpointWaitsForInFlightHeal covers the Codex r2 blocker: an +// unlocked existence check would see 404 while a heal is mid-transfer and +// give up, only for the heal to publish moments later — resurrecting a +// checkpoint the delete had just answered "not here" for. Delete must +// instead wait on the same record lock the heal holds, then genuinely +// delete what the heal just published. +func TestDeleteCheckpointWaitsForInFlightHeal(t *testing.T) { + id := "ck_00000000000000bb" + ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} + m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) + puller.release = make(chan struct{}) + + healDone := make(chan error, 1) + go func() { + _, err := m.ClaimCheckpointHeal(t.Context(), id, time.Hour, "") + healDone <- err + }() + waitFor(t, func() bool { return puller.count() >= 1 }) // heal is mid-pull, unpublished + + deleteDone := make(chan error, 1) + go func() { deleteDone <- m.DeleteCheckpoint(t.Context(), id, "", DeleteLocal) }() + // Give a buggy unlocked check every chance to (wrongly) return early + // before the heal ever publishes. + time.Sleep(50 * time.Millisecond) + select { + case err := <-deleteDone: + t.Fatalf("DeleteCheckpoint returned early (%v) while the heal was unpublished — it must wait for the record lock", err) + default: + } + + close(puller.release) + if err := <-healDone; err != nil { + t.Fatalf("heal: %v", err) + } + if err := <-deleteDone; err != nil { + t.Errorf("delete after heal published: %v, want nil (the record existed once delete's lock was granted)", err) + } + if m.HasCheckpoint(t.Context(), id) { + t.Error("checkpoint still present after delete finished") + } +} + +// TestRecLockEvictionWaitsForAllHolders covers the lock-identity-split half +// of the same Codex r2 blocker: evicting an id's entry while another holder +// still references it would let that holder's (or a new caller's) next +// recLock for the same id return a different mutex, splitting mutual +// exclusion between them. Eviction must wait until every holder has +// released, not just the one that happened to delete the record. +func TestRecLockEvictionWaitsForAllHolders(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + const id = "ck_00000000000000aa" + + l1 := m.recLock(id) + l1.Lock() + l2 := m.recLock(id) // a second, concurrent holder registers interest first + if l2 != l1 { + t.Fatalf("recLock returned a different mutex while the first holder is still outstanding") + } + l1.Unlock() + m.recDoneEvict(id) // first holder's release must not evict: l2 is still outstanding + if _, ok := m.recLocks.Load(id); !ok { + t.Fatal("entry evicted while a second holder still references it") + } + + l3 := m.recLock(id) // a third caller arriving now must still find the SAME mutex + if l3 != l1 { + t.Fatal("a concurrent recLock diverged onto a different mutex for the same id — the lock-identity split") + } + + m.recDone(id) // l2's release + m.recDoneEvict(id) // l3's release, last one out + if _, ok := m.recLocks.Load(id); ok { + t.Error("entry not evicted once every holder released") + } +} + // TestFetchCheckpointNeverPulls: the peer-transfer blob endpoint must never // trigger a recursive pull, even with a healer installed that could serve it. func TestFetchCheckpointNeverPulls(t *testing.T) { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index d5b9aef..c6b141f 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -246,7 +246,13 @@ type Manager struct { // recLocks serializes same-id store record mutations and holds off a // re-publish swap while a clone reads the old generation (per id, RW). - recLocks sync.Map + // recLocksMu guards recRefs, the live-holder count that gates eviction: a + // checkpoint id can become live again after a delete (a peer's heal can + // republish one), so an entry is only safe to evict once nothing still + // holds or awaits it — never on a bare "record deleted" signal. + recLocks sync.Map + recLocksMu sync.Mutex + recRefs map[string]int // notifyTemplates, when set (before serving starts), fires after a // promote or template delete so the mesh republishes immediately @@ -303,6 +309,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg pendingCks: map[string]struct{}{}, egressListeners: map[string]*egressListener{}, egressTaps: map[string]string{}, + recRefs: map[string]int{}, egressSecrets: secrets, dial: egressDialer.DialContext, sweep: netfilter.SweepExcept, diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 3ff92d9..f1b6a80 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -86,7 +86,7 @@ func (m *Manager) DeleteTemplate(ctx context.Context, key types.PoolKey, tenant id := store.TemplateID(key.Hash()) l := m.recLock(id) l.Lock() - defer l.Unlock() + defer func() { l.Unlock(); m.recDone(id) }() raw, err := m.tpls.ReadMeta(ctx, id) if err != nil { if errors.Is(err, store.ErrNotFound) { @@ -173,22 +173,43 @@ func (m *Manager) pooledHash(hash string) bool { // recLock is the per-record mutation lock: template publish/delete serialize // on it directly; checkpoints take it only for delete/fetch (their ids are // fresh and unguessable pre-publish). A clone or wake holds it shared, so a -// delete or re-publish swap never runs under an in-flight read. +// delete or re-publish swap never runs under an in-flight read. Every call +// counts as a live reference — pair it with recDone (or recDoneEvict) once +// the returned lock is unlocked, so a concurrent evict can never hand a +// waiter a different mutex for the same id. func (m *Manager) recLock(id string) *sync.RWMutex { - if l, ok := m.recLocks.Load(id); ok { - return l.(*sync.RWMutex) - } + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + m.recRefs[id]++ l, _ := m.recLocks.LoadOrStore(id, &sync.RWMutex{}) return l.(*sync.RWMutex) } -// dropRecLock evicts a deleted record's lock so recLocks can't grow per -// checkpoint. Safe only for single-use ck ids and only after the store record -// is deleted: a fresh lock is obtainable only after eviction, i.e. after the -// record is gone, so no two live ops diverge onto different locks. Template ids -// (tp_, reused on re-promote) must not be evicted; they are bounded. -func (m *Manager) dropRecLock(id string) { - m.recLocks.Delete(id) +// recDone releases a reference taken by recLock; call after Unlock/RUnlock, +// never before — releasing early is what lets a concurrent recLock observe +// a fresh mutex while this call's lock is still logically held. +func (m *Manager) recDone(id string) { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + m.recRefs[id]-- + if m.recRefs[id] <= 0 { + delete(m.recRefs, id) + } +} + +// recDoneEvict is recDone for a caller that just deleted id's record: once +// the reference count drops to zero — no one else holds or awaits this id's +// lock — its recLocks slot is removed so the map can't grow per checkpoint. +// Template ids (tp_, reused on re-promote) never call this; they stay +// cached forever, bounded by the promoted set. +func (m *Manager) recDoneEvict(id string) { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + m.recRefs[id]-- + if m.recRefs[id] <= 0 { + delete(m.recRefs, id) + m.recLocks.Delete(id) + } } // checkTemplateOwner rejects publishing or deleting over another tenant's @@ -234,12 +255,13 @@ func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey) (string, dir, _, release, err := m.tpls.Fetch(ctx, id) if err != nil { l.RUnlock() + m.recDone(id) if errors.Is(err, store.ErrNotFound) { return "", func() {}, nil } return "", func() {}, err } - return dir, func() { release(); l.RUnlock() }, nil + return dir, func() { release(); l.RUnlock(); m.recDone(id) }, nil } // publishTemplate exports snap into the store under the key's template id. @@ -269,7 +291,7 @@ func (m *Manager) commitTemplate(ctx context.Context, staging, id, tenant string } l := m.recLock(id) l.Lock() - defer l.Unlock() + defer func() { l.Unlock(); m.recDone(id) }() if err := m.checkTemplateOwner(ctx, id, tenant); err != nil { return err } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 4169119..6311e7e 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -362,7 +362,9 @@ func (s *Server) handleCheckpoint(w http.ResponseWriter, r *http.Request) { // handleClaimCheckpoint claims a fresh sandbox branched from a checkpoint. // Tier order: the local claim, then a redirect to a probed owner (zero bytes -// moved, never a stale one), then one peer transfer if neither answered. +// moved, but not authoritative — a peer that missed a delete broadcast still +// answers it holds the record until its own TTL sweep runs), then one peer +// transfer if neither answered. func (s *Server) handleClaimCheckpoint(w http.ResponseWriter, r *http.Request) { req, ok := decodeBody[types.CheckpointClaimRequest](w, r) if !ok { diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index 5b4e3cc..7c7f581 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -15,6 +15,11 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/utils" ) +// retryAfterSeconds hints when to retry a 503. A heal slot frees when a +// transfer finishes, which can be seconds or minutes, so this bounds a waiting +// client to a few probes a minute rather than inviting a 1-second hammer. +const retryAfterSeconds = "10" + // tenantKey carries the resolved tenant scope ("" = root) on the request // context, from requireToken to the handlers. type tenantKey struct{} @@ -74,6 +79,12 @@ func decodeBodyStrict[T any](w http.ResponseWriter, r *http.Request) (T, bool) { func writePoolErr(w http.ResponseWriter, err error) bool { for _, m := range poolErrHTTP { if errors.Is(err, m.err) { + // 503 here means the node's own resources are momentarily exhausted + // (e.g. the concurrent-heal budget), not a caller error — Retry-After + // tells the client this is worth retrying, not failing the request. + if m.code == http.StatusServiceUnavailable { + w.Header().Set("Retry-After", retryAfterSeconds) + } writeErr(w, m.code, cmp.Or(m.msg, err.Error())) return true } diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index a0403aa..9922166 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -1116,6 +1116,28 @@ func TestCheckpointClaimNoRedirectGoesStraightToHeal(t *testing.T) { } } +// TestCheckpointClaimHealBusyIs503WithRetryAfter: a full heal budget is the +// node's own transient exhaustion, not a caller error — the client should +// retry, and Retry-After says so instead of leaving it to guess. +func TestCheckpointClaimHealBusyIs503WithRetryAfter(t *testing.T) { + mgr := &fakeManager{ + healCheckpoint: func(string) (*types.Sandbox, error) { + return nil, pool.ErrHealBusy + }, + } + ts := newPlacerTestServer(t, "sekret", mgr, nil) + + resp := postJSON(t, ts.URL+"/v1/checkpoints/ck_00000000000000aa/claim", "sekret", `{}`) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503", resp.StatusCode) + } + if got := resp.Header.Get("Retry-After"); got == "" { + t.Error("Retry-After header missing on a 503") + } +} + // TestCheckpointBlobUnknownIs404 lets a puller move on to the next owner // instead of treating a stale gossip entry as a transfer failure. func TestCheckpointBlobUnknownIs404(t *testing.T) { diff --git a/sdk/go/checkpoint.go b/sdk/go/checkpoint.go index 8e53845..b8dbdf4 100644 --- a/sdk/go/checkpoint.go +++ b/sdk/go/checkpoint.go @@ -24,8 +24,9 @@ type Checkpoint struct { // New claims a sandbox cloned from the checkpoint, on the checkpoint's // node. The snapshot pins the key axes; WithTimeout may set the TTL. A node -// that no longer holds the checkpoint redirects, which New follows -// transparently, mirroring Client.New. +// that no longer holds the checkpoint redirects to a probed owner, which New +// follows transparently; if every candidate fails transiently, New falls +// back to the origin once so it heals (pulls the checkpoint) locally. func (ck *Checkpoint) New(ctx context.Context, opts ...Option) (*Sandbox, error) { var claim claimRequest for _, opt := range opts { @@ -42,32 +43,29 @@ func (ck *Checkpoint) New(ctx context.Context, opts ...Option) (*Sandbox, error) if err != nil { return nil, err } - if len(cr.Redirect) > 0 { - body, err = encodeBody("checkpoint claim", checkpointClaimRequest{TTLSeconds: claim.TTLSeconds, NoRedirect: true}) - if err != nil { - return nil, err - } - var sb *Sandbox - retryAny := func(error) bool { return true } - if tryErr := tryEach(cr.Redirect, func(addr string) error { - target, claimErr := ck.claimAt(ctx, addr, body) - if claimErr != nil { - return claimErr - } - if len(target.Redirect) > 0 { - return fmt.Errorf("%s redirected again despite no_redirect", addr) - } - sb = ck.c.handleFrom(addr, target) - return nil - }, retryAny); tryErr != nil { - return nil, fmt.Errorf("claim checkpoint: all redirect targets failed: %w", tryErr) - } - return sb, nil + if len(cr.Redirect) == 0 { + return ck.c.handleFrom(ck.addr, cr), nil } - return ck.c.handleFrom(ck.addr, cr), nil + body, err = encodeBody("checkpoint claim", checkpointClaimRequest{TTLSeconds: claim.TTLSeconds, NoRedirect: true}) + if err != nil { + return nil, err + } + addr, target, err := redirectFallback(ck.addr, cr.Redirect, func(a string) (claimResponse, error) { + return ck.claimAt(ctx, a, body) + }) + if err != nil { + return nil, fmt.Errorf("claim checkpoint: %w", err) + } + return ck.c.handleFrom(addr, target), nil } -// Delete removes the checkpoint from its node. +// Delete removes the checkpoint from its node and asks every peer that node +// currently sees to drop any replica a heal pulled — best-effort eventual +// cleanup, not a fleet-wide revocation. A peer that misses that broadcast +// (offline, partitioned, or joined later) keeps serving branches from its +// replica until the node's checkpoint_ttl_hours ages it out; with that TTL +// at its default of 0 (keep forever), an unreachable peer's replica has no +// cleanup bound at all. See the cluster docs' placement lifecycle. func (ck *Checkpoint) Delete(ctx context.Context) error { return doNoContent(ctx, ck.c, http.MethodDelete, ck.addr, "/v1/checkpoints/"+ck.ID, nil, ck.c.apiToken, "delete checkpoint") } diff --git a/sdk/go/checkpoint_test.go b/sdk/go/checkpoint_test.go index 65cd74f..46a5495 100644 --- a/sdk/go/checkpoint_test.go +++ b/sdk/go/checkpoint_test.go @@ -48,15 +48,24 @@ func TestCheckpointNewFollowsRedirect(t *testing.T) { } func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { + // The probed owner transiently fails (500); once exhausted, New falls + // back to the origin, which this time fails definitively too. broken := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) t.Cleanup(broken.Close) + var entryCalls int entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ - Redirect: []string{strings.TrimPrefix(broken.URL, "http://")}, - }) + entryCalls++ + if entryCalls == 1 { + _ = json.NewEncoder(w).Encode(claimResponse{ + Redirect: []string{strings.TrimPrefix(broken.URL, "http://")}, + }) + return + } + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(errorResponse{Error: "origin also failed"}) })) t.Cleanup(entry.Close) @@ -66,12 +75,53 @@ func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { if err == nil { t.Fatal("New: want error, got nil") } - if !strings.Contains(err.Error(), "http 500") { - t.Errorf("err %v, want the 500 surfaced", err) + if !strings.Contains(err.Error(), "origin also failed") { + t.Errorf("err %v, want the origin's failure surfaced", err) } if sb != nil { t.Errorf("sb %+v, want nil on failure", sb) } + if entryCalls != 2 { + t.Errorf("entry called %d times, want 2 (redirect + fallback)", entryCalls) + } +} + +func TestCheckpointNewRedirectFallbackHeals(t *testing.T) { + // The only probed owner is mid-heal (503); once exhausted, New falls + // back to the origin, which heals (pulls the checkpoint) locally. + busy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(busy.Close) + + var entryCalls int + entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + entryCalls++ + if entryCalls == 1 { + _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(busy.URL, "http://")}}) + return + } + var req checkpointClaimRequest + _ = json.NewDecoder(r.Body).Decode(&req) + if !req.NoRedirect { + t.Error("origin fallback did not carry no_redirect") + } + _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_healed", Token: "t"}) + })) + t.Cleanup(entry.Close) + + c := testClient(t, entry) + ck := checkpointHandle(c, c.addr, checkpointRecord{ID: "ck_5"}) + sb, err := ck.New(t.Context()) + if err != nil { + t.Fatalf("New: %v", err) + } + if sb.ID != "sb_healed" { + t.Errorf("id %q, want sb_healed", sb.ID) + } + if entryCalls != 2 { + t.Errorf("entry called %d times, want 2 (redirect + fallback)", entryCalls) + } } // TestCheckpointNewRedirectNeverYieldsEmptyID is a regression test: New used diff --git a/sdk/go/client.go b/sdk/go/client.go index 4539213..33cfb15 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -38,7 +38,9 @@ type Client struct { // defaults: the no-network lane and the smallest size tier. New returns when // the sandbox's silkd is reachable; a warm pool hit is milliseconds, a cold // key can take the full boot. Against a cluster, a warm miss redirects to a -// peer that holds one, which New follows transparently. +// peer that holds one, which New follows transparently; if every candidate +// fails transiently (full, mid-heal, unreachable), New falls back to the +// origin once so it provisions or heals locally. func (c *Client) New(ctx context.Context, template string, opts ...Option) (*Sandbox, error) { claim := claimRequest{Template: template} for _, opt := range opts { @@ -53,30 +55,21 @@ func (c *Client) New(ctx context.Context, template string, opts ...Option) (*San if err != nil { return nil, err } - if len(cr.Redirect) > 0 { - // Retry at a peer with no_redirect set: it warm-or-provisions and - // cannot bounce us again. Try each candidate so one dead/stale peer - // (its addr lingering in a gossip view) doesn't fail the claim. - claim.NoRedirect = true - body, err = encodeBody("claim", claim) - if err != nil { - return nil, err - } - var sb *Sandbox - retryAny := func(error) bool { return true } - if tryErr := tryEach(cr.Redirect, func(addr string) error { - target, claimErr := c.claimAt(ctx, addr, body) - if claimErr != nil { - return claimErr - } - sb = c.handleFrom(addr, target) - return nil - }, retryAny); tryErr != nil { - return nil, fmt.Errorf("claim: all redirect targets failed: %w", tryErr) - } - return sb, nil + if len(cr.Redirect) == 0 { + return c.handleFrom(c.addr, cr), nil + } + claim.NoRedirect = true + body, err = encodeBody("claim", claim) + if err != nil { + return nil, err } - return c.handleFrom(c.addr, cr), nil + addr, target, err := redirectFallback(c.addr, cr.Redirect, func(a string) (claimResponse, error) { + return c.claimAt(ctx, a, body) + }) + if err != nil { + return nil, fmt.Errorf("claim: %w", err) + } + return c.handleFrom(addr, target), nil } // Lookup relocates a sandbox handle whose owner address was lost, given its @@ -267,6 +260,79 @@ func retryMiss(err error) bool { return !errors.As(err, &he) || he.status == http.StatusNotFound } +// retryAny retries a redirect candidate's failure unconditionally: one +// candidate being wrong for the claim (no egress, a stale name) must not +// stop the walk from reaching a candidate that would still succeed. +func retryAny(error) bool { return true } + +// retryTransient reports whether an origin-fallback is worth the round-trip: +// true for a transport failure (unreachable peer), a miss (404, stale +// gossip), full (429), mid-heal (503), or an engine/proxy failure +// (500/502/504 — the last two covering a load balancer in front of +// sandboxd) — any of which the origin's own attempt may not hit. A served +// 4xx like a bad request, an unauthorized/forbidden token, or an egress +// conflict is definitive: the origin would fail the same way, so falling +// back would only hide the real error. +func retryTransient(err error) bool { + var he *httpError + if !errors.As(err, &he) { + return true + } + switch he.status { + case http.StatusNotFound, http.StatusTooManyRequests, http.StatusServiceUnavailable, + http.StatusInternalServerError, http.StatusBadGateway, http.StatusGatewayTimeout: + return true + default: + return false + } +} + +// redirectFallback walks candidates via claimAt, retrying broadly (retryAny) +// so one wrong candidate doesn't cost a candidate that would still succeed. +// If every candidate is exhausted and the last failure was transient +// (retryTransient), it gives origin one more no_redirect attempt — the node +// that issued the redirect provisions or heals locally instead of leaving +// the claim stuck on stale gossip. A definitive last failure (a bad request, +// an auth rejection, a conflict) skips the fallback: origin would fail the +// same way. A second-level redirect (a compliant server never sends one +// once no_redirect is set) fails the candidate rather than being followed. +func redirectFallback(origin string, candidates []string, claimAt func(addr string) (claimResponse, error)) (string, claimResponse, error) { + claimNoRedirect := func(target string) (claimResponse, error) { + cr, err := claimAt(target) + if err != nil { + return claimResponse{}, err + } + if len(cr.Redirect) > 0 { + return claimResponse{}, fmt.Errorf("%s redirected again despite no_redirect", target) + } + return cr, nil + } + + var won string + var cr claimResponse + tryErr := tryEach(candidates, func(addr string) error { + target, err := claimNoRedirect(addr) + if err != nil { + return err + } + won, cr = addr, target + return nil + }, retryAny) + if tryErr == nil { + return won, cr, nil + } + if !retryTransient(tryErr) { + return "", claimResponse{}, fmt.Errorf("all redirect targets failed: %w", tryErr) + } + cr, err := claimNoRedirect(origin) + if err != nil { + // Both halves matter to whoever reads this: the peers' failures say why + // the claim left the origin, the origin's says why coming back did not help. + return "", claimResponse{}, fmt.Errorf("all redirect targets failed, origin fallback failed: %w", errors.Join(tryErr, err)) + } + return origin, cr, nil +} + // scatter probes addrs concurrently, returning the first success and // canceling the losers — one hung peer must not stall the caller. When // every probe fails (or addrs is empty) ok is false. diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index c83140d..86890c5 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -279,6 +279,158 @@ func TestRedirectTriesAllCandidates(t *testing.T) { } } +func TestRedirectCandidateDefinitiveErrorStillTriesNextCandidate(t *testing.T) { + // Candidate one is wrong for this claim (403, definitive) but candidate + // two would still succeed — the per-candidate walk retries broadly, so + // one ill-suited candidate must not cost a candidate that would answer. + forbidden := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(forbidden.Close) + + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_5", Token: "t"}) + })) + t.Cleanup(good.Close) + + entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(claimResponse{ + Redirect: []string{strings.TrimPrefix(forbidden.URL, "http://"), strings.TrimPrefix(good.URL, "http://")}, + }) + })) + t.Cleanup(entry.Close) + + sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") + if err != nil { + t.Fatalf("New: %v", err) + } + if sb.ID != "sb_5" { + t.Errorf("id %q, want sb_5 (second candidate, reached despite the first's 403)", sb.ID) + } +} + +func TestRedirectFallbackHeals(t *testing.T) { + // The only candidate holds the record but is full (429); once it's + // exhausted, New must fall back to the origin itself, which heals. + var fullCalls int + full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fullCalls++ + w.WriteHeader(http.StatusTooManyRequests) + })) + t.Cleanup(full.Close) + + var entryCalls int + entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + entryCalls++ + if entryCalls == 1 { + _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) + return + } + var req claimRequest + _ = json.NewDecoder(r.Body).Decode(&req) + if !req.NoRedirect { + t.Error("origin fallback did not carry no_redirect") + } + _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_healed", Token: "t"}) + })) + t.Cleanup(entry.Close) + + sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") + if err != nil { + t.Fatalf("New: %v", err) + } + if sb.ID != "sb_healed" { + t.Errorf("id %q, want sb_healed", sb.ID) + } + if sb.owner != strings.TrimPrefix(entry.URL, "http://") { + t.Errorf("owner %q, want the origin", sb.owner) + } + if entryCalls != 2 { + t.Errorf("origin called %d times, want 2 (initial + fallback)", entryCalls) + } + if fullCalls != 1 { + t.Errorf("candidate called %d times, want 1", fullCalls) + } +} + +func TestRedirectFallbackAlsoFails(t *testing.T) { + // Every candidate is full (429); the origin fallback answers too, but + // with a definitive error, which must surface as the final error. + var fullCalls int + full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fullCalls++ + w.WriteHeader(http.StatusTooManyRequests) + })) + t.Cleanup(full.Close) + + var entryCalls int + entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + entryCalls++ + if entryCalls == 1 { + _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) + return + } + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(errorResponse{Error: "origin also unavailable"}) + })) + t.Cleanup(entry.Close) + + sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") + if err == nil { + t.Fatal("New: want error, got nil") + } + if !strings.Contains(err.Error(), "origin also unavailable") { + t.Errorf("err %v, want the origin's failure surfaced", err) + } + if sb != nil { + t.Errorf("sb %+v, want nil", sb) + } + if entryCalls != 2 { + t.Errorf("origin called %d times, want 2 (initial + fallback)", entryCalls) + } + if fullCalls != 1 { + t.Errorf("candidate called %d times, want 1", fullCalls) + } +} + +func TestRedirectDefinitiveErrorSkipsFallback(t *testing.T) { + tests := []struct { + name string + code int + want string + }{ + {"conflict", http.StatusConflict, "no egress attachment"}, + {"forbidden", http.StatusForbidden, "tenant not allowed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // A definitive 4xx is not worth trying another candidate, and + // not worth falling back to the origin either — the origin + // would fail the exact same way. + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tt.code) + _ = json.NewEncoder(w).Encode(errorResponse{Error: tt.want}) + })) + t.Cleanup(bad.Close) + + var entryCalls int + entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + entryCalls++ + _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(bad.URL, "http://")}}) + })) + t.Cleanup(entry.Close) + + _, err := testClient(t, entry).New(t.Context(), "rt:24.04") + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Errorf("err %v, want %q surfaced", err, tt.want) + } + if entryCalls != 1 { + t.Errorf("entry called %d times, want 1 (no origin fallback for a definitive error)", entryCalls) + } + }) + } +} + func TestLookupScatter(t *testing.T) { // The owner is node B; the entry node A doesn't own the sandbox but lists // B as a peer. Lookup must scatter to B and bind the handle there. diff --git a/sdk/python/cocoonsandbox/checkpoint.py b/sdk/python/cocoonsandbox/checkpoint.py index 2b1cc9a..13fa09c 100644 --- a/sdk/python/cocoonsandbox/checkpoint.py +++ b/sdk/python/cocoonsandbox/checkpoint.py @@ -6,8 +6,6 @@ from typing import TYPE_CHECKING -from .errors import APIError - if TYPE_CHECKING: from .sandbox import Sandbox @@ -25,10 +23,12 @@ def __init__(self, client, addr: str, rec: dict): def new(self, ttl_seconds: int = 0) -> Sandbox: """Claims a fresh sandbox branched from the checkpoint, following a - redirect to the node that actually holds it.""" + redirect to the node that actually holds it; if every candidate + fails transiently, the claim falls back to the origin once so it + heals (pulls the checkpoint) locally.""" # Local import: a top-level one would close the client -> sandbox -> # checkpoint cycle. - from .client import _try_each + from .client import _redirect_fallback claim = {"ttl_seconds": ttl_seconds} if ttl_seconds else {} path = f"/v1/checkpoints/{self.id}/claim" @@ -38,14 +38,18 @@ def new(self, ttl_seconds: int = 0) -> Sandbox: return self._client._handle_from(self._addr, reply) claim["no_redirect"] = True - def claim_at(peer: str) -> Sandbox: - r = self._client._post_json(peer, path, claim, "claim checkpoint") - if r.get("redirect"): - raise APIError("claim checkpoint", 0, f"{peer} redirected again despite no_redirect") - return self._client._handle_from(peer, r) + def post(peer): + return self._client._post_json(peer, path, claim, "claim checkpoint") - return _try_each(redirect, claim_at, retry=lambda _: True) + addr, reply = _redirect_fallback(self._addr, redirect, post, "claim checkpoint") + return self._client._handle_from(addr, reply) def delete(self) -> None: - """Removes the checkpoint from its node.""" + """Removes the checkpoint from its node and asks every peer that node + currently sees to drop any replica a heal pulled — best-effort eventual + cleanup, not a fleet-wide revocation. A peer that misses that broadcast + (offline, partitioned, or joined later) keeps serving branches from its + replica until the node's checkpoint_ttl_hours ages it out; with that + TTL at its default of 0 (keep forever), an unreachable peer's replica + has no cleanup bound at all.""" self._client._request(self._addr, "DELETE", f"/v1/checkpoints/{self.id}", None, "delete checkpoint") diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 8bfb149..1c6bcad 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -29,20 +29,21 @@ def __init__(self, addr: str, api_token: str = "", timeout: float = 120.0): def new(self, template: str, net: str = "", size: str = "", ttl_seconds: int = 0) -> Sandbox: """Claims a sandbox; a warm hit is milliseconds. On a cluster a warm - miss may redirect to a peer, followed transparently.""" + miss may redirect to a peer, followed transparently; if every + candidate fails transiently, the claim falls back to the origin + once so it provisions or heals locally.""" claim = _claim_body(template, net, size, ttl_seconds) reply = self._post_json(self.addr, "/v1/claim", claim, "claim") redirect = reply.get("redirect") or [] - if redirect: - # Retry at a peer with no_redirect set: it warm-or-provisions and - # cannot bounce us again. - claim["no_redirect"] = True - # A candidate that is full (429) or erroring must not abort the - # follow — try every one until a node answers, matching the Go SDK. - return _try_each(redirect, lambda peer: self._handle_from( - peer, self._post_json(peer, "/v1/claim", claim, "claim")), - retry=lambda _: True) - return self._handle_from(self.addr, reply) + if not redirect: + return self._handle_from(self.addr, reply) + claim["no_redirect"] = True + + def post(peer): + return self._post_json(peer, "/v1/claim", claim, "claim") + + addr, reply = _redirect_fallback(self.addr, redirect, post, "claim") + return self._handle_from(addr, reply) def delete_template(self, template: str, net: str = "", size: str = "") -> None: """Removes a promoted template by name; on a cluster the delete @@ -191,6 +192,49 @@ def _try_each(candidates, call, retry=lambda exc: exc.status in (404, 0)): raise last_error +def _retry_transient(exc: APIError) -> bool: + """Origin-fallback policy: worth the round-trip for a transport failure + (status 0), a miss (404, stale gossip), full (429), mid-heal (503), or an + engine/proxy failure (500/502/504 -- the last two covering a load + balancer in front of sandboxd) -- any of which the origin's own attempt + may not hit. A served 4xx like a bad request, an unauthorized/forbidden + token, or an egress conflict is definitive: the origin would fail the + same way, so falling back would only hide the real error.""" + return exc.status in (0, 404, 429, 503, 500, 502, 504) + + +def _redirect_fallback(origin: str, candidates: list, post, verb: str): + """Walks candidates via post(addr) -> raw reply dict, retrying broadly + (any candidate failure moves to the next) so one wrong candidate doesn't + cost a candidate that would still succeed. If every candidate is + exhausted and the last failure was transient (_retry_transient), gives + the origin one more no_redirect attempt -- the node that issued the + redirect provisions or heals locally instead of leaving the claim stuck + on stale gossip. A definitive last failure skips the fallback: the origin + would fail the same way. A second-level redirect (a compliant server + never sends one once no_redirect is set) fails the candidate rather than + being followed. Returns (addr, reply).""" + def attempt(addr): + reply = post(addr) + if reply.get("redirect"): + raise APIError(verb, 0, f"{addr} redirected again despite no_redirect") + return addr, reply + + try: + return _try_each(candidates, attempt, retry=lambda _: True) + except APIError as exc: + if not _retry_transient(exc): + raise + try: + return attempt(origin) + except APIError as origin_exc: + # Both halves matter to whoever reads this: the peers' failure says + # why the claim left the origin, the origin's why returning did not help. + origin_exc.message = f"{origin_exc.message} (after redirect targets failed: {exc.message})" + origin_exc.args = (f"{verb}: {origin_exc.message} (HTTP {origin_exc.status})",) + raise + + def _scatter(addrs, probe): """Probes every addr concurrently and returns the first success; when all probes fail the last error propagates. Loser threads are daemons diff --git a/sdk/python/tests/test_checkpoint.py b/sdk/python/tests/test_checkpoint.py index 56f3a04..66a9d4a 100644 --- a/sdk/python/tests/test_checkpoint.py +++ b/sdk/python/tests/test_checkpoint.py @@ -61,12 +61,44 @@ def redirect(body, path): def test_checkpoint_new_all_candidates_fail(spawn_node): - broken = spawn_node({("POST", "/v1/checkpoints/ck_2/claim"): lambda body, path: (500, {"error": "boom"})}) - entry = spawn_node({("POST", "/v1/checkpoints/ck_2/claim"): lambda body, path: (200, {"redirect": [broken]})}) + # The probed owner transiently fails (500); once exhausted, new() falls + # back to the origin, which this time fails definitively too. + path = "/v1/checkpoints/ck_2/claim" + broken = spawn_node({("POST", path): lambda body, p: (500, {"error": "boom"})}) + calls = [] + + def entry_claim(body, p): + calls.append(body) + if len(calls) == 1: + return 200, {"redirect": [broken]} + return 409, {"error": "origin also failed"} + + entry = spawn_node({("POST", path): entry_claim}) with pytest.raises(APIError) as exc: Client(entry).checkpoint("ck_2").new() - assert exc.value.status == 500 and "boom" in exc.value.message + assert exc.value.status == 409 and "origin also failed" in exc.value.message + assert len(calls) == 2 and calls[1]["no_redirect"] is True + + +def test_checkpoint_new_redirect_fallback_heals(spawn_node): + # The only probed owner is mid-heal (503); once exhausted, new() falls + # back to the origin, which heals (pulls the checkpoint) locally. + path = "/v1/checkpoints/ck_5/claim" + busy = spawn_node({("POST", path): lambda body, p: (503, {"error": "healing"})}) + + calls = [] + + def entry_claim(body, p): + calls.append(body) + if len(calls) == 1: + return 200, {"redirect": [busy]} + return 200, {"id": "sb_healed", "token": "t"} + + entry = spawn_node({("POST", path): entry_claim}) + sb = Client(entry).checkpoint("ck_5").new() + assert sb.id == "sb_healed" and sb.owner == entry + assert len(calls) == 2 and calls[1]["no_redirect"] is True def test_checkpoint_new_redirect_never_yields_empty_id(spawn_node, dead_addr): diff --git a/sdk/python/tests/test_fault_matrix.py b/sdk/python/tests/test_fault_matrix.py index 2bc48b3..c2dc307 100644 --- a/sdk/python/tests/test_fault_matrix.py +++ b/sdk/python/tests/test_fault_matrix.py @@ -1,6 +1,8 @@ """Fault matrix for the cluster redirect follow and the lookup scatter: -dead first candidate (connection refused), 404 first candidate, all-404 — -and lookup must not pay a hung or dead peer's full timeout.""" +dead first candidate (connection refused), 404 first candidate, a definitive +candidate error that still tries the next one, every candidate exhausted +(then the origin fallback heals, fails, or is skipped for a definitive +error) — and lookup must not pay a hung or dead peer's full timeout.""" import socket import threading @@ -61,13 +63,83 @@ def test_claim_redirect_skips_404_candidate(spawn_node): assert Client(entry).new("rt:24.04").id == "sb_2" -def test_claim_redirect_all_404_propagates_last(spawn_node): +def test_claim_redirect_all_candidates_fail_then_origin_fallback_fails(spawn_node): + # Every candidate is a stale miss (404); once exhausted, new() falls + # back to the origin, which this time fails definitively too. a = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone a"})}) b = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone b"})}) - entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [a, b]})}) + + calls = [] + + def entry_claim(body, path): + calls.append(body) + if len(calls) == 1: + return 200, {"redirect": [a, b]} + return 409, {"error": "origin also failed"} + + entry = spawn_node({("POST", "/v1/claim"): entry_claim}) + with pytest.raises(APIError) as exc: + Client(entry).new("rt:24.04") + assert exc.value.status == 409 and "origin also failed" in exc.value.message + # The last candidate's failure must survive into the message: it is what + # says why the claim left the origin in the first place. + assert "gone b" in exc.value.message + assert len(calls) == 2 and calls[1]["no_redirect"] is True + + +def test_claim_redirect_all_candidates_fail_then_origin_heals(spawn_node): + # The only candidate holds the record but is full (429); once exhausted, + # new() falls back to the origin, which heals (provisions locally). + full_calls = [] + + def full_claim(body, path): + full_calls.append(body) + return 429, {"error": "full"} + + full = spawn_node({("POST", "/v1/claim"): full_claim}) + + calls = [] + + def entry_claim(body, path): + calls.append(body) + if len(calls) == 1: + return 200, {"redirect": [full]} + return 200, {"id": "sb_healed", "token": "t"} + + entry = spawn_node({("POST", "/v1/claim"): entry_claim}) + sb = Client(entry).new("rt:24.04") + assert sb.id == "sb_healed" and sb.owner == entry + assert len(calls) == 2 and calls[1]["no_redirect"] is True + assert len(full_calls) == 1 + + +@pytest.mark.parametrize(("status", "message"), [(409, "no egress"), (403, "tenant not allowed")]) +def test_claim_redirect_definitive_error_skips_origin_fallback(spawn_node, status, message): + # A definitive 4xx is not worth trying another candidate, and not worth + # falling back to the origin either -- the origin would fail the same way. + bad = spawn_node({("POST", "/v1/claim"): lambda body, path: (status, {"error": message})}) + + calls = [] + + def entry_claim(body, path): + calls.append(body) + return 200, {"redirect": [bad]} + + entry = spawn_node({("POST", "/v1/claim"): entry_claim}) with pytest.raises(APIError) as exc: Client(entry).new("rt:24.04") - assert exc.value.status == 404 and "gone b" in exc.value.message + assert exc.value.status == status + assert len(calls) == 1 + + +def test_claim_redirect_candidate_definitive_error_still_tries_next_candidate(spawn_node): + # Candidate one is wrong for this claim (403, definitive) but candidate + # two would still succeed -- the per-candidate walk retries broadly, so + # one ill-suited candidate must not cost a candidate that would answer. + forbidden = spawn_node({("POST", "/v1/claim"): lambda body, path: (403, {"error": "tenant not allowed"})}) + good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_3", "token": "t"})}) + entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [forbidden, good]})}) + assert Client(entry).new("rt:24.04").id == "sb_3" def test_delete_template_skips_dead_owner(spawn_node, dead_addr): From b3a7ec9cfd5878a5cc4544ae416f54e8f7643bee Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 18:34:04 +0800 Subject: [PATCH 18/31] sdk: name the candidate's failure in the fallback test The assertion checked both halves of the combined error but the fixture gave the candidate an unlabelled 429, so a regression that dropped the candidate's message would still have matched on the origin's alone. --- sdk/go/client_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 86890c5..13baafc 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -360,6 +360,7 @@ func TestRedirectFallbackAlsoFails(t *testing.T) { full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fullCalls++ w.WriteHeader(http.StatusTooManyRequests) + _ = json.NewEncoder(w).Encode(errorResponse{Error: "candidate full"}) })) t.Cleanup(full.Close) @@ -382,6 +383,11 @@ func TestRedirectFallbackAlsoFails(t *testing.T) { if !strings.Contains(err.Error(), "origin also unavailable") { t.Errorf("err %v, want the origin's failure surfaced", err) } + // The candidate's failure must survive into the message too: it is what + // says why the claim left the origin in the first place. + if !strings.Contains(err.Error(), "candidate full") { + t.Errorf("err %v, want the candidate's failure surfaced too", err) + } if sb != nil { t.Errorf("sb %+v, want nil", sb) } From 1f9bc40b56b0e47db76361a190c19936d33e41d9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 18:38:05 +0800 Subject: [PATCH 19/31] sandboxd: pull outside the record lock, and let a delete veto the publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Holding the id's write lock across a multi-GB transfer put every other caller for that id behind up to the whole heal budget, on a plain mutex no cancelled client could leave. The transfer now runs unlocked inside a manager-owned flight: waiters share one pull and leave the moment their own context is done, while the transfer itself is detached and still finishes, so the record is paid for once regardless of who stays. Moving the lock off the transfer reopens resurrection in a subtler form. A delete arriving mid-staging correctly finds nothing and answers not-found, but the heal's locked finish would find nothing either and publish anyway — absence alone cannot say whether nothing happened or whether a delete just answered for this id. So a delete that finds an id absent vetoes any heal pending for it, and the heal reads that veto under the same lock it decides to publish under. Both maps are bounded by the concurrent-heal cap. The healer no longer dedups pulls of its own. Its caller supplies the staging directory, so two concurrent pulls are two destinations and sharing one result leaves a caller publishing a directory nothing ever wrote — dedup belongs to whoever owns the destination, which is now the manager. Each fix is pinned by a test that fails without it: dropping the veto resurrects a deleted checkpoint, a blocking receive strands a cancelled caller, and restoring the healer's own dedup leaves the second caller's staging directory empty. --- sandboxd/pool/checkpoint.go | 107 ++++++++++++++++++++++---- sandboxd/pool/checkpoint_heal_test.go | 89 ++++++++++++++------- sandboxd/pool/pool.go | 17 +++- sandboxd/store/peer/peer.go | 22 +++--- sandboxd/store/peer/peer_test.go | 38 +++++---- 5 files changed, 201 insertions(+), 72 deletions(-) diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 4b249c7..f8fe767 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -165,42 +165,113 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl ti return out, err } -// healCheckpoint pulls ckptID under its record lock, so the transfer, -// validation, and publish race no concurrent heal, delete, or TTL sweep of the -// same id. It releases before returning: claimLoaded takes an RLock on the -// same lock, and handles a delete landing in between. +// healCheckpoint dedups concurrent heals of ckptID onto one flight (which +// owns its own staging dir — never a caller's) and lets THIS call abandon +// waiting the moment ctx is done; the flight itself runs detached from any +// one caller (context.Background() in runHeal), so a client hanging up never +// abandons a transfer already paid for — the next call would just pay again. func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { - l := m.recLock(ckptID) - l.Lock() - defer func() { l.Unlock(); m.recDone(ckptID) }() if ckpt, err := m.loadCheckpoint(ctx, ckptID); err == nil { return ckpt, nil } + resCh := m.healFlights.DoChan(ckptID, func() (any, error) { + return m.runHeal(ckptID) + }) + select { + case res := <-resCh: + if res.Err != nil { + return types.Checkpoint{}, res.Err + } + return res.Val.(types.Checkpoint), nil + case <-ctx.Done(): + return types.Checkpoint{}, ctx.Err() + } +} + +// runHeal is healCheckpoint's flight body: it stages and pulls WITHOUT +// holding ckptID's record lock — a heal budget runs up to 30 minutes, and +// holding the lock across it would pin every other operation on the same id +// (a delete, a claim, another heal) behind an uncancellable wait for that +// long. The lock is taken only for the fast, final steps: check for a +// concurrent veto (see vetoIfHealPending), re-validate, and publish. +func (m *Manager) runHeal(ckptID string) (types.Checkpoint, error) { select { case m.healSem <- struct{}{}: defer func() { <-m.healSem }() default: return types.Checkpoint{}, ErrHealBusy } + m.markHealPending(ckptID) staging, err := m.ckpts.Stage(ckptID) if err != nil { + m.clearHealPending(ckptID) return types.Checkpoint{}, fmt.Errorf("stage healed checkpoint: %w", err) } defer func() { _ = os.RemoveAll(staging) }() + ctx := context.Background() validate := func(dir string) error { return validateHealedCheckpoint(dir, ckptID) } if err := m.healer.Pull(ctx, ckptID, staging, validate); err != nil { + m.clearHealPending(ckptID) if errors.Is(err, store.ErrNotFound) { return types.Checkpoint{}, ErrUnknownCheckpoint } return types.Checkpoint{}, fmt.Errorf("heal checkpoint: %w", err) } - // A client hanging up must not abort a transfer that already landed. - if err := m.ckpts.Publish(context.WithoutCancel(ctx), staging, ckptID); err != nil { + + l := m.recLock(ckptID) + l.Lock() + defer func() { l.Unlock(); m.recDone(ckptID) }() + if aborted := m.clearHealPending(ckptID); aborted { + return types.Checkpoint{}, ErrUnknownCheckpoint // a concurrent delete vetoed this heal + } + if ckpt, err := m.loadCheckpoint(ctx, ckptID); err == nil { + return ckpt, nil // published by another path while this one staged + } + if err := validate(staging); err != nil { + return types.Checkpoint{}, fmt.Errorf("validate healed checkpoint: %w", err) + } + if err := m.ckpts.Publish(ctx, staging, ckptID); err != nil { return types.Checkpoint{}, fmt.Errorf("publish healed checkpoint: %w", err) } return m.loadCheckpoint(ctx, ckptID) } +// markHealPending records ckptID as staging or pulling, unlocked, so +// vetoIfHealPending knows a concurrent delete must veto rather than ignore +// it. Bounded by healSem: at most maxConcurrentHeals entries ever exist. +func (m *Manager) markHealPending(ckptID string) { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + m.healPending[ckptID] = struct{}{} +} + +// clearHealPending un-marks ckptID and reports whether vetoIfHealPending +// vetoed it in the meantime; call under ckptID's recLock, so the check and +// the eventual publish decide together. +func (m *Manager) clearHealPending(ckptID string) (aborted bool) { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + delete(m.healPending, ckptID) + if _, aborted = m.healAbort[ckptID]; aborted { + delete(m.healAbort, ckptID) + } + return aborted +} + +// vetoIfHealPending marks ckptID aborted when a heal is currently pending +// for it, so that heal's locked decide phase (clearHealPending) sees the +// veto instead of publishing a checkpoint this call just answered "not +// here" for — a delete otherwise racing an unlocked, still-staging heal +// would return 404 only for the checkpoint to reappear moments later. A +// no-op when no heal is pending, so an unrelated miss leaves no residue. +func (m *Manager) vetoIfHealPending(ckptID string) { + m.recLocksMu.Lock() + defer m.recLocksMu.Unlock() + if _, pending := m.healPending[ckptID]; pending { + m.healAbort[ckptID] = struct{}{} + } +} + // validateHealedCheckpoint checks a staged pull's shape before publishing it: // an unreadable or misattributed record would suppress every later heal. func validateHealedCheckpoint(staging, wantID string) error { @@ -282,14 +353,15 @@ func (m *Manager) pinnedArchiveCks() map[string]struct{} { // delete only its own records — anything else answers ErrUnknownCheckpoint, // never a hint that the id exists; root (empty tenant) deletes anything. // Existence is checked under the record lock, not before it: heal broke the -// old assumption that a local miss means the id is truly gone — a concurrent -// heal can be mid-transfer, unpublished, when an unlocked check would see -// 404, and it would then publish right after this call gave up thinking -// there was nothing to delete. Taking the same lock heal takes makes the two -// wait on each other instead of racing. Every exit evicts the lock entry -// (recDoneEvict, not recDone) — unlike a template id, a checkpoint id is -// effectively one-shot, so nothing is lost keeping the map from growing -// per rejected or successful call alike. +// old assumption that a local miss means the id is truly gone. That alone is +// not enough, though — a heal's transfer runs unlocked (runHeal), so a +// concurrent delete can take the lock, find the checkpoint absent, and +// release it before the heal ever reaches its own locked decide phase; +// vetoIfHealPending closes that gap by telling a pending heal to abandon its +// publish instead of resurrecting what this call just answered "not here" +// for. Every exit evicts the lock entry (recDoneEvict, not recDone) — unlike +// a template id, a checkpoint id is effectively one-shot, so nothing is lost +// keeping the map from growing per rejected or successful call alike. func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope DeleteScope) error { // Reject a bad id before recLock: a rejected id must not leave a // lock-map entry. @@ -301,6 +373,7 @@ func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, s defer func() { l.Unlock(); m.recDoneEvict(ckptID) }() ckpt, err := m.loadCheckpoint(ctx, ckptID) if err != nil { + m.vetoIfHealPending(ckptID) return err } if !tenantOwns(tenant, ckpt.Tenant) { diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index aa0f396..fa7e54e 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -72,8 +72,10 @@ func TestClaimCheckpointAfterHealStaysLocal(t *testing.T) { // TestClaimCheckpointHealDedupsConcurrentPulls: two branches racing to heal // the same missing checkpoint must share one transfer, not each pay for it — -// the id's recLock now serializes them, so the second waits for the first's -// publish and never reaches the puller at all. +// healFlights (not the record lock, which no longer spans the transfer) +// dedups them onto one Manager-owned staging dir, and each branch must +// still get a genuinely valid, non-empty checkpoint out of it (the exact +// case a caller-owned-staging-dir singleflight used to get wrong). func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} @@ -81,21 +83,25 @@ func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { puller.release = make(chan struct{}) var wg sync.WaitGroup + sbs := make([]*types.Sandbox, 2) errs := make([]error, 2) for i := range 2 { wg.Go(func() { - _, err := m.ClaimCheckpointHeal(t.Context(), id, time.Hour, "") - errs[i] = err + sbs[i], errs[i] = m.ClaimCheckpointHeal(t.Context(), id, time.Hour, "") }) } waitFor(t, func() bool { return puller.count() >= 1 }) - time.Sleep(50 * time.Millisecond) // let the second goroutine block on the recLock + time.Sleep(50 * time.Millisecond) // let the second goroutine join the same flight close(puller.release) wg.Wait() for i, err := range errs { if err != nil { t.Errorf("goroutine %d: %v", i, err) + continue + } + if sbs[i] == nil || sbs[i].FromCheckpoint != id { + t.Errorf("goroutine %d: sandbox = %+v, want a valid branch of %s", i, sbs[i], id) } } if got := puller.count(); got != 1 { @@ -103,13 +109,15 @@ func TestClaimCheckpointHealDedupsConcurrentPulls(t *testing.T) { } } -// TestDeleteCheckpointWaitsForInFlightHeal covers the Codex r2 blocker: an -// unlocked existence check would see 404 while a heal is mid-transfer and -// give up, only for the heal to publish moments later — resurrecting a -// checkpoint the delete had just answered "not here" for. Delete must -// instead wait on the same record lock the heal holds, then genuinely -// delete what the heal just published. -func TestDeleteCheckpointWaitsForInFlightHeal(t *testing.T) { +// TestDeleteVetoesInFlightHeal covers the Codex r2 blocker: a heal's transfer +// runs unlocked (holding the lock across a 30-minute budget would pin every +// other operation on the id behind it), so a delete arriving while it stages +// must not block on that transfer — but it must also not let the heal +// resurrect the checkpoint moments after the delete answered "not here" for +// it. The delete returns immediately (nothing local to delete yet) and +// vetoes the pending heal, so the heal's own locked decide phase abandons +// its publish instead of racing the delete to a stale conclusion. +func TestDeleteVetoesInFlightHeal(t *testing.T) { id := "ck_00000000000000bb" ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) @@ -120,29 +128,54 @@ func TestDeleteCheckpointWaitsForInFlightHeal(t *testing.T) { _, err := m.ClaimCheckpointHeal(t.Context(), id, time.Hour, "") healDone <- err }() - waitFor(t, func() bool { return puller.count() >= 1 }) // heal is mid-pull, unpublished + waitFor(t, func() bool { return puller.count() >= 1 }) // heal is staging, unpublished - deleteDone := make(chan error, 1) - go func() { deleteDone <- m.DeleteCheckpoint(t.Context(), id, "", DeleteLocal) }() - // Give a buggy unlocked check every chance to (wrongly) return early - // before the heal ever publishes. - time.Sleep(50 * time.Millisecond) - select { - case err := <-deleteDone: - t.Fatalf("DeleteCheckpoint returned early (%v) while the heal was unpublished — it must wait for the record lock", err) - default: + if err := m.DeleteCheckpoint(t.Context(), id, "", DeleteLocal); !errors.Is(err, ErrUnknownCheckpoint) { + t.Fatalf("delete while heal is staging: %v, want ErrUnknownCheckpoint (nothing local yet, and it must not block)", err) } close(puller.release) - if err := <-healDone; err != nil { - t.Fatalf("heal: %v", err) - } - if err := <-deleteDone; err != nil { - t.Errorf("delete after heal published: %v, want nil (the record existed once delete's lock was granted)", err) + if err := <-healDone; !errors.Is(err, ErrUnknownCheckpoint) { + t.Errorf("heal after a concurrent delete: %v, want ErrUnknownCheckpoint (vetoed, must not publish)", err) } if m.HasCheckpoint(t.Context(), id) { - t.Error("checkpoint still present after delete finished") + t.Error("checkpoint present after a delete vetoed the pending heal — resurrection") + } +} + +// TestClaimCheckpointHealCtxCancelReturnsPromptly: a caller that gives up +// waiting must not be stuck behind the whole heal budget — the flight runs +// detached (context.Background() in runHeal), so this call returns +// ctx.Err() as soon as its own ctx is canceled, while the transfer keeps +// running and still lands. +func TestClaimCheckpointHealCtxCancelReturnsPromptly(t *testing.T) { + id := "ck_00000000000000bb" + ckpt := types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()} + m, puller := newHealManager(t, ckpt, []string{"peer-a:7777"}) + puller.release = make(chan struct{}) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := m.ClaimCheckpointHeal(ctx, id, time.Hour, "") + done <- err + }() + waitFor(t, func() bool { return puller.count() >= 1 }) // transfer started, unlocked + + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("ClaimCheckpointHeal after cancel: %v, want context.Canceled", err) + } + case <-time.After(2 * time.Second): + t.Fatal("ClaimCheckpointHeal did not return promptly after ctx cancellation") } + + // Detached from the canceled caller, the transfer must still complete + // and publish. + close(puller.release) + waitFor(t, func() bool { return m.HasCheckpoint(t.Context(), id) }) } // TestRecLockEvictionWaitsForAllHolders covers the lock-identity-split half diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index c6b141f..d794053 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -22,6 +22,7 @@ import ( "time" "github.com/projecteru2/core/log" + "golang.org/x/sync/singleflight" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/egress" @@ -230,9 +231,19 @@ type Manager struct { ckpts store.Store // A cluster-wide backend resolves every record from every node, so heal // and the delete broadcast are both no-ops against it. - ckptsShared bool - healer *peer.Healer + ckptsShared bool + healer *peer.Healer + // healSem bounds concurrent transfers node-wide; healFlights dedups + // concurrent heals of the same id to one transfer (keyed by checkpoint + // id, owns its own staging dir — callers never see one). healPending and + // healAbort (guarded by recLocksMu) let a delete that finds a checkpoint + // absent veto a heal already staging for that same id, so the heal's + // locked decide phase does not publish moments after a delete answered + // "not here" for it. healSem chan struct{} + healFlights singleflight.Group + healPending map[string]struct{} + healAbort map[string]struct{} peerDelete PeerDeleteFunc tpls store.Store ckptTTL time.Duration @@ -310,6 +321,8 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg egressListeners: map[string]*egressListener{}, egressTaps: map[string]string{}, recRefs: map[string]int{}, + healPending: map[string]struct{}{}, + healAbort: map[string]struct{}{}, egressSecrets: secrets, dial: egressDialer.DialContext, sweep: netfilter.SweepExcept, diff --git a/sandboxd/store/peer/peer.go b/sandboxd/store/peer/peer.go index 70cd4e7..feb93ee 100644 --- a/sandboxd/store/peer/peer.go +++ b/sandboxd/store/peer/peer.go @@ -9,8 +9,6 @@ import ( "path/filepath" "time" - "golang.org/x/sync/singleflight" - "github.com/cocoonstack/sandbox/sandboxd/store" ) @@ -26,15 +24,19 @@ type Validate func(staging string) error // Healer pulls a record this node does not hold into a caller-provided staging // directory; the caller validates and publishes it. A nil owners or puller -// makes every Pull a miss, so a node with no mesh simply cannot heal. +// makes every Pull a miss, so a node with no mesh simply cannot heal. Pull +// does not dedup concurrent calls: a caller-owned staging dir means two +// concurrent Pulls for the same id are two independent destinations, so +// sharing one result (as a Healer-internal singleflight once did) would +// leave whichever caller did not "win" the flight with an untouched, +// published-empty directory. Dedup belongs to whoever owns the destination — +// pool.Manager's per-id heal flight, which owns one staging dir per flight. type Healer struct { owners Owners puller Puller // budget overrides healBudget; a test seam, 30 minutes is unwaitable. budget time.Duration - - flights singleflight.Group } // NewHealer builds a Healer wired to owners and puller. @@ -43,7 +45,7 @@ func NewHealer(owners Owners, puller Puller) *Healer { } // Pull fetches id into staging from the first owner whose transfer validates, -// bounding every owner tried to one budget. Concurrent pulls share a flight. +// bounding every owner tried to one budget. func (h *Healer) Pull(ctx context.Context, id, staging string, validate Validate) error { if h.owners == nil || h.puller == nil { return store.ErrNotFound @@ -56,14 +58,10 @@ func (h *Healer) Pull(ctx context.Context, id, staging string, validate Validate // A client hanging up must not abandon a started pull; the budget bounds it. ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), budget) defer cancel() - _, err, _ := h.flights.Do(id, func() (any, error) { - return nil, h.pullFrom(ctx, id, staging, addrs, budget, validate) - }) - return err + return h.pullFrom(ctx, id, staging, addrs, budget, validate) } -// pullFrom tries each owner in turn, giving each an even slice of budget; -// callers hold id's singleflight slot. +// pullFrom tries each owner in turn, giving each an even slice of budget. func (h *Healer) pullFrom(ctx context.Context, id, staging string, addrs []string, budget time.Duration, validate Validate) error { perOwner := budget / time.Duration(len(addrs)) var errs []error diff --git a/sandboxd/store/peer/peer_test.go b/sandboxd/store/peer/peer_test.go index 7b2e9ff..50f1554 100644 --- a/sandboxd/store/peer/peer_test.go +++ b/sandboxd/store/peer/peer_test.go @@ -98,25 +98,37 @@ func TestHealErrorIsReported(t *testing.T) { } } -// TestHealDedupsConcurrentMisses: N concurrent Pulls of the same id sharing -// one staging destination must trigger exactly one transfer — a stampede of -// branches racing to heal the same checkpoint must not each pay for it. -func TestHealDedupsConcurrentMisses(t *testing.T) { +// TestPullDoesNotDedupConcurrentCalls: Pull used to share one singleflight +// result across concurrent callers, which was only safe because every +// caller happened to pass the SAME staging dir. With independent dirs (the +// only safe assumption once dedup is not Pull's job), sharing a result would +// leave whichever caller did not trigger the shared call with an untouched, +// empty directory published as if it were a real record. Two concurrent +// Pulls for one id but different dirs must each run their own full transfer. +func TestPullDoesNotDedupConcurrentCalls(t *testing.T) { puller := &fakePuller{records: map[string]map[string]string{"peer-a:7777": record()}} h := NewHealer(func(string) []string { return []string{"peer-a:7777"} }, puller) - dst := t.TempDir() + dst1, dst2 := t.TempDir(), t.TempDir() var wg sync.WaitGroup - for range 8 { - wg.Go(func() { - if err := h.Pull(t.Context(), testID, dst, nil); err != nil { - t.Errorf("Pull: %v", err) - } - }) + errs := make([]error, 2) + for i, dst := range []string{dst1, dst2} { + wg.Go(func() { errs[i] = h.Pull(t.Context(), testID, dst, nil) }) } wg.Wait() - if len(puller.asked) != 1 { - t.Errorf("puller called %d times (%v); concurrent misses must dedup to one pull", len(puller.asked), puller.asked) + + for i, err := range errs { + if err != nil { + t.Fatalf("Pull %d: %v", i, err) + } + } + if len(puller.asked) != 2 { + t.Errorf("puller called %d times, want 2: Pull must not dedup — that's the caller's job now", len(puller.asked)) + } + for _, dst := range []string{dst1, dst2} { + if got, err := os.ReadFile(filepath.Join(dst, store.ExportDir, "mem")); err != nil || string(got) != "guest-pages" { + t.Errorf("staged export at %s = %q, %v; want every caller's own dir populated", dst, got, err) + } } } From 60ee4744e73cd9269cad3db368c9c753f179baf5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 19:06:52 +0800 Subject: [PATCH 20/31] mesh: a redirect spends the warmth it was promised MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The redirect target came from a gossiped warm count refreshed once a second, and nothing recorded how many claims this node had already sent against it. Every claim inside one refresh window therefore chased the same stale number: a peer advertising four warm sandboxes received the whole burst, served four instantly, and cold-provisioned the rest one by one — each retry arrives with no_redirect set and cannot bounce again. Observed in production as sandbox allocation slowing roughly tenfold the moment the mesh was enabled. Candidates now debits the winner's count in the local view, so a window's redirects are bounded by what the fleet actually advertised; the peer's next adopted state resets the debit. Only the winner pays — the client walks the list in order and stops at the first success. The debit also rides this node's own outgoing gossip, which is at worst conservative and is corrected by the peer's next epoch. Undebited, the regression test redirects all forty claims of a burst at a peer that advertised four; debited, exactly four leave the node. --- sandboxd/mesh/mesh.go | 16 ++++++++--- sandboxd/mesh/mesh_test.go | 54 +++++++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 34a37b8..8c91a19 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -160,10 +160,17 @@ func (m *Mesh) ConfigMismatches() int { // Candidates returns up to two peer addresses that report warm(keyHash) > 0, // chosen power-of-two-choices to avoid herding every waiter onto one node. -// Self is never a candidate — the caller has already missed locally. +// Self is never a candidate — the caller has already missed locally. The +// winner's count is debited locally: a redirect is a claim in flight, and +// undebited, every claim in one gossip window would chase the same stale +// count, forcing the target to provision the whole burst once its warm pool +// ran out (each retry arrives with no_redirect set). The peer's next adopted +// state resets the debit. func (m *Mesh) Candidates(keyHash string) []string { m.mu.Lock() + defer m.mu.Unlock() type cand struct { + id string addr string warm int } @@ -173,15 +180,15 @@ func (m *Mesh) Candidates(keyHash string) []string { continue } if st.Pools[keyHash] > 0 { - pool = append(pool, cand{st.Addr, st.Pools[keyHash]}) + pool = append(pool, cand{id, st.Addr, st.Pools[keyHash]}) } } - m.mu.Unlock() switch len(pool) { case 0: return nil case 1: + m.view[pool[0].id].Pools[keyHash]-- return []string{pool[0].addr} } // Power-of-two-choices: sample two, order by warmer. This is load @@ -195,6 +202,9 @@ func (m *Mesh) Candidates(keyHash string) []string { if b.warm > a.warm { a, b = b, a } + // Only the winner is debited: the client walks the list in order and + // stops at the first success, so the runner-up's capacity stays intact. + m.view[a.id].Pools[keyHash]-- return []string{a.addr, b.addr} } diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index 17e327d..287aaab 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -96,10 +96,12 @@ func TestForgetPrunesDeadNode(t *testing.T) { func TestCandidatesPowerOfTwo(t *testing.T) { m := newTestMesh(t, "a") + // warm=100 each: 20 draws debit the winners but must never exhaust a + // peer, so the two-distinct-candidates property is what this test sees. m.merge([]NodeState{ - {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, - {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, - {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, + {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 100}}, + {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 100}}, + {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"k": 100}}, }) // With ≥2 warm peers, exactly two distinct candidates come back. for range 20 { @@ -115,6 +117,52 @@ func TestCandidatesPowerOfTwo(t *testing.T) { } } +// TestCandidatesDebitWarmCredit: a redirect consumes one unit of the peer's +// advertised warmth, so a burst inside one gossip window is bounded by what +// the peer actually promised instead of herding onto a stale count. +func TestCandidatesDebitWarmCredit(t *testing.T) { + m := newTestMesh(t, "a") + m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 4}}}) + + var sent int + for range 40 { + if m.Candidates("k") != nil { + sent++ + } + } + if sent != 4 { + t.Errorf("40 claims in one gossip window: %d redirected, want the 4 advertised", sent) + } + + // A fresh adopted state resets the credit. + m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 1}}}) + if m.Candidates("k") == nil { + t.Error("no candidate after a fresh gossip refresh") + } + if m.Candidates("k") != nil { + t.Error("credit outlived the refreshed count") + } +} + +// TestCandidatesDebitSumsAcrossPeers: the window's redirect budget is the +// fleet's advertised total, not per-call luck. +func TestCandidatesDebitSumsAcrossPeers(t *testing.T) { + m := newTestMesh(t, "a") + m.merge([]NodeState{ + {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, + {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, + }) + var sent int + for range 20 { + if m.Candidates("k") != nil { + sent++ + } + } + if sent != 4 { + t.Errorf("redirected %d, want 4 (the fleet's advertised warm total)", sent) + } +} + func TestTwoNodeClusterGossipsPools(t *testing.T) { a := startNode(t, "127.0.0.1", 0, "node-a") b := startNode(t, "127.0.0.1", 0, "node-b") From c020050414b186617708d8fbe5803bca67c103d8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 19:11:39 +0800 Subject: [PATCH 21/31] sandboxd: give the ownership probe a boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HEAD probe was an open metadata oracle: ids carry 64 bits of randomness, every syntactically valid request cost the target a disk read, and nothing rate-limited it. On an encrypted mesh the probe now carries an HMAC over the id and a coarse time bucket, keyed off a probe-specific derivation of cluster_key, verified before any disk is touched; a keyless mesh keeps the old capability-only posture, since a redirect-only fleet has no shared secret to sign with and losing the probe outright would break it. Peer heal now also requires a nonempty api_token: an open node with heal enabled left the raw blob GET reachable with no credential at all. The fan-out stops as soon as enough owners answered, so one hung peer no longer taxes a redirect that already has candidates. Concurrent probes for one id share a single fan-out, and a short positive cache serves a hot id's repeat redirects without re-asking the fleet — measured before, ten misses of one id against twenty peers cost two hundred HEADs; now the first fan-out answers them all until the entry expires or a local delete evicts it. Misses are never cached, so a record that appears a moment later is found. A redirect wants a couple of live candidates but a heal wants the widest source list a few bad owners cannot hide, so the two callers stop sharing one capped answer. --- e2e/e2e_test.go | 2 +- sandboxd/config/config.go | 3 + sandboxd/config/config_test.go | 1 + sandboxd/main.go | 20 +-- sandboxd/server/relay_test.go | 2 +- sandboxd/server/server.go | 35 +++-- sandboxd/server/server_test.go | 88 ++++++++++--- sandboxd/store/peer/probe.go | 206 ++++++++++++++++++++++++++--- sandboxd/store/peer/probe_test.go | 212 ++++++++++++++++++++++++++++++ 9 files changed, 520 insertions(+), 49 deletions(-) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 52efbdc..d2af823 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -304,7 +304,7 @@ func startTenantStack(t *testing.T, apiToken string, tenants []config.TenantSpec } go mgr.Run(t.Context()) - ts := httptest.NewServer(server.New(apiToken, tenants, "", mgr, eng.real, nil, nil, nil).Handler()) + ts := httptest.NewServer(server.New(apiToken, tenants, "", mgr, eng.real, nil, nil, nil, nil).Handler()) t.Cleanup(ts.Close) addr := strings.TrimPrefix(ts.URL, "http://") client, err := sandbox.Connect(addr, sandbox.WithAPIToken(apiToken)) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 7e6ace7..574e7a1 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -377,6 +377,9 @@ func (c *Config) validate() error { if c.CheckpointPeerHeal && c.CheckpointTTLHours == 0 { return fmt.Errorf("checkpoint_peer_heal requires checkpoint_ttl_hours > 0: the ttl is what ages out a healed replica a delete broadcast missed") } + if c.CheckpointPeerHeal && c.APIToken == "" { + return fmt.Errorf("checkpoint_peer_heal requires api_token: without it resolveScope leaves the raw checkpoint blob GET reachable with no credential") + } if err := c.validateTenants(); err != nil { return err } diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index cbedef7..1b8fb27 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -124,6 +124,7 @@ func TestLoadRejectsInvalid(t *testing.T) { {"checkpoint peer heal without mesh", `{"checkpoint_peer_heal":true,"pools":[]}`, "requires an encrypted mesh"}, {"checkpoint peer heal without cluster key", `{"checkpoint_peer_heal":true,"pools":[],"mesh":{"bind":"node1:7946"}}`, "requires an encrypted mesh"}, {"checkpoint peer heal without ttl", `{"checkpoint_peer_heal":true,"pools":[],"mesh":{"bind":"node1:7946","cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="}}`, "requires checkpoint_ttl_hours"}, + {"checkpoint peer heal without api_token", `{"checkpoint_peer_heal":true,"pools":[],"mesh":{"bind":"node1:7946","cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="},"checkpoint_ttl_hours":1}`, "requires api_token"}, } { t.Run(tt.name, func(t *testing.T) { _, err := Load(writeConfig(t, tt.body)) diff --git a/sandboxd/main.go b/sandboxd/main.go index d1c0beb..bafd431 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -34,9 +34,6 @@ import ( const ( shutdownGrace = 5 * time.Second gossipInterval = time.Second - // probeBudget bounds a checkpoint-owner probe fan-out for WithPeerHeal's - // resolver; the HTTPProber itself times out each individual peer sooner. - probeBudget = 5 * time.Second // Slowloris protection; ReadTimeout/WriteTimeout must stay zero — cold // claims block up to the cold probe timeout and relays stream forever. readHeaderTimeout = 5 * time.Second @@ -99,6 +96,7 @@ func main() { var placer server.Placer var prober server.CheckpointProber + var probeKey []byte if cfg.Mesh != nil { msh, err := startMesh(ctx, cfg, mgr) if err != nil { @@ -107,11 +105,17 @@ func main() { defer func() { _ = msh.Shutdown() }() placer = msh mgr.SetTemplateNotifier(func() { msh.UpdateSelf(ctx, mgr.WarmCounts(), mgr.TemplateHashes()) }) - prober = &peer.HTTPProber{Peers: msh.PeerAddrs} + clusterKey, err := cfg.Mesh.DecodedKey() + if err != nil { + logger.Fatalf(ctx, err, "decode mesh cluster key") + } + if len(clusterKey) > 0 { + probeKey = peer.DeriveProbeKey(clusterKey) + } + httpProber := &peer.HTTPProber{Peers: msh.PeerAddrs, ProbeKey: probeKey} + prober = httpProber mgr.WithPeerHeal(cfg.CheckpointPeerHeal, func(id string) []string { - probeCtx, cancel := context.WithTimeout(ctx, probeBudget) - defer cancel() - return prober.Owners(probeCtx, id) + return httpProber.HealOwners(ctx, id) }, cfg.APIToken) broadcaster := &peer.Broadcaster{Peers: msh.PeerAddrs, Token: cfg.APIToken} mgr.WithPeerDelete(broadcaster.Delete) @@ -123,7 +127,7 @@ func main() { if cfg.PreviewListen != "" { preview = server.NewPreviewServer(cfg.PreviewSecret, cfg.PreviewAdvertise, mgr) } - srv := server.New(cfg.APIToken, cfg.Tenants, cfg.AdvertiseAddr, mgr, eng, placer, prober, preview) + srv := server.New(cfg.APIToken, cfg.Tenants, cfg.AdvertiseAddr, mgr, eng, placer, prober, probeKey, preview) httpSrv := &http.Server{ Addr: cfg.Listen, Handler: srv.Handler(), diff --git a/sandboxd/server/relay_test.go b/sandboxd/server/relay_test.go index e770f7c..3f2833c 100644 --- a/sandboxd/server/relay_test.go +++ b/sandboxd/server/relay_test.go @@ -156,7 +156,7 @@ func newRelayServer(t *testing.T, guest func(net.Conn)) (*httptest.Server, *Serv go guest(guestEnd) return relayEnd, nil }} - srv := New("", nil, "node:7777", &fakeManager{}, dialer, nil, nil, nil) + srv := New("", nil, "node:7777", &fakeManager{}, dialer, nil, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { srv.CloseRelays() diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 6311e7e..ce7b988 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -108,6 +108,7 @@ type Placer interface { // redirect decision rather than gossiped. type CheckpointProber interface { Owners(ctx context.Context, id string) []string + Forget(id string) } // InfoResponse is the wire reply of GET /v1/info. Peers lists the other nodes' @@ -135,6 +136,7 @@ type Server struct { dialer Dialer placer Placer prober CheckpointProber + probeKey []byte apiToken string tenants []config.TenantSpec advertise string @@ -151,13 +153,17 @@ type Server struct { // calls). tenants adds per-tenant tokens next to the root apiToken. // advertise is this node's data-plane address, returned as a claim's owner // address. A nil placer disables mesh redirects and a nil prober disables -// checkpoint-owner probing (both single-node). -func New(apiToken string, tenants []config.TenantSpec, advertise string, mgr Manager, dialer Dialer, placer Placer, prober CheckpointProber, preview *PreviewServer) *Server { +// checkpoint-owner probing (both single-node). probeKey (see +// peer.DeriveProbeKey) authenticates the checkpoint HEAD probe; empty +// leaves it unauthenticated (an unencrypted mesh has no shared secret to +// build one from). +func New(apiToken string, tenants []config.TenantSpec, advertise string, mgr Manager, dialer Dialer, placer Placer, prober CheckpointProber, probeKey []byte, preview *PreviewServer) *Server { return &Server{ mgr: mgr, dialer: dialer, placer: placer, prober: prober, + probeKey: probeKey, apiToken: apiToken, tenants: tenants, advertise: advertise, @@ -407,10 +413,17 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { } } -// handleCheckpointProbe answers a peer's HEAD probe. Unauthenticated: the id -// is the capability, so this tells a caller only what it already knows. +// handleCheckpointProbe answers a peer's HEAD probe. With a probeKey +// configured (an encrypted mesh), the caller must present a fresh MAC over +// the id (peer.ProbeHeader) or the probe is rejected before the metadata +// read; without one, the id remains the only capability, same as before. func (s *Server) handleCheckpointProbe(w http.ResponseWriter, r *http.Request) { - if s.mgr.HasCheckpoint(r.Context(), r.PathValue("id")) { + id := r.PathValue("id") + if len(s.probeKey) > 0 && !peer.VerifyProbeMAC(s.probeKey, id, r.Header.Get(peer.ProbeHeader)) { + w.WriteHeader(http.StatusUnauthorized) + return + } + if s.mgr.HasCheckpoint(r.Context(), id) { w.WriteHeader(http.StatusOK) return } @@ -432,14 +445,20 @@ func (s *Server) handleListCheckpoints(w http.ResponseWriter, r *http.Request) { // handleDeleteCheckpoint removes a checkpoint; a tenant caller may delete // only its own records (anything else is 404). no_forward marks a delete // arriving from another node's own broadcast, so this one does not -// re-broadcast and loop the fleet forever. +// re-broadcast and loop the fleet forever. Either way, any cached probe +// answer for id is forgotten unconditionally: a node that never held a +// replica still learns from the broadcast that id's ownership changed. func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) { scope := pool.DeleteFleet if r.URL.Query().Get("no_forward") != "" { scope = pool.DeleteLocal } - err := s.mgr.DeleteCheckpoint(r.Context(), r.PathValue("id"), tenantFrom(r.Context()), scope) - writeResult(w, r, "delete checkpoint", r.PathValue("id"), "delete checkpoint failed", err, func() { + id := r.PathValue("id") + if s.prober != nil { + s.prober.Forget(id) + } + err := s.mgr.DeleteCheckpoint(r.Context(), id, tenantFrom(r.Context()), scope) + writeResult(w, r, "delete checkpoint", id, "delete checkpoint failed", err, func() { w.WriteHeader(http.StatusNoContent) }) } diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 9922166..196b136 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -749,7 +749,7 @@ func TestClaimRedirectsOnWarmMiss(t *testing.T) { provisioned = true return &types.Sandbox{ID: "sb_local"}, nil }} - srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: []string{"node-b:7777", "node-c:7777"}}, nil, nil) + srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: []string{"node-b:7777", "node-c:7777"}}, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -794,7 +794,7 @@ func TestClaimRedirectsToTemplateOwner(t *testing.T) { return &types.Sandbox{ID: "sb_local", Token: "tok"}, nil }, } - srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil) + srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -822,7 +822,7 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { // Unknown locally + gossip names an owner → the claim redirect shape; // unknown everywhere stays 404. mgr := &fakeManager{} // DeleteTemplate → ErrUnknownTemplate - srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil) + srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{owners: []string{"node-b:7777"}}, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -854,7 +854,7 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { t.Errorf("status %d, want 404 with no_redirect despite known owners", resp2.StatusCode) } - srvNoOwner := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{}, nil, nil) + srvNoOwner := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{}, nil, nil, nil) ts2 := httptest.NewServer(srvNoOwner.Handler()) t.Cleanup(func() { ts2.Close(); srvNoOwner.CloseRelays() }) req3, _ := http.NewRequest(http.MethodDelete, ts2.URL+"/v1/templates?template=tpl", nil) @@ -873,7 +873,7 @@ func TestClaimProvisionsWhenNoCandidate(t *testing.T) { mgr := &fakeManager{claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { return &types.Sandbox{ID: "sb_local", Token: "tok"}, nil }} - srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: nil}, nil, nil) + srv := New("", nil, "node-a:7777", mgr, &fakeDialer{}, &fakePlacer{addrs: nil}, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -901,7 +901,7 @@ func TestOwnerEndpoint(t *testing.T) { } return "", pool.ErrUnknownSandbox }} - srv := New("", nil, "node-b:7777", mgr, &fakeDialer{}, nil, nil, nil) + srv := New("", nil, "node-b:7777", mgr, &fakeDialer{}, nil, nil, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -943,7 +943,7 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { return time.Time{}, nil }} ps := NewPreviewServer("secret", "node:7777", &fakePreviewMgr{}) - srv := New("", nil, "node:7777", mgr, &fakeDialer{}, nil, nil, ps) + srv := New("", nil, "node:7777", mgr, &fakeDialer{}, nil, nil, nil, ps) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -1201,8 +1201,9 @@ func TestCheckpointBlobStreamsRecord(t *testing.T) { } // TestCheckpointBlobHeadUnauthenticated proves the probe route bypasses -// requireToken while GET still enforces it: the id is the unguessable -// capability, so an unauthenticated HEAD only confirms existence. +// requireToken while GET still enforces it. With no probe key configured +// (no cluster_key -- an unencrypted mesh has no shared secret to build one +// from), the id remains the only capability, same as before this batch. func TestCheckpointBlobHeadUnauthenticated(t *testing.T) { const ckptID = "ck_00000000000000aa" mgr := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} @@ -1233,6 +1234,45 @@ func TestCheckpointBlobHeadUnauthenticated(t *testing.T) { } } +// TestCheckpointProbeRequiresValidMAC proves that with a probe key +// configured (an encrypted mesh), handleCheckpointProbe rejects a missing or +// wrongly-signed HEAD before it ever reads HasCheckpoint, and accepts one +// signed with the matching key. +func TestCheckpointProbeRequiresValidMAC(t *testing.T) { + const ckptID = "ck_00000000000000aa" + key := peer.DeriveProbeKey([]byte("cluster-secret")) + wrongKey := peer.DeriveProbeKey([]byte("other-secret")) + mgr := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} + ts := newProbeAuthTestServer(t, mgr, key) + + head := func(sig string) int { + req, _ := http.NewRequestWithContext(t.Context(), http.MethodHead, ts.URL+"/v1/checkpoints/"+ckptID+"/blob", nil) + if sig != "" { + req.Header.Set(peer.ProbeHeader, sig) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("head: %v", err) + } + defer resp.Body.Close() + return resp.StatusCode + } + + if got := head(""); got != http.StatusUnauthorized { + t.Errorf("no signature: status = %d, want 401", got) + } + if got := head(peer.SignProbe(wrongKey, ckptID)); got != http.StatusUnauthorized { + t.Errorf("wrong-key signature: status = %d, want 401", got) + } + if got := head(peer.SignProbe(key, ckptID)); got != http.StatusOK { + t.Errorf("valid signature: status = %d, want 200", got) + } + // A valid signature for a DIFFERENT checkpoint must not authorize this one. + if got := head(peer.SignProbe(key, "ck_00000000000000ff")); got != http.StatusUnauthorized { + t.Errorf("signature for a different id: status = %d, want 401", got) + } +} + // TestCheckpointClaimProbesRealPeerAndRedirects is the end-to-end probe: a // live HTTPProber HEADs an actual peer server rather than a fake, proving the // probe wire protocol (unauthenticated HEAD, addr scheme defaulting) against @@ -1240,13 +1280,13 @@ func TestCheckpointBlobHeadUnauthenticated(t *testing.T) { func TestCheckpointClaimProbesRealPeerAndRedirects(t *testing.T) { const ckptID = "ck_00000000000000aa" mgrB := &fakeManager{hasCheckpoint: map[string]bool{ckptID: true}} - srvB := New("sekret", nil, "node-b:7777", mgrB, &fakeDialer{}, nil, nil, nil) + srvB := New("sekret", nil, "node-b:7777", mgrB, &fakeDialer{}, nil, nil, nil, nil) tsB := httptest.NewServer(srvB.Handler()) t.Cleanup(tsB.Close) prober := &peer.HTTPProber{Peers: func() []string { return []string{tsB.URL} }} mgrA := &fakeManager{} // ClaimCheckpoint defaults to ErrUnknownCheckpoint - srvA := New("sekret", nil, "node-a:7777", mgrA, &fakeDialer{}, nil, prober, nil) + srvA := New("sekret", nil, "node-a:7777", mgrA, &fakeDialer{}, nil, prober, nil, nil) tsA := httptest.NewServer(srvA.Handler()) t.Cleanup(tsA.Close) @@ -1275,7 +1315,20 @@ func newTenantTestServer(t *testing.T, apiToken string, tenants []config.TenantS if dialer == nil { dialer = &fakeDialer{} } - srv := New(apiToken, tenants, "node:7777", mgr, dialer, nil, nil, nil) + srv := New(apiToken, tenants, "node:7777", mgr, dialer, nil, nil, nil, nil) + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(func() { + ts.Close() + srv.CloseRelays() + }) + return ts +} + +// newProbeAuthTestServer builds a server with a probe key configured, for +// tests of handleCheckpointProbe's MAC verification specifically. +func newProbeAuthTestServer(t *testing.T, mgr Manager, probeKey []byte) *httptest.Server { + t.Helper() + srv := New("", nil, "node:7777", mgr, &fakeDialer{}, nil, nil, probeKey, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close() @@ -1527,10 +1580,15 @@ func (f *fakePlacer) PeerAddrs() []string { return f.addrs } func (f *fakePlacer) ConfigMismatches() int { return 0 } // fakeProber stubs CheckpointProber with a fixed answer, standing in for a -// live HEAD fan-out. -type fakeProber struct{ owners []string } +// live HEAD fan-out. forgotten records every id Forget was called with, so +// tests can assert a delete evicted the right cache entry. +type fakeProber struct { + owners []string + forgotten []string +} func (f *fakeProber) Owners(context.Context, string) []string { return f.owners } +func (f *fakeProber) Forget(id string) { f.forgotten = append(f.forgotten, id) } // FetchCheckpoint serves the peer-transfer read; the fake reports every record // missing unless a test supplies a directory. @@ -1553,7 +1611,7 @@ func (f *fakeManager) HasCheckpoint(_ context.Context, ckptID string) bool { // only. func newPlacerTestServer(t *testing.T, apiToken string, mgr Manager, prober CheckpointProber) *httptest.Server { t.Helper() - srv := New(apiToken, nil, "node:7777", mgr, &fakeDialer{}, nil, prober, nil) + srv := New(apiToken, nil, "node:7777", mgr, &fakeDialer{}, nil, prober, nil, nil) ts := httptest.NewServer(srv.Handler()) t.Cleanup(ts.Close) return ts diff --git a/sandboxd/store/peer/probe.go b/sandboxd/store/peer/probe.go index a051509..5c30928 100644 --- a/sandboxd/store/peer/probe.go +++ b/sandboxd/store/peer/probe.go @@ -1,20 +1,43 @@ package peer import ( + "cmp" "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/binary" "net/http" "net/url" "strings" "sync" "time" + + "golang.org/x/sync/singleflight" ) const ( probeTimeout = time.Second defaultProbeClientTimeout = 2 * time.Second - maxProbeOwners = 3 + maxRedirectOwners = 3 // a redirect decision needs a couple of candidates, not the whole fleet + maxHealOwners = 8 // a heal wants the widest source list a few full/corrupt owners can't hide behind + redirectCacheTTL = 5 * time.Second + + // ProbeHeader carries the base64 probe MAC when a ProbeKey is configured. + ProbeHeader = "X-Cocoon-Probe" + + // probeBucketSeconds is the coarse time step the probe MAC is bound to; + // VerifyProbeMAC accepts the current bucket plus one on either side, so + // clock skew up to a bucket is tolerated without an unbounded replay window. + probeBucketSeconds = 30 ) +// redirectCacheEntry is Owners' positive-cache record. +type redirectCacheEntry struct { + owners []string + expires time.Time +} + // HTTPProber finds which peers hold a checkpoint by probing them directly: // ownership is stable but unbounded, so it is queried on the rare cross-node // miss instead of gossiped every second by every node. @@ -23,11 +46,73 @@ type HTTPProber struct { // hang on a wedged peer. Client *http.Client Peers func() []string + + // ProbeKey signs and verifies HEAD probes (see DeriveProbeKey); empty + // means an unencrypted mesh, which keeps the older capability-only + // posture rather than losing the probe outright. + ProbeKey []byte + + redirectFlight singleflight.Group + healFlight singleflight.Group + + cacheMu sync.Mutex + cache map[string]redirectCacheEntry + + // cacheTTL overrides redirectCacheTTL; a test seam, 5 seconds is slow to wait out. + cacheTTL time.Duration } -// Owners fans a HEAD out to every peer in parallel and returns up to 3 -// addresses that answered 200, in no particular order. +// Owners fans a HEAD out to every peer in parallel and returns up to +// maxRedirectOwners addresses that answered 200 -- enough candidates for a +// redirect, not the whole fleet. Concurrent Owners calls for the same id +// share one fan-out; a short positive cache then serves a hot id's repeat +// redirects without re-probing the fleet at all. The cache accepts up to +// redirectCacheTTL of staleness: a redirect to a peer that deleted the +// record moments ago fails safely (404, then the client's own no_redirect +// fallback heals from the origin), it never serves a wrong answer silently. func (p *HTTPProber) Owners(ctx context.Context, id string) []string { + if owners, ok := p.cached(id); ok { + return owners + } + v, _, _ := p.redirectFlight.Do(id, func() (any, error) { + owners := p.fanOut(ctx, id, maxRedirectOwners) + p.cachePut(id, owners) + return owners, nil + }) + return v.([]string) +} + +// HealOwners is Owners' wide-fan-out twin: a heal wants the largest +// available source list, so a few full or corrupt owners cannot hide a +// usable one from it. Concurrent calls for the same id share one fan-out; +// results are not cached -- a heal source list should reflect the fleet at +// the moment it is needed, not a redirect-tuned snapshot. +func (p *HTTPProber) HealOwners(ctx context.Context, id string) []string { + v, _, _ := p.healFlight.Do(id, func() (any, error) { + return p.fanOut(ctx, id, maxHealOwners), nil + }) + return v.([]string) +} + +// Forget evicts any cached Owners result for id -- called after a delete +// (original or forwarded from another node's broadcast) touches id here, +// since a stale positive would redirect a claim to a node that no longer +// holds the record. A miss was never cached, so a delete that finds nothing +// locally has nothing to do beyond this. +func (p *HTTPProber) Forget(id string) { + p.cacheMu.Lock() + defer p.cacheMu.Unlock() + delete(p.cache, id) +} + +// fanOut probes every peer concurrently, stopping and canceling the +// stragglers as soon as maxOwners have answered 200 -- a hung or slow peer +// must not add its own timeout to a redirect or heal that already has +// enough candidates. The context is detached from the caller and rebounded +// internally: singleflight shares this fan-out across concurrent callers +// whose own contexts may cancel independently (one client disconnecting +// must not abort another's still-live probe). +func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int) []string { addrs := dedupAddrs(p.Peers()) if len(addrs) == 0 { return nil @@ -36,30 +121,67 @@ func (p *HTTPProber) Owners(ctx context.Context, id string) []string { if client == nil { client = &http.Client{Timeout: defaultProbeClientTimeout} } + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultProbeClientTimeout) + defer cancel() - var mu sync.Mutex - var owners []string + wins := make(chan string, len(addrs)) var wg sync.WaitGroup for _, addr := range addrs { wg.Go(func() { - if probeOwner(ctx, client, addr, id) { - mu.Lock() - owners = append(owners, addr) - mu.Unlock() + if probeOwner(ctx, client, p.ProbeKey, addr, id) { + wins <- addr } }) } - wg.Wait() - if len(owners) > maxProbeOwners { - owners = owners[:maxProbeOwners] + go func() { wg.Wait(); close(wins) }() + + var owners []string + for addr := range wins { + owners = append(owners, addr) + if len(owners) >= maxOwners { + cancel() // enough candidates; a straggler's answer would be discarded anyway + break + } } return owners } -// probeOwner sends no Authorization header: the id itself is the -// unguessable capability, so an unauthenticated HEAD leaks only existence -// to a caller who already holds it. -func probeOwner(ctx context.Context, client *http.Client, addr, id string) bool { +// cached returns id's owners if a still-live positive cache entry exists. +func (p *HTTPProber) cached(id string) ([]string, bool) { + p.cacheMu.Lock() + defer p.cacheMu.Unlock() + entry, ok := p.cache[id] + if !ok || time.Now().After(entry.expires) { + return nil, false + } + return entry.owners, true +} + +// cachePut records a positive result. A miss is never cached: a checkpoint +// that appears moments later (still propagating, or just created) must not +// be hidden behind a stale negative for the TTL. +func (p *HTTPProber) cachePut(id string, owners []string) { + if len(owners) == 0 { + return + } + p.cacheMu.Lock() + defer p.cacheMu.Unlock() + if p.cache == nil { + p.cache = make(map[string]redirectCacheEntry) + } + now := time.Now() + for k, e := range p.cache { + if now.After(e.expires) { + delete(p.cache, k) + } + } + p.cache[id] = redirectCacheEntry{owners: owners, expires: now.Add(cmp.Or(p.cacheTTL, redirectCacheTTL))} +} + +// probeOwner sends a probe MAC when probeKey is set (see DeriveProbeKey); +// an unencrypted mesh (empty probeKey) falls back to the older +// capability-only posture -- the id is unguessable, but nothing else guards it. +func probeOwner(ctx context.Context, client *http.Client, probeKey []byte, addr, id string) bool { ctx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() @@ -72,6 +194,9 @@ func probeOwner(ctx context.Context, client *http.Client, addr, id string) bool if err != nil { return false } + if len(probeKey) > 0 { + req.Header.Set(ProbeHeader, SignProbe(probeKey, id)) + } resp, err := client.Do(req) //nolint:gosec // addr comes from the mesh's own member view if err != nil { return false @@ -94,3 +219,52 @@ func dedupAddrs(addrs []string) []string { } return out } + +// DeriveProbeKey derives the probe-MAC key from the mesh's cluster_key via +// one HMAC step (domain separation): cluster_key already encrypts gossip +// traffic, and reusing that key unmodified for a second, unrelated +// construction is avoidable at negligible cost. Call only when clusterKey +// is non-empty; an empty result leaves probing unauthenticated. +func DeriveProbeKey(clusterKey []byte) []byte { + mac := hmac.New(sha256.New, clusterKey) + mac.Write([]byte("cocoon-checkpoint-probe-v1")) + return mac.Sum(nil) +} + +// SignProbe returns the current probe MAC for id, keyed by key -- the value +// probeOwner sends in ProbeHeader. Exported for callers (and tests) that +// need to construct a valid probe request without reaching into this +// package's internals. +func SignProbe(key []byte, id string) string { + return probeMAC(key, id, currentBucket()) +} + +// VerifyProbeMAC checks sig against id's current time bucket and the one on +// either side, tolerating clock skew up to one bucket without an unbounded +// replay window. +func VerifyProbeMAC(key []byte, id, sig string) bool { + if sig == "" { + return false + } + now := currentBucket() + for _, bucket := range [3]int64{now - 1, now, now + 1} { + if hmac.Equal([]byte(sig), []byte(probeMAC(key, id, bucket))) { + return true + } + } + return false +} + +// probeMAC returns the base64 HMAC-SHA256 tag for id in bucket, keyed by key. +func probeMAC(key []byte, id string, bucket int64) string { + mac := hmac.New(sha256.New, key) + mac.Write([]byte(id)) + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], uint64(bucket)) //nolint:gosec // unix-seconds bucket, positive for any current clock + mac.Write(buf[:]) + return base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +} + +func currentBucket() int64 { + return time.Now().Unix() / probeBucketSeconds +} diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index ba42933..787b85e 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -3,6 +3,7 @@ package peer import ( "net/http" "net/http/httptest" + "sync" "sync/atomic" "testing" "time" @@ -102,3 +103,214 @@ func TestOwnersCapsAtThree(t *testing.T) { t.Errorf("owners = %v, want capped at 3", owners) } } + +// TestOwnersCancelsStragglersOnceCapReached: once the redirect cap is met by +// fast peers, a straggler beyond it must not add its own timeout to the result. +func TestOwnersCancelsStragglersOnceCapReached(t *testing.T) { + var addrs []string + for range maxRedirectOwners { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + addrs = append(addrs, srv.URL) + } + block := make(chan struct{}) + hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + <-block + })) + defer hung.Close() + defer close(block) + addrs = append(addrs, hung.URL) + + start := time.Now() + p := &HTTPProber{Peers: func() []string { return addrs }} + owners := p.Owners(t.Context(), testID) + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { + t.Errorf("Owners took %v, want fast: the cap was met without the straggler", elapsed) + } + if len(owners) != maxRedirectOwners { + t.Errorf("owners = %v, want %d", owners, maxRedirectOwners) + } +} + +// TestHealOwnersReturnsMoreThanRedirectCap: a heal wants the widest source +// list, not the redirect's couple of candidates. +func TestHealOwnersReturnsMoreThanRedirectCap(t *testing.T) { + const want = maxRedirectOwners + 2 + var addrs []string + for range want { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + addrs = append(addrs, srv.URL) + } + + p := &HTTPProber{Peers: func() []string { return addrs }} + if owners := p.HealOwners(t.Context(), testID); len(owners) != want { + t.Errorf("HealOwners = %v (%d), want all %d peers, not capped at the redirect limit", owners, len(owners), want) + } +} + +// TestOwnersCoalescesConcurrentProbes: concurrent redirect probes for one id +// must fan out to each peer once, not once per caller. +func TestOwnersCoalescesConcurrentProbes(t *testing.T) { + var hits atomic.Int32 + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + time.Sleep(50 * time.Millisecond) // widen the window for concurrent callers to overlap + w.WriteHeader(http.StatusOK) + })) + defer ok.Close() + + p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }} + var wg sync.WaitGroup + for range 10 { + wg.Go(func() { p.Owners(t.Context(), testID) }) + } + wg.Wait() + if got := hits.Load(); got != 1 { + t.Errorf("peer hit %d times by 10 concurrent callers, want 1 (coalesced)", got) + } +} + +// TestOwnersCacheCollapsesRepeatedRedirects is the hot-checkpoint-behind-a- +// load-balancer case: a fleet of peers, one of them the owner. Ten +// sequential redirect probes for the same id, well within the TTL, must +// fan out to the fleet once total, not once per call. +func TestOwnersCacheCollapsesRepeatedRedirects(t *testing.T) { + const peers = 5 + var hits atomic.Int32 + var addrs []string + for i := range peers { + owns := i == 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + if owns { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + addrs = append(addrs, srv.URL) + } + + p := &HTTPProber{Peers: func() []string { return addrs }} + for range 10 { + p.Owners(t.Context(), testID) + } + if got := hits.Load(); got != peers { + t.Errorf("peers hit %d times across 10 redirects for one id, want %d (once, not 10x%d)", got, peers, peers) + } +} + +// TestOwnersCacheExpires: past the TTL, a redirect must re-probe rather than +// serve a result that could be stale forever. +func TestOwnersCacheExpires(t *testing.T) { + var hits atomic.Int32 + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer ok.Close() + + p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }, cacheTTL: 10 * time.Millisecond} + p.Owners(t.Context(), testID) + time.Sleep(20 * time.Millisecond) + p.Owners(t.Context(), testID) + if got := hits.Load(); got != 2 { + t.Errorf("peer hit %d times across two Owners calls spanning the TTL, want 2 (expired)", got) + } +} + +// TestForgetEvictsCachedEntry: after Forget, the next Owners call must +// re-probe rather than serve the (possibly now-stale) cached answer. +func TestForgetEvictsCachedEntry(t *testing.T) { + var hits atomic.Int32 + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer ok.Close() + + p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }} + p.Owners(t.Context(), testID) + p.Forget(testID) + p.Owners(t.Context(), testID) + if got := hits.Load(); got != 2 { + t.Errorf("peer hit %d times across two Owners calls with a Forget between, want 2 (re-probed)", got) + } +} + +// TestForgetOtherIDLeavesCacheAlone: Forget must evict only the named id, +// not the whole cache. +func TestForgetOtherIDLeavesCacheAlone(t *testing.T) { + var hits atomic.Int32 + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer ok.Close() + + p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }} + p.Owners(t.Context(), testID) + p.Forget("ck_ffffffffffffffff") + p.Owners(t.Context(), testID) + if got := hits.Load(); got != 1 { + t.Errorf("peer hit %d times, want 1 (Forget of a different id must not evict testID)", got) + } +} + +// TestVerifyProbeMACRejectsWrongKey: a MAC signed with a different key must +// not verify. +func TestVerifyProbeMACRejectsWrongKey(t *testing.T) { + key := DeriveProbeKey([]byte("cluster-secret")) + other := DeriveProbeKey([]byte("different-secret")) + sig := probeMAC(other, testID, currentBucket()) + if VerifyProbeMAC(key, testID, sig) { + t.Error("VerifyProbeMAC accepted a MAC signed with a different key") + } +} + +// TestVerifyProbeMACRejectsWrongID: a MAC signed for a different id must not +// verify -- the id itself must be bound into the signature. +func TestVerifyProbeMACRejectsWrongID(t *testing.T) { + key := DeriveProbeKey([]byte("cluster-secret")) + sig := probeMAC(key, testID, currentBucket()) + if VerifyProbeMAC(key, "ck_ffffffffffffffff", sig) { + t.Error("VerifyProbeMAC accepted a MAC signed for a different id") + } +} + +// TestVerifyProbeMACRejectsEmptySignature: a missing header must not verify. +func TestVerifyProbeMACRejectsEmptySignature(t *testing.T) { + key := DeriveProbeKey([]byte("cluster-secret")) + if VerifyProbeMAC(key, testID, "") { + t.Error("VerifyProbeMAC accepted an empty signature") + } +} + +// TestVerifyProbeMACToleratesAdjacentBucket: a signature from the bucket +// just before or after now must still verify, tolerating clock skew. +func TestVerifyProbeMACToleratesAdjacentBucket(t *testing.T) { + key := DeriveProbeKey([]byte("cluster-secret")) + for _, delta := range []int64{-1, 1} { + sig := probeMAC(key, testID, currentBucket()+delta) + if !VerifyProbeMAC(key, testID, sig) { + t.Errorf("VerifyProbeMAC rejected bucket delta %d, want tolerated clock skew", delta) + } + } +} + +// TestVerifyProbeMACRejectsOutsideWindow: a signature two buckets away is +// outside the tolerated skew and must be rejected -- the window is bounded, +// not an unlimited replay allowance. +func TestVerifyProbeMACRejectsOutsideWindow(t *testing.T) { + key := DeriveProbeKey([]byte("cluster-secret")) + sig := probeMAC(key, testID, currentBucket()-2) + if VerifyProbeMAC(key, testID, sig) { + t.Error("VerifyProbeMAC accepted a bucket two steps away, want rejected") + } +} From 1e78e1f4102f3545d2c35b20f064842acd77abfc Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 19:16:49 +0800 Subject: [PATCH 22/31] test: a delete must evict the probe cache, forwarded or not The eviction landed with the probe boundary; this is the server-level pin that was written for it and lost to a worktree cleanup race. --- sandboxd/server/server_test.go | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 196b136..f8d813e 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -741,6 +741,46 @@ func TestDeleteCheckpointNoForwardQueryParam(t *testing.T) { } } +// TestDeleteCheckpointForgetsProbeCache proves a delete evicts any cached +// probe answer for the id, whether the delete is the original (fleet scope) +// or a forwarded broadcast (no_forward) that finds nothing to delete locally +// — both cases mean this node just learned id's ownership changed. +func TestDeleteCheckpointForgetsProbeCache(t *testing.T) { + prober := &fakeProber{} + + t.Run("original delete", func(t *testing.T) { + prober.forgotten = nil + mgr := &fakeManager{deleteCheckpoint: func(string) error { return nil }} + ts := newPlacerTestServer(t, "api", mgr, prober) + req, _ := http.NewRequest(http.MethodDelete, ts.URL+"/v1/checkpoints/ck_0011223344556677", nil) + req.Header.Set("Authorization", "Bearer api") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("delete: %v", err) + } + defer resp.Body.Close() + if len(prober.forgotten) != 1 || prober.forgotten[0] != "ck_0011223344556677" { + t.Errorf("forgotten = %v, want [ck_0011223344556677]", prober.forgotten) + } + }) + + t.Run("forwarded delete finding nothing locally", func(t *testing.T) { + prober.forgotten = nil + mgr := &fakeManager{deleteCheckpoint: func(string) error { return pool.ErrUnknownCheckpoint }} + ts := newPlacerTestServer(t, "api", mgr, prober) + req, _ := http.NewRequest(http.MethodDelete, ts.URL+"/v1/checkpoints/ck_0011223344556677?no_forward=1", nil) + req.Header.Set("Authorization", "Bearer api") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("delete: %v", err) + } + defer resp.Body.Close() + if len(prober.forgotten) != 1 || prober.forgotten[0] != "ck_0011223344556677" { + t.Errorf("forgotten = %v, want [ck_0011223344556677] even though nothing was held locally", prober.forgotten) + } + }) +} + func TestClaimRedirectsOnWarmMiss(t *testing.T) { // Warm miss (fakeManager.ClaimWarm always misses) + a placer with // candidates → a redirect response, and the local manager never provisions. From e61268e2e0af33c7b0e729bb5b212bd5e2a33e5b Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 19:17:40 +0800 Subject: [PATCH 23/31] docs: the checkpoint placement protocol, as it actually behaves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The claim, blob, probe, and delete surfaces this branch added were undocumented or described mid-flight: the probe is signed on an encrypted mesh (and the "hardening is tracked" placeholder goes away with the gap), peer heal needs a token, a TTL, and an encrypted mesh, the heal-cap 503 carries a Retry-After, and a delete is eventual best-effort cleanup bounded by the checkpoint TTL — not a fleet-wide revocation, and the docs now say so instead of implying otherwise. --- docs/cluster.md | 3 ++- docs/deploy.md | 2 +- docs/sandboxd-api.md | 23 ++++++++++++++--------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 0cac5ab..252aa37 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -135,7 +135,8 @@ On the default per-node store, a branch claim (`Checkpoint.New` / at a node that does not hold the record runs a tier order: 1. **Local claim** — the fast path when the claiming node already holds it. -2. **Probe + redirect** — a live, unauthenticated `HEAD` fan-out to up to 3 +2. **Probe + redirect** — a live `HEAD` fan-out (HMAC-signed when the mesh + carries a `cluster_key`) to up to 3 mesh peers in parallel, redirecting to whoever answers — the same claim-redirect contract a warm miss already uses. The cap means the answer is a hint, not an exhaustive list of every owner. The probe and diff --git a/docs/deploy.md b/docs/deploy.md index 64ef414..12a53bd 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -80,7 +80,7 @@ sandboxd reads one JSON file (`-config`, default | `checkpoint_dir` | `/checkpoints` | where checkpoints and promoted templates live. Point it at a shared FUSE mount (JuiceFS over object storage, NFS) and every node sharing the mount can branch every checkpoint — the path's filesystem is the operator's choice. One contract on any shared root (mount or bucket): a template key has a single writer — promotes go to the sandbox's owner node, and operators must not race promotes of one name from different nodes (checkpoint ids are node-generated and never collide). A checkpoint deleted on one node while another is mid-branch from it fails that branch visibly | | `checkpoint_store` | dir | checkpoint AND promoted-template backend (both live in one store root, id-namespaced ck_/tp_): `{"kind": "s3", "s3": {"bucket": "…", "prefix": "ck/", "endpoint": "…", "region": "…", "force_path_style": true}}` stores checkpoints in object storage (any node claims any checkpoint, no shared mount needed). Credentials come from the standard AWS chain (env/IAM role), never this file. A crash between upload and the meta.json commit marker leaves orphan objects invisible to listings — add an S3 lifecycle rule to reclaim them. Absent = the dir backend at `checkpoint_dir` | | `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the bound on a healed replica a delete broadcast missed | -| `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Two requirements, both enforced at config load: `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (the bound on how long a replica can outlive a delete that failed to reach it). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | +| `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (the bound on how long a replica can outlive a delete that failed to reach it). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, a claim is first redirected to a warm peer) | | `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`); preview accesses record as op `preview_dial`. A request frame whose first line exceeds 4 KiB is skipped, never truncated | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 8d7253e..89904ea 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -214,8 +214,9 @@ Checkpoints are node-local (unless the store is shared — see [Configuration](deploy.md#configuration)), so a miss here runs a tier order: 1. This node checks its own store first. -2. On a miss, it probes up to 3 peers directly — a parallel, unauthenticated - `HEAD` to each (see below) — and answers exactly like a warm-miss +2. On a miss, it probes up to 3 peers directly — a parallel `HEAD` to each + (authenticated on an encrypted mesh, see below) — and answers exactly like + a warm-miss [`POST /v1/claim`](#post-v1claim): `200 {"redirect": ["10.0.0.6:7777", "10.0.0.7:7777"]}`, retry the same body (+`no_redirect: true`) at each candidate until one answers. The probe and the follow-up claim are not @@ -232,7 +233,8 @@ Checkpoints are node-local (unless the store is shared — see 404 for an unknown checkpoint (locally, and after redirect and heal both miss), 409 for an egress-lane checkpoint (see [egress](egress.md)), 429 node or calling tenant at `max_claims` or the node draining, 503 when the node's -concurrent-heal cap is already full — the request is retryable. +concurrent-heal cap is already full — retryable, and the response carries a +`Retry-After` hint. ## GET/HEAD /v1/checkpoints/{id}/blob @@ -244,12 +246,15 @@ part of the public API; an SDK caller has no reason to call it directly. real caller is a peer's heal pull, which presents the fleet `api_token`. 401 missing or unrecognized token, 403 a valid tenant token (authenticated but not the operator), 404 unknown checkpoint. -- `HEAD` is the ownership probe and is currently **unauthenticated** — the id - itself is treated as the capability, so this tells a caller only what it - already knows: 200 when this node holds a branchable (non-archive) copy, - 404 otherwise. Hardening this boundary (requiring a token, bounding who can - fan a probe out) is tracked as follow-up work; do not treat it as secured - today. +- `HEAD` is the ownership probe: 200 when this node holds a branchable + (non-archive) copy, 404 otherwise. On a mesh with `cluster_key` set the + request must carry `X-Cocoon-Probe`, an HMAC over the id and a coarse time + bucket keyed off a probe-specific derivation of the cluster key — verified + before any disk is touched, replayable for roughly a minute at most. On a + keyless mesh (redirect-only fleets have no shared secret to sign with) the + id itself remains the only capability, matching the mesh's own posture. + Repeat probes for one id are answered from a short positive cache on the + asking node, which a local delete evicts immediately. ## GET /v1/checkpoints From cf3efbfd96b965c59f61f689a0deeb4c2cea3b05 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 19:47:49 +0800 Subject: [PATCH 24/31] Revert "mesh: a redirect spends the warmth it was promised" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 60ee474. Measured on a real two-node mesh, the debit is a regression, and the reasoning behind it was wrong. A burst of forty claims at a node holding a golden but no warm sandboxes, against a peer advertising six warm: no mesh at all 0 redirected, 40 local, p50 1173ms mesh, before the debit 35 redirected, 5 local, p50 601ms mesh, with the debit 6 redirected, 34 local, p50 1210ms So the mesh was not slowing allocation down — it was halving it, by spreading forty clones across two nodes instead of queueing them on one. The debit bounded redirects to the peer's advertised warmth, which is mechanically what it claimed to do, and thereby hoarded every overflow clone on the origin: thirty-four of them serialized behind that node's provision limit, landing back at the no-mesh number. The premise was the error. A redirect is not only a bid for a ready sandbox; it is also how provisioning load reaches a peer that has the golden and idle capacity to serve it. Capping it at the warm count throws that away. Whatever slowdown motivated this needs to be measured on a real fleet first — this branch is not the place to guess at it again. --- sandboxd/mesh/mesh.go | 16 +++-------- sandboxd/mesh/mesh_test.go | 54 +++----------------------------------- 2 files changed, 6 insertions(+), 64 deletions(-) diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 8c91a19..34a37b8 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -160,17 +160,10 @@ func (m *Mesh) ConfigMismatches() int { // Candidates returns up to two peer addresses that report warm(keyHash) > 0, // chosen power-of-two-choices to avoid herding every waiter onto one node. -// Self is never a candidate — the caller has already missed locally. The -// winner's count is debited locally: a redirect is a claim in flight, and -// undebited, every claim in one gossip window would chase the same stale -// count, forcing the target to provision the whole burst once its warm pool -// ran out (each retry arrives with no_redirect set). The peer's next adopted -// state resets the debit. +// Self is never a candidate — the caller has already missed locally. func (m *Mesh) Candidates(keyHash string) []string { m.mu.Lock() - defer m.mu.Unlock() type cand struct { - id string addr string warm int } @@ -180,15 +173,15 @@ func (m *Mesh) Candidates(keyHash string) []string { continue } if st.Pools[keyHash] > 0 { - pool = append(pool, cand{id, st.Addr, st.Pools[keyHash]}) + pool = append(pool, cand{st.Addr, st.Pools[keyHash]}) } } + m.mu.Unlock() switch len(pool) { case 0: return nil case 1: - m.view[pool[0].id].Pools[keyHash]-- return []string{pool[0].addr} } // Power-of-two-choices: sample two, order by warmer. This is load @@ -202,9 +195,6 @@ func (m *Mesh) Candidates(keyHash string) []string { if b.warm > a.warm { a, b = b, a } - // Only the winner is debited: the client walks the list in order and - // stops at the first success, so the runner-up's capacity stays intact. - m.view[a.id].Pools[keyHash]-- return []string{a.addr, b.addr} } diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index 287aaab..17e327d 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -96,12 +96,10 @@ func TestForgetPrunesDeadNode(t *testing.T) { func TestCandidatesPowerOfTwo(t *testing.T) { m := newTestMesh(t, "a") - // warm=100 each: 20 draws debit the winners but must never exhaust a - // peer, so the two-distinct-candidates property is what this test sees. m.merge([]NodeState{ - {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 100}}, - {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 100}}, - {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"k": 100}}, + {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, + {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, + {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, }) // With ≥2 warm peers, exactly two distinct candidates come back. for range 20 { @@ -117,52 +115,6 @@ func TestCandidatesPowerOfTwo(t *testing.T) { } } -// TestCandidatesDebitWarmCredit: a redirect consumes one unit of the peer's -// advertised warmth, so a burst inside one gossip window is bounded by what -// the peer actually promised instead of herding onto a stale count. -func TestCandidatesDebitWarmCredit(t *testing.T) { - m := newTestMesh(t, "a") - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 4}}}) - - var sent int - for range 40 { - if m.Candidates("k") != nil { - sent++ - } - } - if sent != 4 { - t.Errorf("40 claims in one gossip window: %d redirected, want the 4 advertised", sent) - } - - // A fresh adopted state resets the credit. - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 1}}}) - if m.Candidates("k") == nil { - t.Error("no candidate after a fresh gossip refresh") - } - if m.Candidates("k") != nil { - t.Error("credit outlived the refreshed count") - } -} - -// TestCandidatesDebitSumsAcrossPeers: the window's redirect budget is the -// fleet's advertised total, not per-call luck. -func TestCandidatesDebitSumsAcrossPeers(t *testing.T) { - m := newTestMesh(t, "a") - m.merge([]NodeState{ - {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, - {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, - }) - var sent int - for range 20 { - if m.Candidates("k") != nil { - sent++ - } - } - if sent != 4 { - t.Errorf("redirected %d, want 4 (the fleet's advertised warm total)", sent) - } -} - func TestTwoNodeClusterGossipsPools(t *testing.T) { a := startNode(t, "127.0.0.1", 0, "node-a") b := startNode(t, "127.0.0.1", 0, "node-b") From 489bacfcb91e0cca1b7d835862be0d95dc9302e3 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 21:22:16 +0800 Subject: [PATCH 25/31] sandboxd: bound the single-owner probe and fence the cache against a delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second review round found two gaps in the ownership probe. A checkpoint usually lives on one node, so the three-owner cap is rarely reached and the fan-out would block on every missing peer's full timeout before returning the single owner it already had; it now returns on a short grace window once the first owner answers, while a genuine all-miss still waits out the timeout because it cannot conclude nobody holds the record until every peer has answered. A delete's cache eviction also raced an in-flight probe: Forget cleared an entry the probe had not written yet, then the probe cached a result naming the node that was just deleted, leaving a stale positive for the cache TTL. Forget now bumps an epoch under the cache lock, and a probe only caches its result if that epoch still holds — so a delete landing any time during the probe keeps its result out of the cache. Each fix has a test that fails when reverted. The delete-window docs are corrected to match: expiry is per-replica, best-effort, bounded by the TTL plus a sweep interval, not a single fleet-wide wall-clock moment. --- docs/cluster.md | 19 ++++----- docs/sandboxd-api.md | 10 +++-- sandboxd/store/peer/probe.go | 64 +++++++++++++++++++++++-------- sandboxd/store/peer/probe_test.go | 63 ++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 28 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 252aa37..adfcd5b 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -159,15 +159,16 @@ at a node that does not hold the record runs a tier order: record, then best-effort broadcasts the delete to every peer this node currently sees, so a replica a heal pulled earlier is cleaned up too, not just the original. A peer that is offline or partitioned during that -broadcast keeps its copy until the checkpoint TTL ages it out: the -worst-case window in which a deleted checkpoint remains branchable by an -id-holder is `checkpoint_ttl_hours`. A healed replica carries the source -checkpoint's original `CreatedAt`, so every node's hourly TTL sweep ages it -out at the same wall-clock moment regardless of whether the broadcast -reached it — which is why heal *requires* a nonzero, fleet-matching TTL -(see [cluster-invariant config](#cluster-invariant-config)). With TTL -disabled (the default), that window never closes on a peer the broadcast -never reaches. +broadcast keeps its copy until the checkpoint TTL ages it out. A healed +replica carries the source checkpoint's original `CreatedAt`, so it becomes +eligible for expiry at the same instant everywhere; each node then removes +it on its own hourly sweep, which is independently phased and retries on +failure. The window in which a deleted checkpoint stays branchable by an +id-holder is therefore `checkpoint_ttl_hours` plus up to one sweep interval, +per replica, best-effort — which is why heal *requires* a nonzero, +fleet-matching TTL (see [cluster-invariant config](#cluster-invariant-config)). +With TTL disabled it could never close at all, so that combination is +rejected at config load. ## State ownership diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 89904ea..07560cb 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -271,10 +271,12 @@ success, 404 unknown. so a healed replica does not outlive it — this is eventual best-effort cleanup, not a fleet-wide revocation.** A peer that is offline or partitioned during the broadcast keeps its copy until the checkpoint TTL -ages it out: the worst-case window in which a deleted checkpoint remains -branchable by an id-holder is `checkpoint_ttl_hours`. A healed replica -carries the source's original `CreatedAt`, so every node's sweep expires it -at the same wall-clock moment; the TTL must also match fleet-wide, which the +ages it out. A healed replica carries the source's original `CreatedAt`, so +it becomes eligible for expiry at the same instant on every node; the actual +removal is each node's own hourly sweep, which is independently phased and +retries on failure, so the true bound is `checkpoint_ttl_hours` plus up to +one sweep interval per replica, best-effort. The TTL must also match +fleet-wide, which the [cluster-invariant config](cluster.md#cluster-invariant-config) digest checks. A window always exists because `checkpoint_peer_heal` cannot be enabled with `checkpoint_ttl_hours: 0` — a replica that can outlive a delete diff --git a/sandboxd/store/peer/probe.go b/sandboxd/store/peer/probe.go index 5c30928..1a31260 100644 --- a/sandboxd/store/peer/probe.go +++ b/sandboxd/store/peer/probe.go @@ -23,6 +23,12 @@ const ( maxHealOwners = 8 // a heal wants the widest source list a few full/corrupt owners can't hide behind redirectCacheTTL = 5 * time.Second + // probeGrace bounds how long a fan-out waits for more owners once it has + // its first: a checkpoint usually lives on one node, so the cap of three is + // rarely reached and without this the collector would block on every + // missing peer's timeout before returning the single owner it already has. + probeGrace = 150 * time.Millisecond + // ProbeHeader carries the base64 probe MAC when a ProbeKey is configured. ProbeHeader = "X-Cocoon-Probe" @@ -57,6 +63,10 @@ type HTTPProber struct { cacheMu sync.Mutex cache map[string]redirectCacheEntry + // epoch is bumped by every Forget under cacheMu: a fan-out that started + // before a delete must not write its now-suspect result back into the + // cache, so cachePut only commits when the epoch it read still holds. + epoch uint64 // cacheTTL overrides redirectCacheTTL; a test seam, 5 seconds is slow to wait out. cacheTTL time.Duration @@ -71,12 +81,13 @@ type HTTPProber struct { // record moments ago fails safely (404, then the client's own no_redirect // fallback heals from the origin), it never serves a wrong answer silently. func (p *HTTPProber) Owners(ctx context.Context, id string) []string { - if owners, ok := p.cached(id); ok { + owners, start, hit := p.cacheLookup(id) + if hit { return owners } v, _, _ := p.redirectFlight.Do(id, func() (any, error) { owners := p.fanOut(ctx, id, maxRedirectOwners) - p.cachePut(id, owners) + p.cachePut(id, owners, start) return owners, nil }) return v.([]string) @@ -102,6 +113,7 @@ func (p *HTTPProber) HealOwners(ctx context.Context, id string) []string { func (p *HTTPProber) Forget(id string) { p.cacheMu.Lock() defer p.cacheMu.Unlock() + p.epoch++ delete(p.cache, id) } @@ -135,37 +147,59 @@ func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int) []str } go func() { wg.Wait(); close(wins) }() + // grace opens only after the first owner answers: until then the fan-out + // waits out the full timeout, since a genuine miss must hear from every + // peer before it can conclude nobody holds the record. var owners []string - for addr := range wins { - owners = append(owners, addr) - if len(owners) >= maxOwners { - cancel() // enough candidates; a straggler's answer would be discarded anyway - break + var grace <-chan time.Time + for { + select { + case addr, ok := <-wins: + if !ok { + return owners + } + owners = append(owners, addr) + if len(owners) >= maxOwners { + cancel() + return owners + } + if grace == nil { + grace = time.After(probeGrace) + } + case <-grace: + cancel() + return owners } } - return owners } -// cached returns id's owners if a still-live positive cache entry exists. -func (p *HTTPProber) cached(id string) ([]string, bool) { +// cacheLookup returns id's owners if a still-live positive entry exists, and +// always the current epoch: the caller carries it into the fan-out so cachePut +// can tell whether a Forget landed while the probe was in flight. +func (p *HTTPProber) cacheLookup(id string) (owners []string, epoch uint64, hit bool) { p.cacheMu.Lock() defer p.cacheMu.Unlock() entry, ok := p.cache[id] if !ok || time.Now().After(entry.expires) { - return nil, false + return nil, p.epoch, false } - return entry.owners, true + return entry.owners, p.epoch, true } -// cachePut records a positive result. A miss is never cached: a checkpoint -// that appears moments later (still propagating, or just created) must not +// cachePut records a positive result, unless a Forget bumped the epoch since +// the fan-out began — that delete may have removed the very record this result +// names, so caching it would redirect claims to a node that no longer holds +// it. A miss is never cached: a checkpoint that appears moments later must not // be hidden behind a stale negative for the TTL. -func (p *HTTPProber) cachePut(id string, owners []string) { +func (p *HTTPProber) cachePut(id string, owners []string, start uint64) { if len(owners) == 0 { return } p.cacheMu.Lock() defer p.cacheMu.Unlock() + if p.epoch != start { + return + } if p.cache == nil { p.cache = make(map[string]redirectCacheEntry) } diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index 787b85e..352076d 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -67,6 +67,69 @@ func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { } } +// TestOwnersReturnsPromptlyWithOneOwner: the common case is a single owner, so +// the fan-out must return shortly after that owner answers on the grace window, +// not block on a slow peer's full timeout the way it would waiting for a cap it +// will never reach. +func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { + block := make(chan struct{}) + hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + <-block + })) + defer hung.Close() + defer close(block) + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer ok.Close() + + p := &HTTPProber{Peers: func() []string { return []string{hung.URL, ok.URL} }} + start := time.Now() + owners := p.Owners(t.Context(), testID) + if elapsed := time.Since(start); elapsed > probeGrace+500*time.Millisecond { + t.Errorf("Owners took %v with one owner, want ~grace, not the full probe timeout", elapsed) + } + if len(owners) != 1 || owners[0] != ok.URL { + t.Errorf("owners = %v, want [%s]", owners, ok.URL) + } +} + +// TestForgetDuringFlightPreventsStaleCache: a delete that lands while a probe +// is in flight must keep that probe's result out of the cache — the record it +// names may be the one just deleted, and a cached positive would redirect +// claims to a node that no longer holds it. +func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { + release := make(chan struct{}) + var hits atomic.Int32 + owner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + <-release // hold the probe open so Forget can land mid-flight + w.WriteHeader(http.StatusOK) + })) + defer owner.Close() + + p := &HTTPProber{Peers: func() []string { return []string{owner.URL} }} + var owners []string + done := make(chan struct{}) + go func() { owners = p.Owners(t.Context(), testID); close(done) }() + + for hits.Load() == 0 { // wait until the probe is in flight + time.Sleep(time.Millisecond) + } + p.Forget(testID) // the delete lands while the flight is open + close(release) + <-done + + if len(owners) != 1 { + t.Fatalf("owners = %v, want the one that answered", owners) + } + // The result must not have been cached: a second call re-probes. + p.Owners(t.Context(), testID) + if got := hits.Load(); got != 2 { + t.Errorf("owner probed %d times, want 2: a Forget mid-flight must not leave a stale positive cached", got) + } +} + // TestOwnersDedupsAddrs: a duplicated peer list must probe each address once. func TestOwnersDedupsAddrs(t *testing.T) { var hits atomic.Int32 From 177d5ddab97c2b9aab75fb096b09ff2582145b14 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 21:44:23 +0800 Subject: [PATCH 26/31] sandboxd: close the operator races, heal validation, and lock leak a third review found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A full-diff review pass turned up three defects the feature work introduced and two gaps in the round-3 probe fixes. Operator reads raced lifecycle writes. Sandbox and Stats took a claim pointer from byID, which releases m.mu, then read HibernateSnap, VMName, ArchiveCk and Deadline — fields hibernate and archive rewrite under m.mu. Both now snapshot under the lock; Stats keeps only the /proc read, which touches no claim state, outside it. A -race test that hammers Sandbox/Stats against Hibernate/Wake trips the detector when the lock is removed. Healed-record validation accepted a shell. It checked the meta parsed, the id matched, the record was not an archive image, and the export directory existed — but not that the key was branchable or the export non-empty. An empty export clones to nothing, and because the healer stops at the first accepted owner and publishes immediately, one such copy suppressed every better owner and poisoned later local claims. Validation now requires a valid key and a non-empty export, so a bad copy is rejected and the next owner tried. The record lock could leak. When a delete's recDoneEvict ran while another holder still referenced the id, it left the entry in place but recorded no pending eviction; the ordinary holder's later recDone then reached zero without removing it. The eviction is now deferred to whichever call drops the last reference. Two probe gaps from the previous round: the grace window that bounds a single-owner redirect was also truncating HealOwners, which must stay exhaustive so a fast owner cannot hide a slower valid source — heal now fans out with no grace. And the delete handler evicted the probe cache before the record was gone, so a probe racing the delete could cache a positive the delete then cleared; eviction now runs after the delete. The TTL-window docs are corrected once more: the checkpoint TTL is the expiry eligibility point, and a node whose hourly sweep keeps failing holds its replica until one succeeds — not a hard ceiling. Each code fix has a test that fails when reverted. --- docs/cluster.md | 8 ++-- docs/deploy.md | 2 +- docs/sandboxd-api.md | 8 ++-- sandboxd/pool/checkpoint.go | 14 +++++- sandboxd/pool/checkpoint_heal_test.go | 66 +++++++++++++++++++++++++++ sandboxd/pool/operator.go | 8 +++- sandboxd/pool/pool.go | 5 ++ sandboxd/pool/stats.go | 13 ++++-- sandboxd/pool/stats_test.go | 34 ++++++++++++++ sandboxd/pool/template.go | 15 ++++++ sandboxd/server/server.go | 6 ++- sandboxd/store/peer/probe.go | 19 +++++--- sandboxd/store/peer/probe_test.go | 20 ++++++++ 13 files changed, 196 insertions(+), 22 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index adfcd5b..dbf72b1 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -164,9 +164,11 @@ replica carries the source checkpoint's original `CreatedAt`, so it becomes eligible for expiry at the same instant everywhere; each node then removes it on its own hourly sweep, which is independently phased and retries on failure. The window in which a deleted checkpoint stays branchable by an -id-holder is therefore `checkpoint_ttl_hours` plus up to one sweep interval, -per replica, best-effort — which is why heal *requires* a nonzero, -fleet-matching TTL (see [cluster-invariant config](#cluster-invariant-config)). +id-holder is therefore normally `checkpoint_ttl_hours` plus the wait for the +next hourly sweep; a sweep that fails retries on a later one, so persistent +sweep failure extends retention until one succeeds — the TTL is the eligibility +point, not a hard ceiling. This is why heal *requires* a nonzero, fleet-matching +TTL (see [cluster-invariant config](#cluster-invariant-config)). With TTL disabled it could never close at all, so that combination is rejected at config load. diff --git a/docs/deploy.md b/docs/deploy.md index 12a53bd..a7dd95f 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -80,7 +80,7 @@ sandboxd reads one JSON file (`-config`, default | `checkpoint_dir` | `/checkpoints` | where checkpoints and promoted templates live. Point it at a shared FUSE mount (JuiceFS over object storage, NFS) and every node sharing the mount can branch every checkpoint — the path's filesystem is the operator's choice. One contract on any shared root (mount or bucket): a template key has a single writer — promotes go to the sandbox's owner node, and operators must not race promotes of one name from different nodes (checkpoint ids are node-generated and never collide). A checkpoint deleted on one node while another is mid-branch from it fails that branch visibly | | `checkpoint_store` | dir | checkpoint AND promoted-template backend (both live in one store root, id-namespaced ck_/tp_): `{"kind": "s3", "s3": {"bucket": "…", "prefix": "ck/", "endpoint": "…", "region": "…", "force_path_style": true}}` stores checkpoints in object storage (any node claims any checkpoint, no shared mount needed). Credentials come from the standard AWS chain (env/IAM role), never this file. A crash between upload and the meta.json commit marker leaves orphan objects invisible to listings — add an S3 lifecycle rule to reclaim them. Absent = the dir backend at `checkpoint_dir` | | `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the bound on a healed replica a delete broadcast missed | -| `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (the bound on how long a replica can outlive a delete that failed to reach it). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | +| `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (a replica a delete broadcast missed becomes eligible for expiry after it, and its next successful hourly sweep removes it — so it is the finite eligibility point, not an exact ceiling). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, a claim is first redirected to a warm peer) | | `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`); preview accesses record as op `preview_dial`. A request frame whose first line exceeds 4 KiB is skipped, never truncated | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 07560cb..581c765 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -274,13 +274,15 @@ partitioned during the broadcast keeps its copy until the checkpoint TTL ages it out. A healed replica carries the source's original `CreatedAt`, so it becomes eligible for expiry at the same instant on every node; the actual removal is each node's own hourly sweep, which is independently phased and -retries on failure, so the true bound is `checkpoint_ttl_hours` plus up to -one sweep interval per replica, best-effort. The TTL must also match +retries on a later sweep if one fails. So a deleted checkpoint normally stops +being branchable within `checkpoint_ttl_hours` plus a sweep interval, but a +node whose sweeps keep failing holds its replica until one succeeds — the TTL +is the eligibility point, not a hard ceiling. The TTL must also match fleet-wide, which the [cluster-invariant config](cluster.md#cluster-invariant-config) digest checks. A window always exists because `checkpoint_peer_heal` cannot be enabled with `checkpoint_ttl_hours: 0` — a replica that can outlive a delete -must have a bound on how long. A shared +must have a finite eligibility point. A shared checkpoint store skips the broadcast: every node already resolves every record directly, so there is no replica to chase. `?no_forward=1` marks a delete already arriving from another node's own broadcast, so it is not diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index f8fe767..69ea319 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -289,8 +289,18 @@ func validateHealedCheckpoint(staging, wantID string) error { if ckpt.Archive { return fmt.Errorf("healed record %s is a wake image, not a checkpoint", wantID) } - if !dirExists(filepath.Join(staging, store.ExportDir)) { - return fmt.Errorf("healed record %s missing export dir", wantID) + if keyErr := ckpt.Key.Validate(); keyErr != nil { + return fmt.Errorf("healed record %s has an invalid key: %w", wantID, keyErr) + } + export, err := os.ReadDir(filepath.Join(staging, store.ExportDir)) + if err != nil { + return fmt.Errorf("healed record %s missing export dir: %w", wantID, err) + } + if len(export) == 0 { + // An empty but present export passes a shape check yet clones to + // nothing; accepting it would publish a dead record, suppress trying a + // good owner, and fail every later local claim of this id. + return fmt.Errorf("healed record %s has an empty export", wantID) } entries, err := os.ReadDir(staging) if err != nil { diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index fa7e54e..f365847 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -212,6 +212,29 @@ func TestRecLockEvictionWaitsForAllHolders(t *testing.T) { } } +// TestRecLockEvictDeferredToLastHolder is the production sequence the test +// above sidesteps: a delete (recDoneEvict) releases while an ordinary holder +// (recDone) still references the id, so the ordinary holder is the last one +// out. The eviction the delete asked for must not be lost. +func TestRecLockEvictDeferredToLastHolder(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + const id = "ck_00000000000000aa" + + l1 := m.recLock(id) // the delete + l2 := m.recLock(id) // a concurrent claim/fetch + if l2 != l1 { + t.Fatal("recLock diverged for the same id") + } + m.recDoneEvict(id) // delete releases first, wants eviction, but l2 still holds + if _, ok := m.recLocks.Load(id); !ok { + t.Fatal("entry evicted while an ordinary holder still references it") + } + m.recDone(id) // the ordinary holder, last one out + if _, ok := m.recLocks.Load(id); ok { + t.Error("recLocks entry leaked: a delete's deferred eviction was lost when a plain recDone dropped the last reference") + } +} + // TestFetchCheckpointNeverPulls: the peer-transfer blob endpoint must never // trigger a recursive pull, even with a healer installed that could serve it. func TestFetchCheckpointNeverPulls(t *testing.T) { @@ -376,6 +399,37 @@ func TestValidateHealedCheckpointRejectsArchive(t *testing.T) { } } +// TestValidateHealedCheckpointRejectsEmptyExport: an export directory that is +// present but empty clones to nothing — publishing it would suppress a good +// owner and fail every later local claim, so it must be rejected and the next +// owner tried. +func TestValidateHealedCheckpointRejectsEmptyExport(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { + t.Fatalf("setup: %v", err) + } + meta, err := json.Marshal(types.Checkpoint{ID: "ck_00000000000000aa", Key: testKey}) + if err != nil { + t.Fatalf("setup: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, store.MetaFile), meta, 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + if err := validateHealedCheckpoint(dir, "ck_00000000000000aa"); err == nil { + t.Error("validate accepted a record with an empty export") + } +} + +// TestValidateHealedCheckpointRejectsInvalidKey: a record whose key does not +// name a branchable pool must not be published. +func TestValidateHealedCheckpointRejectsInvalidKey(t *testing.T) { + dir := t.TempDir() + plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa", Key: types.PoolKey{Net: "bogus"}}) + if err := validateHealedCheckpoint(dir, "ck_00000000000000aa"); err == nil { + t.Error("validate accepted a record with an invalid key") + } +} + // TestDeleteCheckpointBroadcasts: a local delete calls the peer-delete hook // once so a copy healed onto another node is removed too. func TestDeleteCheckpointBroadcasts(t *testing.T) { @@ -473,6 +527,9 @@ func (p *healPuller) Pull(_ context.Context, _, _, dst string) error { if err := os.MkdirAll(filepath.Join(dst, store.ExportDir), 0o750); err != nil { return err } + if err := os.WriteFile(filepath.Join(dst, store.ExportDir, "mem"), []byte("state"), 0o600); err != nil { + return err + } meta, err := json.Marshal(p.ckpt) if err != nil { return err @@ -501,6 +558,9 @@ func (p *blockingPuller) Pull(_ context.Context, _, id, dst string) error { if err := os.MkdirAll(filepath.Join(dst, store.ExportDir), 0o750); err != nil { return err } + if err := os.WriteFile(filepath.Join(dst, store.ExportDir, "mem"), []byte("state"), 0o600); err != nil { + return err + } meta, err := json.Marshal(types.Checkpoint{ID: id, Key: testKey, CreatedAt: time.Now()}) if err != nil { return err @@ -511,9 +571,15 @@ func (p *blockingPuller) Pull(_ context.Context, _, id, dst string) error { // plantHealedRecord writes a valid-shaped staged record for validate tests. func plantHealedRecord(t *testing.T, dir string, ckpt types.Checkpoint) { t.Helper() + if ckpt.Key == (types.PoolKey{}) { + ckpt.Key = testKey // a healed record carries a real, branchable key + } if err := os.MkdirAll(filepath.Join(dir, store.ExportDir), 0o750); err != nil { t.Fatalf("setup: %v", err) } + if err := os.WriteFile(filepath.Join(dir, store.ExportDir, "mem"), []byte("state"), 0o600); err != nil { + t.Fatalf("setup: %v", err) + } meta, err := json.Marshal(ckpt) if err != nil { t.Fatalf("setup: %v", err) diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index fce3012..fbc31ae 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -15,10 +15,14 @@ type Cred struct { // Sandbox reports one live claim's summary. func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { - sb, ok := m.byID(id) - if !ok { + m.mu.Lock() + defer m.mu.Unlock() + sb := m.claimed[id] + if sb == nil { return SandboxSummary{}, false } + // summarize reads fields hibernate and archive mutate under m.mu, so it + // runs here, not after byID has released the lock. return summarize(sb), true } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index d794053..cc55a26 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -264,6 +264,10 @@ type Manager struct { recLocks sync.Map recLocksMu sync.Mutex recRefs map[string]int + // recEvict remembers ids a delete asked to evict while a reference still + // held the lock, so the eviction happens when the last holder leaves rather + // than being lost — otherwise the recLocks entry leaks for that id. + recEvict map[string]struct{} // notifyTemplates, when set (before serving starts), fires after a // promote or template delete so the mesh republishes immediately @@ -321,6 +325,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg egressListeners: map[string]*egressListener{}, egressTaps: map[string]string{}, recRefs: map[string]int{}, + recEvict: map[string]struct{}{}, healPending: map[string]struct{}{}, healAbort: map[string]struct{}{}, egressSecrets: secrets, diff --git a/sandboxd/pool/stats.go b/sandboxd/pool/stats.go index 1e2b633..d10a883 100644 --- a/sandboxd/pool/stats.go +++ b/sandboxd/pool/stats.go @@ -29,8 +29,13 @@ type SandboxStats struct { // Stats reports one live claim's resource usage. func (m *Manager) Stats(ctx context.Context, id string) (SandboxStats, bool) { - sb, ok := m.byID(id) - if !ok { + // Snapshot the mutable fields under m.mu — hibernate and archive rewrite + // HibernateSnap and VMName there — then do the /proc read, which touches no + // sandbox state, outside the lock. + m.mu.Lock() + sb := m.claimed[id] + if sb == nil { + m.mu.Unlock() return SandboxStats{}, false } spec, _ := sb.Key.Size.Spec() @@ -41,8 +46,10 @@ func (m *Manager) Stats(ctx context.Context, id string) (SandboxStats, bool) { Hibernated: sb.HibernateSnap != "", MeasuredAt: time.Now().UTC(), } + vmName := sb.VMName + m.mu.Unlock() if !st.Hibernated { - if rss, ok := m.vmResidentBytes(ctx, sb.VMName); ok { + if rss, ok := m.vmResidentBytes(ctx, vmName); ok { st.MemUsedBytes, st.MemUsedMeasured = rss, true } } diff --git a/sandboxd/pool/stats_test.go b/sandboxd/pool/stats_test.go index f8fb112..c3ff9bb 100644 --- a/sandboxd/pool/stats_test.go +++ b/sandboxd/pool/stats_test.go @@ -3,6 +3,7 @@ package pool import ( "os" "runtime" + "sync" "testing" "github.com/cocoonstack/sandbox/sandboxd/config" @@ -19,6 +20,39 @@ func TestStatsUnknownSandbox(t *testing.T) { // TestStatsHibernatedIsUnmeasured: a hibernated claim has no VMM process, so // MemUsedMeasured must stay false rather than reading a stale or zero RSS — // and Stats must not even ask the engine, which a hibernated claim can't answer. +// TestOperatorReadsRaceFreeUnderHibernate: Sandbox and Stats read fields that +// hibernate rewrites under m.mu, so they must snapshot under the same lock. +// Run with -race; without the lock this trips the detector. +func TestOperatorReadsRaceFreeUnderHibernate(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) + sb := mustClaim(t, m, testKey) + + done := make(chan struct{}) + var wg sync.WaitGroup + wg.Go(func() { + defer close(done) + for range 100 { + _ = m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}) + _ = m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}) + } + }) + // Read continuously for the writer's whole run, so the operator reads and + // the lifecycle mutations actually overlap for the detector to see. + wg.Go(func() { + for { + select { + case <-done: + return + default: + m.Sandbox(sb.ID) + m.Stats(t.Context(), sb.ID) + } + } + }) + wg.Wait() +} + func TestStatsHibernatedIsUnmeasured(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index f1b6a80..528a65c 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -194,6 +194,7 @@ func (m *Manager) recDone(id string) { m.recRefs[id]-- if m.recRefs[id] <= 0 { delete(m.recRefs, id) + m.evictIfPending(id) } } @@ -208,6 +209,20 @@ func (m *Manager) recDoneEvict(id string) { m.recRefs[id]-- if m.recRefs[id] <= 0 { delete(m.recRefs, id) + delete(m.recEvict, id) + m.recLocks.Delete(id) + return + } + // Another holder or waiter is still on this lock; defer the eviction to + // whichever call drops the last reference, so it is not lost here. + m.recEvict[id] = struct{}{} +} + +// evictIfPending drops id's lock entry if a delete asked for eviction while a +// reference still held it; callers hold recLocksMu with the count at zero. +func (m *Manager) evictIfPending(id string) { + if _, ok := m.recEvict[id]; ok { + delete(m.recEvict, id) m.recLocks.Delete(id) } } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index ce7b988..632e59f 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -454,10 +454,14 @@ func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) scope = pool.DeleteLocal } id := r.PathValue("id") + err := s.mgr.DeleteCheckpoint(r.Context(), id, tenantFrom(r.Context()), scope) + // Evict after the record is gone, not before: a probe racing the delete + // would otherwise observe the still-present record and cache it just as the + // pre-delete eviction cleared the entry. Unconditional — a forwarded delete + // that finds nothing here still learned the id's ownership changed. if s.prober != nil { s.prober.Forget(id) } - err := s.mgr.DeleteCheckpoint(r.Context(), id, tenantFrom(r.Context()), scope) writeResult(w, r, "delete checkpoint", id, "delete checkpoint failed", err, func() { w.WriteHeader(http.StatusNoContent) }) diff --git a/sandboxd/store/peer/probe.go b/sandboxd/store/peer/probe.go index 1a31260..0d0d160 100644 --- a/sandboxd/store/peer/probe.go +++ b/sandboxd/store/peer/probe.go @@ -86,7 +86,7 @@ func (p *HTTPProber) Owners(ctx context.Context, id string) []string { return owners } v, _, _ := p.redirectFlight.Do(id, func() (any, error) { - owners := p.fanOut(ctx, id, maxRedirectOwners) + owners := p.fanOut(ctx, id, maxRedirectOwners, probeGrace) p.cachePut(id, owners, start) return owners, nil }) @@ -100,7 +100,9 @@ func (p *HTTPProber) Owners(ctx context.Context, id string) []string { // the moment it is needed, not a redirect-tuned snapshot. func (p *HTTPProber) HealOwners(ctx context.Context, id string) []string { v, _, _ := p.healFlight.Do(id, func() (any, error) { - return p.fanOut(ctx, id, maxHealOwners), nil + // No grace: a heal wants every owner it can find, so it must not stop + // early on a fast one and hide a slower valid source behind it. + return p.fanOut(ctx, id, maxHealOwners, 0), nil }) return v.([]string) } @@ -124,7 +126,7 @@ func (p *HTTPProber) Forget(id string) { // internally: singleflight shares this fan-out across concurrent callers // whose own contexts may cancel independently (one client disconnecting // must not abort another's still-live probe). -func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int) []string { +func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int, grace time.Duration) []string { addrs := dedupAddrs(p.Peers()) if len(addrs) == 0 { return nil @@ -150,8 +152,11 @@ func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int) []str // grace opens only after the first owner answers: until then the fan-out // waits out the full timeout, since a genuine miss must hear from every // peer before it can conclude nobody holds the record. + // graceCh stays nil when grace is 0 (heal wants an exhaustive list), so the + // select then only ever takes a win and runs until the cap or every peer + // has answered. var owners []string - var grace <-chan time.Time + var graceCh <-chan time.Time for { select { case addr, ok := <-wins: @@ -163,10 +168,10 @@ func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int) []str cancel() return owners } - if grace == nil { - grace = time.After(probeGrace) + if grace > 0 && graceCh == nil { + graceCh = time.After(grace) } - case <-grace: + case <-graceCh: cancel() return owners } diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index 352076d..425fd05 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -216,6 +216,26 @@ func TestHealOwnersReturnsMoreThanRedirectCap(t *testing.T) { } } +// TestHealOwnersIsExhaustiveDespiteAFastOwner: a heal must collect every owner +// even when one answers fast, so it does not stop on the grace window a redirect +// uses and hide slower valid sources behind the quick one. +func TestHealOwnersIsExhaustiveDespiteAFastOwner(t *testing.T) { + fast := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer fast.Close() + slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(2 * probeGrace) // answers well after the redirect grace would have closed + w.WriteHeader(http.StatusOK) + })) + defer slow.Close() + + p := &HTTPProber{Peers: func() []string { return []string{fast.URL, slow.URL} }} + if owners := p.HealOwners(t.Context(), testID); len(owners) != 2 { + t.Errorf("HealOwners = %v, want both owners; the grace window must not truncate a heal", owners) + } +} + // TestOwnersCoalescesConcurrentProbes: concurrent redirect probes for one id // must fan out to each peer once, not once per caller. func TestOwnersCoalescesConcurrentProbes(t *testing.T) { From 1dfc1e8547abdd54ef3c00eea79d879c4170a3a0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 22:03:08 +0800 Subject: [PATCH 27/31] sandboxd: reject a non-branchable or contentless healed record, and finish the docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fourth review pass caught the residue of the last round's validation work. An egress-lane key passes Key.Validate but Capturable is false, so claimLoaded refuses every branch of it; a healed record carrying one would publish, poison the id, and suppress a good owner. Validation now rejects it. An export directory with no regular file — empty, or only empty subdirectories — clones to nothing and is rejected too; the byte content beyond that is cocoon's own format, not sandboxd's to validate, so the check stops at "has content". The checkpoint_ttl_hours line in deploy.md still called the TTL a hard bound; it now matches the other docs — the eligibility point, extended by sweep failure. Test and comment fixes the same pass flagged: the missing-export test built a record with an empty key, so it failed at the key check instead of reaching the export branch; the puller fixture's comment still said it wrote an empty export; and the hibernated-stats test's comment had been left above the new operator-race test. New tests cover the egress-key and contentless-export rejections, both mutation-verified. --- docs/deploy.md | 2 +- sandboxd/pool/checkpoint.go | 22 ++++++++++++---- sandboxd/pool/checkpoint_heal_test.go | 38 +++++++++++++++++++++++++-- sandboxd/pool/stats_test.go | 6 ++--- 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index a7dd95f..5c0966b 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -79,7 +79,7 @@ sandboxd reads one JSON file (`-config`, default | `preview_advertise` | = `preview_listen` | the base URL a browser/proxy reaches this node's preview server at | | `checkpoint_dir` | `/checkpoints` | where checkpoints and promoted templates live. Point it at a shared FUSE mount (JuiceFS over object storage, NFS) and every node sharing the mount can branch every checkpoint — the path's filesystem is the operator's choice. One contract on any shared root (mount or bucket): a template key has a single writer — promotes go to the sandbox's owner node, and operators must not race promotes of one name from different nodes (checkpoint ids are node-generated and never collide). A checkpoint deleted on one node while another is mid-branch from it fails that branch visibly | | `checkpoint_store` | dir | checkpoint AND promoted-template backend (both live in one store root, id-namespaced ck_/tp_): `{"kind": "s3", "s3": {"bucket": "…", "prefix": "ck/", "endpoint": "…", "region": "…", "force_path_style": true}}` stores checkpoints in object storage (any node claims any checkpoint, no shared mount needed). Credentials come from the standard AWS chain (env/IAM role), never this file. A crash between upload and the meta.json commit marker leaves orphan objects invisible to listings — add an S3 lifecycle rule to reclaim them. Absent = the dir backend at `checkpoint_dir` | -| `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the bound on a healed replica a delete broadcast missed | +| `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the expiry eligibility point for a healed replica a delete broadcast missed, after which its next successful hourly sweep removes it; persistent sweep failure extends retention until one succeeds, so it is not a hard ceiling | | `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (a replica a delete broadcast missed becomes eligible for expiry after it, and its next successful hourly sweep removes it — so it is the finite eligibility point, not an exact ceiling). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, a claim is first redirected to a warm peer) | diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 69ea319..ec3136a 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -292,15 +292,21 @@ func validateHealedCheckpoint(staging, wantID string) error { if keyErr := ckpt.Key.Validate(); keyErr != nil { return fmt.Errorf("healed record %s has an invalid key: %w", wantID, keyErr) } + if !ckpt.Key.Capturable() { + // An egress-lane key is well-formed but not branchable: claimLoaded + // rejects every branch of it, so publishing one poisons the id and + // suppresses a good owner. Reject it here and try the next owner. + return fmt.Errorf("healed record %s has a non-branchable (egress) key", wantID) + } export, err := os.ReadDir(filepath.Join(staging, store.ExportDir)) if err != nil { return fmt.Errorf("healed record %s missing export dir: %w", wantID, err) } - if len(export) == 0 { - // An empty but present export passes a shape check yet clones to - // nothing; accepting it would publish a dead record, suppress trying a - // good owner, and fail every later local claim of this id. - return fmt.Errorf("healed record %s has an empty export", wantID) + // A present export with no regular file clones to nothing (an empty dir, or + // only empty subdirs); the byte content is cocoon's own format and not + // sandboxd's to validate further. + if !hasRegularFile(export) { + return fmt.Errorf("healed record %s has no export content", wantID) } entries, err := os.ReadDir(staging) if err != nil { @@ -314,6 +320,12 @@ func validateHealedCheckpoint(staging, wantID string) error { return nil } +// hasRegularFile reports whether entries holds at least one regular file, so an +// export of only empty subdirectories is treated as empty. +func hasRegularFile(entries []os.DirEntry) bool { + return slices.ContainsFunc(entries, func(e os.DirEntry) bool { return e.Type().IsRegular() }) +} + // Checkpoints lists the store's checkpoints, newest first — on a shared // backend (a FUSE mount, a bucket), that is the cluster's set, not one // node's. A non-empty tenant filters to that tenant's records; empty (root) diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index f365847..4924c69 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -377,7 +377,9 @@ func TestValidateHealedCheckpointRejectsMissingMeta(t *testing.T) { // dir cannot back a branch. func TestValidateHealedCheckpointRejectsMissingExport(t *testing.T) { dir := t.TempDir() - meta, err := json.Marshal(types.Checkpoint{ID: "ck_00000000000000aa"}) + // A valid key, so the record reaches the export check rather than failing + // earlier on the key. + meta, err := json.Marshal(types.Checkpoint{ID: "ck_00000000000000aa", Key: testKey}) if err != nil { t.Fatalf("setup: %v", err) } @@ -430,6 +432,38 @@ func TestValidateHealedCheckpointRejectsInvalidKey(t *testing.T) { } } +// TestValidateHealedCheckpointRejectsEgressKey: an egress-lane key is valid but +// not branchable (claimLoaded refuses it), so a healed record carrying one must +// be rejected rather than published to poison the id. +func TestValidateHealedCheckpointRejectsEgressKey(t *testing.T) { + dir := t.TempDir() + egress := testKey + egress.Net = types.NetEgress + plantHealedRecord(t, dir, types.Checkpoint{ID: "ck_00000000000000aa", Key: egress}) + if err := validateHealedCheckpoint(dir, "ck_00000000000000aa"); err == nil { + t.Error("validate accepted a non-branchable egress-lane record") + } +} + +// TestValidateHealedCheckpointRejectsContentlessExport: an export directory +// holding only an empty subdirectory clones to nothing and must be rejected. +func TestValidateHealedCheckpointRejectsContentlessExport(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, store.ExportDir, "sub"), 0o750); err != nil { + t.Fatalf("setup: %v", err) + } + meta, err := json.Marshal(types.Checkpoint{ID: "ck_00000000000000aa", Key: testKey}) + if err != nil { + t.Fatalf("setup: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, store.MetaFile), meta, 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + if err := validateHealedCheckpoint(dir, "ck_00000000000000aa"); err == nil { + t.Error("validate accepted an export with no regular file") + } +} + // TestDeleteCheckpointBroadcasts: a local delete calls the peer-delete hook // once so a copy healed onto another node is removed too. func TestDeleteCheckpointBroadcasts(t *testing.T) { @@ -508,7 +542,7 @@ func newHealManager(t *testing.T, ckpt types.Checkpoint, addrs []string) (*Manag } // healPuller fakes peer.Puller: it writes a valid record (meta.json + an -// empty export/ dir the fakeEngine "clones" from) and can block until +// export/ dir the fakeEngine "clones" from) and can block until // release, so a test can prove concurrent misses dedup to one pull. type healPuller struct { mu sync.Mutex diff --git a/sandboxd/pool/stats_test.go b/sandboxd/pool/stats_test.go index c3ff9bb..4680dba 100644 --- a/sandboxd/pool/stats_test.go +++ b/sandboxd/pool/stats_test.go @@ -17,9 +17,6 @@ func TestStatsUnknownSandbox(t *testing.T) { } } -// TestStatsHibernatedIsUnmeasured: a hibernated claim has no VMM process, so -// MemUsedMeasured must stay false rather than reading a stale or zero RSS — -// and Stats must not even ask the engine, which a hibernated claim can't answer. // TestOperatorReadsRaceFreeUnderHibernate: Sandbox and Stats read fields that // hibernate rewrites under m.mu, so they must snapshot under the same lock. // Run with -race; without the lock this trips the detector. @@ -53,6 +50,9 @@ func TestOperatorReadsRaceFreeUnderHibernate(t *testing.T) { wg.Wait() } +// TestStatsHibernatedIsUnmeasured: a hibernated claim has no VMM process, so +// MemUsedMeasured must stay false rather than reading a stale or zero RSS — +// and Stats must not even ask the engine, which a hibernated claim can't answer. func TestStatsHibernatedIsUnmeasured(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) From a25df7d6ce55b0c68776eda658482ebb3e8f4b5a Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 22:23:49 +0800 Subject: [PATCH 28/31] sandboxd: give the record transfer a completion marker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source-side filesystem error between two files leaves TarRecord's deferred Close writing a valid tar footer over a short archive — so the receiver saw a well-formed but incomplete record and published it. On the documented FUSE/NFS checkpoint backends a transient EIO or ESTALE mid-walk is a routine, not adversarial, event, and a published incomplete record then answers probes and serves its own /blob, so the next node to heal from it inherits the same truncated copy — the fault spreads instead of staying put. TarRecord now writes a completion marker as its last entry, after the export has streamed whole; Untar requires it before returning success and never writes it to disk. A transfer cut short lacks the marker and is rejected, so the healer clears its staging and tries the next owner — the transfer converges on a good copy instead of persisting a bad one. The blob handler's comment claimed a mid-stream failure truncates the tar; it does not, and now says what actually guards completeness. This is a transport-completeness signal, not validation of the export's bytes: those are cocoon's snapshot format, and a peer shipping semantically corrupt but well-formed bytes remains out of scope (an authenticated peer would have to be compromised, and the failure is a loud provision error, not silent corruption). --- sandboxd/server/server.go | 5 +-- sandboxd/store/peer/transport.go | 21 +++++++++++ sandboxd/store/peer/transport_test.go | 51 +++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 632e59f..a677cdc 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -405,8 +405,9 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { defer release() w.Header().Set("Content-Type", "application/x-tar") - // Status committed before the walk, so a mid-stream failure truncates the - // tar and the reader fails the pull on the short archive. + // Status is committed before the walk, so a mid-stream failure cannot change + // it; the tar's completion marker is what tells the reader the record + // arrived whole, and a short transfer is rejected for lacking it. w.WriteHeader(http.StatusOK) if err := peer.TarRecord(dir, meta, w); err != nil { log.WithFunc("server.handleCheckpointBlob").Error(r.Context(), err, "stream checkpoint") diff --git a/sandboxd/store/peer/transport.go b/sandboxd/store/peer/transport.go index eba98be..cc9405b 100644 --- a/sandboxd/store/peer/transport.go +++ b/sandboxd/store/peer/transport.go @@ -26,6 +26,12 @@ const ( metaFile = "meta.json" exportPrefix = "export/" + + // recordTrailer is written last, only after the whole export streamed, so a + // receiver that reaches EOF without it knows the transfer stopped early — a + // source-side walk or read error leaves a valid but short tar (Close still + // writes the footer), and that must not be published as a complete record. + recordTrailer = ".record-complete" ) // ErrNotFound reports that a peer does not hold the requested record. Declared @@ -109,6 +115,13 @@ func TarRecord(exportDir string, meta []byte, w io.Writer) error { if err := tarInto(exportDir, exportPrefix, tw); err != nil { return err } + // The completion marker rides last, after the export streamed whole, so its + // absence tells the receiver the transfer was cut short. + if err := tw.WriteHeader(&tar.Header{ + Name: recordTrailer, Mode: 0o600, Size: 0, Typeflag: tar.TypeReg, + }); err != nil { + return fmt.Errorf("tar record trailer: %w", err) + } return tw.Close() } @@ -167,14 +180,22 @@ func tarInto(src, prefix string, tw *tar.Writer) error { func Untar(r io.Reader, dst string) error { tr := tar.NewReader(r) var written int64 + var complete bool for { hdr, err := tr.Next() if errors.Is(err, io.EOF) { + if !complete { + return fmt.Errorf("untar: record incomplete, no completion marker (transfer cut short)") + } return nil } if err != nil { return fmt.Errorf("untar: %w", err) } + if hdr.Name == recordTrailer { + complete = true // a protocol marker, never written to disk + continue + } target, err := safeJoin(dst, hdr.Name) if err != nil { return err diff --git a/sandboxd/store/peer/transport_test.go b/sandboxd/store/peer/transport_test.go index ea6325b..9cee3dc 100644 --- a/sandboxd/store/peer/transport_test.go +++ b/sandboxd/store/peer/transport_test.go @@ -115,6 +115,9 @@ func TestUntarSkipsNonDataEntries(t *testing.T) { if _, err := tw.Write([]byte("{}")); err != nil { t.Fatalf("Write: %v", err) } + if err := tw.WriteHeader(&tar.Header{Name: recordTrailer, Mode: 0o600, Typeflag: tar.TypeReg}); err != nil { + t.Fatalf("WriteHeader trailer: %v", err) + } if err := tw.Close(); err != nil { t.Fatalf("Close: %v", err) } @@ -185,6 +188,7 @@ func TestHTTPPullerPresentsToken(t *testing.T) { tw := tar.NewWriter(&buf) _ = tw.WriteHeader(&tar.Header{Name: "meta.json", Mode: 0o600, Size: 2, Typeflag: tar.TypeReg}) _, _ = tw.Write([]byte("{}")) + _ = tw.WriteHeader(&tar.Header{Name: recordTrailer, Mode: 0o600, Typeflag: tar.TypeReg}) _ = tw.Close() _, _ = w.Write(buf.Bytes()) })) @@ -262,11 +266,58 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { // tarDir streams src's contents with no record layout, exercising tarInto and // Untar on their own. + +// TestUntarRejectsRecordWithoutCompletionMarker: a transfer cut short by a +// source-side walk/read error still yields a valid short tar (Close writes the +// footer regardless), so the receiver must reject a record that arrives without +// the trailer rather than publish an incomplete one. +func TestUntarRejectsRecordWithoutCompletionMarker(t *testing.T) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + // meta plus one complete export file, then the walk "fails" — no trailer. + _ = tw.WriteHeader(&tar.Header{Name: "meta.json", Mode: 0o600, Size: 4, Typeflag: tar.TypeReg}) + _, _ = tw.Write([]byte(`{"a"`)) + _ = tw.WriteHeader(&tar.Header{Name: "export/mem", Mode: 0o600, Size: 5, Typeflag: tar.TypeReg}) + _, _ = tw.Write([]byte("bytes")) + _ = tw.Close() // valid footer, no trailer entry + + if err := Untar(&buf, t.TempDir()); err == nil { + t.Error("Untar accepted a record with no completion marker (an incomplete transfer)") + } +} + +// TestTarRecordUntarRoundTripComplete: the real send/receive pair carries the +// trailer, so a whole record is accepted and the marker itself is not written +// to disk (validateHealedCheckpoint would reject an unexpected entry). +func TestTarRecordUntarRoundTripComplete(t *testing.T) { + exportDir := t.TempDir() + if err := os.WriteFile(filepath.Join(exportDir, "mem"), []byte("state"), 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + var buf bytes.Buffer + if err := TarRecord(exportDir, []byte(`{"id":"ck_1"}`), &buf); err != nil { + t.Fatalf("TarRecord: %v", err) + } + dst := t.TempDir() + if err := Untar(&buf, dst); err != nil { + t.Fatalf("Untar: %v", err) + } + if _, err := os.Stat(filepath.Join(dst, recordTrailer)); !os.IsNotExist(err) { + t.Errorf("completion marker was written to disk (%v); it must be consumed, not stored", err) + } + if _, err := os.Stat(filepath.Join(dst, "export", "mem")); err != nil { + t.Errorf("export content missing after round trip: %v", err) + } +} + func tarDir(src string, w io.Writer) error { tw := tar.NewWriter(w) defer func() { _ = tw.Close() }() if err := tarInto(src, "", tw); err != nil { return err } + if err := tw.WriteHeader(&tar.Header{Name: recordTrailer, Mode: 0o600, Typeflag: tar.TypeReg}); err != nil { + return err + } return tw.Close() } From 171cd0f32fb26b1de6735765cabbddb1bc9ee4d4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 23:14:41 +0800 Subject: [PATCH 29/31] review: tighten comment budget and dedup the peer URL plumbing Comment narratives cut back to their load-bearing invariants (checkpoint delete/heal docs, probe fan-out docs, healer history note); the three scheme-default URL builders collapse into one checkpointURL helper; resolveScope reuses isRootToken; a stranded tarDir doc rejoins its func. --- sandboxd/pool/checkpoint.go | 53 +++++++---------- sandboxd/pool/pool.go | 9 +-- sandboxd/server/server.go | 12 ++-- sandboxd/store/peer/broadcast.go | 9 +-- sandboxd/store/peer/peer.go | 14 ++--- sandboxd/store/peer/probe.go | 83 ++++++++++----------------- sandboxd/store/peer/transport.go | 18 +++--- sandboxd/store/peer/transport_test.go | 5 +- 8 files changed, 75 insertions(+), 128 deletions(-) diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index ec3136a..c9a6a42 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -167,9 +167,8 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, ttl ti // healCheckpoint dedups concurrent heals of ckptID onto one flight (which // owns its own staging dir — never a caller's) and lets THIS call abandon -// waiting the moment ctx is done; the flight itself runs detached from any -// one caller (context.Background() in runHeal), so a client hanging up never -// abandons a transfer already paid for — the next call would just pay again. +// waiting the moment ctx is done; the flight itself runs detached (runHeal), +// so a client hanging up never abandons a transfer already paid for. func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Checkpoint, error) { if ckpt, err := m.loadCheckpoint(ctx, ckptID); err == nil { return ckpt, nil @@ -190,10 +189,9 @@ func (m *Manager) healCheckpoint(ctx context.Context, ckptID string) (types.Chec // runHeal is healCheckpoint's flight body: it stages and pulls WITHOUT // holding ckptID's record lock — a heal budget runs up to 30 minutes, and -// holding the lock across it would pin every other operation on the same id -// (a delete, a claim, another heal) behind an uncancellable wait for that -// long. The lock is taken only for the fast, final steps: check for a -// concurrent veto (see vetoIfHealPending), re-validate, and publish. +// holding the lock across it would pin every same-id operation behind an +// uncancellable wait. The lock covers only the fast final steps: the veto +// check, re-validate, publish. func (m *Manager) runHeal(ckptID string) (types.Checkpoint, error) { select { case m.healSem <- struct{}{}: @@ -258,11 +256,9 @@ func (m *Manager) clearHealPending(ckptID string) (aborted bool) { return aborted } -// vetoIfHealPending marks ckptID aborted when a heal is currently pending -// for it, so that heal's locked decide phase (clearHealPending) sees the -// veto instead of publishing a checkpoint this call just answered "not -// here" for — a delete otherwise racing an unlocked, still-staging heal -// would return 404 only for the checkpoint to reappear moments later. A +// vetoIfHealPending marks ckptID aborted when a heal is pending, so the +// heal's locked decide phase (clearHealPending) abandons its publish instead +// of resurrecting a checkpoint this delete just answered "not here" for. A // no-op when no heal is pending, so an unrelated miss leaves no residue. func (m *Manager) vetoIfHealPending(ckptID string) { m.recLocksMu.Lock() @@ -293,9 +289,8 @@ func validateHealedCheckpoint(staging, wantID string) error { return fmt.Errorf("healed record %s has an invalid key: %w", wantID, keyErr) } if !ckpt.Key.Capturable() { - // An egress-lane key is well-formed but not branchable: claimLoaded - // rejects every branch of it, so publishing one poisons the id and - // suppresses a good owner. Reject it here and try the next owner. + // A well-formed egress key is not branchable: publishing it would + // poison the id and suppress a good owner; reject, try the next owner. return fmt.Errorf("healed record %s has a non-branchable (egress) key", wantID) } export, err := os.ReadDir(filepath.Join(staging, store.ExportDir)) @@ -370,20 +365,14 @@ func (m *Manager) pinnedArchiveCks() map[string]struct{} { return pinned } -// DeleteCheckpoint removes a checkpoint's snapshot and record, then broadcasts -// to peers when fleet-scoped so a healed copy does not outlive it. A tenant may -// delete only its own records — anything else answers ErrUnknownCheckpoint, -// never a hint that the id exists; root (empty tenant) deletes anything. -// Existence is checked under the record lock, not before it: heal broke the -// old assumption that a local miss means the id is truly gone. That alone is -// not enough, though — a heal's transfer runs unlocked (runHeal), so a -// concurrent delete can take the lock, find the checkpoint absent, and -// release it before the heal ever reaches its own locked decide phase; -// vetoIfHealPending closes that gap by telling a pending heal to abandon its -// publish instead of resurrecting what this call just answered "not here" -// for. Every exit evicts the lock entry (recDoneEvict, not recDone) — unlike -// a template id, a checkpoint id is effectively one-shot, so nothing is lost -// keeping the map from growing per rejected or successful call alike. +// DeleteCheckpoint removes a checkpoint's snapshot and record, then +// broadcasts to peers when fleet-scoped so a healed copy does not outlive it. +// A tenant may delete only its own records — anything else answers +// ErrUnknownCheckpoint, never a hint the id exists; root deletes anything. +// Existence is checked under the record lock (heal broke the "local miss +// means truly gone" assumption), plus vetoIfHealPending for a heal whose +// transfer runs unlocked. Every exit evicts the lock entry (recDoneEvict) — +// a checkpoint id is effectively one-shot, so the map must not grow per call. func (m *Manager) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope DeleteScope) error { // Reject a bad id before recLock: a rejected id must not leave a // lock-map entry. @@ -445,10 +434,8 @@ func (m *Manager) sweepExpiredCheckpoints(ctx context.Context) { } // deleteCkLocked removes a checkpoint under its record lock, so the delete -// never runs beneath an in-flight branch clone, heal, or archived wake. The -// lock is unlocked before the entry is considered for eviction: evicting -// while still (logically) held is what would let a concurrent recLock for -// the same id hand out a different mutex and split the serialization. +// never runs beneath an in-flight branch clone, heal, or archived wake; the +// lock is released before eviction is considered (see recDoneEvict). func (m *Manager) deleteCkLocked(ctx context.Context, ckID string) error { l := m.recLock(ckID) l.Lock() diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index cc55a26..36451c8 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -234,12 +234,9 @@ type Manager struct { ckptsShared bool healer *peer.Healer // healSem bounds concurrent transfers node-wide; healFlights dedups - // concurrent heals of the same id to one transfer (keyed by checkpoint - // id, owns its own staging dir — callers never see one). healPending and - // healAbort (guarded by recLocksMu) let a delete that finds a checkpoint - // absent veto a heal already staging for that same id, so the heal's - // locked decide phase does not publish moments after a delete answered - // "not here" for it. + // same-id heals onto one transfer, which owns its own staging dir. + // healPending/healAbort (guarded by recLocksMu) let a delete veto a heal + // still staging the same id (see vetoIfHealPending). healSem chan struct{} healFlights singleflight.Group healPending map[string]struct{} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index a677cdc..27a71f5 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -446,9 +446,7 @@ func (s *Server) handleListCheckpoints(w http.ResponseWriter, r *http.Request) { // handleDeleteCheckpoint removes a checkpoint; a tenant caller may delete // only its own records (anything else is 404). no_forward marks a delete // arriving from another node's own broadcast, so this one does not -// re-broadcast and loop the fleet forever. Either way, any cached probe -// answer for id is forgotten unconditionally: a node that never held a -// replica still learns from the broadcast that id's ownership changed. +// re-broadcast and loop the fleet forever. func (s *Server) handleDeleteCheckpoint(w http.ResponseWriter, r *http.Request) { scope := pool.DeleteFleet if r.URL.Query().Get("no_forward") != "" { @@ -589,9 +587,9 @@ func (s *Server) rootRequest(r *http.Request) bool { } // isRootToken reports whether token is the configured root api_token (the -// operator credential). It mirrors resolveScope's root branch: an unset api -// token matches nothing, so an open node grants no operator elevation on the -// per-sandbox release path. Tenant tokens never match — they live in s.tenants. +// operator credential). An unset api token matches nothing, so an open node +// grants no operator elevation; tenant tokens never match — they live in +// s.tenants. func (s *Server) isRootToken(token string) bool { return s.apiToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.apiToken)) == 1 } @@ -621,7 +619,7 @@ func (s *Server) resolveScope(r *http.Request) (string, bool) { if !ok { return "", false } - if s.apiToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.apiToken)) == 1 { + if s.isRootToken(token) { return "", true } for _, tn := range s.tenants { diff --git a/sandboxd/store/peer/broadcast.go b/sandboxd/store/peer/broadcast.go index 38409ab..f7126be 100644 --- a/sandboxd/store/peer/broadcast.go +++ b/sandboxd/store/peer/broadcast.go @@ -5,8 +5,6 @@ import ( "fmt" "io" "net/http" - "net/url" - "strings" "sync" "time" @@ -56,12 +54,7 @@ func deleteOn(ctx context.Context, client *http.Client, addr, id, token string) ctx, cancel := context.WithTimeout(ctx, deleteTimeout) defer cancel() - base := addr - if !strings.Contains(base, "://") { - base = "http://" + base - } - u := base + "/v1/checkpoints/" + url.PathEscape(id) + "?no_forward=1" - req, err := http.NewRequestWithContext(ctx, http.MethodDelete, u, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, checkpointURL(addr, id, "?no_forward=1"), nil) if err != nil { return err } diff --git a/sandboxd/store/peer/peer.go b/sandboxd/store/peer/peer.go index feb93ee..0a2688e 100644 --- a/sandboxd/store/peer/peer.go +++ b/sandboxd/store/peer/peer.go @@ -22,15 +22,11 @@ type Owners func(id string) []string // error tries the next owner. The healer does not know the record's shape. type Validate func(staging string) error -// Healer pulls a record this node does not hold into a caller-provided staging -// directory; the caller validates and publishes it. A nil owners or puller -// makes every Pull a miss, so a node with no mesh simply cannot heal. Pull -// does not dedup concurrent calls: a caller-owned staging dir means two -// concurrent Pulls for the same id are two independent destinations, so -// sharing one result (as a Healer-internal singleflight once did) would -// leave whichever caller did not "win" the flight with an untouched, -// published-empty directory. Dedup belongs to whoever owns the destination — -// pool.Manager's per-id heal flight, which owns one staging dir per flight. +// Healer pulls a record this node does not hold into a caller-provided +// staging directory; the caller validates and publishes it. A nil owners or +// puller makes every Pull a miss (no mesh, no heal). Pull never dedups +// concurrent calls: each caller owns its destination dir, so dedup belongs to +// whoever owns the staging — pool.Manager's per-id heal flight. type Healer struct { owners Owners puller Puller diff --git a/sandboxd/store/peer/probe.go b/sandboxd/store/peer/probe.go index 0d0d160..b4aac8a 100644 --- a/sandboxd/store/peer/probe.go +++ b/sandboxd/store/peer/probe.go @@ -8,8 +8,6 @@ import ( "encoding/base64" "encoding/binary" "net/http" - "net/url" - "strings" "sync" "time" @@ -24,9 +22,8 @@ const ( redirectCacheTTL = 5 * time.Second // probeGrace bounds how long a fan-out waits for more owners once it has - // its first: a checkpoint usually lives on one node, so the cap of three is - // rarely reached and without this the collector would block on every - // missing peer's timeout before returning the single owner it already has. + // its first: a checkpoint usually lives on one node, and without this the + // collector would wait out every missing peer's timeout to return it. probeGrace = 150 * time.Millisecond // ProbeHeader carries the base64 probe MAC when a ProbeKey is configured. @@ -72,14 +69,11 @@ type HTTPProber struct { cacheTTL time.Duration } -// Owners fans a HEAD out to every peer in parallel and returns up to -// maxRedirectOwners addresses that answered 200 -- enough candidates for a -// redirect, not the whole fleet. Concurrent Owners calls for the same id -// share one fan-out; a short positive cache then serves a hot id's repeat -// redirects without re-probing the fleet at all. The cache accepts up to -// redirectCacheTTL of staleness: a redirect to a peer that deleted the -// record moments ago fails safely (404, then the client's own no_redirect -// fallback heals from the origin), it never serves a wrong answer silently. +// Owners fans a HEAD out to every peer and returns up to maxRedirectOwners +// addresses that answered 200. Concurrent calls for one id share a fan-out; +// a short positive cache serves a hot id's repeat redirects. Cache staleness +// fails safely: a redirect to a peer that just deleted the record 404s, and +// the client's no_redirect fallback heals from the origin. func (p *HTTPProber) Owners(ctx context.Context, id string) []string { owners, start, hit := p.cacheLookup(id) if hit { @@ -93,11 +87,9 @@ func (p *HTTPProber) Owners(ctx context.Context, id string) []string { return v.([]string) } -// HealOwners is Owners' wide-fan-out twin: a heal wants the largest -// available source list, so a few full or corrupt owners cannot hide a -// usable one from it. Concurrent calls for the same id share one fan-out; -// results are not cached -- a heal source list should reflect the fleet at -// the moment it is needed, not a redirect-tuned snapshot. +// HealOwners is Owners' wide twin: a heal wants the largest source list, so +// a few full or corrupt owners cannot hide a usable one. Uncached — a heal's +// source list should reflect the fleet now, not a redirect-tuned snapshot. func (p *HTTPProber) HealOwners(ctx context.Context, id string) []string { v, _, _ := p.healFlight.Do(id, func() (any, error) { // No grace: a heal wants every owner it can find, so it must not stop @@ -107,11 +99,9 @@ func (p *HTTPProber) HealOwners(ctx context.Context, id string) []string { return v.([]string) } -// Forget evicts any cached Owners result for id -- called after a delete -// (original or forwarded from another node's broadcast) touches id here, -// since a stale positive would redirect a claim to a node that no longer -// holds the record. A miss was never cached, so a delete that finds nothing -// locally has nothing to do beyond this. +// Forget evicts any cached Owners result for id -- called after any delete +// (original or forwarded broadcast) touches id here, since a stale positive +// would redirect claims to a node that no longer holds the record. func (p *HTTPProber) Forget(id string) { p.cacheMu.Lock() defer p.cacheMu.Unlock() @@ -119,13 +109,10 @@ func (p *HTTPProber) Forget(id string) { delete(p.cache, id) } -// fanOut probes every peer concurrently, stopping and canceling the -// stragglers as soon as maxOwners have answered 200 -- a hung or slow peer -// must not add its own timeout to a redirect or heal that already has -// enough candidates. The context is detached from the caller and rebounded -// internally: singleflight shares this fan-out across concurrent callers -// whose own contexts may cancel independently (one client disconnecting -// must not abort another's still-live probe). +// fanOut probes every peer concurrently, canceling stragglers once maxOwners +// have answered 200. The context is detached and rebounded internally: +// singleflight shares one fan-out across callers whose own contexts cancel +// independently — one client disconnecting must not abort another's probe. func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int, grace time.Duration) []string { addrs := dedupAddrs(p.Peers()) if len(addrs) == 0 { @@ -149,12 +136,10 @@ func (p *HTTPProber) fanOut(ctx context.Context, id string, maxOwners int, grace } go func() { wg.Wait(); close(wins) }() - // grace opens only after the first owner answers: until then the fan-out - // waits out the full timeout, since a genuine miss must hear from every - // peer before it can conclude nobody holds the record. - // graceCh stays nil when grace is 0 (heal wants an exhaustive list), so the - // select then only ever takes a win and runs until the cap or every peer - // has answered. + // grace opens only after the first win: a genuine miss must hear from + // every peer before concluding nobody holds the record. graceCh stays nil + // when grace is 0 (a heal wants the exhaustive list), so the select then + // runs to the cap or until every peer has answered. var owners []string var graceCh <-chan time.Time for { @@ -192,10 +177,9 @@ func (p *HTTPProber) cacheLookup(id string) (owners []string, epoch uint64, hit } // cachePut records a positive result, unless a Forget bumped the epoch since -// the fan-out began — that delete may have removed the very record this result -// names, so caching it would redirect claims to a node that no longer holds -// it. A miss is never cached: a checkpoint that appears moments later must not -// be hidden behind a stale negative for the TTL. +// the fan-out began — that delete may have removed the very record this +// result names. A miss is never cached: a checkpoint appearing moments later +// must not hide behind a stale negative for the TTL. func (p *HTTPProber) cachePut(id string, owners []string, start uint64) { if len(owners) == 0 { return @@ -224,12 +208,7 @@ func probeOwner(ctx context.Context, client *http.Client, probeKey []byte, addr, ctx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() - base := addr - if !strings.Contains(base, "://") { - base = "http://" + base - } - u := base + "/v1/checkpoints/" + url.PathEscape(id) + "/blob" - req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodHead, checkpointURL(addr, id, "/blob"), nil) if err != nil { return false } @@ -260,20 +239,16 @@ func dedupAddrs(addrs []string) []string { } // DeriveProbeKey derives the probe-MAC key from the mesh's cluster_key via -// one HMAC step (domain separation): cluster_key already encrypts gossip -// traffic, and reusing that key unmodified for a second, unrelated -// construction is avoidable at negligible cost. Call only when clusterKey -// is non-empty; an empty result leaves probing unauthenticated. +// one HMAC step — domain separation from the gossip encryption. Call only +// with a non-empty clusterKey. func DeriveProbeKey(clusterKey []byte) []byte { mac := hmac.New(sha256.New, clusterKey) mac.Write([]byte("cocoon-checkpoint-probe-v1")) return mac.Sum(nil) } -// SignProbe returns the current probe MAC for id, keyed by key -- the value -// probeOwner sends in ProbeHeader. Exported for callers (and tests) that -// need to construct a valid probe request without reaching into this -// package's internals. +// SignProbe returns id's current probe MAC keyed by key — the ProbeHeader +// value; exported for callers and tests that build a valid probe request. func SignProbe(key []byte, id string) string { return probeMAC(key, id, currentBucket()) } diff --git a/sandboxd/store/peer/transport.go b/sandboxd/store/peer/transport.go index cc9405b..b3e06b5 100644 --- a/sandboxd/store/peer/transport.go +++ b/sandboxd/store/peer/transport.go @@ -56,12 +56,7 @@ func (p *HTTPPuller) Pull(ctx context.Context, addr, id, dst string) error { ctx, cancel := context.WithTimeout(ctx, pullTimeout) defer cancel() - base := addr - if !strings.Contains(base, "://") { - base = "http://" + base - } - u := base + "/v1/checkpoints/" + url.PathEscape(id) + "/blob" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, checkpointURL(addr, id, "/blob"), nil) if err != nil { return err } @@ -115,8 +110,6 @@ func TarRecord(exportDir string, meta []byte, w io.Writer) error { if err := tarInto(exportDir, exportPrefix, tw); err != nil { return err } - // The completion marker rides last, after the export streamed whole, so its - // absence tells the receiver the transfer was cut short. if err := tw.WriteHeader(&tar.Header{ Name: recordTrailer, Mode: 0o600, Size: 0, Typeflag: tar.TypeReg, }); err != nil { @@ -237,6 +230,15 @@ func writeFile(target string, r io.Reader, mode os.FileMode) error { return f.Close() } +// checkpointURL builds the control-plane URL for id's record on addr — the +// one home for the scheme default a mesh member view omits. +func checkpointURL(addr, id, suffix string) string { + if !strings.Contains(addr, "://") { + addr = "http://" + addr + } + return addr + "/v1/checkpoints/" + url.PathEscape(id) + suffix +} + // safeJoin resolves name under root, rejecting absolute paths and any name // that would escape root. func safeJoin(root, name string) (string, error) { diff --git a/sandboxd/store/peer/transport_test.go b/sandboxd/store/peer/transport_test.go index 9cee3dc..9356191 100644 --- a/sandboxd/store/peer/transport_test.go +++ b/sandboxd/store/peer/transport_test.go @@ -264,9 +264,6 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { } } -// tarDir streams src's contents with no record layout, exercising tarInto and -// Untar on their own. - // TestUntarRejectsRecordWithoutCompletionMarker: a transfer cut short by a // source-side walk/read error still yields a valid short tar (Close writes the // footer regardless), so the receiver must reject a record that arrives without @@ -310,6 +307,8 @@ func TestTarRecordUntarRoundTripComplete(t *testing.T) { } } +// tarDir streams src's contents with no record layout, exercising tarInto and +// Untar on their own. func tarDir(src string, w io.Writer) error { tw := tar.NewWriter(w) defer func() { _ = tw.Close() }() From 66268da52e75186146ebf2111b04ee64dd00c653 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 23:23:38 +0800 Subject: [PATCH 30/31] docs: the three operator read/wake endpoints, and the probe fan-out as built wake, GET /v1/sandboxes/{id}, and GET /v1/sandboxes/{id}/stats existed only in the route table; hibernate's auth line missed the operator elevation it shares with release; cluster.md described the HEAD fan-out as capped at 3 peers when the cap is on the answer, not the probe. --- docs/cluster.md | 8 ++++---- docs/deploy.md | 2 +- docs/sandboxd-api.md | 38 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index dbf72b1..ff79686 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -136,10 +136,10 @@ at a node that does not hold the record runs a tier order: 1. **Local claim** — the fast path when the claiming node already holds it. 2. **Probe + redirect** — a live `HEAD` fan-out (HMAC-signed when the mesh - carries a `cluster_key`) to up to 3 - mesh peers in parallel, redirecting to whoever answers — the same - claim-redirect contract a warm miss already uses. The cap means the - answer is a hint, not an exhaustive list of every owner. The probe and + carries a `cluster_key`) to every mesh peer in parallel, redirecting to + the first owners that answer — the same claim-redirect contract a warm + miss already uses. The answer is capped at 3 addresses: a hint, not an + exhaustive list of every owner. The probe and the follow-up claim are not atomic: a peer can answer the probe, then lose the record — a delete's broadcast lands, or its own TTL sweep runs (below) — before the retry reaches it, so a redirect can go stale diff --git a/docs/deploy.md b/docs/deploy.md index 5c0966b..f284e1d 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -71,7 +71,7 @@ sandboxd reads one JSON file (`-config`, default | `bridge` / `network` | unset | egress-lane attachment: a host bridge device, or a CNI conflist name. Mutually exclusive; with neither set the node serves only the no-network lane. [Guarded egress](egress.md) needs the bridge form and rejects a CNI network at load | | `egress_ca` | unset | [HTTPS-interception](egress.md#https-interception) PKI: `root_cert` (the cluster root baked into intercepted guests; may bundle old+new roots during rotation) plus this node's `intermediate_cert`/`intermediate_key` from `sandboxd ca issue-intermediate`. Required when any pool rule sets `intercept` | | `api_token` | unset | the operator (root) credential: when set, guards the node-level endpoints (Bearer) with full access, including release-by-id cleanup. Per-sandbox tokens guard ordinary sandbox-scoped calls | -| `tenants` | unset | multi-tenant tokens next to `api_token`: `[{"name": "acme", "token": "…", "max_claims": 50}]`. A tenant token reaches the resource-creating verbs (claim, fork, promote, checkpoint, preview) and everything it creates is stamped with the tenant name; operator surfaces (`GET /v1/sandboxes`, `GET /v1/info`, `PUT /v1/pools`, `POST/DELETE /v1/drain`, `/metrics`) answer it 403. `max_claims` (0 = unlimited) caps that tenant's live claims next to the node-wide cap. Requires `api_token` set (operator surfaces need it). Names and tokens must be unique, tokens distinct from `api_token`. On a cluster all nodes must carry the same tenants set (the SDK replays a tenant token across a redirect; a peer missing that tenant answers 401), and per-node caps mean a tenant's effective cluster limit is `max_claims` × nodes. Empty = exactly the single-token behavior | +| `tenants` | unset | multi-tenant tokens next to `api_token`: `[{"name": "acme", "token": "…", "max_claims": 50}]`. A tenant token reaches the resource-creating verbs (claim, fork, promote, checkpoint, preview) and everything it creates is stamped with the tenant name; operator surfaces (`GET /v1/sandboxes` and the per-id reads under it, `GET /v1/info`, `PUT /v1/pools`, `POST/DELETE /v1/drain`, `/metrics`) answer it 403. `max_claims` (0 = unlimited) caps that tenant's live claims next to the node-wide cap. Requires `api_token` set (operator surfaces need it). Names and tokens must be unique, tokens distinct from `api_token`. On a cluster all nodes must carry the same tenants set (the SDK replays a tenant token across a redirect; a peer missing that tenant answers 401), and per-node caps mean a tenant's effective cluster limit is `max_claims` × nodes. Empty = exactly the single-token behavior | | `max_fork_count` | 16 | children a single `fork` may create; each is a full-RAM VM, so this bounds one request's memory blast radius to the node's capacity | | `refill_concurrency` | 0 (auto) | concurrent VM provisioning budget, shared by warm-pool refills, fork clones, and the reap/hibernate/reconcile engine batches. 0 sizes it from the node: `NumCPU*2/3` clamped to [4, 256] — a 384-core node gets 256; small nodes keep a floor of 4 | | `preview_listen` | (off) | address for a preview HTTP server that serves guest ports under signed URLs; needs `preview_secret` | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 581c765..7e178ce 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -7,7 +7,9 @@ All bodies are JSON. Three token kinds: resource-creating verbs (claim, fork, promote, checkpoint create/claim, preview mint) and the tenant-scoped listings/deletes below; everything a tenant creates is stamped with its name. Operator surfaces - (`GET /v1/sandboxes`, `GET /v1/info`, `PUT /v1/pools`, `POST/DELETE /v1/drain`, `GET /metrics`) + (`GET /v1/sandboxes` and the per-id reads under it, `GET /v1/info`, + `PUT /v1/pools`, `POST/DELETE /v1/drain`, `GET /metrics`, + `GET /v1/checkpoints/{id}/blob`) answer a tenant token `403` — authenticated but not authorized; an unknown token stays `401`. - **sandbox** — every claimed sandbox carries its own bearer token guarding @@ -73,7 +75,8 @@ Releasing an already-gone sandbox is 404 — the SDK treats it as success. ## POST /v1/sandboxes/{id}/hibernate -Auth: the sandbox's own token. Atomically snapshots the VM and stops it, +Auth: the sandbox's own token, or the root token by id (operator, like +release). Atomically snapshots the VM and stops it, freeing its memory; the next agent access restores it transparently (sessions, processes, and memory state intact — cocoon's hibernate keeps the snapshot point and the stop coincident). Idempotent on an already-hibernated @@ -83,6 +86,14 @@ node only provides the transition. 204 on success, 404 unknown id or wrong token, 409 on the egress lane (egress-lane sandboxes never hibernate; see [egress](egress.md)). +## POST /v1/sandboxes/{id}/wake + +Auth: the sandbox's own token, or the root token by id (operator). Restores +a hibernated (or archived) sandbox and leaves it running — waking is +otherwise only a side effect of the next agent access, so this is the +explicit form for warming a sandbox ahead of use. Idempotent on one already +running. 204 on success, 404 unknown id or wrong token. + ## POST /v1/sandboxes/{id}/fork Auth: the node `api_token` (Bearer) — forking creates node resources, like a @@ -295,6 +306,29 @@ Auth: root only (tenant tokens get 403). The operator index: `{"sandboxes": [{id, key, deadline, hibernated, archived?, from_checkpoint?, claim_ref?}]}` — never tokens. +## GET /v1/sandboxes/{id} + +Auth: root only. One live claim in the index-row shape above, so a +reconcile loop can read a single sandbox without scanning the whole node +listing. 404 unknown id. + +## GET /v1/sandboxes/{id}/stats + +Auth: root only. One sandbox's resource usage — the per-sandbox counterpart +to the node-scoped `/metrics`: + +```json +{"id": "sb_…", "cpu_count": 2, "mem_total_bytes": 1073741824, + "mem_used_bytes": 187654144, "mem_used_measured": true, + "hibernated": false, "measured_at": "…"} +``` + +`cpu_count`/`mem_total_bytes` come from the size tier. `mem_used_bytes` is +the host VMM process's resident set — the only usage signal available +without a guest agent; `mem_used_measured` is false when there is no VMM +process to read (hibernated, or the PID is not yet known), so a zero is +never mistaken for idle. 404 unknown id. + ## GET /metrics Auth: root only (tenant tokens get 403). Prometheus text format, From 7f2b97af934695344530db26b07932c811044d35 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 26 Jul 2026 23:47:30 +0800 Subject: [PATCH 31/31] sdk: a redirect candidate's 401 is a rotation transient, not a verdict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mid-rotation a candidate that has not adopted the new token answers 401 before its handler runs, while the origin proved the token valid by issuing the redirect — so the claim falls back to the origin instead of failing for the whole rotation window with a misleading credential error. 403/409 stay definitive. --- docs/cluster.md | 2 +- sdk/go/client.go | 17 ++++++------ sdk/go/client_test.go | 37 +++++++++++++++++++++++++++ sdk/python/cocoonsandbox/client.py | 13 +++++----- sdk/python/tests/test_fault_matrix.py | 24 +++++++++++++++++ 5 files changed, 76 insertions(+), 17 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index ff79686..dee252f 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -212,7 +212,7 @@ Some config must match on every node or the cluster fails in confusing ways: | config | what breaks on mismatch | |---|---| -| `api_token`, `tenants` | the SDK replays the authorizing token across a redirect; a peer missing that tenant/token answers 401 | +| `api_token`, `tenants` | the SDK replays the authorizing token across a redirect; a mid-rotation peer missing that tenant/token answers 401, which the SDK treats as transient — the claim falls back to the origin node (which already authorized it) and resolves there | | `preview_secret` | a preview URL signed on one node fails verification on another | | `mesh.cluster_key` | nodes cannot join / decrypt gossip at all | | `egress_ca` cluster root | a guest checkpointed/redirected across nodes trusts the root; a divergent root fails interception | diff --git a/sdk/go/client.go b/sdk/go/client.go index 33cfb15..5da411f 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -266,21 +266,20 @@ func retryMiss(err error) bool { func retryAny(error) bool { return true } // retryTransient reports whether an origin-fallback is worth the round-trip: -// true for a transport failure (unreachable peer), a miss (404, stale -// gossip), full (429), mid-heal (503), or an engine/proxy failure -// (500/502/504 — the last two covering a load balancer in front of -// sandboxd) — any of which the origin's own attempt may not hit. A served -// 4xx like a bad request, an unauthorized/forbidden token, or an egress -// conflict is definitive: the origin would fail the same way, so falling -// back would only hide the real error. +// true for a transport failure, a miss (404), full (429), mid-heal (503), an +// engine/proxy failure (500/502/504), or a mid-rotation 401 (the origin +// proved the token valid by issuing the redirect). A served 4xx like a bad +// request, a forbidden token, or an egress conflict is definitive: the +// origin would fail the same way. func retryTransient(err error) bool { var he *httpError if !errors.As(err, &he) { return true } switch he.status { - case http.StatusNotFound, http.StatusTooManyRequests, http.StatusServiceUnavailable, - http.StatusInternalServerError, http.StatusBadGateway, http.StatusGatewayTimeout: + case http.StatusUnauthorized, http.StatusNotFound, http.StatusTooManyRequests, + http.StatusServiceUnavailable, http.StatusInternalServerError, + http.StatusBadGateway, http.StatusGatewayTimeout: return true default: return false diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 13baafc..7268903 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -437,6 +437,43 @@ func TestRedirectDefinitiveErrorSkipsFallback(t *testing.T) { } } +func TestRedirectRotating401CandidateFallsBackToOrigin(t *testing.T) { + var staleCalls int + stale := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + staleCalls++ + w.WriteHeader(http.StatusUnauthorized) + _ = json.NewEncoder(w).Encode(errorResponse{Error: "invalid api token"}) + })) + t.Cleanup(stale.Close) + + var entryCalls int + entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + entryCalls++ + if entryCalls == 1 { + _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(stale.URL, "http://")}}) + return + } + var req claimRequest + _ = json.NewDecoder(r.Body).Decode(&req) + if !req.NoRedirect { + t.Error("origin fallback did not carry no_redirect") + } + _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_local", Token: "t"}) + })) + t.Cleanup(entry.Close) + + sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") + if err != nil { + t.Fatalf("New: %v", err) + } + if sb.ID != "sb_local" { + t.Errorf("id %q, want sb_local (provisioned at the already-authorized origin)", sb.ID) + } + if staleCalls != 1 || entryCalls != 2 { + t.Errorf("stale=%d entry=%d calls, want 1 and 2 (redirect + fallback)", staleCalls, entryCalls) + } +} + func TestLookupScatter(t *testing.T) { // The owner is node B; the entry node A doesn't own the sandbox but lists // B as a peer. Lookup must scatter to B and bind the handle there. diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 1c6bcad..cb94243 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -194,13 +194,12 @@ def _try_each(candidates, call, retry=lambda exc: exc.status in (404, 0)): def _retry_transient(exc: APIError) -> bool: """Origin-fallback policy: worth the round-trip for a transport failure - (status 0), a miss (404, stale gossip), full (429), mid-heal (503), or an - engine/proxy failure (500/502/504 -- the last two covering a load - balancer in front of sandboxd) -- any of which the origin's own attempt - may not hit. A served 4xx like a bad request, an unauthorized/forbidden - token, or an egress conflict is definitive: the origin would fail the - same way, so falling back would only hide the real error.""" - return exc.status in (0, 404, 429, 503, 500, 502, 504) + (status 0), a miss (404), full (429), mid-heal (503), an engine/proxy + failure (500/502/504), or a mid-rotation 401 (the origin proved the + token valid by issuing the redirect). A served 4xx like a bad request, + a forbidden token, or an egress conflict is definitive: the origin + would fail the same way.""" + return exc.status in (0, 401, 404, 429, 503, 500, 502, 504) def _redirect_fallback(origin: str, candidates: list, post, verb: str): diff --git a/sdk/python/tests/test_fault_matrix.py b/sdk/python/tests/test_fault_matrix.py index c2dc307..d6e5b22 100644 --- a/sdk/python/tests/test_fault_matrix.py +++ b/sdk/python/tests/test_fault_matrix.py @@ -132,6 +132,30 @@ def entry_claim(body, path): assert len(calls) == 1 +def test_claim_redirect_401_candidate_falls_back_to_origin(spawn_node): + stale_calls = [] + + def stale_claim(body, path): + stale_calls.append(body) + return 401, {"error": "invalid api token"} + + stale = spawn_node({("POST", "/v1/claim"): stale_claim}) + + calls = [] + + def entry_claim(body, path): + calls.append(body) + if len(calls) == 1: + return 200, {"redirect": [stale]} + return 200, {"id": "sb_local", "token": "t"} + + entry = spawn_node({("POST", "/v1/claim"): entry_claim}) + sb = Client(entry).new("rt:24.04") + assert sb.id == "sb_local" and sb.owner == entry + assert len(calls) == 2 and calls[1]["no_redirect"] is True + assert len(stale_calls) == 1 + + def test_claim_redirect_candidate_definitive_error_still_tries_next_candidate(spawn_node): # Candidate one is wrong for this claim (403, definitive) but candidate # two would still succeed -- the per-candidate walk retries broadly, so