From b6de0570308533e7bd56efcef6f1a2f45ca26ca9 Mon Sep 17 00:00:00 2001 From: doge Date: Sat, 25 Jul 2026 03:18:14 +0800 Subject: [PATCH 1/6] apiserver: serve an e2b-compatible API over the same warm pools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopting cocoon meant rewriting against the Kubernetes agent-sandbox API even for callers who already had working e2b code. Add an opt-in REST surface that speaks the e2b wire contract, so an unmodified e2b SDK (JS or Python) claims from these warm pools by pointing E2B_API_URL at the apiserver. It is a translation layer, not a second control plane: every verb lands on the same scale.SandboxStore the aggregated API uses, so an e2b create is the identical node-local claim and what it creates stays visible to `kubectl get sandboxes`. Nothing is stored here — sandbox identity is the sandboxd claim id the owning node already assigns, so DELETE releases exactly the claimed microVM rather than guessing by name. Covered: create (templateID -> pool template, allow_internet_access -> network lane, 503 on a drained pool so the caller retries), list/get, delete, timeout and refreshes. It runs on its own listener behind X-API-KEY and refuses to start unauthenticated unless anonymous access is passed explicitly, so a misconfigured deployment cannot silently expose an open claim endpoint. envdVersion is always reported: the SDK version-compares it and kills the sandbox it just created when it cannot parse one. Limits — envd host routing, unimplemented pause/fork/snapshot verbs — are documented in docs/e2b-compat.md. --- README.md | 19 ++ cmd/sandbox-apiserver/main.go | 110 ++++++++++ docs/e2b-compat.md | 79 +++++++ pkg/e2bcompat/http.go | 58 ++++++ pkg/e2bcompat/server.go | 361 ++++++++++++++++++++++++++++++++ pkg/e2bcompat/server_test.go | 380 ++++++++++++++++++++++++++++++++++ pkg/e2bcompat/types.go | 98 +++++++++ 7 files changed, 1105 insertions(+) create mode 100644 docs/e2b-compat.md create mode 100644 pkg/e2bcompat/http.go create mode 100644 pkg/e2bcompat/server.go create mode 100644 pkg/e2bcompat/server_test.go create mode 100644 pkg/e2bcompat/types.go diff --git a/README.md b/README.md index d672c5e..cccc110 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,25 @@ spec: For low-latency acquisition, define a `SandboxTemplate` + `SandboxWarmPool` and create `SandboxClaim`s — see [examples/](examples/). +## Use it with the e2b SDK + +The aggregated apiserver can also serve an e2b-compatible REST surface, so an +**unmodified e2b SDK** claims from these same warm pools — point `E2B_API_URL` +at it: + +```bash +sandbox-apiserver --enable-e2b-api --e2b-api-key-file=/etc/e2b/keys +``` + +```js +export E2B_API_URL=https://your-apiserver:8080 +const sandbox = await Sandbox.create('registry.example.com/rt:24.04') +``` + +It is a translation layer, not a second control plane: an e2b create is the same +node-local claim, and the sandbox stays visible to `kubectl get sandboxes`. Flags, +endpoint mapping, and limits in [docs/e2b-compat.md](docs/e2b-compat.md). + ## Install Helm: diff --git a/cmd/sandbox-apiserver/main.go b/cmd/sandbox-apiserver/main.go index 8f47dc5..9ce30e5 100644 --- a/cmd/sandbox-apiserver/main.go +++ b/cmd/sandbox-apiserver/main.go @@ -21,7 +21,10 @@ package main import ( "context" + "errors" "fmt" + "net" + "net/http" "os" "strings" "time" @@ -40,6 +43,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" extv1beta1 "github.com/cocoonstack/cocoon-sandbox-operator/extensions/api/v1beta1" + "github.com/cocoonstack/cocoon-sandbox-operator/pkg/e2bcompat" "github.com/cocoonstack/cocoon-sandbox-operator/pkg/scale" sandboxapiserver "github.com/cocoonstack/cocoon-sandbox-operator/pkg/scale/apiserver" "github.com/cocoonstack/cocoon-sandbox-operator/pkg/scale/warmpool" @@ -51,6 +55,14 @@ import ( // empty sandbox lists. const inventoryCacheSyncTimeout = 2 * time.Minute +const ( + // e2bReadHeaderTimeout bounds how long a client may take to send its request + // headers on the e2b surface, so a stalled connection cannot pin a handler. + e2bReadHeaderTimeout = 10 * time.Second + // e2bShutdownTimeout bounds the graceful drain of in-flight e2b requests. + e2bShutdownTimeout = 10 * time.Second +) + // options are the standard aggregated-apiserver options: secure serving plus // delegated authentication/authorization (token/SAR review against the host // kube-apiserver). There is deliberately no etcd option — this server stores @@ -73,6 +85,16 @@ type options struct { // O(pools+nodes); never per-sandbox. WarmPoolInterval is its resync cadence. WarmPoolDriver bool WarmPoolInterval time.Duration + + // E2B* configure the optional e2b-compatible REST surface, which lets an + // unmodified e2b SDK drive the same warm pools. It is off by default and + // serves on its own address, so the aggregated API is never affected. + E2BAPI bool + E2BAddr string + E2BNamespace string + E2BDomain string + E2BAPIKeyFile string + E2BAllowAnonymous bool } func newOptions() *options { @@ -82,6 +104,8 @@ func newOptions() *options { Authorization: genericoptions.NewDelegatingAuthorizationOptions(), Features: genericoptions.NewFeatureOptions(), WarmPoolDriver: true, + E2BAddr: ":8080", + E2BNamespace: "default", } o.SecureServing.BindPort = 6443 // Allow running without a remote kubeconfig (in-cluster service account). @@ -103,6 +127,37 @@ func (o *options) addFlags(fs *pflag.FlagSet) { "Run the in-process SandboxWarmPool → sandboxd pool reconcile loop (control-plane warm-capacity surface; pool-level, never per-sandbox).") fs.DurationVar(&o.WarmPoolInterval, "warm-pool-sync-interval", o.WarmPoolInterval, "Resync cadence for the SandboxWarmPool driver, and with it the sampling period of the warm count in pool status (0 = default 5s).") + fs.BoolVar(&o.E2BAPI, "enable-e2b-api", o.E2BAPI, + "Serve the e2b-compatible REST surface, so an unmodified e2b SDK can claim from the same warm pools (point E2B_API_URL at it).") + fs.StringVar(&o.E2BAddr, "e2b-bind-address", o.E2BAddr, + "Address the e2b-compatible surface listens on.") + fs.StringVar(&o.E2BNamespace, "e2b-namespace", o.E2BNamespace, + "Namespace e2b claims are made in; e2b has no namespace concept, so every compat claim lands here.") + fs.StringVar(&o.E2BDomain, "e2b-domain", o.E2BDomain, + "Base domain reported to the SDK, from which it derives the in-sandbox envd host. Empty leaves the SDK's own E2B_DOMAIN/E2B_SANDBOX_URL in charge.") + fs.StringVar(&o.E2BAPIKeyFile, "e2b-api-key-file", o.E2BAPIKeyFile, + "Path to a file (Secret mount) of accepted e2b API keys, one per line, presented by the SDK as X-API-KEY.") + fs.BoolVar(&o.E2BAllowAnonymous, "e2b-allow-anonymous", o.E2BAllowAnonymous, + "Serve the e2b surface with NO API key. Development only: it leaves the claim endpoint open to anyone who can reach the port.") +} + +// e2bAPIKeys reads the accepted e2b API keys from the key file, one per line. +// Blank lines and #-comments are ignored. +func (o *options) e2bAPIKeys() ([]string, error) { + if o.E2BAPIKeyFile == "" { + return nil, nil + } + b, err := os.ReadFile(o.E2BAPIKeyFile) + if err != nil { + return nil, fmt.Errorf("read e2b api key file %q: %w", o.E2BAPIKeyFile, err) + } + var keys []string + for _, line := range strings.Split(string(b), "\n") { + if line = strings.TrimSpace(line); line != "" && !strings.HasPrefix(line, "#") { + keys = append(keys, line) + } + } + return keys, nil } // resolveSandboxdToken returns the sandboxd token, reading it from the token file @@ -191,6 +246,15 @@ func run() error { } } + // The e2b-compatible surface is a translation layer over the same store, on + // its own listener: an e2b SDK Create becomes the identical node-local claim, + // and what it creates stays visible to `kubectl get sandboxes`. + if o.E2BAPI { + if err := startE2BServer(ctx, o, store); err != nil { + return err + } + } + server, err := cfg.Complete(nil).New("cocoon-sandbox-apiserver", genericapiserver.NewEmptyDelegate()) if err != nil { return fmt.Errorf("build generic apiserver: %w", err) @@ -277,6 +341,52 @@ func startWarmPoolDriver(ctx context.Context, restCfg *restclient.Config, token return nil } +// startE2BServer starts the e2b-compatible REST surface on its own listener and +// stops it when ctx is cancelled. It shares the aggregated apiserver's store, so +// there is no second source of truth: a claim made here is the same node-local +// claim, released the same way, and listed by the same scatter-gather read. +func startE2BServer(ctx context.Context, o *options, store scale.SandboxStore) error { + keys, err := o.e2bAPIKeys() + if err != nil { + return err + } + srv, err := e2bcompat.NewServer(store, e2bcompat.Options{ + Namespace: o.E2BNamespace, + Domain: o.E2BDomain, + APIKeys: keys, + AllowAnonymous: o.E2BAllowAnonymous, + Log: ctrl.Log.WithName("e2b"), + }) + if err != nil { + return err + } + httpSrv := &http.Server{ + Addr: o.E2BAddr, + Handler: srv.Handler(), + ReadHeaderTimeout: e2bReadHeaderTimeout, + } + ln, err := net.Listen("tcp", o.E2BAddr) + if err != nil { + return fmt.Errorf("listen on e2b address %q: %w", o.E2BAddr, err) + } + klog.InfoS("serving e2b-compatible API", "address", o.E2BAddr, "namespace", o.E2BNamespace, + "authenticated", len(keys) > 0) + go func() { + if err := httpSrv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + klog.ErrorS(err, "e2b-compatible API server exited") + } + }() + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), e2bShutdownTimeout) + defer cancel() + if err := httpSrv.Shutdown(shutdownCtx); err != nil { + klog.ErrorS(err, "e2b-compatible API server shutdown") + } + }() + return nil +} + // currentNamespace returns the pod's namespace (for the leader-election lease), // read from the service-account mount, defaulting to the deployment namespace. func currentNamespace() string { diff --git a/docs/e2b-compat.md b/docs/e2b-compat.md new file mode 100644 index 0000000..1f542dc --- /dev/null +++ b/docs/e2b-compat.md @@ -0,0 +1,79 @@ +# e2b-compatible API + +The aggregated apiserver can serve an [e2b](https://e2b.dev)-compatible REST +surface, so an **unmodified e2b SDK** (JS or Python) claims from the same warm +microVM pools this operator already manages. Point `E2B_API_URL` at it and +`Sandbox.create()` works. + +It is a translation layer, not a second control plane. Every request lands on +the same `SandboxStore` the Kubernetes API uses, so an e2b create *is* the +node-local claim a `kubectl create sandbox` performs — and the sandbox it +returns shows up in `kubectl get sandboxes`. Nothing extra is stored: sandbox +identity is the sandboxd claim id the owning node already assigns. + +## Enable it + +```bash +sandbox-apiserver \ + --enable-e2b-api \ + --e2b-bind-address=:8080 \ + --e2b-namespace=sandboxes \ + --e2b-api-key-file=/etc/e2b/keys \ + --e2b-domain=sandbox.example.com +``` + +| Flag | Default | Meaning | +|---|---|---| +| `--enable-e2b-api` | `false` | Serve the surface at all. | +| `--e2b-bind-address` | `:8080` | Its own listener; the aggregated API is untouched. | +| `--e2b-namespace` | `default` | Namespace claims land in — e2b has no namespace concept. | +| `--e2b-api-key-file` | — | File of accepted `X-API-KEY` values, one per line (`#` comments ignored). | +| `--e2b-domain` | — | Base domain reported to the SDK for reaching in-sandbox `envd`. | +| `--e2b-allow-anonymous` | `false` | Serve with **no** API key. Development only. | + +Startup **fails** if neither `--e2b-api-key-file` nor `--e2b-allow-anonymous` is +set, so a misconfiguration cannot silently expose an open claim endpoint. + +## Use it + +```bash +export E2B_API_URL=https://your-apiserver:8080 +export E2B_API_KEY=e2b_yourkey +``` + +```js +import { Sandbox } from '@e2b/code-interpreter' + +// templateID is the pool template — the container image the warm pool is built +// from, the same axis SandboxWarmPool keys on. +const sandbox = await Sandbox.create('registry.example.com/rt:24.04') +``` + +## Endpoint mapping + +| e2b endpoint | Maps to | Notes | +|---|---|---| +| `POST /sandboxes` | `store.Claim` | `templateID` → pool template; `allow_internet_access` → `egress` lane, else the hardened `none` lane. `201` on success, `503` when the pool is drained (retryable). | +| `GET /sandboxes`, `GET /v2/sandboxes` | `store.List` | Live sandboxes in the compat namespace. | +| `GET /sandboxes/{id}` | `store.List` + id match | `404` when no live sandbox carries the id. | +| `DELETE /sandboxes/{id}` | `store.Release` | Releases the node-local claim id, never by Kubernetes name. `204`. | +| `POST /sandboxes/{id}/timeout` | existence check | TTL is fixed by the node at claim time; the call is verified and acknowledged, not silently faked. | +| `POST /sandboxes/{id}/refreshes` | existence check | Keepalive; liveness is node-owned. | +| `GET /health` | — | Unauthenticated, for probes. | + +## Limits worth knowing + +- **Reaching `envd` (the in-sandbox data plane).** The SDK derives the sandbox + host as `{port}-{sandboxID}.{domain}`. A cocoon `sandboxID` is the sandboxd + claim id, which is not a valid DNS label, so that subdomain form only works + behind a proxy routing on the `E2b-Sandbox-Id` / `E2b-Sandbox-Port` headers + the SDK also sends. Otherwise set `E2B_SANDBOX_URL` explicitly. +- **`envdVersion`** is reported as `0.4.0` by default. The SDK version-compares + it and *kills the sandbox* if it cannot parse it, so it is always sent. +- **Not implemented:** pause/resume, fork, snapshots, templates, metrics, and + team/node admin endpoints. `pause`/`fork` map onto cocoon capabilities and are + the natural next step; the rest are e2b-hosted concerns. +- **Size class** is pinned (`small`) — e2b's `NewSandbox` carries no size + selector. +- `metadata` and `envVars` are accepted so SDK calls do not fail, but the + node-local claim path takes neither today. diff --git a/pkg/e2bcompat/http.go b/pkg/e2bcompat/http.go new file mode 100644 index 0000000..88270d4 --- /dev/null +++ b/pkg/e2bcompat/http.go @@ -0,0 +1,58 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2bcompat + +import ( + "encoding/json" + "net/http" + + "k8s.io/apiserver/pkg/storage/names" +) + +const ( + // maxBodyBytes caps a request body. The e2b bodies are small objects; the + // cap keeps an unbounded upload off the claim path. + maxBodyBytes = 1 << 20 + // tokenAnnotation carries the per-sandbox ownership credential the claim + // returned. It mirrors the aggregated apiserver's annotation of the same + // name, and is surfaced to the SDK as envdAccessToken. + tokenAnnotation = "sandbox.cocoonstack.io/token" + // namePrefix prefixes the Kubernetes object name of a compat claim, so a + // sandbox created through this surface is recognizable in `kubectl get + // sandboxes` and in node inventory. + namePrefix = "e2b-" +) + +// generateName returns a fresh Kubernetes-safe name for a compat claim. e2b +// clients do not name their sandboxes; the identity a caller sees back is the +// node-assigned claim id, not this name. +func generateName() string { + return names.SimpleNameGenerator.GenerateName(namePrefix) +} + +// writeJSON writes v as the response body with the given status. +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + // The body is already committed by WriteHeader, so an encode failure can + // only be logged by the caller's transport; there is no status left to set. + _ = json.NewEncoder(w).Encode(v) +} + +// writeError writes the e2b error envelope, which the SDK surfaces as the +// failure message. +func writeError(w http.ResponseWriter, status int, msg string) { + writeJSON(w, status, APIError{Code: int32(status), Message: msg}) +} diff --git a/pkg/e2bcompat/server.go b/pkg/e2bcompat/server.go new file mode 100644 index 0000000..eccb4a0 --- /dev/null +++ b/pkg/e2bcompat/server.go @@ -0,0 +1,361 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package e2bcompat serves an e2b-compatible REST surface in front of the L3 +// sandbox store, so an unmodified e2b SDK (JS or Python) can drive cocoon warm +// pools by pointing E2B_API_URL at this server. +// +// It is a translation layer, not a second control plane: every request lands on +// the same scale.SandboxStore the aggregated apiserver uses, so an e2b Create is +// the identical node-local claim a `kubectl create sandbox` performs, and the +// sandbox it returns is visible to `kubectl get sandboxes`. Nothing is stored +// here; sandbox identity is the sandboxd claim id the node already assigns. +// +// Mapping to the e2b contract (e2b-dev/E2B spec/openapi.yml): +// +// POST /sandboxes -> store.Claim (templateID -> pool template) +// GET /sandboxes, /v2/sandboxes -> store.List +// GET /sandboxes/{sandboxID} -> store.List + id match +// DELETE /sandboxes/{sandboxID} -> store.Release +// POST /sandboxes/{sandboxID}/timeout -> accepted (TTL is set at claim time) +// POST /sandboxes/{sandboxID}/refreshes -> accepted (keepalive) +// GET /health -> liveness +package e2bcompat + +import ( + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/go-logr/logr" + + cocoonv1 "github.com/cocoonstack/cocoon-sandbox-operator/api/cocoon/v1" + sandboxv1beta1 "github.com/cocoonstack/cocoon-sandbox-operator/api/v1beta1" + "github.com/cocoonstack/cocoon-sandbox-operator/pkg/scale" +) + +const ( + // DefaultEnvdVersion is reported to the SDK when no version is configured. + // The SDK version-compares this before choosing the envd auth style, so it + // must be a real semver at or above the modern-auth cutoff (0.4.0). + DefaultEnvdVersion = "0.4.0" + // DefaultTimeoutSeconds matches the e2b SDK's own default TTL. + DefaultTimeoutSeconds = 15 + // apiKeyHeader is the header the e2b SDKs authenticate with. + apiKeyHeader = "X-API-KEY" +) + +// Options configures the compat server. +type Options struct { + // Namespace is the Kubernetes namespace claims are made in. e2b has no + // namespace concept, so every compat claim lands in this one. + Namespace string + // Domain is echoed as the sandbox `domain`, from which the SDK derives the + // envd host as "{port}-{sandboxID}.{domain}". Empty leaves it unset, which + // makes the SDK fall back to its configured E2B_DOMAIN/E2B_SANDBOX_URL. + // + // Note that a sandboxID here is the node's sandboxd claim id, which is not + // guaranteed to be a valid DNS label (it carries an underscore). The + // subdomain form therefore only works behind a proxy that routes on the + // E2b-Sandbox-Id / E2b-Sandbox-Port headers the SDK also sends; otherwise + // point the SDK at the sandbox with E2B_SANDBOX_URL. + Domain string + // EnvdVersion overrides DefaultEnvdVersion. + EnvdVersion string + // APIKeys, when non-empty, is the set of accepted X-API-KEY values. Empty + // disables authentication and is refused unless AllowAnonymous is set, so a + // misconfigured deployment cannot silently serve an open claim endpoint. + APIKeys []string + // AllowAnonymous permits serving with no API key (local development). + AllowAnonymous bool + // SizeClass pins the warm-pool size axis for compat claims (default + // "small"); e2b's NewSandbox carries no size selector. + SizeClass string + // Log receives request-level errors. + Log logr.Logger +} + +// Server translates e2b REST calls onto a scale.SandboxStore. +type Server struct { + store scale.SandboxStore + opts Options + keys map[string]struct{} +} + +// NewServer builds a compat server. It fails when no API key is configured and +// anonymous access was not explicitly allowed. +func NewServer(store scale.SandboxStore, opts Options) (*Server, error) { + if store == nil { + return nil, errors.New("e2bcompat: store is required") + } + if opts.Namespace == "" { + opts.Namespace = "default" + } + if opts.EnvdVersion == "" { + opts.EnvdVersion = DefaultEnvdVersion + } + if opts.SizeClass == "" { + opts.SizeClass = string(cocoonv1.SizeSmall) + } + keys := make(map[string]struct{}, len(opts.APIKeys)) + for _, k := range opts.APIKeys { + if k = strings.TrimSpace(k); k != "" { + keys[k] = struct{}{} + } + } + if len(keys) == 0 && !opts.AllowAnonymous { + return nil, errors.New("e2bcompat: no API key configured; set one or enable anonymous access explicitly") + } + return &Server{store: store, opts: opts, keys: keys}, nil +} + +// Handler returns the routed, authenticated HTTP handler. +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + // /health is unauthenticated so probes work without a key. + mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + + mux.Handle("POST /sandboxes", s.auth(http.HandlerFunc(s.createSandbox))) + mux.Handle("GET /sandboxes", s.auth(http.HandlerFunc(s.listSandboxes))) + mux.Handle("GET /v2/sandboxes", s.auth(http.HandlerFunc(s.listSandboxes))) + mux.Handle("GET /sandboxes/{sandboxID}", s.auth(http.HandlerFunc(s.getSandbox))) + mux.Handle("DELETE /sandboxes/{sandboxID}", s.auth(http.HandlerFunc(s.deleteSandbox))) + mux.Handle("POST /sandboxes/{sandboxID}/timeout", s.auth(http.HandlerFunc(s.setTimeout))) + mux.Handle("POST /sandboxes/{sandboxID}/refreshes", s.auth(http.HandlerFunc(s.refresh))) + return mux +} + +// auth enforces the X-API-KEY header unless anonymous access is allowed. The +// comparison is constant-time so a valid key cannot be recovered by timing. +func (s *Server) auth(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(s.keys) > 0 { + presented := r.Header.Get(apiKeyHeader) + if !s.validKey(presented) { + writeError(w, http.StatusUnauthorized, "invalid API key") + return + } + } + next.ServeHTTP(w, r) + }) +} + +func (s *Server) validKey(presented string) bool { + if presented == "" { + return false + } + ok := false + for k := range s.keys { + if subtle.ConstantTimeCompare([]byte(k), []byte(presented)) == 1 { + ok = true + } + } + return ok +} + +// createSandbox claims a warm microVM for the requested template. It is the +// same node-local claim the aggregated apiserver's Create performs. +func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { + var req NewSandbox + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + return + } + if strings.TrimSpace(req.TemplateID) == "" { + writeError(w, http.StatusBadRequest, "templateID is required") + return + } + + name := generateName() + pool := scale.PoolKey{ + Template: req.TemplateID, + Net: netFor(req.AllowInternetAccess), + Size: s.opts.SizeClass, + } + assignment, err := s.store.Claim(r.Context(), s.opts.Namespace, name, pool) + if err != nil { + if scale.IsNoWarmCapacity(err) { + // Retryable: warm capacity refills asynchronously on the node. + writeError(w, http.StatusServiceUnavailable, fmt.Sprintf( + "no warm sandbox available for template %q; retry as warm capacity refills", req.TemplateID)) + return + } + s.opts.Log.Error(err, "e2b create: claim failed", "template", req.TemplateID, "name", name) + writeError(w, http.StatusInternalServerError, "failed to claim a sandbox") + return + } + + writeJSON(w, http.StatusCreated, Sandbox{ + TemplateID: req.TemplateID, + SandboxID: assignment.SandboxName, + ClientID: assignment.Node, + EnvdVersion: s.opts.EnvdVersion, + EnvdAccessToken: assignment.Token, + Domain: s.opts.Domain, + }) +} + +// listSandboxes reports the live sandboxes in the compat namespace. +func (s *Server) listSandboxes(w http.ResponseWriter, r *http.Request) { + list, err := s.store.List(r.Context(), scale.ListOptions{Namespace: s.opts.Namespace}) + if err != nil { + s.opts.Log.Error(err, "e2b list: store list failed") + writeError(w, http.StatusInternalServerError, "failed to list sandboxes") + return + } + out := make([]SandboxDetail, 0, len(list.Items)) + for i := range list.Items { + out = append(out, s.detailFor(&list.Items[i])) + } + writeJSON(w, http.StatusOK, out) +} + +// getSandbox resolves one sandbox by its e2b sandboxID (the sandboxd claim id). +func (s *Server) getSandbox(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("sandboxID") + sb, err := s.lookup(r, id) + if err != nil { + s.writeLookupError(w, err, id, "get") + return + } + writeJSON(w, http.StatusOK, s.detailFor(sb)) +} + +// deleteSandbox releases the claim back to its owning node's warm pool. +func (s *Server) deleteSandbox(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("sandboxID") + sb, err := s.lookup(r, id) + if err != nil { + s.writeLookupError(w, err, id, "delete") + return + } + node := sb.Status.NodeName + if node == "" { + // No owning node resolved: nothing to release against. + w.WriteHeader(http.StatusNoContent) + return + } + if err := s.store.Release(r.Context(), node, id); err != nil { + s.opts.Log.Error(err, "e2b delete: release failed", "sandboxID", id, "node", node) + writeError(w, http.StatusInternalServerError, "failed to release the sandbox") + return + } + w.WriteHeader(http.StatusNoContent) +} + +// setTimeout accepts the SDK's TTL update. The node fixes a claim's TTL when it +// hands the microVM over, so this verifies the sandbox exists and acknowledges; +// it does not silently claim to have extended a deadline it cannot move. +func (s *Server) setTimeout(w http.ResponseWriter, r *http.Request) { + var req SandboxTimeoutRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + return + } + if req.Timeout < 0 { + writeError(w, http.StatusBadRequest, "timeout must be >= 0") + return + } + id := r.PathValue("sandboxID") + if _, err := s.lookup(r, id); err != nil { + s.writeLookupError(w, err, id, "timeout") + return + } + w.WriteHeader(http.StatusNoContent) +} + +// refresh is the SDK keepalive. Liveness is node-owned, so this confirms the +// sandbox is still live and acknowledges. +func (s *Server) refresh(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("sandboxID") + if _, err := s.lookup(r, id); err != nil { + s.writeLookupError(w, err, id, "refresh") + return + } + w.WriteHeader(http.StatusNoContent) +} + +// errSandboxNotFound is returned by lookup when no live sandbox carries the id. +var errSandboxNotFound = errors.New("sandbox not found") + +// lookup finds the sandbox whose sandboxd claim id matches id. The id is the +// node-assigned claim id, which the store stamps on each synthesized Sandbox. +func (s *Server) lookup(r *http.Request, id string) (*sandboxv1beta1.Sandbox, error) { + if strings.TrimSpace(id) == "" { + return nil, errSandboxNotFound + } + list, err := s.store.List(r.Context(), scale.ListOptions{Namespace: s.opts.Namespace}) + if err != nil { + return nil, err + } + for i := range list.Items { + if list.Items[i].Annotations[scale.ClaimIDAnnotation] == id { + return &list.Items[i], nil + } + } + return nil, errSandboxNotFound +} + +func (s *Server) writeLookupError(w http.ResponseWriter, err error, id, op string) { + if errors.Is(err, errSandboxNotFound) { + writeError(w, http.StatusNotFound, fmt.Sprintf("sandbox %q not found", id)) + return + } + s.opts.Log.Error(err, "e2b "+op+": lookup failed", "sandboxID", id) + writeError(w, http.StatusInternalServerError, "failed to resolve the sandbox") +} + +// detailFor renders a live Sandbox as the e2b detail shape. Fields e2b requires +// but cocoon does not track per sandbox (disk size, end-of-life) are reported as +// zero/derived values rather than omitted, so the SDK's decoder stays happy. +func (s *Server) detailFor(sb *sandboxv1beta1.Sandbox) SandboxDetail { + started := sb.CreationTimestamp.Time + if started.IsZero() { + started = time.Now() + } + return SandboxDetail{ + TemplateID: templateOf(sb), + SandboxID: sb.Annotations[scale.ClaimIDAnnotation], + ClientID: sb.Status.NodeName, + StartedAt: started.UTC().Format(time.RFC3339), + EndAt: started.Add(DefaultTimeoutSeconds * time.Second).UTC().Format(time.RFC3339), + State: StateRunning, + EnvdVersion: s.opts.EnvdVersion, + EnvdAccessToken: sb.Annotations[tokenAnnotation], + Domain: s.opts.Domain, + } +} + +// templateOf reports the pool template a sandbox was claimed from: the first +// container image, the same axis scale.PoolKeyFor keys on. +func templateOf(sb *sandboxv1beta1.Sandbox) string { + if c := sb.Spec.PodTemplate.Spec.Containers; len(c) > 0 { + return c[0].Image + } + return "" +} + +// netFor maps e2b's allow_internet_access onto the pool's network axis. +func netFor(allowInternet *bool) string { + if allowInternet != nil && *allowInternet { + return string(cocoonv1.NetEgress) + } + return scale.NetDefault +} diff --git a/pkg/e2bcompat/server_test.go b/pkg/e2bcompat/server_test.go new file mode 100644 index 0000000..298eb90 --- /dev/null +++ b/pkg/e2bcompat/server_test.go @@ -0,0 +1,380 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2bcompat + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + + sandboxv1beta1 "github.com/cocoonstack/cocoon-sandbox-operator/api/v1beta1" + "github.com/cocoonstack/cocoon-sandbox-operator/pkg/scale" +) + +const testKey = "e2b_testkey" + +// fakeStore records what the compat layer asked of the store and replays canned +// answers, so the tests assert the translation rather than the node behavior. +type fakeStore struct { + claimPool scale.PoolKey + claimNS string + claimName string + claimErr error + assign scale.Assignment + + items []sandboxv1beta1.Sandbox + + releasedNode string + releasedID string + releaseErr error + listErr error +} + +func (f *fakeStore) List(context.Context, scale.ListOptions) (*sandboxv1beta1.SandboxList, error) { + if f.listErr != nil { + return nil, f.listErr + } + return &sandboxv1beta1.SandboxList{Items: f.items}, nil +} + +func (f *fakeStore) Get(context.Context, string, string) (*sandboxv1beta1.Sandbox, error) { + return nil, nil +} + +func (f *fakeStore) Watch(context.Context, scale.ListOptions) (watch.Interface, error) { + return nil, nil +} + +func (f *fakeStore) Claim(_ context.Context, ns, name string, pool scale.PoolKey) (scale.Assignment, error) { + f.claimNS, f.claimName, f.claimPool = ns, name, pool + if f.claimErr != nil { + return scale.Assignment{}, f.claimErr + } + return f.assign, nil +} + +func (f *fakeStore) Release(_ context.Context, node, id string) error { + f.releasedNode, f.releasedID = node, id + return f.releaseErr +} + +func newTestServer(t *testing.T, store scale.SandboxStore, opts ...func(*Options)) http.Handler { + t.Helper() + o := Options{Namespace: "sandboxes", APIKeys: []string{testKey}, Log: logr.Discard()} + for _, fn := range opts { + fn(&o) + } + s, err := NewServer(store, o) + if err != nil { + t.Fatalf("NewServer: %v", err) + } + return s.Handler() +} + +func do(t *testing.T, h http.Handler, method, path, body, key string) *httptest.ResponseRecorder { + t.Helper() + var r *http.Request + if body == "" { + r = httptest.NewRequest(method, path, nil) + } else { + r = httptest.NewRequest(method, path, strings.NewReader(body)) + } + if key != "" { + r.Header.Set(apiKeyHeader, key) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w +} + +// liveSandbox builds a sandbox as the store's scatter-gather read reports it. +func liveSandbox(name, claimID, node, image, token string) sandboxv1beta1.Sandbox { + sb := sandboxv1beta1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "sandboxes", + CreationTimestamp: metav1.Now(), + Annotations: map[string]string{ + scale.ClaimIDAnnotation: claimID, + tokenAnnotation: token, + }, + }, + } + sb.Spec.PodTemplate.Spec.Containers = []corev1.Container{{Name: "c", Image: image}} + sb.Status.NodeName = node + return sb +} + +// TestCreateClaimsFromTemplatePool is the core contract: an e2b create is the +// same node-local claim, keyed on the template the caller asked for, and the +// response carries the fields the SDK requires. +func TestCreateClaimsFromTemplatePool(t *testing.T) { + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_abc123", Node: "node-a", Token: "tok-1"}} + h := newTestServer(t, store, func(o *Options) { o.Domain = "sandbox.example.com" }) + + w := do(t, h, http.MethodPost, "/sandboxes", `{"templateID":"registry/rt:24.04","timeout":300}`, testKey) + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201 (the e2b create contract): %s", w.Code, w.Body.String()) + } + var got Sandbox + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode response: %v", err) + } + if got.SandboxID != "sb_abc123" { + t.Errorf("sandboxID = %q, want the node-assigned claim id", got.SandboxID) + } + if got.TemplateID != "registry/rt:24.04" { + t.Errorf("templateID = %q, want it echoed back", got.TemplateID) + } + if got.EnvdAccessToken != "tok-1" { + t.Errorf("envdAccessToken = %q, want the claim token", got.EnvdAccessToken) + } + if got.Domain != "sandbox.example.com" { + t.Errorf("domain = %q, want the configured domain", got.Domain) + } + // The SDK version-compares envdVersion and kills the sandbox if it cannot + // parse it or it is below 0.1.0, so it must always be a real version. + if got.EnvdVersion == "" { + t.Error("envdVersion is empty; the e2b SDK kills the sandbox and throws when it cannot parse this") + } + if store.claimPool.Template != "registry/rt:24.04" { + t.Errorf("claimed pool template = %q, want the requested templateID", store.claimPool.Template) + } + if store.claimNS != "sandboxes" { + t.Errorf("claim namespace = %q, want the configured compat namespace", store.claimNS) + } + if !strings.HasPrefix(store.claimName, namePrefix) { + t.Errorf("claim name = %q, want the %q prefix so compat claims are recognizable", store.claimName, namePrefix) + } +} + +// TestCreateNetworkLane checks e2b's internet flag selects the pool's net axis. +func TestCreateNetworkLane(t *testing.T) { + for _, tc := range []struct { + name string + body string + want string + }{ + {"default is the hardened lane", `{"templateID":"t"}`, scale.NetDefault}, + {"internet access picks egress", `{"templateID":"t","allow_internet_access":true}`, "egress"}, + {"explicit false stays hardened", `{"templateID":"t","allow_internet_access":false}`, scale.NetDefault}, + } { + t.Run(tc.name, func(t *testing.T) { + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}} + h := newTestServer(t, store) + if w := do(t, h, http.MethodPost, "/sandboxes", tc.body, testKey); w.Code != http.StatusCreated { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + if store.claimPool.Net != tc.want { + t.Errorf("pool net = %q, want %q", store.claimPool.Net, tc.want) + } + }) + } +} + +// TestCreateNoWarmCapacityIsRetryable keeps the retryable signal retryable: a +// drained pool refills asynchronously, so it must not surface as a 500. +func TestCreateNoWarmCapacityIsRetryable(t *testing.T) { + store := &fakeStore{claimErr: scale.ErrNoWarmCapacity} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes", `{"templateID":"t"}`, testKey) + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 so the client retries as warm capacity refills", w.Code) + } +} + +func TestCreateRejectsMissingTemplate(t *testing.T) { + h := newTestServer(t, &fakeStore{}) + if w := do(t, h, http.MethodPost, "/sandboxes", `{}`, testKey); w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 for a body with no templateID", w.Code) + } +} + +// TestAuthRequiresAPIKey proves the claim endpoint is closed without the key +// the SDK presents. +func TestAuthRequiresAPIKey(t *testing.T) { + store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}} + h := newTestServer(t, store) + + for _, tc := range []struct{ name, key string }{ + {"no key", ""}, + {"wrong key", "e2b_wrong"}, + } { + t.Run(tc.name, func(t *testing.T) { + w := do(t, h, http.MethodPost, "/sandboxes", `{"templateID":"t"}`, tc.key) + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", w.Code) + } + if store.claimName != "" { + t.Fatal("an unauthenticated request reached the claim path") + } + }) + } +} + +// TestNewServerRefusesOpenByDefault: forgetting to configure a key must fail +// loud at startup, not silently serve an open claim endpoint. +func TestNewServerRefusesOpenByDefault(t *testing.T) { + if _, err := NewServer(&fakeStore{}, Options{Namespace: "x"}); err == nil { + t.Fatal("NewServer accepted no API key without explicit anonymous access") + } + if _, err := NewServer(&fakeStore{}, Options{Namespace: "x", AllowAnonymous: true}); err != nil { + t.Fatalf("NewServer rejected explicit anonymous access: %v", err) + } +} + +func TestHealthNeedsNoKey(t *testing.T) { + h := newTestServer(t, &fakeStore{}) + if w := do(t, h, http.MethodGet, "/health", "", ""); w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want 204 without a key so probes work", w.Code) + } +} + +// TestDeleteReleasesTheClaimedID checks release targets the node-local claim id +// rather than the Kubernetes name — releasing by name would free the wrong VM. +func TestDeleteReleasesTheClaimedID(t *testing.T) { + store := &fakeStore{items: []sandboxv1beta1.Sandbox{ + liveSandbox("e2b-aaa", "sb_one", "node-a", "img:1", "tok"), + liveSandbox("e2b-bbb", "sb_two", "node-b", "img:1", "tok"), + }} + h := newTestServer(t, store) + + if w := do(t, h, http.MethodDelete, "/sandboxes/sb_two", "", testKey); w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want 204: %s", w.Code, w.Body.String()) + } + if store.releasedID != "sb_two" || store.releasedNode != "node-b" { + t.Errorf("released (node=%q id=%q), want (node-b, sb_two)", store.releasedNode, store.releasedID) + } +} + +func TestDeleteUnknownSandboxIs404(t *testing.T) { + store := &fakeStore{items: []sandboxv1beta1.Sandbox{liveSandbox("a", "sb_one", "node-a", "i", "t")}} + h := newTestServer(t, store) + + if w := do(t, h, http.MethodDelete, "/sandboxes/sb_missing", "", testKey); w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } + if store.releasedID != "" { + t.Errorf("released %q for an unknown sandbox", store.releasedID) + } +} + +// TestGetReportsDetail checks the detail shape the SDK decodes on getInfo. +func TestGetReportsDetail(t *testing.T) { + store := &fakeStore{items: []sandboxv1beta1.Sandbox{ + liveSandbox("e2b-aaa", "sb_one", "node-a", "registry/rt:24.04", "tok-9"), + }} + h := newTestServer(t, store) + + w := do(t, h, http.MethodGet, "/sandboxes/sb_one", "", testKey) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + var got SandboxDetail + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.SandboxID != "sb_one" || got.TemplateID != "registry/rt:24.04" || got.ClientID != "node-a" { + t.Errorf("detail = %+v, want the live sandbox's id/template/node", got) + } + if got.State != StateRunning { + t.Errorf("state = %q, want %q", got.State, StateRunning) + } + // The SDK parses these as dates; empty strings make it produce Invalid Date. + if got.StartedAt == "" || got.EndAt == "" { + t.Errorf("startedAt/endAt = %q/%q, want RFC3339 timestamps", got.StartedAt, got.EndAt) + } +} + +// TestListReturnsArray: the SDK decodes a bare JSON array, not an object. +func TestListReturnsArray(t *testing.T) { + store := &fakeStore{items: []sandboxv1beta1.Sandbox{ + liveSandbox("a", "sb_one", "node-a", "i:1", "t"), + liveSandbox("b", "sb_two", "node-b", "i:2", "t"), + }} + h := newTestServer(t, store) + + for _, path := range []string{"/sandboxes", "/v2/sandboxes"} { + t.Run(path, func(t *testing.T) { + w := do(t, h, http.MethodGet, path, "", testKey) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + var got []SandboxDetail + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode as array: %v", err) + } + if len(got) != 2 { + t.Fatalf("len = %d, want 2", len(got)) + } + }) + } +} + +// TestListEmptyIsArrayNotNull: an empty pool must encode as [], since `null` +// breaks callers that iterate the result directly. +func TestListEmptyIsArrayNotNull(t *testing.T) { + h := newTestServer(t, &fakeStore{}) + w := do(t, h, http.MethodGet, "/sandboxes", "", testKey) + if body := strings.TrimSpace(w.Body.String()); body != "[]" { + t.Fatalf("body = %s, want []", body) + } +} + +// TestTimeoutAndRefreshAckLiveSandbox: both verbs confirm the sandbox exists +// before acknowledging, so a caller never gets an OK for a dead sandbox. +func TestTimeoutAndRefreshAckLiveSandbox(t *testing.T) { + store := &fakeStore{items: []sandboxv1beta1.Sandbox{liveSandbox("a", "sb_one", "node-a", "i", "t")}} + h := newTestServer(t, store) + + for _, tc := range []struct{ name, path, body string }{ + {"timeout", "/sandboxes/sb_one/timeout", `{"timeout":60}`}, + {"refresh", "/sandboxes/sb_one/refreshes", `{"duration":30}`}, + } { + t.Run(tc.name, func(t *testing.T) { + if w := do(t, h, http.MethodPost, tc.path, tc.body, testKey); w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want 204: %s", w.Code, w.Body.String()) + } + }) + } + t.Run("unknown sandbox is 404", func(t *testing.T) { + w := do(t, h, http.MethodPost, "/sandboxes/sb_gone/timeout", `{"timeout":60}`, testKey) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } + }) +} + +// TestErrorBodyCarriesMessage: the SDK surfaces `message` from the envelope. +func TestErrorBodyCarriesMessage(t *testing.T) { + h := newTestServer(t, &fakeStore{}) + w := do(t, h, http.MethodPost, "/sandboxes", `{}`, testKey) + var got APIError + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode error envelope: %v", err) + } + if got.Message == "" || got.Code != http.StatusBadRequest { + t.Errorf("error = %+v, want code 400 and a message", got) + } +} diff --git a/pkg/e2bcompat/types.go b/pkg/e2bcompat/types.go new file mode 100644 index 0000000..59bf29f --- /dev/null +++ b/pkg/e2bcompat/types.go @@ -0,0 +1,98 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2bcompat + +// The wire types below mirror the e2b REST API schemas the e2b SDKs consume +// (e2b-dev/E2B spec/openapi.yml: NewSandbox, Sandbox, SandboxDetail, +// ListedSandbox, ResumedSandbox). Field names and JSON casing are fixed by that +// contract — the SDK unmarshals them directly — so they are reproduced exactly +// rather than restyled to this repo's own conventions. + +// NewSandbox is the POST /sandboxes request body (spec: NewSandbox). Only +// templateID is required; the rest carry SDK defaults. +type NewSandbox struct { + TemplateID string `json:"templateID"` + // Timeout is the sandbox time-to-live in seconds (SDK default 15). + Timeout *int32 `json:"timeout,omitempty"` + // Metadata and EnvVars are echoed back on reads. They are recorded on the + // compat side only: the node-local claim path takes neither today. + Metadata map[string]string `json:"metadata,omitempty"` + EnvVars map[string]string `json:"envVars,omitempty"` + // AutoPause and Secure are accepted and reported back so an SDK that sets + // them does not fail; neither changes the claim. + AutoPause *bool `json:"autoPause,omitempty"` + Secure *bool `json:"secure,omitempty"` + // AllowInternetAccess selects the warm pool's network lane: true picks the + // egress-capable pool, false/nil the isolated one. + AllowInternetAccess *bool `json:"allow_internet_access,omitempty"` +} + +// Sandbox is the POST /sandboxes response (spec: Sandbox). templateID, +// sandboxID, clientID and envdVersion are required by the schema. +type Sandbox struct { + TemplateID string `json:"templateID"` + SandboxID string `json:"sandboxID"` + ClientID string `json:"clientID"` + // EnvdVersion gates SDK behavior (it version-compares before choosing the + // envd auth style), so it is always reported. + EnvdVersion string `json:"envdVersion"` + EnvdAccessToken string `json:"envdAccessToken,omitempty"` + Domain string `json:"domain,omitempty"` + Alias string `json:"alias,omitempty"` +} + +// SandboxDetail is the GET /sandboxes/{sandboxID} response (spec: +// SandboxDetail), and its field set also satisfies ListedSandbox. +type SandboxDetail struct { + TemplateID string `json:"templateID"` + SandboxID string `json:"sandboxID"` + ClientID string `json:"clientID"` + StartedAt string `json:"startedAt"` + EndAt string `json:"endAt"` + State string `json:"state"` + EnvdVersion string `json:"envdVersion"` + CPUCount int32 `json:"cpuCount"` + MemoryMB int32 `json:"memoryMB"` + DiskSizeMB int32 `json:"diskSizeMB"` + Alias string `json:"alias,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + EnvdAccessToken string `json:"envdAccessToken,omitempty"` + Domain string `json:"domain,omitempty"` + AllowInternetAccess *bool `json:"allowInternetAccess,omitempty"` +} + +// SandboxTimeoutRequest is the POST /sandboxes/{sandboxID}/timeout request body +// (spec: SandboxTimeoutRequest) — the SDK's setTimeout call. +type SandboxTimeoutRequest struct { + Timeout int32 `json:"timeout"` +} + +// SandboxRefreshRequest is the POST /sandboxes/{sandboxID}/refreshes request +// body (spec: SandboxRefreshRequest) — the keepalive. +type SandboxRefreshRequest struct { + Duration *int32 `json:"duration,omitempty"` +} + +// APIError is the e2b error envelope. The SDK surfaces `message` on failures. +type APIError struct { + Code int32 `json:"code"` + Message string `json:"message"` +} + +// Sandbox states reported to the SDK (spec: SandboxState). +const ( + StateRunning = "running" + StatePaused = "paused" +) From ee7d360acf501ecc45b3f728497402806a1731eb Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 26 Jul 2026 11:38:38 +0800 Subject: [PATCH 2/6] scale: route pause, resume, fork and snapshot to the owning node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The store could place and release a sandbox but nothing else, so everything a caller might do to one after delivery had no path through the control plane. Add the post-claim verb set to SandboxStore and the sandboxd client behind it. Every verb addresses an already-delivered sandbox on a known node, so there is no pool selection involved — only routing to the owner — and none of them store anything: the control plane stays O(pools+nodes) however many sandboxes are paused, forked or checkpointed. They authenticate with the fleet api_token over sandboxd's operator path, so the control plane needs no per-sandbox secret. Latency is deliberately documented as non-uniform on the interface: resume takes cocoon's mmap restore fast path and a fork's children clone at 28-75ms, but pause and snapshot write the guest's memory out and cost time proportional to its size. --- pkg/scale/claimgateway_impl.go | 13 ++ pkg/scale/sandboxd/lifecycle.go | 322 +++++++++++++++++++++++++++ pkg/scale/sandboxstore.go | 67 ++++++ pkg/scale/sandboxstore_claim_test.go | 24 ++ pkg/scale/sandboxstore_lifecycle.go | 205 +++++++++++++++++ 5 files changed, 631 insertions(+) create mode 100644 pkg/scale/sandboxd/lifecycle.go create mode 100644 pkg/scale/sandboxstore_lifecycle.go diff --git a/pkg/scale/claimgateway_impl.go b/pkg/scale/claimgateway_impl.go index 2fa564a..76f7a6e 100644 --- a/pkg/scale/claimgateway_impl.go +++ b/pkg/scale/claimgateway_impl.go @@ -49,6 +49,19 @@ func IsFallback(err error) bool { return errors.Is(err, ErrNoNodeCapacity) } type SandboxdClient interface { Claim(ctx context.Context, spec sandboxd.ClaimSpec) (sandboxd.ClaimResult, error) Release(ctx context.Context, id, token string) error + + // The lifecycle verbs address an already-delivered sandbox by id. They all + // take sandboxd's operator path, authorized by the fleet api_token the + // client already carries, so the control plane needs no per-sandbox secret. + Hibernate(ctx context.Context, id string) error + Wake(ctx context.Context, id string) error + Fork(ctx context.Context, id string, spec sandboxd.ForkSpec) (sandboxd.ForkResult, error) + Checkpoint(ctx context.Context, id string, spec sandboxd.CheckpointSpec) (sandboxd.Checkpoint, error) + Checkpoints(ctx context.Context) ([]sandboxd.Checkpoint, error) + DeleteCheckpoint(ctx context.Context, checkpointID string) error + ClaimCheckpoint(ctx context.Context, checkpointID string, spec sandboxd.CheckpointClaimSpec) (sandboxd.ClaimResult, error) + Promote(ctx context.Context, id string, spec sandboxd.PromoteSpec) (sandboxd.PoolKey, error) + Stats(ctx context.Context, id string) (sandboxd.SandboxStats, error) } // Authorizer runs the inline policy check (SubjectAccessReview + ResourceQuota) diff --git a/pkg/scale/sandboxd/lifecycle.go b/pkg/scale/sandboxd/lifecycle.go new file mode 100644 index 0000000..9f8e009 --- /dev/null +++ b/pkg/scale/sandboxd/lifecycle.go @@ -0,0 +1,322 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sandboxd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// The lifecycle verbs below all authenticate with the node's root api_token +// (the fleet token this client already holds) and take sandboxd's operator +// path, which resolves the sandbox by id without a per-sandbox token. That is +// deliberate: keeping one secret per sandbox in the control plane would turn +// its O(nodes) storage into O(sandboxes), the property the whole design rests +// on. sandboxd authorizes these by the root token exactly as it does release. + +// ForkSpec is the POST /v1/sandboxes/{id}/fork body. Token stays empty on the +// operator path; Count must be within the node's max_fork_count. +type ForkSpec struct { + Token string `json:"token,omitempty"` + Count int `json:"count"` + TTLSeconds int `json:"ttl_seconds,omitempty"` +} + +// ForkResult carries one claim per child; children are fresh sandboxes with +// their own ids and leases, and the parent keeps running. +type ForkResult struct { + Children []ClaimResult `json:"children"` +} + +// CheckpointSpec is the POST /v1/sandboxes/{id}/checkpoint body. +type CheckpointSpec struct { + Token string `json:"token,omitempty"` + Name string `json:"name,omitempty"` +} + +// Checkpoint is a captured sandbox state that branches can be claimed from. +type Checkpoint struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + SandboxID string `json:"sandbox_id"` + Key PoolKey `json:"key"` + CreatedAt time.Time `json:"created_at"` +} + +// PoolKey is a checkpoint's or template's pool identity. +type PoolKey struct { + Template string `json:"template"` + Net string `json:"net,omitempty"` + Size string `json:"size,omitempty"` + Engine string `json:"engine,omitempty"` +} + +// CheckpointClaimSpec is the POST /v1/checkpoints/{id}/claim body. +type CheckpointClaimSpec struct { + TTLSeconds int `json:"ttl_seconds,omitempty"` +} + +// PromoteSpec is the POST /v1/sandboxes/{id}/promote body: it publishes the +// sandbox's state as a node-local template future claims clone from. +type PromoteSpec struct { + Token string `json:"token,omitempty"` + Template string `json:"template"` +} + +// PromoteResult returns the promoted template's full key. +type PromoteResult struct { + Key PoolKey `json:"key"` +} + +// SandboxSummary is one live claim as the owning node reports it. +type SandboxSummary struct { + ID string `json:"id"` + Key PoolKey `json:"key"` + Deadline time.Time `json:"deadline"` + Hibernated bool `json:"hibernated"` + Archived bool `json:"archived,omitempty"` + FromCheckpoint string `json:"from_checkpoint,omitempty"` + ClaimRef string `json:"claim_ref,omitempty"` +} + +// SandboxStats is one sandbox's resource usage. CPUCount and MemTotalBytes are +// the tier the VM was booted with (authoritative); MemUsedBytes is the host +// VMM's resident set and is only meaningful when MemUsedMeasured is true — a +// hibernated sandbox has no process to measure. +type SandboxStats struct { + ID string `json:"id"` + CPUCount int `json:"cpu_count"` + MemTotalBytes int64 `json:"mem_total_bytes"` + MemUsedBytes int64 `json:"mem_used_bytes"` + Hibernated bool `json:"hibernated"` + MeasuredAt time.Time `json:"measured_at"` + MemUsedMeasured bool `json:"mem_used_measured"` +} + +// Hibernate performs POST /v1/sandboxes/{id}/hibernate: it snapshots the +// sandbox and stops its VM, freeing the node's memory. This is the pause verb; +// its cost is proportional to guest RAM because the memory is written out. +func (c *Client) Hibernate(ctx context.Context, id string) error { + return c.sandboxVerb(ctx, id, "hibernate") +} + +// Wake performs POST /v1/sandboxes/{id}/wake, restoring a hibernated sandbox +// through cocoon's mmap fast path (~55 ms) and leaving it running. Idempotent +// on a sandbox that is already awake. +func (c *Client) Wake(ctx context.Context, id string) error { + return c.sandboxVerb(ctx, id, "wake") +} + +// sandboxVerb posts a body-less sandbox-scoped verb and expects 204. +func (c *Client) sandboxVerb(ctx context.Context, id, verb string) error { + if id == "" { + return fmt.Errorf("sandboxd: %s requires a sandbox id", verb) + } + u := c.baseURL + "/v1/sandboxes/" + url.PathEscape(id) + "/" + verb + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, nil) + if err != nil { + return err + } + c.authenticate(req, c.token) + + resp, err := c.hc.Do(req) + if err != nil { + return fmt.Errorf("sandboxd: %s: %w", verb, err) + } + defer drainAndClose(resp) + + if resp.StatusCode != http.StatusNoContent { + return statusError(resp) + } + return nil +} + +// Fork performs POST /v1/sandboxes/{id}/fork, branching the sandbox into count +// children. The parent is checkpointed in place and keeps running; each child +// is a fresh claim with its own id and lease. +func (c *Client) Fork(ctx context.Context, id string, spec ForkSpec) (ForkResult, error) { + var out ForkResult + if id == "" { + return out, fmt.Errorf("sandboxd: fork requires a sandbox id") + } + err := c.postJSON(ctx, "/v1/sandboxes/"+url.PathEscape(id)+"/fork", spec, &out) + return out, err +} + +// Checkpoint performs POST /v1/sandboxes/{id}/checkpoint, capturing the +// sandbox's state under a fresh id. The source keeps running. +func (c *Client) Checkpoint(ctx context.Context, id string, spec CheckpointSpec) (Checkpoint, error) { + if id == "" { + return Checkpoint{}, fmt.Errorf("sandboxd: checkpoint requires a sandbox id") + } + var out struct { + Checkpoint Checkpoint `json:"checkpoint"` + } + err := c.postJSON(ctx, "/v1/sandboxes/"+url.PathEscape(id)+"/checkpoint", spec, &out) + return out.Checkpoint, err +} + +// ClaimCheckpoint performs POST /v1/checkpoints/{id}/claim, delivering a fresh +// sandbox branched from the checkpoint's exact state. +func (c *Client) ClaimCheckpoint(ctx context.Context, checkpointID string, spec CheckpointClaimSpec) (ClaimResult, error) { + var out ClaimResult + if checkpointID == "" { + return out, fmt.Errorf("sandboxd: claim checkpoint requires a checkpoint id") + } + err := c.postJSON(ctx, "/v1/checkpoints/"+url.PathEscape(checkpointID)+"/claim", spec, &out) + return out, err +} + +// Checkpoints performs GET /v1/checkpoints, newest first. +func (c *Client) Checkpoints(ctx context.Context) ([]Checkpoint, error) { + var out struct { + Checkpoints []Checkpoint `json:"checkpoints"` + } + err := c.getJSON(ctx, "/v1/checkpoints", &out) + return out.Checkpoints, err +} + +// DeleteCheckpoint performs DELETE /v1/checkpoints/{id}. A 404 is success. +func (c *Client) DeleteCheckpoint(ctx context.Context, checkpointID string) error { + if checkpointID == "" { + return fmt.Errorf("sandboxd: delete checkpoint requires a checkpoint id") + } + u := c.baseURL + "/v1/checkpoints/" + url.PathEscape(checkpointID) + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, u, nil) + if err != nil { + return err + } + c.authenticate(req, c.token) + + resp, err := c.hc.Do(req) + if err != nil { + return fmt.Errorf("sandboxd: delete checkpoint: %w", err) + } + defer drainAndClose(resp) + + switch resp.StatusCode { + case http.StatusNoContent, http.StatusNotFound: + return nil + default: + return statusError(resp) + } +} + +// Promote performs POST /v1/sandboxes/{id}/promote, publishing the sandbox as +// a node-local template that later claims for that key clone from. +func (c *Client) Promote(ctx context.Context, id string, spec PromoteSpec) (PoolKey, error) { + if id == "" { + return PoolKey{}, fmt.Errorf("sandboxd: promote requires a sandbox id") + } + var out PromoteResult + err := c.postJSON(ctx, "/v1/sandboxes/"+url.PathEscape(id)+"/promote", spec, &out) + return out.Key, err +} + +// Sandbox performs GET /v1/sandboxes/{id}, the single-sandbox read that avoids +// scanning the whole-node listing. +func (c *Client) Sandbox(ctx context.Context, id string) (SandboxSummary, error) { + var out SandboxSummary + if id == "" { + return out, fmt.Errorf("sandboxd: sandbox read requires an id") + } + err := c.getJSON(ctx, "/v1/sandboxes/"+url.PathEscape(id), &out) + return out, err +} + +// Stats performs GET /v1/sandboxes/{id}/stats. +func (c *Client) Stats(ctx context.Context, id string) (SandboxStats, error) { + var out SandboxStats + if id == "" { + return out, fmt.Errorf("sandboxd: stats requires a sandbox id") + } + err := c.getJSON(ctx, "/v1/sandboxes/"+url.PathEscape(id)+"/stats", &out) + return out, err +} + +// Sandboxes performs GET /v1/sandboxes, this node's live claims. +func (c *Client) Sandboxes(ctx context.Context) ([]SandboxSummary, error) { + var out struct { + Sandboxes []SandboxSummary `json:"sandboxes"` + } + err := c.getJSON(ctx, "/v1/sandboxes", &out) + return out.Sandboxes, err +} + +// postJSON sends body as JSON and decodes a 2xx reply into out. +func (c *Client) postJSON(ctx context.Context, path string, body, out any) error { + payload, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("sandboxd: encode %s: %w", path, err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + c.authenticate(req, c.token) + + resp, err := c.hc.Do(req) + if err != nil { + return fmt.Errorf("sandboxd: %s: %w", path, err) + } + defer drainAndClose(resp) + + if resp.StatusCode/100 != 2 { + return statusError(resp) + } + return decodeInto(resp, out, path) +} + +// getJSON decodes a 2xx GET reply into out. +func (c *Client) getJSON(ctx context.Context, path string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil) + if err != nil { + return err + } + c.authenticate(req, c.token) + + resp, err := c.hc.Do(req) + if err != nil { + return fmt.Errorf("sandboxd: %s: %w", path, err) + } + defer drainAndClose(resp) + + if resp.StatusCode != http.StatusOK { + return statusError(resp) + } + return decodeInto(resp, out, path) +} + +// decodeInto reads a bounded reply body into out. +func decodeInto(resp *http.Response, out any, path string) error { + if out == nil { + return nil + } + b, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return fmt.Errorf("sandboxd: read %s reply: %w", path, err) + } + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("sandboxd: decode %s reply: %w", path, err) + } + return nil +} diff --git a/pkg/scale/sandboxstore.go b/pkg/scale/sandboxstore.go index c37fa6b..11a5a26 100644 --- a/pkg/scale/sandboxstore.go +++ b/pkg/scale/sandboxstore.go @@ -16,6 +16,7 @@ package scale import ( "context" + "time" "k8s.io/apimachinery/pkg/watch" @@ -72,6 +73,72 @@ type SandboxStore interface { // it never destroys a VM on pod state alone. The node's sandboxd address is // resolved from its NodeInventory. Release(ctx context.Context, node, id string) error + + // SandboxLifecycle is the post-claim verb set. Every verb routes to the + // owning node and stores nothing: the control plane stays O(pools+nodes) + // however many sandboxes are paused, forked, or checkpointed. + SandboxLifecycle +} + +// SandboxLifecycle is the verb set a claimed sandbox supports after delivery. +// It is separate from SandboxStore's placement verbs because these all address +// an existing sandbox on a known node — there is no pool selection involved, +// only routing to the owner. +// +// Latency is not uniform across these verbs and callers should not assume it +// is: Resume takes cocoon's mmap restore fast path (~55 ms) and Fork's children +// clone at 28–75 ms each, but Pause and Snapshot write the guest's memory out +// and therefore cost time proportional to its size. +type SandboxLifecycle interface { + // Pause hibernates the sandbox: its state is snapshotted and the VM stops, + // freeing the node's memory. Idempotent on an already-paused sandbox. + Pause(ctx context.Context, node, id string) error + // Resume restores a paused sandbox and leaves it running. Idempotent on a + // running one. + Resume(ctx context.Context, node, id string) error + // Fork branches the sandbox into count children, each a fresh claim with + // its own id and lease. The parent is checkpointed in place and keeps + // running. + Fork(ctx context.Context, node, id string, count int, ttlSeconds int) ([]Assignment, error) + // Snapshot captures the sandbox's state as a named checkpoint that later + // claims can branch from. The source keeps running. + Snapshot(ctx context.Context, node, id, name string) (Snapshot, error) + // Snapshots lists the checkpoints on a node, newest first. + Snapshots(ctx context.Context, node string) ([]Snapshot, error) + // DeleteSnapshot removes a checkpoint. A missing checkpoint is success. + DeleteSnapshot(ctx context.Context, node, snapshotID string) error + // ClaimSnapshot delivers a fresh sandbox branched from a checkpoint. + ClaimSnapshot(ctx context.Context, node, snapshotID string, ttlSeconds int) (Assignment, error) + // Promote publishes the sandbox as a node-local template that later claims + // for that key clone from. + Promote(ctx context.Context, node, id, template string) (PoolKey, error) + // Stats reports one sandbox's resource usage. + Stats(ctx context.Context, node, id string) (SandboxStats, error) +} + +// Snapshot is a captured sandbox state that new sandboxes can branch from. +type Snapshot struct { + ID string + Name string + SandboxID string + Pool PoolKey + CreatedAt time.Time + // Node is the node holding the checkpoint. Checkpoints are node-local, so + // a caller needs it to branch from or delete this snapshot later. + Node string +} + +// SandboxStats is one sandbox's resource usage. CPUCount and MemTotalBytes are +// the tier the VM was booted with and are authoritative; MemUsedBytes is only +// meaningful when MemUsedMeasured is true (a paused sandbox has no process to +// measure), so callers must not read zero as "idle". +type SandboxStats struct { + CPUCount int + MemTotalBytes int64 + MemUsedBytes int64 + MemUsedMeasured bool + Paused bool + MeasuredAt time.Time } // InventoryEntry is one live sandbox as summarized by its owning node. diff --git a/pkg/scale/sandboxstore_claim_test.go b/pkg/scale/sandboxstore_claim_test.go index 5537ed9..a9d288f 100644 --- a/pkg/scale/sandboxstore_claim_test.go +++ b/pkg/scale/sandboxstore_claim_test.go @@ -168,3 +168,27 @@ func TestStoreClaimRelease_FailClosedWithoutRouting(t *testing.T) { require.Error(t, store.Release(context.Background(), "n1", "sb-1")) } + +// The lifecycle verbs are not exercised here; they satisfy the SandboxdClient +// port so the recorder stays a drop-in. +func (c *recordingClient) Hibernate(context.Context, string) error { return nil } +func (c *recordingClient) Wake(context.Context, string) error { return nil } +func (c *recordingClient) Fork(context.Context, string, sandboxd.ForkSpec) (sandboxd.ForkResult, error) { + return sandboxd.ForkResult{}, nil +} +func (c *recordingClient) Checkpoint(context.Context, string, sandboxd.CheckpointSpec) (sandboxd.Checkpoint, error) { + return sandboxd.Checkpoint{}, nil +} +func (c *recordingClient) Checkpoints(context.Context) ([]sandboxd.Checkpoint, error) { + return nil, nil +} +func (c *recordingClient) DeleteCheckpoint(context.Context, string) error { return nil } +func (c *recordingClient) ClaimCheckpoint(context.Context, string, sandboxd.CheckpointClaimSpec) (sandboxd.ClaimResult, error) { + return sandboxd.ClaimResult{}, nil +} +func (c *recordingClient) Promote(context.Context, string, sandboxd.PromoteSpec) (sandboxd.PoolKey, error) { + return sandboxd.PoolKey{}, nil +} +func (c *recordingClient) Stats(context.Context, string) (sandboxd.SandboxStats, error) { + return sandboxd.SandboxStats{}, nil +} diff --git a/pkg/scale/sandboxstore_lifecycle.go b/pkg/scale/sandboxstore_lifecycle.go new file mode 100644 index 0000000..571fbc5 --- /dev/null +++ b/pkg/scale/sandboxstore_lifecycle.go @@ -0,0 +1,205 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scale + +import ( + "context" + "fmt" + + "github.com/cocoonstack/cocoon-sandbox-operator/pkg/scale/sandboxd" +) + +// The lifecycle verbs route to the sandbox's owning node and write nothing: +// like Claim and Release, they are synchronous node-local transactions, so the +// control plane's storage stays O(pools+nodes) no matter how many sandboxes are +// paused, forked, or checkpointed. + +// Pause hibernates a delivered sandbox on its owning node. +func (s *scatterGatherStore) Pause(ctx context.Context, node, id string) error { + cl, err := s.nodeClient(ctx, node, "pause", id) + if err != nil { + return err + } + if err := cl.Hibernate(ctx, id); err != nil { + return fmt.Errorf("scale: sandboxd pause of %q on node %q: %w", id, node, err) + } + return nil +} + +// Resume restores a paused sandbox through cocoon's mmap fast path. +func (s *scatterGatherStore) Resume(ctx context.Context, node, id string) error { + cl, err := s.nodeClient(ctx, node, "resume", id) + if err != nil { + return err + } + if err := cl.Wake(ctx, id); err != nil { + return fmt.Errorf("scale: sandboxd resume of %q on node %q: %w", id, node, err) + } + return nil +} + +// Fork branches a sandbox into count children on its owning node. Children are +// fresh claims with their own ids; the parent keeps running. +func (s *scatterGatherStore) Fork(ctx context.Context, node, id string, count, ttlSeconds int) ([]Assignment, error) { + if count < 1 { + return nil, fmt.Errorf("scale: fork count must be >= 1, got %d", count) + } + cl, err := s.nodeClient(ctx, node, "fork", id) + if err != nil { + return nil, err + } + res, err := cl.Fork(ctx, id, sandboxd.ForkSpec{Count: count, TTLSeconds: ttlSeconds}) + if err != nil { + return nil, fmt.Errorf("scale: sandboxd fork of %q on node %q: %w", id, node, err) + } + out := make([]Assignment, 0, len(res.Children)) + for _, c := range res.Children { + out = append(out, Assignment{ + SandboxName: c.ID, + Node: node, + Address: c.OwnerAddr, + Token: c.Token, + }) + } + return out, nil +} + +// Snapshot captures a sandbox's state as a checkpoint on its owning node. +func (s *scatterGatherStore) Snapshot(ctx context.Context, node, id, name string) (Snapshot, error) { + cl, err := s.nodeClient(ctx, node, "snapshot", id) + if err != nil { + return Snapshot{}, err + } + ck, err := cl.Checkpoint(ctx, id, sandboxd.CheckpointSpec{Name: name}) + if err != nil { + return Snapshot{}, fmt.Errorf("scale: sandboxd snapshot of %q on node %q: %w", id, node, err) + } + return snapshotFrom(ck, node), nil +} + +// Snapshots lists a node's checkpoints. +func (s *scatterGatherStore) Snapshots(ctx context.Context, node string) ([]Snapshot, error) { + cl, err := s.nodeClient(ctx, node, "list snapshots", "") + if err != nil { + return nil, err + } + cks, err := cl.Checkpoints(ctx) + if err != nil { + return nil, fmt.Errorf("scale: sandboxd list snapshots on node %q: %w", node, err) + } + out := make([]Snapshot, 0, len(cks)) + for _, ck := range cks { + out = append(out, snapshotFrom(ck, node)) + } + return out, nil +} + +// DeleteSnapshot removes a checkpoint from its node. +func (s *scatterGatherStore) DeleteSnapshot(ctx context.Context, node, snapshotID string) error { + cl, err := s.nodeClient(ctx, node, "delete snapshot", snapshotID) + if err != nil { + return err + } + if err := cl.DeleteCheckpoint(ctx, snapshotID); err != nil { + return fmt.Errorf("scale: sandboxd delete snapshot %q on node %q: %w", snapshotID, node, err) + } + return nil +} + +// ClaimSnapshot delivers a fresh sandbox branched from a checkpoint. +func (s *scatterGatherStore) ClaimSnapshot(ctx context.Context, node, snapshotID string, ttlSeconds int) (Assignment, error) { + cl, err := s.nodeClient(ctx, node, "claim snapshot", snapshotID) + if err != nil { + return Assignment{}, err + } + res, err := cl.ClaimCheckpoint(ctx, snapshotID, sandboxd.CheckpointClaimSpec{TTLSeconds: ttlSeconds}) + if err != nil { + return Assignment{}, fmt.Errorf("scale: sandboxd claim snapshot %q on node %q: %w", snapshotID, node, err) + } + return Assignment{ + SandboxName: res.ID, + Node: node, + Address: res.OwnerAddr, + Token: res.Token, + }, nil +} + +// Promote publishes a sandbox as a node-local template. +func (s *scatterGatherStore) Promote(ctx context.Context, node, id, template string) (PoolKey, error) { + if template == "" { + return PoolKey{}, fmt.Errorf("scale: promote requires a template name") + } + cl, err := s.nodeClient(ctx, node, "promote", id) + if err != nil { + return PoolKey{}, err + } + key, err := cl.Promote(ctx, id, sandboxd.PromoteSpec{Template: template}) + if err != nil { + return PoolKey{}, fmt.Errorf("scale: sandboxd promote of %q on node %q: %w", id, node, err) + } + return PoolKey{Template: key.Template, Net: key.Net, Size: key.Size}, nil +} + +// Stats reports a sandbox's resource usage from its owning node. +func (s *scatterGatherStore) Stats(ctx context.Context, node, id string) (SandboxStats, error) { + cl, err := s.nodeClient(ctx, node, "stats", id) + if err != nil { + return SandboxStats{}, err + } + st, err := cl.Stats(ctx, id) + if err != nil { + return SandboxStats{}, fmt.Errorf("scale: sandboxd stats of %q on node %q: %w", id, node, err) + } + return SandboxStats{ + CPUCount: st.CPUCount, + MemTotalBytes: st.MemTotalBytes, + MemUsedBytes: st.MemUsedBytes, + MemUsedMeasured: st.MemUsedMeasured, + Paused: st.Hibernated, + MeasuredAt: st.MeasuredAt, + }, nil +} + +// nodeClient resolves a node's advertised sandboxd address and returns a client +// for it, failing closed when claim routing was never configured. verb and id +// name the operation in the error so a routing failure is diagnosable. +func (s *scatterGatherStore) nodeClient(ctx context.Context, node, verb, id string) (SandboxdClient, error) { + if s.sandboxdFactory == nil { + return nil, fmt.Errorf("scale: claim routing not configured (call WithClaimRouting)") + } + if node == "" { + return nil, fmt.Errorf("scale: %s requires an owning node", verb) + } + inv, err := s.src.NodeInventory(ctx, node) + if err != nil { + return nil, fmt.Errorf("scale: resolve node %q for %s of %q: %w", node, verb, id, err) + } + if inv.Address == "" { + return nil, fmt.Errorf("scale: node %q advertises no sandboxd address for %s of %q", node, verb, id) + } + return s.sandboxdFactory(inv.Address, s.sandboxdToken), nil +} + +// snapshotFrom converts a node's checkpoint record to the store's shape. +func snapshotFrom(ck sandboxd.Checkpoint, node string) Snapshot { + return Snapshot{ + ID: ck.ID, + Name: ck.Name, + SandboxID: ck.SandboxID, + Pool: PoolKey{Template: ck.Key.Template, Net: ck.Key.Net, Size: ck.Key.Size}, + CreatedAt: ck.CreatedAt, + Node: node, + } +} From 49f485af479a7ffda43ea979897c272b55796f65 Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 26 Jul 2026 11:38:38 +0800 Subject: [PATCH 3/6] apiserver: serve pause, resume, fork and snapshot as subresources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream agent-sandbox has no pause/fork/snapshot concept, and the value of this operator is that an unmodified upstream client keeps working. Adding fields to the standard Sandbox schema would fork it; a subresource is a verb the standard type never has to know about. They are POST-only action verbs (the pods/eviction shape, not pods/status): each is a synchronous node-local transaction, so there is nothing to GET and nothing to reconcile. Every served kind also needs an OpenAPI model, including metav1.TypeMeta under BOTH its Go package path and its canonical name — without them InstallAPIGroup refuses to start the server at all, which is a crash loop on rollout rather than a degraded feature. TestInstallSandboxAPISucceeds exercises that exact startup path so the next missing model fails locally instead of in the cluster. --- api/v1beta1/sandbox_lifecycle_types.go | 138 ++++++++++++++++ api/v1beta1/zz_generated.deepcopy.go | 165 +++++++++++++++++++ pkg/scale/apiserver/apiserver.go | 6 + pkg/scale/apiserver/lifecycle_storage.go | 192 +++++++++++++++++++++++ pkg/scale/apiserver/openapi.go | 66 +++++++- pkg/scale/apiserver/openapi_test.go | 71 +++++++++ pkg/scale/apiserver/scheme.go | 10 ++ pkg/scale/apiserver/storage_test.go | 22 +++ 8 files changed, 669 insertions(+), 1 deletion(-) create mode 100644 api/v1beta1/sandbox_lifecycle_types.go create mode 100644 pkg/scale/apiserver/lifecycle_storage.go diff --git a/api/v1beta1/sandbox_lifecycle_types.go b/api/v1beta1/sandbox_lifecycle_types.go new file mode 100644 index 0000000..c44299c --- /dev/null +++ b/api/v1beta1/sandbox_lifecycle_types.go @@ -0,0 +1,138 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// The lifecycle verbs below are served as SUBRESOURCES of sandboxes +// (sandboxes/pause, /resume, /fork, /snapshot), never as fields on SandboxSpec. +// Upstream agent-sandbox has no pause/fork/snapshot concept, and the value of +// this operator is that an unmodified upstream client keeps working: adding +// fields to the standard schema would fork it, whereas a subresource is a verb +// the standard type does not have to know about. +// +// They are actions, not desired state, which is also why they are POSTed rather +// than reconciled — each one is a synchronous node-local transaction, the same +// shape as the claim that delivered the sandbox. + +// +kubebuilder:object:root=true + +// SandboxPauseOptions is the body of POST sandboxes/{name}/pause. Pausing +// snapshots the guest's memory and stops its VM, so it costs time proportional +// to that memory — unlike resume, which takes the mmap restore fast path. +type SandboxPauseOptions struct { + metav1.TypeMeta `json:",inline"` +} + +// +kubebuilder:object:root=true + +// SandboxResumeOptions is the body of POST sandboxes/{name}/resume. Resuming a +// paused sandbox restores it through cocoon's mmap fast path and is idempotent +// on one that is already running. +type SandboxResumeOptions struct { + metav1.TypeMeta `json:",inline"` +} + +// +kubebuilder:object:root=true + +// SandboxForkOptions is the body of POST sandboxes/{name}/fork. The source is +// checkpointed in place and keeps running; each child is a brand-new sandbox +// with its own id and lease, not a replica of the source's identity. +type SandboxForkOptions struct { + metav1.TypeMeta `json:",inline"` + + // count is how many children to branch. Defaults to 1, and is bounded by + // the owning node's configured fork limit. + // +optional + Count int32 `json:"count,omitempty"` + + // ttlSeconds is each child's lease. Children never inherit the parent's + // remaining lease — a lease is a per-sandbox resource bound. Zero takes the + // node's default. + // +optional + TTLSeconds int32 `json:"ttlSeconds,omitempty"` +} + +// +kubebuilder:object:root=true + +// SandboxForkResult is the reply to a fork: one entry per child, in request +// order. +type SandboxForkResult struct { + metav1.TypeMeta `json:",inline"` + + // children are the branched sandboxes. + Children []ForkedSandbox `json:"children"` +} + +// ForkedSandbox identifies one child of a fork. +type ForkedSandbox struct { + // sandboxID is the child's node-local claim id. + SandboxID string `json:"sandboxID"` + // nodeName is the node that owns the child. A fork is node-local, so every + // child lands on the source's node. + NodeName string `json:"nodeName"` + // address is the child's connection address, when the node published one. + // +optional + Address string `json:"address,omitempty"` +} + +// +kubebuilder:object:root=true + +// SandboxSnapshotOptions is the body of POST sandboxes/{name}/snapshot. The +// source keeps running; the checkpoint is an immutable state later sandboxes +// can branch from. +type SandboxSnapshotOptions struct { + metav1.TypeMeta `json:",inline"` + + // name labels the checkpoint. Optional; the node assigns an id regardless. + // +optional + Name string `json:"name,omitempty"` +} + +// +kubebuilder:object:root=true + +// SandboxSnapshotResult is the reply to a snapshot. +type SandboxSnapshotResult struct { + metav1.TypeMeta `json:",inline"` + + // snapshotID is the checkpoint's node-local id. + SnapshotID string `json:"snapshotID"` + // name echoes the requested label, when one was given. + // +optional + Name string `json:"name,omitempty"` + // nodeName is the node holding the checkpoint. Checkpoints are node-local, + // so branching from or deleting one requires knowing its node. + NodeName string `json:"nodeName"` + // creationTimestamp is when the node captured the checkpoint. + // +optional + CreationTimestamp metav1.Time `json:"creationTimestamp,omitempty"` +} + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, + &SandboxPauseOptions{}, + &SandboxResumeOptions{}, + &SandboxForkOptions{}, + &SandboxForkResult{}, + &SandboxSnapshotOptions{}, + &SandboxSnapshotResult{}, + ) + return nil + }) +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 47d44fb..35e72c5 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -38,6 +38,21 @@ func (in *EmbeddedObjectMetadata) DeepCopy() *EmbeddedObjectMetadata { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ForkedSandbox) DeepCopyInto(out *ForkedSandbox) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ForkedSandbox. +func (in *ForkedSandbox) DeepCopy() *ForkedSandbox { + if in == nil { + return nil + } + out := new(ForkedSandbox) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Lifecycle) DeepCopyInto(out *Lifecycle) { *out = *in @@ -180,6 +195,59 @@ func (in *SandboxBlueprint) DeepCopy() *SandboxBlueprint { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxForkOptions) DeepCopyInto(out *SandboxForkOptions) { + *out = *in + out.TypeMeta = in.TypeMeta +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxForkOptions. +func (in *SandboxForkOptions) DeepCopy() *SandboxForkOptions { + if in == nil { + return nil + } + out := new(SandboxForkOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxForkOptions) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxForkResult) DeepCopyInto(out *SandboxForkResult) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Children != nil { + in, out := &in.Children, &out.Children + *out = make([]ForkedSandbox, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxForkResult. +func (in *SandboxForkResult) DeepCopy() *SandboxForkResult { + if in == nil { + return nil + } + out := new(SandboxForkResult) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxForkResult) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SandboxList) DeepCopyInto(out *SandboxList) { *out = *in @@ -212,6 +280,103 @@ func (in *SandboxList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxPauseOptions) DeepCopyInto(out *SandboxPauseOptions) { + *out = *in + out.TypeMeta = in.TypeMeta +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxPauseOptions. +func (in *SandboxPauseOptions) DeepCopy() *SandboxPauseOptions { + if in == nil { + return nil + } + out := new(SandboxPauseOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxPauseOptions) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxResumeOptions) DeepCopyInto(out *SandboxResumeOptions) { + *out = *in + out.TypeMeta = in.TypeMeta +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxResumeOptions. +func (in *SandboxResumeOptions) DeepCopy() *SandboxResumeOptions { + if in == nil { + return nil + } + out := new(SandboxResumeOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxResumeOptions) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSnapshotOptions) DeepCopyInto(out *SandboxSnapshotOptions) { + *out = *in + out.TypeMeta = in.TypeMeta +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSnapshotOptions. +func (in *SandboxSnapshotOptions) DeepCopy() *SandboxSnapshotOptions { + if in == nil { + return nil + } + out := new(SandboxSnapshotOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxSnapshotOptions) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSnapshotResult) DeepCopyInto(out *SandboxSnapshotResult) { + *out = *in + out.TypeMeta = in.TypeMeta + in.CreationTimestamp.DeepCopyInto(&out.CreationTimestamp) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSnapshotResult. +func (in *SandboxSnapshotResult) DeepCopy() *SandboxSnapshotResult { + if in == nil { + return nil + } + out := new(SandboxSnapshotResult) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxSnapshotResult) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SandboxSpec) DeepCopyInto(out *SandboxSpec) { *out = *in diff --git a/pkg/scale/apiserver/apiserver.go b/pkg/scale/apiserver/apiserver.go index 643820c..18e2825 100644 --- a/pkg/scale/apiserver/apiserver.go +++ b/pkg/scale/apiserver/apiserver.go @@ -51,6 +51,12 @@ func InstallSandboxAPI(server *genericapiserver.GenericAPIServer, store scale.Sa apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(sandboxv1beta1.GroupVersion.Group, Scheme, ParameterCodec, Codecs) apiGroupInfo.VersionedResourcesStorageMap[sandboxv1beta1.GroupVersion.Version] = map[string]rest.Storage{ "sandboxes": NewSandboxREST(store), + // Action subresources. A map key with a slash installs the tail as a + // subresource of the head, so the standard Sandbox schema is untouched. + "sandboxes/pause": NewSandboxPauseREST(store), + "sandboxes/resume": NewSandboxResumeREST(store), + "sandboxes/fork": NewSandboxForkREST(store), + "sandboxes/snapshot": NewSandboxSnapshotREST(store), } if err := server.InstallAPIGroup(&apiGroupInfo); err != nil { return fmt.Errorf("apiserver: install sandboxes API group: %w", err) diff --git a/pkg/scale/apiserver/lifecycle_storage.go b/pkg/scale/apiserver/lifecycle_storage.go new file mode 100644 index 0000000..13bb285 --- /dev/null +++ b/pkg/scale/apiserver/lifecycle_storage.go @@ -0,0 +1,192 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiserver + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/registry/rest" + + sandboxv1beta1 "github.com/cocoonstack/cocoon-sandbox-operator/api/v1beta1" + "github.com/cocoonstack/cocoon-sandbox-operator/pkg/scale" +) + +// The lifecycle subresources are POST-only action verbs, the pods/eviction +// shape rather than the pods/status one: each is a synchronous node-local +// transaction against an already-delivered sandbox, so there is nothing to GET +// and nothing to reconcile. Keeping them as subresources leaves the standard +// Sandbox schema untouched, which is what lets an unmodified upstream +// agent-sandbox client keep working against this server. + +// lifecycleREST is the shared plumbing of the action subresources: resolve the +// named sandbox through the store, then run one verb against its owning node. +type lifecycleREST struct { + store scale.SandboxStore + // newOptions builds the request body type this subresource accepts. + newOptions func() runtime.Object + // verb performs the action and returns the response object. + verb func(ctx context.Context, store scale.SandboxStore, sb *sandboxv1beta1.Sandbox, opts runtime.Object) (runtime.Object, error) +} + +var ( + _ rest.Storage = &lifecycleREST{} + _ rest.Scoper = &lifecycleREST{} + _ rest.NamedCreater = &lifecycleREST{} + _ rest.GroupVersionKindProvider = &lifecycleREST{} +) + +func (r *lifecycleREST) New() runtime.Object { return r.newOptions() } + +func (r *lifecycleREST) Destroy() {} + +func (r *lifecycleREST) NamespaceScoped() bool { return true } + +// GroupVersionKind pins the kind the request pipeline decodes the body as. +func (r *lifecycleREST) GroupVersionKind(schema.GroupVersion) schema.GroupVersionKind { + gvks, _, err := Scheme.ObjectKinds(r.newOptions()) + if err != nil || len(gvks) == 0 { + return schema.GroupVersionKind{} + } + return gvks[0] +} + +// Create runs the verb. The subresource's parent name arrives separately from +// the body, which is why this is a NamedCreater. +func (r *lifecycleREST) Create( + ctx context.Context, + name string, + obj runtime.Object, + createValidation rest.ValidateObjectFunc, + _ *metav1.CreateOptions, +) (runtime.Object, error) { + if createValidation != nil { + if err := createValidation(ctx, obj); err != nil { + return nil, err + } + } + namespace := genericapirequest.NamespaceValue(ctx) + sb, err := r.store.Get(ctx, namespace, name) + if err != nil { + // NotFound propagates unchanged so IsNotFound stays true for the caller. + return nil, err + } + if sb.Status.NodeName == "" { + return nil, apierrors.NewInternalError(fmt.Errorf( + "sandbox %s/%s has no owning node; cannot run a lifecycle verb against it", namespace, name)) + } + if claimID(sb) == "" { + // Releasing or pausing by k8s name would target the wrong claim, so a + // sandbox whose node has not published its claim id fails loud. + return nil, apierrors.NewInternalError(fmt.Errorf( + "sandbox %s/%s carries no %s (node-local claim id); refusing to act by name", + namespace, name, ClaimIDAnnotation)) + } + return r.verb(ctx, r.store, sb, obj) +} + +// claimID reports the node-local claim id the store's verbs address. +func claimID(sb *sandboxv1beta1.Sandbox) string { return sb.Annotations[ClaimIDAnnotation] } + +// NewSandboxPauseREST serves sandboxes/pause. +func NewSandboxPauseREST(store scale.SandboxStore) rest.Storage { + return &lifecycleREST{ + store: store, + newOptions: func() runtime.Object { return &sandboxv1beta1.SandboxPauseOptions{} }, + verb: func(ctx context.Context, st scale.SandboxStore, sb *sandboxv1beta1.Sandbox, _ runtime.Object) (runtime.Object, error) { + if err := st.Pause(ctx, sb.Status.NodeName, claimID(sb)); err != nil { + return nil, apierrors.NewInternalError(fmt.Errorf("pause sandbox %s/%s: %w", sb.Namespace, sb.Name, err)) + } + return &sandboxv1beta1.SandboxPauseOptions{}, nil + }, + } +} + +// NewSandboxResumeREST serves sandboxes/resume. +func NewSandboxResumeREST(store scale.SandboxStore) rest.Storage { + return &lifecycleREST{ + store: store, + newOptions: func() runtime.Object { return &sandboxv1beta1.SandboxResumeOptions{} }, + verb: func(ctx context.Context, st scale.SandboxStore, sb *sandboxv1beta1.Sandbox, _ runtime.Object) (runtime.Object, error) { + if err := st.Resume(ctx, sb.Status.NodeName, claimID(sb)); err != nil { + return nil, apierrors.NewInternalError(fmt.Errorf("resume sandbox %s/%s: %w", sb.Namespace, sb.Name, err)) + } + return &sandboxv1beta1.SandboxResumeOptions{}, nil + }, + } +} + +// NewSandboxForkREST serves sandboxes/fork. +func NewSandboxForkREST(store scale.SandboxStore) rest.Storage { + return &lifecycleREST{ + store: store, + newOptions: func() runtime.Object { return &sandboxv1beta1.SandboxForkOptions{} }, + verb: func(ctx context.Context, st scale.SandboxStore, sb *sandboxv1beta1.Sandbox, obj runtime.Object) (runtime.Object, error) { + opts, ok := obj.(*sandboxv1beta1.SandboxForkOptions) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected SandboxForkOptions, got %T", obj)) + } + count := int(opts.Count) + if count == 0 { + count = 1 + } + if count < 0 { + return nil, apierrors.NewBadRequest(fmt.Sprintf("count must be >= 1, got %d", count)) + } + children, err := st.Fork(ctx, sb.Status.NodeName, claimID(sb), count, int(opts.TTLSeconds)) + if err != nil { + return nil, apierrors.NewInternalError(fmt.Errorf("fork sandbox %s/%s: %w", sb.Namespace, sb.Name, err)) + } + out := &sandboxv1beta1.SandboxForkResult{Children: make([]sandboxv1beta1.ForkedSandbox, 0, len(children))} + for _, c := range children { + out.Children = append(out.Children, sandboxv1beta1.ForkedSandbox{ + SandboxID: c.SandboxName, + NodeName: c.Node, + Address: c.Address, + }) + } + return out, nil + }, + } +} + +// NewSandboxSnapshotREST serves sandboxes/snapshot. +func NewSandboxSnapshotREST(store scale.SandboxStore) rest.Storage { + return &lifecycleREST{ + store: store, + newOptions: func() runtime.Object { return &sandboxv1beta1.SandboxSnapshotOptions{} }, + verb: func(ctx context.Context, st scale.SandboxStore, sb *sandboxv1beta1.Sandbox, obj runtime.Object) (runtime.Object, error) { + opts, ok := obj.(*sandboxv1beta1.SandboxSnapshotOptions) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected SandboxSnapshotOptions, got %T", obj)) + } + snap, err := st.Snapshot(ctx, sb.Status.NodeName, claimID(sb), opts.Name) + if err != nil { + return nil, apierrors.NewInternalError(fmt.Errorf("snapshot sandbox %s/%s: %w", sb.Namespace, sb.Name, err)) + } + return &sandboxv1beta1.SandboxSnapshotResult{ + SnapshotID: snap.ID, + Name: snap.Name, + NodeName: snap.Node, + CreationTimestamp: metav1.NewTime(snap.CreatedAt), + }, nil + }, + } +} diff --git a/pkg/scale/apiserver/openapi.go b/pkg/scale/apiserver/openapi.go index b7352f6..cbdc61b 100644 --- a/pkg/scale/apiserver/openapi.go +++ b/pkg/scale/apiserver/openapi.go @@ -104,8 +104,72 @@ func sandboxOpenAPIDefinitions(ref openapicommon.ReferenceCallback) map[string]o }, Dependencies: []string{sandboxDefPrefix + "Sandbox"}, } - return map[string]openapicommon.OpenAPIDefinition{ + defs := map[string]openapicommon.OpenAPIDefinition{ sandboxDefPrefix + "Sandbox": sandbox, sandboxDefPrefix + "SandboxList": list, } + // The action subresources (sandboxes/pause, /resume, /fork, /snapshot) + // exchange their own request and reply types. Every type reachable from a + // served resource needs a model here or InstallAPIGroup fails outright with + // "cannot find model definition for ...TypeMeta" — the subresource bodies + // embed metav1.TypeMeta, and openapinamer resolves them through this map, + // not through the Scheme. + // The action bodies embed metav1.TypeMeta, and the definition builder walks + // embedded field types looking for a model. Without this entry it aborts + // the whole group install with "cannot find model definition for + // ...meta.v1.TypeMeta" — a crash loop on rollout, not a degraded feature. + typeMeta := openapicommon.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: spec.StringOrArray{"object"}, + Properties: map[string]spec.Schema{ + "kind": stringSchema(), + "apiVersion": stringSchema(), + }, + }, + }, + } + // Registered under BOTH spellings: the namer keys models by Go package path, + // while the definition builder resolves embedded fields by their canonical + // ("friendly") OpenAPI name. Only one of them being present still aborts the + // group install. + defs["k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta"] = typeMeta + defs["io.k8s.apimachinery.pkg.apis.meta.v1.TypeMeta"] = typeMeta + for _, kind := range []string{ + "SandboxPauseOptions", + "SandboxResumeOptions", + "SandboxForkOptions", + "SandboxForkResult", + "SandboxSnapshotOptions", + "SandboxSnapshotResult", + } { + defs[sandboxDefPrefix+kind] = actionDefinition(kind) + } + return defs +} + +// actionDefinition is the model for one action-subresource body. Like the +// Sandbox model it is deliberately coarse: these types are never persisted, so +// per-field server-side-apply tracking buys nothing — the model exists so the +// TypeConverter can resolve the GVK at all. +func actionDefinition(kind string) openapicommon.OpenAPIDefinition { + ext := gvkExtension(kind) + // These bodies carry their own fields (count, ttlSeconds, children, ...). + // Declaring only kind/apiVersion would make every other field "unknown" to + // the field manager, so the interior is preserved rather than enumerated — + // the same trade the Sandbox model makes, and for the same reason: nothing + // here is persisted, so per-field tracking buys nothing. + ext["x-kubernetes-preserve-unknown-fields"] = true + return openapicommon.OpenAPIDefinition{ + Schema: spec.Schema{ + VendorExtensible: spec.VendorExtensible{Extensions: ext}, + SchemaProps: spec.SchemaProps{ + Type: spec.StringOrArray{"object"}, + Properties: map[string]spec.Schema{ + "kind": stringSchema(), + "apiVersion": stringSchema(), + }, + }, + }, + } } diff --git a/pkg/scale/apiserver/openapi_test.go b/pkg/scale/apiserver/openapi_test.go index a75388e..b2d2f92 100644 --- a/pkg/scale/apiserver/openapi_test.go +++ b/pkg/scale/apiserver/openapi_test.go @@ -18,7 +18,11 @@ import ( "testing" "k8s.io/apimachinery/pkg/util/managedfields" + genericapiserver "k8s.io/apiserver/pkg/server" + apiservercompatibility "k8s.io/apiserver/pkg/util/compatibility" + restclient "k8s.io/client-go/rest" "k8s.io/kube-openapi/pkg/builder3" + "k8s.io/kube-openapi/pkg/validation/spec" sandboxv1beta1 "github.com/cocoonstack/cocoon-sandbox-operator/api/v1beta1" ) @@ -57,3 +61,70 @@ func TestManagedFieldsTypeConverterResolvesSandbox(t *testing.T) { t.Fatalf("ObjectToTyped(SandboxList) must succeed: %v", err) } } + +// TestEveryServedTypeHasAnOpenAPIModel reproduces a real deployment failure: +// the action subresources were registered in the Scheme but had no OpenAPI +// model, and InstallAPIGroup then refused to start the server outright with +// +// unable to get openapi models: cannot find model definition for +// io.k8s.apimachinery.pkg.apis.meta.v1.TypeMeta +// +// because the subresource bodies embed metav1.TypeMeta and openapinamer +// resolves them through this map, not the Scheme. A missing model is not a +// degraded feature — it is a crash loop on rollout, so every served kind must +// be present here. +func TestEveryServedTypeHasAnOpenAPIModel(t *testing.T) { + defs := sandboxOpenAPIDefinitions(func(path string) spec.Ref { return spec.Ref{} }) + + for _, kind := range []string{ + "Sandbox", + "SandboxList", + "SandboxPauseOptions", + "SandboxResumeOptions", + "SandboxForkOptions", + "SandboxForkResult", + "SandboxSnapshotOptions", + "SandboxSnapshotResult", + } { + key := sandboxDefPrefix + kind + def, ok := defs[key] + if !ok { + t.Errorf("no OpenAPI model for %s; InstallAPIGroup will fail to start the apiserver", kind) + continue + } + gvks, ok := def.Schema.Extensions["x-kubernetes-group-version-kind"] + if !ok { + t.Errorf("%s has no x-kubernetes-group-version-kind; the managed-fields TypeConverter cannot map it", kind) + continue + } + entries, ok := gvks.([]interface{}) + if !ok || len(entries) != 1 { + t.Errorf("%s gvk extension = %v, want exactly one entry", kind, gvks) + continue + } + m, _ := entries[0].(map[string]interface{}) + if m["kind"] != kind { + t.Errorf("%s declares kind %v, want %s", kind, m["kind"], kind) + } + } +} + +// TestInstallSandboxAPISucceeds is the production startup path: main.go builds +// a GenericAPIServer and calls InstallSandboxAPI. An incomplete OpenAPI model +// graph fails HERE, and the binary then crash-loops on rollout — so this must +// be exercised locally, not discovered in the cluster. +func TestInstallSandboxAPISucceeds(t *testing.T) { + cfg := genericapiserver.NewConfig(Codecs) + cfg.EffectiveVersion = apiservercompatibility.DefaultBuildEffectiveVersion() + cfg.OpenAPIV3Config = NewOpenAPIV3Config() + cfg.ExternalAddress = "localhost:6443" + cfg.LoopbackClientConfig = &restclient.Config{Host: "localhost:6443"} + + server, err := cfg.Complete(nil).New("test-apiserver", genericapiserver.NewEmptyDelegate()) + if err != nil { + t.Fatalf("build generic apiserver: %v", err) + } + if err := InstallSandboxAPI(server, nil); err != nil { + t.Fatalf("InstallSandboxAPI: %v", err) + } +} diff --git a/pkg/scale/apiserver/scheme.go b/pkg/scale/apiserver/scheme.go index f320ead..27af73c 100644 --- a/pkg/scale/apiserver/scheme.go +++ b/pkg/scale/apiserver/scheme.go @@ -48,6 +48,16 @@ func init() { // conversions while keeping v1beta1 the served, prioritized version. internalGV := schema.GroupVersion{Group: sandboxv1beta1.GroupVersion.Group, Version: runtime.APIVersionInternal} Scheme.AddKnownTypes(internalGV, &sandboxv1beta1.Sandbox{}, &sandboxv1beta1.SandboxList{}) + // The action subresources' request/response bodies round-trip through the + // same pipeline, so they need the identity internal version too. + Scheme.AddKnownTypes(internalGV, + &sandboxv1beta1.SandboxPauseOptions{}, + &sandboxv1beta1.SandboxResumeOptions{}, + &sandboxv1beta1.SandboxForkOptions{}, + &sandboxv1beta1.SandboxForkResult{}, + &sandboxv1beta1.SandboxSnapshotOptions{}, + &sandboxv1beta1.SandboxSnapshotResult{}, + ) metav1.AddToGroupVersion(Scheme, sandboxv1beta1.GroupVersion) // The common request/response meta types (ListOptions, GetOptions, Status, // WatchEvent, ...) live at the "v1" options version the request pipeline diff --git a/pkg/scale/apiserver/storage_test.go b/pkg/scale/apiserver/storage_test.go index e49996d..74dcabf 100644 --- a/pkg/scale/apiserver/storage_test.go +++ b/pkg/scale/apiserver/storage_test.go @@ -107,3 +107,25 @@ func TestDelete_FailsLoudWithoutClaimID(t *testing.T) { assert.True(t, apierrors.IsInternalError(err), "expected an internal error, got %v", err) assert.False(t, store.released, "must not release when the sandboxd claim id is unknown") } + +// The lifecycle verbs are not exercised by these tests; they satisfy the +// SandboxStore contract so the fake stays a drop-in. +func (f *fakeStore) Pause(context.Context, string, string) error { return nil } +func (f *fakeStore) Resume(context.Context, string, string) error { return nil } +func (f *fakeStore) Fork(context.Context, string, string, int, int) ([]scale.Assignment, error) { + return nil, nil +} +func (f *fakeStore) Snapshot(context.Context, string, string, string) (scale.Snapshot, error) { + return scale.Snapshot{}, nil +} +func (f *fakeStore) Snapshots(context.Context, string) ([]scale.Snapshot, error) { return nil, nil } +func (f *fakeStore) DeleteSnapshot(context.Context, string, string) error { return nil } +func (f *fakeStore) ClaimSnapshot(context.Context, string, string, int) (scale.Assignment, error) { + return scale.Assignment{}, nil +} +func (f *fakeStore) Promote(context.Context, string, string, string) (scale.PoolKey, error) { + return scale.PoolKey{}, nil +} +func (f *fakeStore) Stats(context.Context, string, string) (scale.SandboxStats, error) { + return scale.SandboxStats{}, nil +} From 177132b13fd5a6844ff1829778b61840e99183b1 Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 26 Jul 2026 11:47:05 +0800 Subject: [PATCH 4/6] e2bcompat: serve the full sandbox lifecycle, and publish DNS-safe ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compat layer could create, list and delete; every other e2b verb 404'd, so an SDK could start a sandbox and then do nothing with it. Add pause, connect, fork, snapshots, metrics and templates. The status codes here are the contract, not decoration: the SDK derives pause()'s boolean from a 409 ("was already paused"), and connect returns 201 when it actually restored a paused sandbox versus 200 when it was already running. Both are asserted. "Already paused" is answered by the owning node, not by the synthesized read view. That view is republished on a ~30s cadence, so for up to half a minute after a pause the listed sandbox still reads Running — deciding from it let a second pause through with 204, telling the caller it had paused a sandbox that was already down. e2b's memory=false (a filesystem-only pause whose resume cold-boots) is rejected with a 400 rather than silently taking a full memory snapshot: the caller would get back a different sandbox than it asked for. Sandbox ids are published DNS-label safe. A sandboxd claim id is "sb_"+hex, and the SDK builds the in-sandbox envd host as "{port}-{sandboxID}.{domain}" — an underscore there yields a host that cannot resolve, so the sandbox would be created but unreachable. Both spellings are accepted on the way back in, and release always uses the raw claim id, which is the only one the node knows. --- pkg/e2bcompat/lifecycle.go | 375 ++++++++++++++++++++++++++++++++ pkg/e2bcompat/lifecycle_test.go | 326 +++++++++++++++++++++++++++ pkg/e2bcompat/sandboxid.go | 71 ++++++ pkg/e2bcompat/sandboxid_test.go | 79 +++++++ pkg/e2bcompat/server.go | 43 +++- pkg/e2bcompat/server_test.go | 31 ++- pkg/e2bcompat/types.go | 67 ++++++ 7 files changed, 979 insertions(+), 13 deletions(-) create mode 100644 pkg/e2bcompat/lifecycle.go create mode 100644 pkg/e2bcompat/lifecycle_test.go create mode 100644 pkg/e2bcompat/sandboxid.go create mode 100644 pkg/e2bcompat/sandboxid_test.go diff --git a/pkg/e2bcompat/lifecycle.go b/pkg/e2bcompat/lifecycle.go new file mode 100644 index 0000000..48974dc --- /dev/null +++ b/pkg/e2bcompat/lifecycle.go @@ -0,0 +1,375 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2bcompat + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" + + sandboxv1beta1 "github.com/cocoonstack/cocoon-sandbox-operator/api/v1beta1" + "github.com/cocoonstack/cocoon-sandbox-operator/pkg/scale" +) + +// pauseSandbox hibernates the sandbox: its memory is written out and the VM +// stops, so the cost is proportional to guest RAM. e2b's contract is specific +// about the already-paused case — the SDK reads 409 as "already paused" and +// returns false rather than raising — so that state is reported, not retried. +func (s *Server) pauseSandbox(w http.ResponseWriter, r *http.Request) { + var req SandboxPauseRequest + if err := decodeOptional(w, r, &req); err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + return + } + id := r.PathValue("sandboxID") + sb, err := s.lookup(r, id) + if err != nil { + s.writeLookupError(w, err, id, "pause") + return + } + if s.isPaused(r.Context(), sb) { + writeError(w, http.StatusConflict, fmt.Sprintf("sandbox %q is already paused", id)) + return + } + // memory=false asks for a filesystem-only snapshot whose resume cold-boots. + // The node's hibernate always captures memory, so honoring it would mean + // silently giving back a different sandbox than asked for. + if req.Memory != nil && !*req.Memory { + writeError(w, http.StatusBadRequest, + "filesystem-only pause (memory=false) is not supported; this backend always snapshots memory") + return + } + if err := s.store.Pause(r.Context(), sb.Status.NodeName, claimIDOf(sb)); err != nil { + s.opts.Log.Error(err, "e2b pause failed", "sandboxID", id) + writeError(w, http.StatusInternalServerError, "failed to pause the sandbox") + return + } + w.WriteHeader(http.StatusNoContent) +} + +// connectSandbox is the SDK's resume: it returns the sandbox's connection +// details, restoring it first when paused. 200 means it was already running, +// 201 that it was paused and got resumed — the SDK accepts either, and the +// distinction is what tells an operator whether a restore actually happened. +// A resume takes cocoon's mmap fast path (~55 ms), unlike the pause that +// preceded it. +func (s *Server) connectSandbox(w http.ResponseWriter, r *http.Request) { + var req ConnectSandbox + if err := decodeOptional(w, r, &req); err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + return + } + id := r.PathValue("sandboxID") + sb, err := s.lookup(r, id) + if err != nil { + s.writeLookupError(w, err, id, "connect") + return + } + status := http.StatusOK + if s.isPaused(r.Context(), sb) { + if err := s.store.Resume(r.Context(), sb.Status.NodeName, claimIDOf(sb)); err != nil { + s.opts.Log.Error(err, "e2b connect: resume failed", "sandboxID", id) + writeError(w, http.StatusInternalServerError, "failed to resume the sandbox") + return + } + status = http.StatusCreated + } + writeJSON(w, status, Sandbox{ + TemplateID: templateOf(sb), + SandboxID: publicID(claimIDOf(sb)), + ClientID: sb.Status.NodeName, + EnvdVersion: s.opts.EnvdVersion, + EnvdAccessToken: sb.Annotations[tokenAnnotation], + Domain: s.opts.Domain, + }) +} + +// forkSandbox branches the sandbox into count children. The parent is +// checkpointed in place and keeps running; every child is a fresh sandbox with +// its own id and lease. Per e2b's contract a partial failure is still a 201 +// carrying per-child detail — a non-201 means nothing was attempted — so the +// node's all-or-nothing fork is reported as a whole-request failure only when +// it rejects the request outright. +func (s *Server) forkSandbox(w http.ResponseWriter, r *http.Request) { + var req SandboxForkRequest + if err := decodeOptional(w, r, &req); err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + return + } + count := int32(1) + if req.Count != nil { + count = *req.Count + } + if count < 1 { + writeError(w, http.StatusBadRequest, fmt.Sprintf("count must be >= 1, got %d", count)) + return + } + id := r.PathValue("sandboxID") + sb, err := s.lookup(r, id) + if err != nil { + s.writeLookupError(w, err, id, "fork") + return + } + if s.isPaused(r.Context(), sb) { + writeError(w, http.StatusConflict, + fmt.Sprintf("sandbox %q is paused and cannot be forked; resume it first", id)) + return + } + children, err := s.store.Fork(r.Context(), sb.Status.NodeName, claimIDOf(sb), int(count), timeoutSeconds(req.Timeout)) + if err != nil { + s.opts.Log.Error(err, "e2b fork failed", "sandboxID", id, "count", count) + writeError(w, http.StatusInternalServerError, "failed to fork the sandbox") + return + } + template := templateOf(sb) + out := make([]SandboxForkResult, 0, len(children)) + for i := range children { + child := children[i] + out = append(out, SandboxForkResult{Sandbox: &Sandbox{ + TemplateID: template, + SandboxID: publicID(child.SandboxName), + ClientID: child.Node, + EnvdVersion: s.opts.EnvdVersion, + EnvdAccessToken: child.Token, + Domain: s.opts.Domain, + }}) + } + writeJSON(w, http.StatusCreated, out) +} + +// createSnapshot captures the sandbox's state as a checkpoint later sandboxes +// can branch from. The source keeps running. +func (s *Server) createSnapshot(w http.ResponseWriter, r *http.Request) { + var req SandboxSnapshotRequest + if err := decodeOptional(w, r, &req); err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + return + } + id := r.PathValue("sandboxID") + sb, err := s.lookup(r, id) + if err != nil { + s.writeLookupError(w, err, id, "snapshot") + return + } + snap, err := s.store.Snapshot(r.Context(), sb.Status.NodeName, claimIDOf(sb), req.Name) + if err != nil { + s.opts.Log.Error(err, "e2b snapshot failed", "sandboxID", id) + writeError(w, http.StatusInternalServerError, "failed to snapshot the sandbox") + return + } + writeJSON(w, http.StatusCreated, snapshotInfo(snap)) +} + +// listSnapshots reports the checkpoints across the fleet's nodes. +func (s *Server) listSnapshots(w http.ResponseWriter, r *http.Request) { + nodes, err := s.nodesWithSandboxes(r) + if err != nil { + s.opts.Log.Error(err, "e2b list snapshots: resolve nodes failed") + writeError(w, http.StatusInternalServerError, "failed to list snapshots") + return + } + out := []SnapshotInfo{} + for _, node := range nodes { + snaps, err := s.store.Snapshots(r.Context(), node) + if err != nil { + // One unreachable node must not blank the whole listing. + s.opts.Log.Error(err, "e2b list snapshots: node failed", "node", node) + continue + } + for _, snap := range snaps { + out = append(out, snapshotInfo(snap)) + } + } + writeJSON(w, http.StatusOK, out) +} + +// deleteSnapshot removes a checkpoint. e2b addresses snapshots as templates on +// delete, so this serves DELETE /templates/{templateID} too. +func (s *Server) deleteSnapshot(w http.ResponseWriter, r *http.Request) { + snapshotID := r.PathValue("snapshotID") + nodes, err := s.nodesWithSandboxes(r) + if err != nil { + s.opts.Log.Error(err, "e2b delete snapshot: resolve nodes failed") + writeError(w, http.StatusInternalServerError, "failed to delete the snapshot") + return + } + // Checkpoints are node-local and the id does not name its node, so the + // delete is offered to each node; a node that does not hold it reports + // success (delete is idempotent), which keeps this safe to fan out. + for _, node := range nodes { + if err := s.store.DeleteSnapshot(r.Context(), node, snapshotID); err != nil { + s.opts.Log.Error(err, "e2b delete snapshot: node failed", "node", node, "snapshotID", snapshotID) + } + } + w.WriteHeader(http.StatusNoContent) +} + +// sandboxMetrics reports one sandbox's resource usage. e2b's schema requires +// every field, so all are emitted; the ones this backend cannot measure are +// reported as zero rather than invented (see SandboxStats). +func (s *Server) sandboxMetrics(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("sandboxID") + sb, err := s.lookup(r, id) + if err != nil { + s.writeLookupError(w, err, id, "metrics") + return + } + st, err := s.store.Stats(r.Context(), sb.Status.NodeName, claimIDOf(sb)) + if err != nil { + s.opts.Log.Error(err, "e2b metrics failed", "sandboxID", id) + writeError(w, http.StatusInternalServerError, "failed to read sandbox metrics") + return + } + at := st.MeasuredAt + if at.IsZero() { + at = time.Now() + } + writeJSON(w, http.StatusOK, []SandboxMetric{{ + Timestamp: at.UTC().Format(time.RFC3339), + TimestampUnix: at.Unix(), + CPUCount: int32(st.CPUCount), //nolint:gosec // a size tier's CPU count is single digits + MemUsed: st.MemUsedBytes, + MemTotal: st.MemTotalBytes, + }}) +} + +// listTemplates reports the pools this fleet can serve claims from. e2b's +// templates are build artifacts with their own lifecycle; the equivalent here +// is the set of warm-pool keys nodes advertise, which is what a caller can +// actually pass as templateID on create. +func (s *Server) listTemplates(w http.ResponseWriter, r *http.Request) { + nodes, err := s.inventories(r) + if err != nil { + s.opts.Log.Error(err, "e2b list templates failed") + writeError(w, http.StatusInternalServerError, "failed to list templates") + return + } + seen := map[string]struct{}{} + out := []Template{} + for _, inv := range nodes { + for _, pc := range inv.Pools { + if pc.Template == "" { + continue + } + if _, dup := seen[pc.Template]; dup { + continue + } + seen[pc.Template] = struct{}{} + out = append(out, Template{ + TemplateID: pc.Template, + BuildID: pc.Template, + Public: true, + Aliases: []string{}, + Names: []string{pc.Template}, + EnvdVersion: s.opts.EnvdVersion, + }) + } + } + writeJSON(w, http.StatusOK, out) +} + +// inventories returns every node's published inventory, the fleet view the +// pool-derived surfaces (templates, snapshot listing) are assembled from. +func (s *Server) inventories(r *http.Request) ([]*scale.NodeInventory, error) { + if s.opts.Inventory == nil { + return nil, errors.New("e2bcompat: no inventory source configured") + } + nodes, err := s.opts.Inventory.ListNodes(r.Context()) + if err != nil { + return nil, err + } + out := make([]*scale.NodeInventory, 0, len(nodes)) + for _, node := range nodes { + inv, err := s.opts.Inventory.NodeInventory(r.Context(), node) + if err != nil { + // A partitioned node is skipped, not fatal — the same rule the + // aggregated read path applies. + s.opts.Log.Error(err, "e2b: node inventory unavailable", "node", node) + continue + } + out = append(out, inv) + } + return out, nil +} + +// nodesWithSandboxes lists the nodes a checkpoint could live on. +func (s *Server) nodesWithSandboxes(r *http.Request) ([]string, error) { + if s.opts.Inventory == nil { + return nil, errors.New("e2bcompat: no inventory source configured") + } + return s.opts.Inventory.ListNodes(r.Context()) +} + +// decodeOptional decodes a JSON body that the schema allows to be absent. An +// empty body leaves the target at its zero value rather than failing, which is +// what pause and fork require. +func decodeOptional(w http.ResponseWriter, r *http.Request, out any) error { + if r.Body == nil { + return nil + } + err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)).Decode(out) + if errors.Is(err, io.EOF) { + return nil + } + return err +} + +// isPaused reports whether the sandbox is hibernated, asking its owning node +// rather than trusting the synthesized read view. +// +// The label on a listed sandbox comes from NodeInventory, which the node +// republishes on a ~30 s cadence, so it lags a pause by up to that long. e2b's +// contract needs a strong answer here — the SDK reads 409 as "already paused" +// and returns false — and a stale Running label turns that into a second 204, +// telling the caller it paused a sandbox that was already down. The per-sandbox +// node read is authoritative and costs one round trip to a node we are about to +// call anyway. +// +// A node that cannot be reached falls back to the cached label: degrading to the +// eventually-consistent answer is better than failing the request outright. +func (s *Server) isPaused(ctx context.Context, sb *sandboxv1beta1.Sandbox) bool { + if node, id := sb.Status.NodeName, claimIDOf(sb); node != "" && id != "" { + if st, err := s.store.Stats(ctx, node, id); err == nil { + return st.Paused + } + } + return sb.Labels[scale.PhaseLabel] == phaseHibernated +} + +// claimIDOf reports the node-local claim id the store's verbs address. +func claimIDOf(sb *sandboxv1beta1.Sandbox) string { + return sb.Annotations[scale.ClaimIDAnnotation] +} + +// timeoutSeconds resolves an optional TTL to the e2b default. +func timeoutSeconds(v *int32) int { + if v == nil { + return DefaultTimeoutSeconds + } + return int(*v) +} + +// snapshotInfo renders a checkpoint in e2b's snapshot shape. +func snapshotInfo(snap scale.Snapshot) SnapshotInfo { + names := []string{} + if snap.Name != "" { + names = append(names, snap.Name) + } + return SnapshotInfo{SnapshotID: snap.ID, Names: names} +} diff --git a/pkg/e2bcompat/lifecycle_test.go b/pkg/e2bcompat/lifecycle_test.go new file mode 100644 index 0000000..7fa4ff5 --- /dev/null +++ b/pkg/e2bcompat/lifecycle_test.go @@ -0,0 +1,326 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2bcompat + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + sandboxv1beta1 "github.com/cocoonstack/cocoon-sandbox-operator/api/v1beta1" + "github.com/cocoonstack/cocoon-sandbox-operator/pkg/scale" +) + +// lifecycleStore records which verb the compat layer routed where, so the tests +// assert the translation rather than any node behavior. +type lifecycleStore struct { + fakeStore + + pausedNode, pausedID string + resumedNode, resumedID string + forkedID string + forkCount int + forkChildren []scale.Assignment + snapshotName string + snapshot scale.Snapshot + err error + + // nodePaused is what the owning node reports for Stats().Paused — the + // authoritative answer isPaused now consults instead of the cached label. + nodePaused bool +} + +func (f *lifecycleStore) Stats(context.Context, string, string) (scale.SandboxStats, error) { + return scale.SandboxStats{Paused: f.nodePaused, CPUCount: 1, MemTotalBytes: 512 << 20}, nil +} + +func (f *lifecycleStore) Pause(_ context.Context, node, id string) error { + f.pausedNode, f.pausedID = node, id + return f.err +} + +func (f *lifecycleStore) Resume(_ context.Context, node, id string) error { + f.resumedNode, f.resumedID = node, id + return f.err +} + +func (f *lifecycleStore) Fork(_ context.Context, _, id string, count, _ int) ([]scale.Assignment, error) { + f.forkedID, f.forkCount = id, count + return f.forkChildren, f.err +} + +func (f *lifecycleStore) Snapshot(_ context.Context, _, _, name string) (scale.Snapshot, error) { + f.snapshotName = name + return f.snapshot, f.err +} + +// pausedSandbox is a sandbox as the node publishes it while hibernated. +func pausedSandbox(name, claimID, node, image, token string) sandboxv1beta1.Sandbox { + sb := liveSandbox(name, claimID, node, image, token) + sb.Labels = map[string]string{scale.PhaseLabel: phaseHibernated} + return sb +} + +// nodeReportsPaused makes the fake's owning node report the sandbox hibernated, +// which is what isPaused actually consults. +func nodeReportsPaused(s *lifecycleStore) { s.nodePaused = true } + +// TestPauseAlreadyPausedIs409 is load-bearing for the SDK, not cosmetic: +// e2b's pause() returns a boolean, and it derives false ("was already paused") +// from a 409. Reporting anything else turns an idempotent no-op into an error +// the caller has to interpret. +func TestPauseAlreadyPausedIs409(t *testing.T) { + store := &lifecycleStore{} + nodeReportsPaused(store) + store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img", "tok")} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/pause", ``, testKey) + if w.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 (the SDK reads it as already-paused): %s", w.Code, w.Body.String()) + } + if store.pausedID != "" { + t.Errorf("Pause was routed to the node for an already-paused sandbox (id %q)", store.pausedID) + } +} + +// TestPauseRoutesToOwningNode: the verb must address the node-local claim id on +// the owning node, never the public id the client spelled. +func TestPauseRoutesToOwningNode(t *testing.T) { + store := &lifecycleStore{} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/pause", ``, testKey) + if w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want 204: %s", w.Code, w.Body.String()) + } + if store.pausedNode != "node-a" || store.pausedID != "sb_abc" { + t.Errorf("Pause(%q, %q), want (node-a, sb_abc) — the raw claim id, not the published one", + store.pausedNode, store.pausedID) + } +} + +// TestPauseFilesystemOnlyIsRejected: e2b's memory=false asks for a +// filesystem-only snapshot whose resume cold-boots. The node always captures +// memory, so honoring the flag would hand back a different sandbox than the +// caller asked for — say so instead of silently doing the other thing. +func TestPauseFilesystemOnlyIsRejected(t *testing.T) { + store := &lifecycleStore{} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/pause", `{"memory":false}`, testKey) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 for an unsupported pause mode", w.Code) + } + if store.pausedID != "" { + t.Error("the sandbox was paused anyway; an unsupported mode must not fall through") + } +} + +// TestConnectRunningIs200 / TestConnectPausedIs201: e2b's connect IS the SDK's +// resume. 200 means it was already running, 201 that a restore actually +// happened — the SDK accepts either, and the distinction is what tells an +// operator whether the mmap restore path ran. +func TestConnectRunningIs200(t *testing.T) { + store := &lifecycleStore{} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/connect", `{"timeout":30}`, testKey) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 for a running sandbox: %s", w.Code, w.Body.String()) + } + if store.resumedID != "" { + t.Errorf("a running sandbox was resumed (id %q); connect must be a no-op then", store.resumedID) + } + var got Sandbox + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.SandboxID != "sb-abc" { + t.Errorf("sandboxID = %q, want the DNS-safe published id", got.SandboxID) + } +} + +func TestConnectPausedIs201AndResumes(t *testing.T) { + store := &lifecycleStore{} + nodeReportsPaused(store) + store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img", "tok")} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/connect", `{"timeout":30}`, testKey) + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201 when a paused sandbox is resumed: %s", w.Code, w.Body.String()) + } + if store.resumedNode != "node-a" || store.resumedID != "sb_abc" { + t.Errorf("Resume(%q, %q), want (node-a, sb_abc)", store.resumedNode, store.resumedID) + } +} + +// TestForkPausedIs409 mirrors e2b: a paused source cannot be forked, and the +// message must say to resume it first rather than failing opaquely. +func TestForkPausedIs409(t *testing.T) { + store := &lifecycleStore{} + nodeReportsPaused(store) + store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img", "tok")} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/fork", `{"count":2}`, testKey) + if w.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 for forking a paused sandbox", w.Code) + } + if store.forkedID != "" { + t.Error("fork was routed to the node for a paused source") + } +} + +// TestForkReturnsPerChildResults: e2b's fork reply is one entry per child, each +// carrying its own new sandbox id — children are fresh sandboxes, not replicas +// of the parent's identity. +func TestForkReturnsPerChildResults(t *testing.T) { + store := &lifecycleStore{ + forkChildren: []scale.Assignment{ + {SandboxName: "sb_c1", Node: "node-a", Token: "t1"}, + {SandboxName: "sb_c2", Node: "node-a", Token: "t2"}, + }, + } + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/fork", `{"count":2}`, testKey) + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201: %s", w.Code, w.Body.String()) + } + var got []SandboxForkResult + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d results, want one per child", len(got)) + } + if store.forkCount != 2 { + t.Errorf("fork count routed = %d, want 2", store.forkCount) + } + for i, want := range []string{"sb-c1", "sb-c2"} { + if got[i].Sandbox == nil || got[i].Sandbox.SandboxID != want { + t.Errorf("child %d = %+v, want published id %q", i, got[i].Sandbox, want) + } + } +} + +// TestForkDefaultsToOneChild: count is optional in e2b's schema. +func TestForkDefaultsToOneChild(t *testing.T) { + store := &lifecycleStore{forkChildren: []scale.Assignment{{SandboxName: "sb_c1", Node: "node-a"}}} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + h := newTestServer(t, store) + + if w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/fork", ``, testKey); w.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201 for a body-less fork: %s", w.Code, w.Body.String()) + } + if store.forkCount != 1 { + t.Errorf("fork count = %d, want the schema default of 1", store.forkCount) + } +} + +// TestSnapshotReturns201WithID: the source keeps running; the reply carries the +// checkpoint id a later branch is taken from. +func TestSnapshotReturns201WithID(t *testing.T) { + store := &lifecycleStore{snapshot: scale.Snapshot{ID: "ck_1234", Name: "before-migration", Node: "node-a"}} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/snapshots", `{"name":"before-migration"}`, testKey) + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201: %s", w.Code, w.Body.String()) + } + var got SnapshotInfo + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.SnapshotID != "ck_1234" { + t.Errorf("snapshotID = %q, want the checkpoint id", got.SnapshotID) + } + if len(got.Names) != 1 || got.Names[0] != "before-migration" { + t.Errorf("names = %v, want the requested label echoed", got.Names) + } + if store.snapshotName != "before-migration" { + t.Errorf("name routed = %q, want it passed through", store.snapshotName) + } +} + +// TestLifecycleVerbsOnUnknownSandboxAre404 keeps the SDK's not-found branches +// working: it raises SandboxNotFound on 404 rather than a generic error. +func TestLifecycleVerbsOnUnknownSandboxAre404(t *testing.T) { + for _, path := range []string{ + "/sandboxes/sb-missing/pause", + "/sandboxes/sb-missing/connect", + "/sandboxes/sb-missing/fork", + "/sandboxes/sb-missing/snapshots", + } { + t.Run(path, func(t *testing.T) { + h := newTestServer(t, &lifecycleStore{}) + if w := do(t, h, http.MethodPost, path, `{}`, testKey); w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404: %s", w.Code, w.Body.String()) + } + }) + } +} + +// TestPauseTrustsTheNodeNotTheStaleView reproduces a bug found in the cluster: +// the read view is synthesized from NodeInventory, which the node republishes +// on a ~30 s cadence, so for up to half a minute after a pause the listed +// sandbox still carries phase=Running. Deciding "already paused" from that +// label let a second pause through with 204, telling the caller it had paused a +// sandbox that was already down — and the e2b SDK turns that 204 into "yes, I +// paused it". The owning node is authoritative and must win. +func TestPauseTrustsTheNodeNotTheStaleView(t *testing.T) { + store := &lifecycleStore{} + store.nodePaused = true // the node has it hibernated ... + sb := liveSandbox("s1", "sb_abc", "node-a", "img", "tok") + sb.Labels = map[string]string{scale.PhaseLabel: "Running"} // ... but the view still says Running + store.items = []sandboxv1beta1.Sandbox{sb} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/pause", ``, testKey) + if w.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409: a stale Running label must not mask the node's hibernated state", w.Code) + } + if store.pausedID != "" { + t.Error("pause was routed to the node for an already-hibernated sandbox") + } +} + +// TestConnectTrustsTheNodeNotTheStaleView is the same hazard on resume: a stale +// Running label would return 200 without restoring, handing the caller +// connection details for a VM that is not running. +func TestConnectTrustsTheNodeNotTheStaleView(t *testing.T) { + store := &lifecycleStore{} + store.nodePaused = true + sb := liveSandbox("s1", "sb_abc", "node-a", "img", "tok") + sb.Labels = map[string]string{scale.PhaseLabel: "Running"} + store.items = []sandboxv1beta1.Sandbox{sb} + h := newTestServer(t, store) + + w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/connect", `{"timeout":30}`, testKey) + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201: the node says paused, so connect must actually resume", w.Code) + } + if store.resumedID != "sb_abc" { + t.Errorf("resumed %q, want sb_abc — a stale label must not skip the restore", store.resumedID) + } +} diff --git a/pkg/e2bcompat/sandboxid.go b/pkg/e2bcompat/sandboxid.go new file mode 100644 index 0000000..e267b29 --- /dev/null +++ b/pkg/e2bcompat/sandboxid.go @@ -0,0 +1,71 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2bcompat + +import "strings" + +// A sandboxd claim id is "sb_" + hex (sandboxd pool/claim.go), whose underscore +// is not legal in a DNS label. The e2b SDK derives the in-sandbox envd host as +// "{port}-{sandboxID}.{domain}", so an id carrying an underscore produces a host +// that cannot resolve — the sandbox would be created but unreachable. +// +// The compat surface therefore publishes a DNS-safe rendering of the claim id +// and accepts either form on the way back in. The mapping only rewrites +// characters that are illegal in a DNS label, so it is stable, and it round +// trips for every id sandboxd actually mints (whose only illegal character is +// that one underscore). + +// publicID renders a node-local claim id as a DNS-label-safe sandbox id, the +// form handed to e2b clients. +func publicID(claimID string) string { + if !needsRewrite(claimID) { + return claimID + } + var b strings.Builder + b.Grow(len(claimID)) + for _, r := range strings.ToLower(claimID) { + if isDNSSafe(r) { + b.WriteRune(r) + continue + } + b.WriteByte('-') + } + return b.String() +} + +// matchesID reports whether a live sandbox's claim id is the one a client asked +// for, accepting both the raw claim id and its published DNS-safe rendering so +// an id observed through either surface keeps working. +func matchesID(claimID, requested string) bool { + if claimID == "" || requested == "" { + return false + } + return claimID == requested || publicID(claimID) == requested +} + +func needsRewrite(s string) bool { + for _, r := range s { + if !isDNSSafe(r) { + return true + } + } + return false +} + +// isDNSSafe reports whether r is legal inside a DNS label (RFC 1123): lowercase +// alphanumerics and the hyphen. +func isDNSSafe(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' +} diff --git a/pkg/e2bcompat/sandboxid_test.go b/pkg/e2bcompat/sandboxid_test.go new file mode 100644 index 0000000..1c2bc68 --- /dev/null +++ b/pkg/e2bcompat/sandboxid_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2bcompat + +import "testing" + +// TestPublicIDIsDNSLabelSafe is the reason this mapping exists: the e2b SDK +// builds the in-sandbox envd host as "{port}-{sandboxID}.{domain}". A sandboxd +// claim id is "sb_"+hex, and that underscore is illegal in a DNS label — an id +// published raw yields a host that cannot resolve, so the sandbox is created +// but unreachable. +func TestPublicIDIsDNSLabelSafe(t *testing.T) { + for _, tt := range []struct { + name string + in string + want string + }{ + {"sandboxd claim id", "sb_0123456789abcdef", "sb-0123456789abcdef"}, + {"already safe is untouched", "sb-0123456789abcdef", "sb-0123456789abcdef"}, + {"uppercase is lowered", "SB_ABCDEF", "sb-abcdef"}, + {"empty stays empty", "", ""}, + } { + t.Run(tt.name, func(t *testing.T) { + got := publicID(tt.in) + if got != tt.want { + t.Fatalf("publicID(%q) = %q, want %q", tt.in, got, tt.want) + } + for _, r := range got { + if !isDNSSafe(r) { + t.Errorf("publicID(%q) = %q contains %q, illegal in a DNS label", tt.in, got, r) + } + } + }) + } +} + +// TestMatchesIDAcceptsBothForms: an id observed through either surface has to +// keep working. A client that saw the published (DNS-safe) id sends that back; +// something reading the raw claim id from the node sends the original. +func TestMatchesIDAcceptsBothForms(t *testing.T) { + const claim = "sb_0123456789abcdef" + + if !matchesID(claim, claim) { + t.Error("raw claim id must match itself") + } + if !matchesID(claim, "sb-0123456789abcdef") { + t.Error("published DNS-safe id must match its claim id") + } + if matchesID(claim, "sb-ffffffffffffffff") { + t.Error("a different id must not match") + } +} + +// TestMatchesIDRejectsEmpty guards the lookup loop: a sandbox whose node has +// not published a claim id has an empty annotation, and an empty request path +// is equally meaningless. Matching those would return an arbitrary sandbox. +func TestMatchesIDRejectsEmpty(t *testing.T) { + if matchesID("", "") { + t.Error("empty must not match empty") + } + if matchesID("sb_0123456789abcdef", "") { + t.Error("empty request must not match a real id") + } + if matchesID("", "sb-0123456789abcdef") { + t.Error("a sandbox with no claim id must not match a real request") + } +} diff --git a/pkg/e2bcompat/server.go b/pkg/e2bcompat/server.go index eccb4a0..5d2d3e8 100644 --- a/pkg/e2bcompat/server.go +++ b/pkg/e2bcompat/server.go @@ -58,6 +58,9 @@ const ( DefaultTimeoutSeconds = 15 // apiKeyHeader is the header the e2b SDKs authenticate with. apiKeyHeader = "X-API-KEY" + // phaseHibernated is the phase label value a node publishes for a paused + // sandbox (vk-cocoon-sandbox inventory publisher). + phaseHibernated = "Hibernated" ) // Options configures the compat server. @@ -69,11 +72,10 @@ type Options struct { // envd host as "{port}-{sandboxID}.{domain}". Empty leaves it unset, which // makes the SDK fall back to its configured E2B_DOMAIN/E2B_SANDBOX_URL. // - // Note that a sandboxID here is the node's sandboxd claim id, which is not - // guaranteed to be a valid DNS label (it carries an underscore). The - // subdomain form therefore only works behind a proxy that routes on the - // E2b-Sandbox-Id / E2b-Sandbox-Port headers the SDK also sends; otherwise - // point the SDK at the sandbox with E2B_SANDBOX_URL. + // The sandbox ids published here are DNS-label safe (see sandboxid.go), so + // that host form resolves; the proxy in front of the sandboxes still has to + // route it, either on the host or on the E2b-Sandbox-Id / E2b-Sandbox-Port + // headers the SDK also sends. Domain string // EnvdVersion overrides DefaultEnvdVersion. EnvdVersion string @@ -86,6 +88,11 @@ type Options struct { // SizeClass pins the warm-pool size axis for compat claims (default // "small"); e2b's NewSandbox carries no size selector. SizeClass string + // Inventory enumerates the fleet's nodes and their advertised pools. It is + // required by the surfaces that are fleet-wide rather than sandbox-scoped + // (template listing, snapshot listing); without it those report an error + // instead of an empty list, so a missing dependency cannot read as "none". + Inventory scale.InventorySource // Log receives request-level errors. Log logr.Logger } @@ -139,6 +146,18 @@ func (s *Server) Handler() http.Handler { mux.Handle("DELETE /sandboxes/{sandboxID}", s.auth(http.HandlerFunc(s.deleteSandbox))) mux.Handle("POST /sandboxes/{sandboxID}/timeout", s.auth(http.HandlerFunc(s.setTimeout))) mux.Handle("POST /sandboxes/{sandboxID}/refreshes", s.auth(http.HandlerFunc(s.refresh))) + + // Lifecycle verbs. + mux.Handle("POST /sandboxes/{sandboxID}/pause", s.auth(http.HandlerFunc(s.pauseSandbox))) + mux.Handle("POST /sandboxes/{sandboxID}/connect", s.auth(http.HandlerFunc(s.connectSandbox))) + mux.Handle("POST /sandboxes/{sandboxID}/fork", s.auth(http.HandlerFunc(s.forkSandbox))) + mux.Handle("POST /sandboxes/{sandboxID}/snapshots", s.auth(http.HandlerFunc(s.createSnapshot))) + mux.Handle("GET /snapshots", s.auth(http.HandlerFunc(s.listSnapshots))) + mux.Handle("GET /sandboxes/{sandboxID}/metrics", s.auth(http.HandlerFunc(s.sandboxMetrics))) + mux.Handle("GET /templates", s.auth(http.HandlerFunc(s.listTemplates))) + mux.Handle("GET /v2/templates", s.auth(http.HandlerFunc(s.listTemplates))) + // e2b addresses a snapshot as a template on delete. + mux.Handle("DELETE /templates/{snapshotID}", s.auth(http.HandlerFunc(s.deleteSnapshot))) return mux } @@ -204,7 +223,7 @@ func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusCreated, Sandbox{ TemplateID: req.TemplateID, - SandboxID: assignment.SandboxName, + SandboxID: publicID(assignment.SandboxName), ClientID: assignment.Node, EnvdVersion: s.opts.EnvdVersion, EnvdAccessToken: assignment.Token, @@ -252,8 +271,12 @@ func (s *Server) deleteSandbox(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) return } - if err := s.store.Release(r.Context(), node, id); err != nil { - s.opts.Log.Error(err, "e2b delete: release failed", "sandboxID", id, "node", node) + // Release against the raw node-local claim id, never the id as the client + // spelled it: the published id is a DNS-safe rendering, and sandboxd knows + // only the original. + claimID := sb.Annotations[scale.ClaimIDAnnotation] + if err := s.store.Release(r.Context(), node, claimID); err != nil { + s.opts.Log.Error(err, "e2b delete: release failed", "sandboxID", id, "claimID", claimID, "node", node) writeError(w, http.StatusInternalServerError, "failed to release the sandbox") return } @@ -306,7 +329,7 @@ func (s *Server) lookup(r *http.Request, id string) (*sandboxv1beta1.Sandbox, er return nil, err } for i := range list.Items { - if list.Items[i].Annotations[scale.ClaimIDAnnotation] == id { + if matchesID(list.Items[i].Annotations[scale.ClaimIDAnnotation], id) { return &list.Items[i], nil } } @@ -332,7 +355,7 @@ func (s *Server) detailFor(sb *sandboxv1beta1.Sandbox) SandboxDetail { } return SandboxDetail{ TemplateID: templateOf(sb), - SandboxID: sb.Annotations[scale.ClaimIDAnnotation], + SandboxID: publicID(sb.Annotations[scale.ClaimIDAnnotation]), ClientID: sb.Status.NodeName, StartedAt: started.UTC().Format(time.RFC3339), EndAt: started.Add(DefaultTimeoutSeconds * time.Second).UTC().Format(time.RFC3339), diff --git a/pkg/e2bcompat/server_test.go b/pkg/e2bcompat/server_test.go index 298eb90..afa416f 100644 --- a/pkg/e2bcompat/server_test.go +++ b/pkg/e2bcompat/server_test.go @@ -140,8 +140,11 @@ func TestCreateClaimsFromTemplatePool(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { t.Fatalf("decode response: %v", err) } - if got.SandboxID != "sb_abc123" { - t.Errorf("sandboxID = %q, want the node-assigned claim id", got.SandboxID) + // The published id is the DNS-label-safe rendering of the node's claim id, + // not the raw one: the SDK builds "{port}-{sandboxID}.{domain}", and the + // claim id's underscore would make that host unresolvable. + if got.SandboxID != "sb-abc123" { + t.Errorf("sandboxID = %q, want the DNS-safe rendering of the claim id", got.SandboxID) } if got.TemplateID != "registry/rt:24.04" { t.Errorf("templateID = %q, want it echoed back", got.TemplateID) @@ -295,7 +298,7 @@ func TestGetReportsDetail(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { t.Fatalf("decode: %v", err) } - if got.SandboxID != "sb_one" || got.TemplateID != "registry/rt:24.04" || got.ClientID != "node-a" { + if got.SandboxID != "sb-one" || got.TemplateID != "registry/rt:24.04" || got.ClientID != "node-a" { t.Errorf("detail = %+v, want the live sandbox's id/template/node", got) } if got.State != StateRunning { @@ -378,3 +381,25 @@ func TestErrorBodyCarriesMessage(t *testing.T) { t.Errorf("error = %+v, want code 400 and a message", got) } } + +// The lifecycle verbs are not exercised by these tests; they satisfy the +// SandboxStore contract so the fake stays a drop-in. +func (f *fakeStore) Pause(context.Context, string, string) error { return nil } +func (f *fakeStore) Resume(context.Context, string, string) error { return nil } +func (f *fakeStore) Fork(context.Context, string, string, int, int) ([]scale.Assignment, error) { + return nil, nil +} +func (f *fakeStore) Snapshot(context.Context, string, string, string) (scale.Snapshot, error) { + return scale.Snapshot{}, nil +} +func (f *fakeStore) Snapshots(context.Context, string) ([]scale.Snapshot, error) { return nil, nil } +func (f *fakeStore) DeleteSnapshot(context.Context, string, string) error { return nil } +func (f *fakeStore) ClaimSnapshot(context.Context, string, string, int) (scale.Assignment, error) { + return scale.Assignment{}, nil +} +func (f *fakeStore) Promote(context.Context, string, string, string) (scale.PoolKey, error) { + return scale.PoolKey{}, nil +} +func (f *fakeStore) Stats(context.Context, string, string) (scale.SandboxStats, error) { + return scale.SandboxStats{}, nil +} diff --git a/pkg/e2bcompat/types.go b/pkg/e2bcompat/types.go index 59bf29f..4bd9644 100644 --- a/pkg/e2bcompat/types.go +++ b/pkg/e2bcompat/types.go @@ -85,6 +85,73 @@ type SandboxRefreshRequest struct { Duration *int32 `json:"duration,omitempty"` } +// SandboxPauseRequest is the POST /sandboxes/{id}/pause body. memory=false +// takes a filesystem-only snapshot, whose resume cold-boots. +type SandboxPauseRequest struct { + Memory *bool `json:"memory,omitempty"` +} + +// ConnectSandbox is the POST /sandboxes/{id}/connect body — the SDK's resume. +// timeout is required by the schema; TTL is only ever extended. +type ConnectSandbox struct { + Timeout int32 `json:"timeout"` +} + +// SandboxForkRequest is the POST /sandboxes/{id}/fork body. +type SandboxForkRequest struct { + Timeout *int32 `json:"timeout,omitempty"` + Count *int32 `json:"count,omitempty"` +} + +// SandboxForkResult is one entry of the fork reply: exactly one of Sandbox or +// Error is set, so a partial failure still returns 201 with per-child detail. +type SandboxForkResult struct { + Sandbox *Sandbox `json:"sandbox,omitempty"` + Error *APIError `json:"error,omitempty"` +} + +// SandboxSnapshotRequest is the POST /sandboxes/{id}/snapshots body. +type SandboxSnapshotRequest struct { + Name string `json:"name,omitempty"` +} + +// SnapshotInfo is the snapshot create/list reply. +type SnapshotInfo struct { + SnapshotID string `json:"snapshotID"` + Names []string `json:"names"` +} + +// SandboxMetric is one metrics sample. Every field is required by the schema, +// so all are always emitted; the SDK reads the deprecated `timestamp`. +type SandboxMetric struct { + Timestamp string `json:"timestamp"` + TimestampUnix int64 `json:"timestampUnix"` + CPUCount int32 `json:"cpuCount"` + CPUUsedPct float32 `json:"cpuUsedPct"` + MemUsed int64 `json:"memUsed"` + MemTotal int64 `json:"memTotal"` + MemCache int64 `json:"memCache"` + DiskUsed int64 `json:"diskUsed"` + DiskTotal int64 `json:"diskTotal"` +} + +// Template is the templates-listing entry. +type Template struct { + TemplateID string `json:"templateID"` + BuildID string `json:"buildID"` + CPUCount int32 `json:"cpuCount"` + MemoryMB int32 `json:"memoryMB"` + DiskSizeMB int32 `json:"diskSizeMB"` + Public bool `json:"public"` + Aliases []string `json:"aliases"` + Names []string `json:"names"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + SpawnCount int64 `json:"spawnCount"` + BuildCount int32 `json:"buildCount"` + EnvdVersion string `json:"envdVersion"` +} + // APIError is the e2b error envelope. The SDK surfaces `message` on failures. type APIError struct { Code int32 `json:"code"` From e20e7fdeb5224258ec5e28207bb9775031117059 Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 26 Jul 2026 11:47:05 +0800 Subject: [PATCH 5/6] apiserver: wire the e2b surface, and stop one drain from taking both replicas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2b server needs the fleet's node inventory to answer the surfaces that are fleet-wide rather than sandbox-scoped (template and snapshot listing); without it those would report an error rather than an empty list, so a missing dependency cannot read as "none". Also add a PodDisruptionBudget and a hostname topology spread. An unavailable APIService does not just fail `kubectl get sandboxes` — it degrades discovery for the whole cluster — and the two replicas had nothing stopping a single node drain from evicting both, nor anything stopping the scheduler from placing them on one node to begin with. The spread is DoNotSchedule rather than preferred: both replicas on one node is worse than one replica, because it turns a routine drain into a cluster-wide discovery outage. minAvailable rather than maxUnavailable so the guarantee survives a scale to 1, where maxUnavailable: 1 would still permit evicting the only replica. --- cmd/sandbox-apiserver/main.go | 5 ++-- config/apiserver/deployment.yaml | 12 ++++++++ config/apiserver/poddisruptionbudget.yaml | 35 +++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 config/apiserver/poddisruptionbudget.yaml diff --git a/cmd/sandbox-apiserver/main.go b/cmd/sandbox-apiserver/main.go index 9ce30e5..7d0cb9e 100644 --- a/cmd/sandbox-apiserver/main.go +++ b/cmd/sandbox-apiserver/main.go @@ -250,7 +250,7 @@ func run() error { // its own listener: an e2b SDK Create becomes the identical node-local claim, // and what it creates stays visible to `kubectl get sandboxes`. if o.E2BAPI { - if err := startE2BServer(ctx, o, store); err != nil { + if err := startE2BServer(ctx, o, store, invSource); err != nil { return err } } @@ -345,7 +345,7 @@ func startWarmPoolDriver(ctx context.Context, restCfg *restclient.Config, token // stops it when ctx is cancelled. It shares the aggregated apiserver's store, so // there is no second source of truth: a claim made here is the same node-local // claim, released the same way, and listed by the same scatter-gather read. -func startE2BServer(ctx context.Context, o *options, store scale.SandboxStore) error { +func startE2BServer(ctx context.Context, o *options, store scale.SandboxStore, inv scale.InventorySource) error { keys, err := o.e2bAPIKeys() if err != nil { return err @@ -353,6 +353,7 @@ func startE2BServer(ctx context.Context, o *options, store scale.SandboxStore) e srv, err := e2bcompat.NewServer(store, e2bcompat.Options{ Namespace: o.E2BNamespace, Domain: o.E2BDomain, + Inventory: inv, APIKeys: keys, AllowAnonymous: o.E2BAllowAnonymous, Log: ctrl.Log.WithName("e2b"), diff --git a/config/apiserver/deployment.yaml b/config/apiserver/deployment.yaml index 70b1858..53e1f1d 100644 --- a/config/apiserver/deployment.yaml +++ b/config/apiserver/deployment.yaml @@ -42,6 +42,18 @@ spec: app.kubernetes.io/component: aggregated-apiserver spec: serviceAccountName: cocoon-sandbox-apiserver + # An unavailable APIService does not just fail `kubectl get sandboxes`: it + # fails discovery for the whole cluster, so the two replicas must never + # share a failure domain. Spreading is required, not preferred — running + # both on one node is worse than running one, because it turns a single + # drain into a cluster-wide discovery outage. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: cocoon-sandbox-apiserver containers: - name: cocoon-sandbox-apiserver image: ko://github.com/cocoonstack/cocoon-sandbox-operator/cmd/sandbox-apiserver # placeholder value, replaced by deployment scripts diff --git a/config/apiserver/poddisruptionbudget.yaml b/config/apiserver/poddisruptionbudget.yaml new file mode 100644 index 0000000..4999c0e --- /dev/null +++ b/config/apiserver/poddisruptionbudget.yaml @@ -0,0 +1,35 @@ +# Copyright 2026 The CocoonStack Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The aggregated apiserver backs an APIService, and an unavailable APIService +# degrades discovery for the entire cluster — not just this group. A node drain +# (an upgrade, a scale-in, a maintenance cordon) must therefore never be allowed +# to take both replicas at once. +# +# minAvailable: 1 rather than maxUnavailable: 1 on purpose: it holds even if the +# Deployment is ever scaled to 1, where maxUnavailable: 1 would still permit +# evicting the only replica. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: cocoon-sandbox-apiserver + namespace: cocoon-sandbox-system + labels: + app.kubernetes.io/name: cocoon-sandbox-apiserver + app.kubernetes.io/component: aggregated-apiserver +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: cocoon-sandbox-apiserver From cfcf6508cb8b258aac21b94af45484498b774f0f Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 26 Jul 2026 11:47:05 +0800 Subject: [PATCH 6/6] docs: explain where a snapshot lives, and ship a runnable walk-through MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds docs/snapshot-placement.md: the three placement tiers, why the fast path IS the storage choice (hardlink + reflink + mmap all require the checkpoint and the new VM to share one local filesystem), and why JuiceFS is the wrong answer twice over — it has no reflink, so every clone becomes a network read, and it needs object storage anyway, at which point the existing s3 backend is simpler. States the durability boundary plainly, in the doc, the roadmap and the SSOT: a checkpoint lives on one node and is lost with that node's disk. Peer healing addresses "cannot reach", not "lost". Replication is recorded in ROADMAP.md as explicitly not built — checkpoints are branch points for agent workloads, not backups, and replicating them would cost write amplification on every capture. examples/lifecycle/example.go exercises every operation on both surfaces against a live cluster, so the output doubles as acceptance evidence. It uses a REST client rather than controller-runtime's SubResource helper because the action subresources have different request and response types, which that helper cannot express. It also demonstrates the two behaviours callers must handle: reads are eventually consistent, and the published sandbox id is DNS-safe. --- README.md | 73 ++++- ROADMAP.md | 13 + docs/snapshot-placement.md | 138 ++++++++ examples/lifecycle/example.go | 577 ++++++++++++++++++++++++++++++++++ 4 files changed, 800 insertions(+), 1 deletion(-) create mode 100644 docs/snapshot-placement.md create mode 100644 examples/lifecycle/example.go diff --git a/README.md b/README.md index cccc110..9cd2849 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,75 @@ spec: For low-latency acquisition, define a `SandboxTemplate` + `SandboxWarmPool` and create `SandboxClaim`s — see [examples/](examples/). +### Lifecycle: pause, resume, fork, snapshot + +Beyond create/delete, a delivered sandbox supports four action verbs, served as +**subresources** so the standard `agents.x-k8s.io` schema stays untouched (an +unmodified upstream client keeps working): + +| subresource | body | effect | +|---|---|---| +| `sandboxes/pause` | `SandboxPauseOptions` | snapshots guest memory and stops the VM | +| `sandboxes/resume` | `SandboxResumeOptions` | restores it via cocoon's mmap fast path | +| `sandboxes/fork` | `SandboxForkOptions{count,ttlSeconds}` | branches N children; the source keeps running | +| `sandboxes/snapshot` | `SandboxSnapshotOptions{name}` | captures a checkpoint later sandboxes branch from | + +```bash +kubectl create --raw \ + /apis/agents.x-k8s.io/v1beta1/namespaces/default/sandboxes/my-sandbox/snapshot \ + -f - <<<'{"apiVersion":"agents.x-k8s.io/v1beta1","kind":"SandboxSnapshotOptions","name":"before-migration"}' +``` + +**These verbs are not uniformly fast.** `resume` takes cocoon's mmap restore +path and a fork's children clone node-locally, but `pause` and `snapshot` write +the guest's memory out, so they cost time proportional to its size. Where a +checkpoint lives, and what happens when its node cannot serve a branch, is in +[docs/snapshot-placement.md](docs/snapshot-placement.md). + +### Runnable walk-through + +[`examples/lifecycle/example.go`](examples/lifecycle/example.go) exercises +**every** operation on **both** surfaces against a live cluster — create, get, +list, snapshot, fork, pause, resume, delete over the Kubernetes API, then the +same lifecycle plus templates, metrics, snapshot listing, timeout and keepalive +over the e2b REST API: + +```bash +go run ./examples/lifecycle \ + -kubeconfig ~/.kube/config -namespace default \ + -e2b-url http://localhost:8080 -e2b-key "$E2B_API_KEY" +``` + +It discovers a template from the fleet's advertised warm pools, so no image +argument is needed. Omit `-e2b-url` to run only the Kubernetes half. Each step +prints what it did, so the output doubles as acceptance evidence: + +``` +=== Kubernetes API === + create Sandbox default/example-219526000 + get node=cocoon-bd25-sandboxd claimID=sb_77ace349e7cc1db6 + snapshot snapshotID=ck_92799687f14e8fea on node=cocoon-bd25-sandboxd + fork child[0] sandboxID=sb_06d1b2438dcc1d1b node=cocoon-bd25-sandboxd + pause took 315ms (proportional to guest memory) + resume took 109ms (mmap restore fast path) + +=== e2b-compatible REST API === + create sandboxID=sb-175124667cfa1281 envdVersion=0.4.0 + metrics cpuCount=1 memTotal=5.36870912e+08 + pause 409 on repeat — the already-paused contract holds + connect 201 — restored via the mmap fast path +``` + +Two behaviors it demonstrates deliberately, because callers must handle them: + +- **Reads are eventually consistent.** `Create` returns as soon as the + node-local claim completes, but `List`/`Get` are served from `NodeInventory`, + which nodes republish on a ~30s cadence — so a read immediately after a + create legitimately returns `NotFound`. The example polls; so should you. +- **The published sandbox id is DNS-label safe.** The e2b SDK builds the + in-sandbox host as `{port}-{sandboxID}.{domain}`, so the node's raw claim id + (`sb_...`) is rendered as `sb-...` on the e2b surface. + ## Use it with the e2b SDK The aggregated apiserver can also serve an e2b-compatible REST surface, so an @@ -564,7 +633,9 @@ make test-race make generate # CRDs, RBAC, deepcopy (idempotent) ``` -See [docs/](docs/) for API, configuration, and runtime details. +See [docs/](docs/) for API, configuration, and runtime details, including +[snapshot placement](docs/snapshot-placement.md) — where a checkpoint lives, +how a branch reaches it from another node, and what that costs. ## Community diff --git a/ROADMAP.md b/ROADMAP.md index 5ebc07e..d61b0d5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -27,6 +27,19 @@ move any of these forward are welcome. `sandboxes.agents.x-k8s.io` from live per-node inventory streams rather than inventory re-list. +## Medium term (continued) + +- **Checkpoints that survive node loss.** A checkpoint currently lives on the + node that took it: peer healing moves a record to a node that cannot reach + it, but nothing replicates one, so losing that node's disk loses the + checkpoint (see [docs/snapshot-placement.md](docs/snapshot-placement.md)). + Making them durable needs asynchronous replication to N peers plus a + placement policy that tracks replica sets and repairs under-replication. + **Not built, and deliberately so** — checkpoints are branch points for agent + workloads, not backups, and replication would cost write amplification on + every capture. Until this lands, promote anything that must outlive its node + to a template and distribute it as one. + ## Longer term - **Million-sandbox validation.** Exercise the full L0–L3 design at fleet diff --git a/docs/snapshot-placement.md b/docs/snapshot-placement.md new file mode 100644 index 0000000..3b6453f --- /dev/null +++ b/docs/snapshot-placement.md @@ -0,0 +1,138 @@ +# Snapshot placement: where a checkpoint lives, and what happens when its node cannot serve it + +A snapshot in this system is a **checkpoint**: an immutable capture of a +sandbox's state that new sandboxes branch from. Taking one is +`POST sandboxes/{name}/snapshot` (or e2b's `POST /sandboxes/{id}/snapshots`); +branching one produces a fresh sandbox with its own id and lease. + +This document is about the part that is not obvious: a checkpoint is **created +on one node's local disk**, and the node that eventually has to serve a branch +of it may not be that node. + +## The constraint everything else follows from + +Cloning a sandbox is fast — 28–75 ms — because nothing is copied: + +| step | mechanism | +|---|---| +| guest memory | `os.Link` hardlink of the snapshot's `memory-range-*` files | +| writable disks | reflink (`FICLONE`), concurrent, `NoSync` | +| restore | mmap copy-on-write — the guest maps the file, pages privatize on write | + +All three require the checkpoint and the new VM's run directory to be **on the +same local filesystem**, and that filesystem to support reflink (XFS/btrfs). +Hardlinks do not cross filesystems. `FICLONE` does not work on NFS, Lustre, or +any FUSE object-storage mount. mmap over a network filesystem turns a page +fault into a network round trip. + +So the fast path is not an optimization layered on top of the storage choice — +**it is the storage choice**. Any design that puts checkpoints on shared network +storage has already given up the number that makes the system worth using. + +That is the whole reason the design below prefers *moving the placement to the +data* over *moving the data to the placement*. + +## Three tiers + +### L1 — local hit (the normal case) + +A checkpoint stays on the node that took it. A branch of it issued to that node +reads it from the local store and clones on the local reflink path. + +Nothing in this tier is new; it is what a single-node deployment already does. +It is listed because it is the case that must stay fast, and the two tiers below +exist only to avoid degrading it. + +### L2 — gossip + redirect (the cross-node case) + +Nodes gossip the checkpoint ids they hold, alongside the warm-pool counts and +promoted-template hashes they already gossip. A node asked to branch a +checkpoint it does not hold answers with a **redirect** to a node that does — +the same `200` + `redirect: [addrs]` contract a warm-miss claim already uses, +and the client retries there with `no_redirect: true`. + +The record does not move. The clone still happens on a node whose disk already +holds the data, on its local fast path. Cross-node correctness is bought with +one extra round trip and **zero** bytes transferred. + +This is the tier that makes "snapshot on node A, branch from anywhere" work. + +### L3 — peer heal (the last resort) + +Redirect cannot answer when no owner can serve: the owning node is gone, +draining, or out of capacity. The choice then is between paying one transfer +and failing the request. + +With `checkpoint_peer_heal` enabled, the node pulls the record from an owner +over sandboxd's own peer transport, publishes it into its local store through +the same atomic staging path a locally-created checkpoint takes, and then +serves the branch locally. + +The cost is paid **once**: afterwards this node is itself an owner, gossips the +record, and serves later branches of it at L1 speed. + +It is **off by default**. Trading a transfer for availability is an operator's +decision, not a default. + +#### Why the transport is sandboxd's own + +The peer transport is deliberately **not** cocoon's image/snapshot mover. +cocoon's transfer is bound to its VM store layout and its own addressing; +reusing it would tie a checkpoint's mobility to the engine's release cycle and +to a layout that has nothing to do with the record format being moved. + +sandboxd's transport moves exactly one thing — a store record: an `export/` +directory plus its `meta.json` — between two sandboxd nodes that already +authenticate to each other with the fleet `api_token`, over the control-plane +port they already share. It is a tar stream (`GET /v1/checkpoints/{id}/blob`), +so a pull is one round trip and never materializes the record twice. Entry +names are validated against path traversal and the stream is size-capped: a +peer is authenticated, but an authenticated peer is still not allowed to write +outside the destination this node chose. + +## Why not JuiceFS (or any shared filesystem) + +Two independent reasons, either of which is sufficient: + +1. **It destroys the fast path.** JuiceFS does not support reflink. Putting + checkpoints on it turns every clone from "hardlink + reflink, 28 ms, zero + bytes moved" into a full read over the network. The system's headline number + is the thing being traded away. + +2. **It does not solve the stated problem.** JuiceFS is a metadata engine plus + an object store. With no S3 available, running JuiceFS means first running + MinIO/SeaweedFS — at which point object storage exists, and the + already-implemented `s3` store backend is strictly simpler than adding a + POSIX layer on top of it. + +The same reasoning rules out NFS and Lustre mounts for the checkpoint directory. +If a shared backend is ever required, use the `s3` backend (which keeps a local +cache generation and therefore preserves the local-clone step) rather than a +network POSIX mount. + +## Durability: what this design does *not* give you + +A checkpoint lives on one node. **If that node's disk is lost, the checkpoint is +lost.** L3 heals a node that cannot *reach* a record; it does not replicate one. + +This is a deliberate, scoped decision, not an oversight — checkpoints here are +branch points for agent workloads, not backups. Making them survive node loss +needs asynchronous replication to N peers and a placement policy that tracks +replica sets; that work is recorded in [ROADMAP.md](../ROADMAP.md) and is +explicitly not built. + +Operationally: **treat a checkpoint as durable only while its node is.** If a +checkpoint matters beyond that, promote it to a template +(`POST /v1/sandboxes/{id}/promote`) and distribute it as one, or export it out +of the fleet. + +## Configuration + +| setting | default | effect | +|---|---|---| +| `checkpoint_dir` | `/checkpoints` | where records live. Keep it on the same local reflink filesystem as the VM run directory. | +| `checkpoint_store.kind` | `dir` | `s3` switches to object storage with a local cache generation. | +| `checkpoint_peer_heal` | `false` | enables L3. Requires a mesh; ignored without one. | + +L1 and L2 need no configuration: gossip and redirect are active whenever a mesh +is configured. diff --git a/examples/lifecycle/example.go b/examples/lifecycle/example.go new file mode 100644 index 0000000..b1fbd45 --- /dev/null +++ b/examples/lifecycle/example.go @@ -0,0 +1,577 @@ +// Copyright 2026 The CocoonStack Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command example exercises every sandbox operation this operator serves, +// through BOTH of its surfaces, against a live cluster: +// +// - the Kubernetes API (controller-runtime client + the action subresources) +// - the e2b-compatible REST API (the same warm pools, no Kubernetes client) +// +// It is a runnable acceptance walk-through, not a unit test: each step prints +// what it did so the output doubles as evidence. +// +// go run ./examples/lifecycle \ +// -kubeconfig ~/.kube/config -namespace default \ +// -template \ +// -e2b-url http://localhost:8080 -e2b-key +// +// Omit -e2b-url to run only the Kubernetes half; omit -template and it is +// discovered from the fleet's advertised warm pools. +// +// Two behaviors of this system shape the code below and are worth reading +// before copying it: +// +// - Reads are eventually consistent. Sandbox objects are synthesized from +// per-node NodeInventory, which nodes republish on a ~30s cadence, so a +// just-created sandbox is not immediately in List/Get. waitVisible below is +// how a caller is expected to handle that. +// - Latency is not uniform. resume takes cocoon's mmap restore fast path and +// fork clones a node-local snapshot, but pause and snapshot write the +// guest's memory out and therefore cost time proportional to its size. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + + sandboxv1beta1 "github.com/cocoonstack/cocoon-sandbox-operator/api/v1beta1" + extv1beta1 "github.com/cocoonstack/cocoon-sandbox-operator/extensions/api/v1beta1" +) + +// visibilityTimeout bounds the wait for the read view to publish a sandbox. +// It is generous relative to the ~30s inventory cadence so a single slow +// publish does not fail the walk-through. +const visibilityTimeout = 90 * time.Second + +type options struct { + kubeconfig string + namespace string + template string + e2bURL string + e2bKey string + keep bool +} + +func main() { + var o options + flag.StringVar(&o.kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "path to a kubeconfig; empty uses in-cluster config") + flag.StringVar(&o.namespace, "namespace", "default", "namespace to create sandboxes in") + flag.StringVar(&o.template, "template", "", "container image selecting the warm pool; empty discovers one from NodeInventory") + flag.StringVar(&o.e2bURL, "e2b-url", "", "base URL of the e2b-compatible surface; empty skips the e2b half") + flag.StringVar(&o.e2bKey, "e2b-key", "", "X-API-KEY for the e2b surface") + flag.BoolVar(&o.keep, "keep", false, "leave the created sandboxes running instead of deleting them") + flag.Parse() + + if err := run(context.Background(), o); err != nil { + fmt.Fprintln(os.Stderr, "FAILED:", err) + os.Exit(1) + } + fmt.Println("\nAll operations completed.") +} + +func run(ctx context.Context, o options) error { + scheme, err := newScheme() + if err != nil { + return err + } + c, err := newClient(o.kubeconfig, scheme) + if err != nil { + return fmt.Errorf("build Kubernetes client: %w", err) + } + rc, err := newRESTClient(o.kubeconfig, scheme) + if err != nil { + return fmt.Errorf("build REST client: %w", err) + } + if o.template == "" { + if o.template, err = discoverTemplate(ctx, c); err != nil { + return err + } + fmt.Printf("discovered template from the fleet: %s\n", o.template) + } + if err := runKubernetes(ctx, c, rc, o); err != nil { + return fmt.Errorf("kubernetes surface: %w", err) + } + if o.e2bURL == "" { + fmt.Println("\n-e2b-url not set; skipping the e2b surface.") + return nil + } + if err := runE2B(ctx, o); err != nil { + return fmt.Errorf("e2b surface: %w", err) + } + return nil +} + +// ---------------------------------------------------------------- Kubernetes + +// newClient builds a controller-runtime client that knows this operator's +// types. Any Kubernetes client works — client-go, the dynamic client, or +// kubectl; nothing here is specific to controller-runtime. +func loadConfig(kubeconfig string) (*rest.Config, error) { + if kubeconfig == "" { + return rest.InClusterConfig() + } + return clientcmd.BuildConfigFromFlags("", kubeconfig) +} + +func newScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + if err := sandboxv1beta1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := extv1beta1.AddToScheme(scheme); err != nil { + return nil, err + } + return scheme, nil +} + +func newClient(kubeconfig string, scheme *runtime.Scheme) (client.Client, error) { + cfg, err := loadConfig(kubeconfig) + if err != nil { + return nil, err + } + return client.New(cfg, client.Options{Scheme: scheme}) +} + +// discoverTemplate reads the fleet's advertised warm pools and returns a +// template that actually has capacity, so the walk-through does not depend on +// a hard-coded image. +func discoverTemplate(ctx context.Context, c client.Client) (string, error) { + var inventories extv1beta1.NodeInventoryList + if err := c.List(ctx, &inventories); err != nil { + return "", fmt.Errorf("list NodeInventory: %w", err) + } + for _, inv := range inventories.Items { + for _, pool := range inv.Pools { + if pool.Template != "" { + return pool.Template, nil + } + } + } + return "", errors.New("no node advertises a warm pool; pass -template explicitly") +} + +func runKubernetes(ctx context.Context, c client.Client, rc rest.Interface, o options) error { + section("Kubernetes API") + + name := fmt.Sprintf("example-%d", time.Now().UnixNano()%1e9) + sb := &sandboxv1beta1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: o.namespace}, + } + sb.Spec.PodTemplate.Spec.Containers = []corev1.Container{{Name: "agent", Image: o.template}} + + // 1. Create — a claim against a warm pool, not a scheduling decision. + if err := c.Create(ctx, sb); err != nil { + return fmt.Errorf("create Sandbox: %w", err) + } + step("create", "Sandbox %s/%s", o.namespace, name) + + // 2. Wait until the read view publishes it, then Get and List. + live, err := waitVisible(ctx, c, o.namespace, name) + if err != nil { + return err + } + step("get", "node=%s claimID=%s", live.Status.NodeName, live.Annotations[claimIDAnnotation]) + + var list sandboxv1beta1.SandboxList + if err := c.List(ctx, &list, client.InNamespace(o.namespace)); err != nil { + return fmt.Errorf("list Sandboxes: %w", err) + } + step("list", "%d sandbox(es) in %s", len(list.Items), o.namespace) + + // 3. snapshot — an immutable checkpoint; the source keeps running. + snap := &sandboxv1beta1.SandboxSnapshotResult{} + if err := post(ctx, rc, o.namespace, name, "snapshot", + &sandboxv1beta1.SandboxSnapshotOptions{Name: "example-checkpoint"}, snap); err != nil { + return fmt.Errorf("snapshot: %w", err) + } + step("snapshot", "snapshotID=%s on node=%s", snap.SnapshotID, snap.NodeName) + + // 4. fork — the source is checkpointed in place and keeps running; each + // child is a brand-new sandbox with its own id and lease. + forked := &sandboxv1beta1.SandboxForkResult{} + if err := post(ctx, rc, o.namespace, name, "fork", + &sandboxv1beta1.SandboxForkOptions{Count: 2, TTLSeconds: 600}, forked); err != nil { + return fmt.Errorf("fork: %w", err) + } + for i, child := range forked.Children { + step("fork", "child[%d] sandboxID=%s node=%s", i, child.SandboxID, child.NodeName) + } + + // 5. pause — writes the guest's memory out and stops the VM, so this is + // the slow verb: its cost is proportional to guest RAM. + start := time.Now() + if err := post(ctx, rc, o.namespace, name, "pause", &sandboxv1beta1.SandboxPauseOptions{}, nil); err != nil { + return fmt.Errorf("pause: %w", err) + } + step("pause", "took %s (proportional to guest memory)", time.Since(start).Round(time.Millisecond)) + + // 6. resume — cocoon's mmap restore fast path, and idempotent on a + // sandbox that is already running. + start = time.Now() + if err := post(ctx, rc, o.namespace, name, "resume", &sandboxv1beta1.SandboxResumeOptions{}, nil); err != nil { + return fmt.Errorf("resume: %w", err) + } + step("resume", "took %s (mmap restore fast path)", time.Since(start).Round(time.Millisecond)) + + // 7. Delete — releases the claim back to the node's pool. + if o.keep { + step("delete", "skipped (-keep)") + return nil + } + if err := c.Delete(ctx, live); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete Sandbox: %w", err) + } + step("delete", "released %s/%s", o.namespace, name) + return nil +} + +// claimIDAnnotation carries the node-local claim id of a delivered sandbox. +const claimIDAnnotation = "sandbox.cocoonstack.io/claim-id" + +// waitVisible polls until the synthesized read view publishes the sandbox. +// Create returns as soon as the node-local claim completes, but List/Get are +// served from NodeInventory, which is republished on a ~30s cadence — so a +// caller that reads immediately after creating must expect a NotFound. +func waitVisible(ctx context.Context, c client.Client, ns, name string) (*sandboxv1beta1.Sandbox, error) { + deadline := time.Now().Add(visibilityTimeout) + for { + var sb sandboxv1beta1.Sandbox + err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &sb) + if err == nil { + return &sb, nil + } + if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("get Sandbox: %w", err) + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("sandbox %s/%s not visible within %s", ns, name, visibilityTimeout) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(3 * time.Second): + } + } +} + +// post invokes an action subresource. These are POST-only verbs (the +// pods/eviction shape), which is why they are not fields on SandboxSpec: the +// standard agent-sandbox schema stays untouched, so an unmodified upstream +// client keeps working against this server. +// +// A raw REST client is used rather than controller-runtime's SubResource +// helper because these actions have DIFFERENT request and response types +// (SandboxForkOptions in, SandboxForkResult out); the helper decodes the reply +// back into the object it was given, which cannot express that. +func post(ctx context.Context, rc rest.Interface, ns, name, sub string, body, out runtime.Object) error { + req := rc.Post(). + Namespace(ns). + Resource("sandboxes"). + Name(name). + SubResource(sub). + Body(body) + if out == nil { + return req.Do(ctx).Error() + } + return req.Do(ctx).Into(out) +} + +// newRESTClient builds a REST client for the sandboxes group, the transport the +// action subresources are invoked over. +func newRESTClient(kubeconfig string, scheme *runtime.Scheme) (rest.Interface, error) { + cfg, err := loadConfig(kubeconfig) + if err != nil { + return nil, err + } + gv := sandboxv1beta1.GroupVersion + cfg.GroupVersion = &gv + cfg.APIPath = "/apis" + cfg.NegotiatedSerializer = serializer.NewCodecFactory(scheme).WithoutConversion() + return rest.RESTClientFor(cfg) +} + +// ----------------------------------------------------------------- e2b REST + +func runE2B(ctx context.Context, o options) error { + section("e2b-compatible REST API") + e := &e2bClient{base: strings.TrimRight(o.e2bURL, "/"), key: o.e2bKey} + + if err := e.health(ctx); err != nil { + return err + } + step("health", "reachable") + + // 1. Templates — the pools this fleet can serve claims from; a templateID + // from here is what create accepts. + var templates []map[string]any + if err := e.do(ctx, http.MethodGet, "/templates", nil, &templates); err != nil { + return fmt.Errorf("list templates: %w", err) + } + step("templates", "%d available", len(templates)) + + // 2. Create. + var created map[string]any + if err := e.do(ctx, http.MethodPost, "/sandboxes", + map[string]any{"templateID": o.template, "timeout": 600}, &created); err != nil { + return fmt.Errorf("create: %w", err) + } + id, _ := created["sandboxID"].(string) + step("create", "sandboxID=%s envdVersion=%v", id, created["envdVersion"]) + + // The published id is a DNS-label-safe rendering of the node's claim id: + // the SDK builds "{port}-{sandboxID}.{domain}", and an underscore there + // would produce a host that cannot resolve. + if strings.Contains(id, "_") { + return fmt.Errorf("sandboxID %q is not DNS-label safe", id) + } + + // 3. List and Get. + var listed []map[string]any + if err := e.do(ctx, http.MethodGet, "/sandboxes", nil, &listed); err != nil { + return fmt.Errorf("list: %w", err) + } + step("list", "%d sandbox(es)", len(listed)) + + if err := e.waitVisible(ctx, id); err != nil { + return err + } + step("get", "%s is in the read view", id) + + // 4. Metrics. + var metrics []map[string]any + if err := e.do(ctx, http.MethodGet, "/sandboxes/"+id+"/metrics", nil, &metrics); err != nil { + return fmt.Errorf("metrics: %w", err) + } + if len(metrics) > 0 { + step("metrics", "cpuCount=%v memTotal=%v", metrics[0]["cpuCount"], metrics[0]["memTotal"]) + } + + // 5. Snapshot, then list snapshots. + var snap map[string]any + if err := e.do(ctx, http.MethodPost, "/sandboxes/"+id+"/snapshots", + map[string]any{"name": "example-e2b-snap"}, &snap); err != nil { + return fmt.Errorf("snapshot: %w", err) + } + step("snapshot", "snapshotID=%v", snap["snapshotID"]) + + var snaps []map[string]any + if err := e.do(ctx, http.MethodGet, "/snapshots", nil, &snaps); err != nil { + return fmt.Errorf("list snapshots: %w", err) + } + step("snapshots", "%d checkpoint(s) fleet-wide", len(snaps)) + + // 6. Fork — one result per child, each a new sandbox. + var forks []map[string]any + if err := e.do(ctx, http.MethodPost, "/sandboxes/"+id+"/fork", + map[string]any{"count": 2, "timeout": 600}, &forks); err != nil { + return fmt.Errorf("fork: %w", err) + } + step("fork", "%d child sandbox(es)", len(forks)) + + // 7. Pause, and prove the already-paused contract: the SDK reads 409 as + // "was already paused" and returns false rather than raising. + if code, err := e.status(ctx, http.MethodPost, "/sandboxes/"+id+"/pause", nil); err != nil { + return err + } else if code != http.StatusNoContent { + return fmt.Errorf("pause returned %d, want 204", code) + } + step("pause", "204") + + if code, err := e.status(ctx, http.MethodPost, "/sandboxes/"+id+"/pause", nil); err != nil { + return err + } else if code != http.StatusConflict { + return fmt.Errorf("repeated pause returned %d, want 409 (already paused)", code) + } + step("pause", "409 on repeat — the already-paused contract holds") + + // 8. Connect is the SDK's resume: 201 when it actually restored a paused + // sandbox, 200 when it was already running. + if code, err := e.status(ctx, http.MethodPost, "/sandboxes/"+id+"/connect", + map[string]any{"timeout": 600}); err != nil { + return err + } else if code != http.StatusCreated { + return fmt.Errorf("connect on a paused sandbox returned %d, want 201", code) + } + step("connect", "201 — restored via the mmap fast path") + + if code, err := e.status(ctx, http.MethodPost, "/sandboxes/"+id+"/connect", + map[string]any{"timeout": 600}); err != nil { + return err + } else if code != http.StatusOK { + return fmt.Errorf("connect on a running sandbox returned %d, want 200", code) + } + step("connect", "200 — already running, no restore") + + // 9. setTimeout and the keepalive. + if code, err := e.status(ctx, http.MethodPost, "/sandboxes/"+id+"/timeout", + map[string]any{"timeout": 900}); err != nil { + return err + } else if code/100 != 2 { + return fmt.Errorf("timeout returned %d", code) + } + step("timeout", "extended") + + if code, err := e.status(ctx, http.MethodPost, "/sandboxes/"+id+"/refreshes", + map[string]any{"duration": 60}); err != nil { + return err + } else if code/100 != 2 { + return fmt.Errorf("refreshes returned %d", code) + } + step("refreshes", "keepalive accepted") + + // 10. Delete. + if o.keep { + step("delete", "skipped (-keep)") + return nil + } + if code, err := e.status(ctx, http.MethodDelete, "/sandboxes/"+id, nil); err != nil { + return err + } else if code != http.StatusNoContent { + return fmt.Errorf("delete returned %d, want 204", code) + } + step("delete", "released %s", id) + return nil +} + +// e2bClient is a minimal client for the e2b REST contract. The real e2b SDKs +// (JS and Python) speak exactly this and need no changes — point E2B_API_URL at +// the server. This exists because there is no official Go SDK. +type e2bClient struct { + base string + key string + hc http.Client +} + +func (e *e2bClient) health(ctx context.Context) error { + code, err := e.status(ctx, http.MethodGet, "/health", nil) + if err != nil { + return err + } + if code/100 != 2 { + return fmt.Errorf("health returned %d", code) + } + return nil +} + +// waitVisible polls until the read view publishes the sandbox — the same +// eventual consistency the Kubernetes surface has, for the same reason. +func (e *e2bClient) waitVisible(ctx context.Context, id string) error { + deadline := time.Now().Add(visibilityTimeout) + for { + code, err := e.status(ctx, http.MethodGet, "/sandboxes/"+id, nil) + if err != nil { + return err + } + if code == http.StatusOK { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("sandbox %s not visible within %s", id, visibilityTimeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(3 * time.Second): + } + } +} + +func (e *e2bClient) request(ctx context.Context, method, path string, body any) (*http.Request, error) { + var rdr io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return nil, err + } + rdr = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, method, e.base+path, rdr) + if err != nil { + return nil, err + } + if e.key != "" { + req.Header.Set("X-API-KEY", e.key) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return req, nil +} + +// do performs a request and decodes a 2xx JSON reply into out. +func (e *e2bClient) do(ctx context.Context, method, path string, body, out any) error { + req, err := e.request(ctx, method, path, body) + if err != nil { + return err + } + resp, err := e.hc.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + payload, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return err + } + if resp.StatusCode/100 != 2 { + return fmt.Errorf("%s %s: HTTP %d: %s", method, path, resp.StatusCode, strings.TrimSpace(string(payload))) + } + if out == nil || len(payload) == 0 { + return nil + } + return json.Unmarshal(payload, out) +} + +// status performs a request and returns only its status code, for the verbs +// whose contract IS the status code. +func (e *e2bClient) status(ctx context.Context, method, path string, body any) (int, error) { + req, err := e.request(ctx, method, path, body) + if err != nil { + return 0, err + } + resp, err := e.hc.Do(req) + if err != nil { + return 0, err + } + defer func() { _ = resp.Body.Close() }() + _, _ = io.Copy(io.Discard, resp.Body) + return resp.StatusCode, nil +} + +// -------------------------------------------------------------------- output + +func section(name string) { fmt.Printf("\n=== %s ===\n", name) } + +func step(verb, format string, args ...any) { + fmt.Printf(" %-10s %s\n", verb, fmt.Sprintf(format, args...)) +}