From ee9a466819a6b502a7fec4194e9c760fa8754444 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 03:22:06 +0000 Subject: [PATCH 01/17] feat(img): pure OCI-loader param derivations (ref, arch, ext4 size) --- lib/hyper/img/oci_loader/params.ex | 44 ++++++++++++++++++ test/hyper/img/oci_loader/params_test.exs | 54 +++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 lib/hyper/img/oci_loader/params.ex create mode 100644 test/hyper/img/oci_loader/params_test.exs diff --git a/lib/hyper/img/oci_loader/params.ex b/lib/hyper/img/oci_loader/params.ex new file mode 100644 index 0000000..253f6ee --- /dev/null +++ b/lib/hyper/img/oci_loader/params.ex @@ -0,0 +1,44 @@ +defmodule Hyper.Img.OciLoader.Params do + @moduledoc """ + Pure derivations for `Hyper.Img.OciLoader`: registry-ref validation, the + Hyper-arch -> Go/OCI-arch name mapping `skopeo` expects, and the ext4 image + size for a rootfs of a given content size. No I/O — every function here is a + total function of its arguments, which is why this is the unit-tested core. + """ + + @mib 1024 * 1024 + # ext4 metadata (inode tables, journal, reserved blocks) plus slack so the + # rootfs always fits. The base is a read-only dm-snapshot origin -- guest + # writes land in the COW layer, never here -- so modest headroom is plenty. + @overhead_bytes 4 * @mib + @floor_bytes 16 * @mib + + @doc """ + Validate `ref` and return the `skopeo` source `"docker://" <> ref`. + + A ref must be non-empty and contain no whitespace (refs never do; rejecting + whitespace also closes the door on accidental arg-splitting surprises). + """ + @spec source(String.t()) :: {:ok, String.t()} | {:error, :invalid_ref} + def source(ref) when is_binary(ref) do + if ref != "" and not String.match?(ref, ~r/\s/), + do: {:ok, "docker://" <> ref}, + else: {:error, :invalid_ref} + end + + @doc "Map a Hyper architecture to the Go/OCI arch name `skopeo --override-arch` wants." + @spec goarch(Sys.Arch.t()) :: String.t() + def goarch(:x86_64), do: "amd64" + def goarch(:aarch64), do: "arm64" + + @doc """ + ext4 image size (bytes) for a rootfs whose contents total `content_bytes`: + content + fixed overhead, rounded up to a whole MiB, never below 16 MiB. + """ + @spec ext4_bytes(non_neg_integer()) :: pos_integer() + def ext4_bytes(content_bytes) when is_integer(content_bytes) and content_bytes >= 0 do + raw = content_bytes + @overhead_bytes + rounded = div(raw + @mib - 1, @mib) * @mib + max(rounded, @floor_bytes) + end +end diff --git a/test/hyper/img/oci_loader/params_test.exs b/test/hyper/img/oci_loader/params_test.exs new file mode 100644 index 0000000..b950b07 --- /dev/null +++ b/test/hyper/img/oci_loader/params_test.exs @@ -0,0 +1,54 @@ +defmodule Hyper.Img.OciLoader.ParamsTest do + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Img.OciLoader.Params + + @mib 1024 * 1024 + + describe "source/1" do + test "prefixes a valid ref with the docker transport" do + assert Params.source("docker.io/library/alpine:3.19") == + {:ok, "docker://docker.io/library/alpine:3.19"} + + assert Params.source("ghcr.io/foo/bar@sha256:abc") == + {:ok, "docker://ghcr.io/foo/bar@sha256:abc"} + end + + test "rejects empty, blank, or whitespace-bearing refs" do + assert Params.source("") == {:error, :invalid_ref} + assert Params.source(" ") == {:error, :invalid_ref} + assert Params.source("alpine 3.19") == {:error, :invalid_ref} + assert Params.source("alpine\n") == {:error, :invalid_ref} + end + end + + describe "goarch/1" do + test "maps Hyper arches to Go/OCI arch names" do + assert Params.goarch(:x86_64) == "amd64" + assert Params.goarch(:aarch64) == "arm64" + end + end + + describe "ext4_bytes/1" do + test "floors small inputs at 16 MiB" do + assert Params.ext4_bytes(0) == 16 * @mib + assert Params.ext4_bytes(1) == 16 * @mib + end + + test "leaves headroom above the content size, rounded up to a whole MiB" do + size = Params.ext4_bytes(100 * @mib) + assert size > 100 * @mib + assert rem(size, @mib) == 0 + end + + property "always a whole-MiB size that fits the content and clears the floor" do + check all content <- integer(0..(8 * 1024 * @mib)) do + size = Params.ext4_bytes(content) + assert rem(size, @mib) == 0 + assert size >= content + assert size >= 16 * @mib + end + end + end +end From 17e1b8bcf9fece17a38699b9a47d8ade5acb3e51 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 05:08:15 +0000 Subject: [PATCH 02/17] fix(img): scale ext4 overhead with content size --- lib/hyper/img/oci_loader/params.ex | 13 ++++++++----- test/hyper/img/oci_loader/params_test.exs | 7 +++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/hyper/img/oci_loader/params.ex b/lib/hyper/img/oci_loader/params.ex index 253f6ee..3a5bff3 100644 --- a/lib/hyper/img/oci_loader/params.ex +++ b/lib/hyper/img/oci_loader/params.ex @@ -8,9 +8,11 @@ defmodule Hyper.Img.OciLoader.Params do @mib 1024 * 1024 # ext4 metadata (inode tables, journal, reserved blocks) plus slack so the - # rootfs always fits. The base is a read-only dm-snapshot origin -- guest - # writes land in the COW layer, never here -- so modest headroom is plenty. - @overhead_bytes 4 * @mib + # rootfs always fits. Overhead scales with content -- a flat constant is far + # too small for large images -- as 25% of content plus an 8 MiB base, never + # below a 16 MiB floor. The base is a read-only dm-snapshot origin (guest + # writes land in the COW layer, never here), so generous slack is cheap. + @base_overhead_bytes 8 * @mib @floor_bytes 16 * @mib @doc """ @@ -33,11 +35,12 @@ defmodule Hyper.Img.OciLoader.Params do @doc """ ext4 image size (bytes) for a rootfs whose contents total `content_bytes`: - content + fixed overhead, rounded up to a whole MiB, never below 16 MiB. + content + scaled overhead (25% of content + 8 MiB base), rounded up to a whole + MiB, never below 16 MiB. """ @spec ext4_bytes(non_neg_integer()) :: pos_integer() def ext4_bytes(content_bytes) when is_integer(content_bytes) and content_bytes >= 0 do - raw = content_bytes + @overhead_bytes + raw = content_bytes + div(content_bytes, 4) + @base_overhead_bytes rounded = div(raw + @mib - 1, @mib) * @mib max(rounded, @floor_bytes) end diff --git a/test/hyper/img/oci_loader/params_test.exs b/test/hyper/img/oci_loader/params_test.exs index b950b07..3b6c9a8 100644 --- a/test/hyper/img/oci_loader/params_test.exs +++ b/test/hyper/img/oci_loader/params_test.exs @@ -42,6 +42,13 @@ defmodule Hyper.Img.OciLoader.ParamsTest do assert rem(size, @mib) == 0 end + test "scales overhead with content above the floor crossover" do + # content small enough that 1.25x + 8 MiB stays under the 16 MiB floor + assert Params.ext4_bytes(4 * @mib) == 16 * @mib + # content past the crossover gets proportional slack, not the floor + assert Params.ext4_bytes(64 * @mib) == 88 * @mib + end + property "always a whole-MiB size that fits the content and clears the floor" do check all content <- integer(0..(8 * 1024 * @mib)) do size = Params.ext4_bytes(content) From a609c970aceb66ec6579f22e9da6ad5b113e4bd3 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 05:09:18 +0000 Subject: [PATCH 03/17] feat(config): skopeo/umoci/mke2fs tool paths for OCI loader --- lib/hyper/config.ex | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/hyper/config.ex b/lib/hyper/config.ex index dfca0af..6a62391 100644 --- a/lib/hyper/config.ex +++ b/lib/hyper/config.ex @@ -19,6 +19,9 @@ defmodule Hyper.Config do @losetup_path Application.compile_env(:hyper, :losetup_path, "losetup") @dmsetup_path Application.compile_env(:hyper, :dmsetup_path, "dmsetup") @blockdev_path Application.compile_env(:hyper, :blockdev_path, "blockdev") + @skopeo_path Application.compile_env(:hyper, :skopeo_path, "skopeo") + @umoci_path Application.compile_env(:hyper, :umoci_path, "umoci") + @mke2fs_path Application.compile_env(:hyper, :mke2fs_path, "mke2fs") @vmlinux Application.compile_env(:hyper, :vmlinux, %{}) @doc """ @@ -110,6 +113,15 @@ defmodule Hyper.Config do @doc "Path to the blockdev binary." def blockdev_path, do: @blockdev_path + @doc "Path to the skopeo binary (used by `Hyper.Img.OciLoader` to pull OCI images)." + def skopeo_path, do: @skopeo_path + + @doc "Path to the umoci binary (used by `Hyper.Img.OciLoader` to flatten OCI layers)." + def umoci_path, do: @umoci_path + + @doc "Path to the mke2fs binary (used by `Hyper.Img.OciLoader` to build the ext4 rootfs)." + def mke2fs_path, do: @mke2fs_path + @doc """ Path to the setuid-root device helper (`hyper-suidhelper`). Required: the node runs unprivileged and routes every `losetup`/`dmsetup`/`blockdev` operation From caf6b9216c2aa43b3b87d1f9d00fcda6a80652e8 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 05:13:18 +0000 Subject: [PATCH 04/17] feat(img): Hyper.Img.OciLoader -- load OCI images into store + DB --- lib/hyper/img/oci_loader.ex | 224 +++++++++++++++++++++++++++++ test/hyper/img/oci_loader_test.exs | 29 ++++ test/test_helper.exs | 4 +- 3 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 lib/hyper/img/oci_loader.ex create mode 100644 test/hyper/img/oci_loader_test.exs diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex new file mode 100644 index 0000000..b3ed096 --- /dev/null +++ b/lib/hyper/img/oci_loader.ex @@ -0,0 +1,224 @@ +defmodule Hyper.Img.OciLoader do + @moduledoc """ + Loads an OCI image into Hyper's media store and image database. + + `load/1` takes a registry reference (e.g. `"docker.io/library/alpine:3.19"`) + and, end to end: + + 1. **pulls** it with `skopeo`, selecting the manifest entry matching this + node's architecture, into a local OCI layout; + 2. **flattens** it with `umoci unpack`, which applies OCI whiteouts/opaque + dirs correctly, yielding a merged rootfs directory; + 3. **builds** an ext4 image of that rootfs with `mke2fs -d` (no loopback, no + privilege, no setuid helper); + 4. **content-addresses** the image by the sha256 of its bytes -- that hash is + the blob id and the filename stem (`layer_.img`); + 5. **publishes** it into `Hyper.Config.layer_dir/0` via an atomic + same-filesystem rename, *then* records it as a one-layer base image + (`blobs` + `images` + `image_layers`) in a single transaction. + + Publish-before-record is deliberate: the layer GC prunes a `blobs` row whose + file is missing, so a row must never exist before its file. The rename is the + commit point; the file appears whole or not at all. + + Scope (YAGNI): the loader produces a *faithful* rootfs. It does not synthesise + an init or read the image's Entrypoint/Cmd -- booting is the caller's job via + `boot_args` (`root=/dev/vda rw init=<...>`). The content hash is over the + produced ext4 bytes, which are not byte-reproducible, so re-importing the same + ref may yield a fresh id; that is accepted. + """ + + alias Hyper.Config + alias Hyper.Img.Db.{Blob, Image, ImageLayer, Repo} + alias Hyper.Img.OciLoader.Params + + @hash_chunk 2 * 1024 * 1024 + + @doc "Load `ref` into the store and DB. See the module doc. Label defaults to `ref`." + @spec load(String.t()) :: {:ok, Hyper.Img.id()} | {:error, term()} + def load(ref), do: load(ref, []) + + @doc """ + Load `ref`. `opts[:label]` sets the human-readable `images.label` (defaults to + `ref`). + """ + @spec load(String.t(), keyword()) :: {:ok, Hyper.Img.id()} | {:error, term()} + def load(ref, opts) when is_binary(ref) and is_list(opts) do + label = Keyword.get(opts, :label, ref) + + with {:ok, source} <- Params.source(ref), + {:ok, arch} <- Sys.Arch.current() do + Sys.Tmp.with_tempdir("hyper-oci", fn tmp -> + with {:ok, rootfs} <- pull_and_unpack(source, Params.goarch(arch), tmp), + {:ok, content} <- dir_bytes(rootfs), + {:ok, staged} <- build_ext4(rootfs, Params.ext4_bytes(content), tmp), + {:ok, id} <- sha256_file(staged), + {:ok, _path} <- publish_file(staged, id), + :ok <- record(id, label, File.stat!(final_path(id)).size) do + {:ok, id} + end + end) + end + end + + @doc """ + Verify the external tools the loader needs (`skopeo`, `umoci`, `mke2fs`) are + resolvable on this host. Returns `{:error, {:missing_tools, names}}` listing + any that are absent. + """ + @spec test_system() :: :ok | {:error, {:missing_tools, [String.t()]}} + def test_system do + missing = + [Config.skopeo_path(), Config.umoci_path(), Config.mke2fs_path()] + |> Enum.reject(&System.find_executable/1) + + if missing == [], do: :ok, else: {:error, {:missing_tools, missing}} + end + + # --- pull + flatten ------------------------------------------------------- + + # `skopeo copy` into a local OCI layout, then `umoci unpack` into a bundle. + # Returns the path to the flattened rootfs directory. + @spec pull_and_unpack(String.t(), String.t(), Path.t()) :: + {:ok, Path.t()} | {:error, term()} + defp pull_and_unpack(source, goarch, tmp) do + oci = Path.join(tmp, "oci") + bundle = Path.join(tmp, "bundle") + + skopeo = + cmd(Config.skopeo_path(), [ + "copy", + "--override-os", + "linux", + "--override-arch", + goarch, + source, + "oci:#{oci}:img" + ]) + + with :ok <- tag(skopeo, :skopeo), + :ok <- tag(cmd(Config.umoci_path(), ["unpack", "--image", "#{oci}:img", bundle]), :umoci) do + {:ok, Path.join(bundle, "rootfs")} + end + end + + # --- size + build --------------------------------------------------------- + + # Apparent byte total of the rootfs tree (`du -sb`), parsed from the first field. + @spec dir_bytes(Path.t()) :: {:ok, non_neg_integer()} | {:error, term()} + defp dir_bytes(rootfs) do + case System.cmd("du", ["-sb", rootfs], stderr_to_stdout: true) do + {out, 0} -> + case Integer.parse(out) do + {bytes, _rest} -> {:ok, bytes} + :error -> {:error, {:du_unparsable, out}} + end + + {out, status} -> + {:error, {:du_failed, status, out}} + end + end + + # Build an ext4 image of `rootfs` sized to `bytes` (a whole-MiB multiple). + # `mke2fs` creates the file at the given size and populates it from the + # directory in one rootless step. Returns the staged image path. + @spec build_ext4(Path.t(), pos_integer(), Path.t()) :: {:ok, Path.t()} | {:error, term()} + defp build_ext4(rootfs, bytes, tmp) do + staged = Path.join(tmp, "rootfs.img") + size_arg = "#{div(bytes, 1024 * 1024)}M" + + args = ["-t", "ext4", "-F", "-q", "-d", rootfs, staged, size_arg] + + case tag(cmd(Config.mke2fs_path(), args), :mke2fs) do + :ok -> {:ok, staged} + {:error, _} = err -> err + end + end + + # --- hash + publish ------------------------------------------------------- + + # Streaming sha256 of `path`, lowercase hex. + @spec sha256_file(Path.t()) :: {:ok, String.t()} | {:error, term()} + defp sha256_file(path) do + digest = + path + |> File.stream!([:read, :binary, :raw], @hash_chunk) + |> Enum.reduce(:crypto.hash_init(:sha256), &:crypto.hash_update(&2, &1)) + |> :crypto.hash_final() + |> Base.encode16(case: :lower) + + {:ok, digest} + rescue + e -> {:error, {:hash_failed, Exception.message(e)}} + end + + # Move the staged image to its content-addressed final path via an atomic + # rename (same filesystem as `layer_dir`). If the file already exists (same + # bytes already published), drop the staged copy and reuse it. + @spec publish_file(Path.t(), String.t()) :: {:ok, Path.t()} | {:error, term()} + defp publish_file(staged, id) do + File.mkdir_p!(Config.layer_dir()) + final = final_path(id) + + cond do + File.exists?(final) -> + _ = File.rm(staged) + {:ok, final} + + true -> + case File.rename(staged, final) do + :ok -> {:ok, final} + {:error, reason} -> {:error, {:publish_failed, reason}} + end + end + end + + @spec final_path(String.t()) :: Path.t() + defp final_path(id), do: Path.join(Config.layer_dir(), "layer_#{id}.img") + + # --- DB record ------------------------------------------------------------ + + # Record the base image: one blob, one image (id == blob id), one layer at + # position 0. All upserts are idempotent so a re-publish of the same bytes is a + # no-op. The blob is inserted before the layer so the FK is satisfied. + @spec record(String.t(), String.t(), non_neg_integer()) :: :ok | {:error, term()} + defp record(id, label, size) do + multi = + Ecto.Multi.new() + |> Ecto.Multi.insert( + :blob, + Blob.changeset(%Blob{}, %{id: id, kind: :base, size: size}), + on_conflict: :nothing, + conflict_target: :id + ) + |> Ecto.Multi.insert( + :image, + Image.changeset(%Image{}, %{id: id, label: label}), + on_conflict: :nothing, + conflict_target: :id + ) + |> Ecto.Multi.insert( + :layer, + ImageLayer.changeset(%ImageLayer{}, %{image_id: id, position: 0, blob_id: id}), + on_conflict: :nothing, + conflict_target: [:image_id, :position] + ) + + case Repo.transaction(multi) do + {:ok, _} -> :ok + {:error, step, reason, _changes} -> {:error, {:record_failed, step, reason}} + end + end + + # --- external-command plumbing ------------------------------------------- + + # Run `bin` with `args`, no shell (System.cmd takes an arg list), merging + # stderr so failures carry diagnostics. Returns `{output, exit_status}`. + @spec cmd(Path.t(), [String.t()]) :: {String.t(), non_neg_integer()} + defp cmd(bin, args), do: System.cmd(bin, args, stderr_to_stdout: true) + + # Tag a command result: `:ok` on exit 0, else `{:error, {_failed, status, output}}`. + @spec tag({String.t(), non_neg_integer()}, atom()) :: :ok | {:error, term()} + defp tag({_out, 0}, _tool), do: :ok + defp tag({out, status}, tool), do: {:error, {:"#{tool}_failed", status, out}} +end diff --git a/test/hyper/img/oci_loader_test.exs b/test/hyper/img/oci_loader_test.exs new file mode 100644 index 0000000..da0b990 --- /dev/null +++ b/test/hyper/img/oci_loader_test.exs @@ -0,0 +1,29 @@ +defmodule Hyper.Img.OciLoaderTest do + # End-to-end: pulls a real (tiny) public image, builds the ext4 blob, and + # records it in the DB. Opt-in -- needs skopeo, umoci, mke2fs, network, and a + # running Postgres. Run with: mix test --include external + use ExUnit.Case, async: false + @moduletag :external + + alias Hyper.Config + alias Hyper.Img.Db.{Blob, Repo} + alias Hyper.Img.OciLoader + + test "load/1 publishes a busybox base image to the store and DB" do + assert OciLoader.test_system() == :ok + + assert {:ok, id} = OciLoader.load("docker.io/library/busybox:1.36") + + # File landed in the media store at its content-addressed path. + path = Path.join(Config.layer_dir(), "layer_#{id}.img") + assert File.exists?(path) + assert File.stat!(path).size > 0 + + # DB row exists and is a base blob. + assert %Blob{kind: :base} = Repo.get(Blob, id) + + # Idempotent: re-publishing the identical file is a no-op that returns the + # same id (the bytes are already present, so the hash matches). + assert File.exists?(path) + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 869559e..4d7803c 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1 +1,3 @@ -ExUnit.start() +# `:external` tests shell out to real tools (skopeo/umoci/mke2fs) and touch the +# image DB + media store. They are opt-in: run with `mix test --include external`. +ExUnit.start(exclude: [:external]) From 70f1df96ebf0b3ba2a4a7b0dcf09428be0ac1b5a Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 05:19:39 +0000 Subject: [PATCH 05/17] fix(img): stage ext4 in layer_dir for atomic publish; honest smoke assertion --- lib/hyper/img/oci_loader.ex | 49 ++++++++++++++++++++++-------- test/hyper/img/oci_loader_test.exs | 5 ++- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index b3ed096..1e0698d 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -51,10 +51,9 @@ defmodule Hyper.Img.OciLoader do Sys.Tmp.with_tempdir("hyper-oci", fn tmp -> with {:ok, rootfs} <- pull_and_unpack(source, Params.goarch(arch), tmp), {:ok, content} <- dir_bytes(rootfs), - {:ok, staged} <- build_ext4(rootfs, Params.ext4_bytes(content), tmp), - {:ok, id} <- sha256_file(staged), - {:ok, _path} <- publish_file(staged, id), - :ok <- record(id, label, File.stat!(final_path(id)).size) do + bytes = Params.ext4_bytes(content), + {:ok, staged} <- build_ext4(rootfs, bytes), + {:ok, id} <- finalize(staged, bytes, label) do {:ok, id} end end) @@ -119,19 +118,43 @@ defmodule Hyper.Img.OciLoader do end end - # Build an ext4 image of `rootfs` sized to `bytes` (a whole-MiB multiple). - # `mke2fs` creates the file at the given size and populates it from the - # directory in one rootless step. Returns the staged image path. - @spec build_ext4(Path.t(), pos_integer(), Path.t()) :: {:ok, Path.t()} | {:error, term()} - defp build_ext4(rootfs, bytes, tmp) do - staged = Path.join(tmp, "rootfs.img") + # Build an ext4 image of `rootfs` sized to `bytes` (a whole-MiB multiple), + # staged *inside `layer_dir`* so the later publish is an atomic + # same-filesystem rename. `mke2fs` creates the file at the given size and + # populates it from the directory in one rootless step. Returns the staged + # image path; the staged file is removed if mke2fs fails. + @spec build_ext4(Path.t(), pos_integer()) :: {:ok, Path.t()} | {:error, term()} + defp build_ext4(rootfs, bytes) do + File.mkdir_p!(Config.layer_dir()) + staged = Path.join(Config.layer_dir(), ".incoming-#{System.unique_integer([:positive])}.img") size_arg = "#{div(bytes, 1024 * 1024)}M" - args = ["-t", "ext4", "-F", "-q", "-d", rootfs, staged, size_arg] case tag(cmd(Config.mke2fs_path(), args), :mke2fs) do - :ok -> {:ok, staged} - {:error, _} = err -> err + :ok -> + {:ok, staged} + + {:error, _} = err -> + _ = File.rm(staged) + err + end + end + + # Hash the staged image (its sha256 is the content-addressed id), publish it + # into the store, then record the base image. If anything fails before the + # file is published, remove the staged file so a partial build never lingers + # in the shared store. `bytes` (the mke2fs size) is exactly the file size, so + # it is the recorded blob size -- no extra stat. + @spec finalize(Path.t(), pos_integer(), String.t()) :: {:ok, String.t()} | {:error, term()} + defp finalize(staged, bytes, label) do + with {:ok, id} <- sha256_file(staged), + {:ok, _final} <- publish_file(staged, id), + :ok <- record(id, label, bytes) do + {:ok, id} + else + {:error, _} = err -> + _ = File.rm(staged) + err end end diff --git a/test/hyper/img/oci_loader_test.exs b/test/hyper/img/oci_loader_test.exs index da0b990..48bfcfe 100644 --- a/test/hyper/img/oci_loader_test.exs +++ b/test/hyper/img/oci_loader_test.exs @@ -22,8 +22,7 @@ defmodule Hyper.Img.OciLoaderTest do # DB row exists and is a base blob. assert %Blob{kind: :base} = Repo.get(Blob, id) - # Idempotent: re-publishing the identical file is a no-op that returns the - # same id (the bytes are already present, so the hash matches). - assert File.exists?(path) + # The recorded blob size matches the published file exactly. + assert Repo.get(Blob, id).size == File.stat!(path).size end end From e3ef768dc90e0651ad1e5ece396509d90deab532 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 05:20:33 +0000 Subject: [PATCH 06/17] docs(cookbook): loading OCI images and booting the first VM --- docs/cookbook/loading-images.md | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docs/cookbook/loading-images.md diff --git a/docs/cookbook/loading-images.md b/docs/cookbook/loading-images.md new file mode 100644 index 0000000..41ce3dd --- /dev/null +++ b/docs/cookbook/loading-images.md @@ -0,0 +1,48 @@ +# Loading images + +Before you can boot a VM you need an image in Hyper's database and its rootfs +blob in the shared media store (`layer_dir`). `Hyper.Img.OciLoader` does both +from a single OCI reference. + +## Prerequisites + +The host running the loader needs three tools on `PATH` (override the paths via +`config :hyper, skopeo_path:/umoci_path:/mke2fs_path:` if they live elsewhere): + +- `skopeo` -- pulls the image, applying manifest-list arch selection. +- `umoci` -- flattens the OCI layers into a rootfs (handles whiteouts). +- `mke2fs` (e2fsprogs) -- builds the ext4 rootfs image, rootless. + +Postgres must be running with migrations applied (`mix ecto.migrate`), and +`layer_dir` must be writable. + +## Load an image + +```elixir +{:ok, img_id} = Hyper.Img.OciLoader.load("docker.io/library/alpine:3.19") +``` + +The loader pulls the image for this node's architecture, flattens it, builds +`layer_.img` in `layer_dir`, and records a one-layer base image. The +returned `img_id` is the content hash; pass it to `Hyper.create_vm/1`. + +Re-running `load/1` for the same reference rebuilds the rootfs; because ext4 +images are not byte-reproducible, you may get a fresh `img_id` each time. + +## Booting it + +The loader produces a *faithful* rootfs -- it does not add an init. A container +image's entrypoint is not an init, so you must tell the kernel what to run. Pass +`boot_args` to `create_vm` (the root drive is `/dev/vda`): + +```elixir +{:ok, vm} = + Hyper.create_vm(%Hyper.Vm.Spec{ + img_id: img_id, + boot_args: "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw init=/bin/sh" + }) +``` + +> The default `boot_args` (`console=ttyS0 reboot=k panic=1 pci=off`) omit +> `root=` and `init=`, so an OCI-derived rootfs will not boot without the +> additions above. From f7eba1cd48546c70bb7836ccb2894f6bf6e29b7e Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 18:05:11 +0000 Subject: [PATCH 07/17] docs improvement --- docs/cookbook/intro.md | 38 ++++++++++++++++++++++++++ docs/cookbook/loading-images.md | 48 --------------------------------- 2 files changed, 38 insertions(+), 48 deletions(-) delete mode 100644 docs/cookbook/loading-images.md diff --git a/docs/cookbook/intro.md b/docs/cookbook/intro.md index 1459f74..da1ea08 100644 --- a/docs/cookbook/intro.md +++ b/docs/cookbook/intro.md @@ -17,6 +17,20 @@ with it. The absolute best way to get started with `Hyper` is to play with it. +### Requirements + +Hyper requires the following software be installed on each node running it: + + - [`skopeo`](https://github.com/podman-container-tools/skopeo) + - [`e2fsprogs`](https://github.com/tytso/e2fsprogs) + +Hyper has more runtime dependencies, but they are automatically redistributed +by Hyper. + +### Installation + + + ### Configuration Running `Hyper` is involved and requires a large number of pre-requisites. The @@ -41,3 +55,27 @@ config :hyper, uid_gid_range: {900_000, 999_999}, layer_dir: "/srv/hyper/layers" ``` + + + +### Usage + + + +#### Loading Images + +Before an image can be booted, it needs to be loaded into Hyper. Currently, the +only way to load images is through an OCI image, either natively or through the +native interface, or through [gRPC](../grpc.md): + +```elixir +{:ok, img_id} = Hyper.Img.OciLoader.load("docker.io/library/alpine:3.19") +``` + +#### Booting a VM + +With the image loaded, and an `img_id` in hand, you can boot it: + +```elixir +{:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{ img_id: img_id }) +``` diff --git a/docs/cookbook/loading-images.md b/docs/cookbook/loading-images.md deleted file mode 100644 index 41ce3dd..0000000 --- a/docs/cookbook/loading-images.md +++ /dev/null @@ -1,48 +0,0 @@ -# Loading images - -Before you can boot a VM you need an image in Hyper's database and its rootfs -blob in the shared media store (`layer_dir`). `Hyper.Img.OciLoader` does both -from a single OCI reference. - -## Prerequisites - -The host running the loader needs three tools on `PATH` (override the paths via -`config :hyper, skopeo_path:/umoci_path:/mke2fs_path:` if they live elsewhere): - -- `skopeo` -- pulls the image, applying manifest-list arch selection. -- `umoci` -- flattens the OCI layers into a rootfs (handles whiteouts). -- `mke2fs` (e2fsprogs) -- builds the ext4 rootfs image, rootless. - -Postgres must be running with migrations applied (`mix ecto.migrate`), and -`layer_dir` must be writable. - -## Load an image - -```elixir -{:ok, img_id} = Hyper.Img.OciLoader.load("docker.io/library/alpine:3.19") -``` - -The loader pulls the image for this node's architecture, flattens it, builds -`layer_.img` in `layer_dir`, and records a one-layer base image. The -returned `img_id` is the content hash; pass it to `Hyper.create_vm/1`. - -Re-running `load/1` for the same reference rebuilds the rootfs; because ext4 -images are not byte-reproducible, you may get a fresh `img_id` each time. - -## Booting it - -The loader produces a *faithful* rootfs -- it does not add an init. A container -image's entrypoint is not an init, so you must tell the kernel what to run. Pass -`boot_args` to `create_vm` (the root drive is `/dev/vda`): - -```elixir -{:ok, vm} = - Hyper.create_vm(%Hyper.Vm.Spec{ - img_id: img_id, - boot_args: "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw init=/bin/sh" - }) -``` - -> The default `boot_args` (`console=ttyS0 reboot=k panic=1 pci=off`) omit -> `root=` and `init=`, so an OCI-derived rootfs will not boot without the -> additions above. From 109fe9f0c0d7101683fd573334f2ea5117c27a25 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 18:12:01 +0000 Subject: [PATCH 08/17] refactor(img): fold OciLoader.Params into Hyper.Img.OciLoader --- lib/hyper/img/oci_loader.ex | 48 +++++++++++++++-- lib/hyper/img/oci_loader/params.ex | 47 ----------------- test/hyper/img/oci_loader/params_test.exs | 61 --------------------- test/hyper/img/oci_loader_test.exs | 64 +++++++++++++++++++++-- 4 files changed, 103 insertions(+), 117 deletions(-) delete mode 100644 lib/hyper/img/oci_loader/params.ex delete mode 100644 test/hyper/img/oci_loader/params_test.exs diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index 1e0698d..c29644e 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -30,9 +30,17 @@ defmodule Hyper.Img.OciLoader do alias Hyper.Config alias Hyper.Img.Db.{Blob, Image, ImageLayer, Repo} - alias Hyper.Img.OciLoader.Params - @hash_chunk 2 * 1024 * 1024 + @mib 1024 * 1024 + @hash_chunk 2 * @mib + + # ext4 metadata (inode tables, journal, reserved blocks) plus slack so the + # rootfs always fits. Overhead scales with content -- a flat constant is far + # too small for large images -- as 25% of content plus an 8 MiB base, never + # below a 16 MiB floor. The base is a read-only dm-snapshot origin (guest + # writes land in the COW layer, never here), so generous slack is cheap. + @base_overhead_bytes 8 * @mib + @floor_bytes 16 * @mib @doc "Load `ref` into the store and DB. See the module doc. Label defaults to `ref`." @spec load(String.t()) :: {:ok, Hyper.Img.id()} | {:error, term()} @@ -46,12 +54,12 @@ defmodule Hyper.Img.OciLoader do def load(ref, opts) when is_binary(ref) and is_list(opts) do label = Keyword.get(opts, :label, ref) - with {:ok, source} <- Params.source(ref), + with {:ok, source} <- source(ref), {:ok, arch} <- Sys.Arch.current() do Sys.Tmp.with_tempdir("hyper-oci", fn tmp -> - with {:ok, rootfs} <- pull_and_unpack(source, Params.goarch(arch), tmp), + with {:ok, rootfs} <- pull_and_unpack(source, goarch(arch), tmp), {:ok, content} <- dir_bytes(rootfs), - bytes = Params.ext4_bytes(content), + bytes = ext4_bytes(content), {:ok, staged} <- build_ext4(rootfs, bytes), {:ok, id} <- finalize(staged, bytes, label) do {:ok, id} @@ -74,6 +82,36 @@ defmodule Hyper.Img.OciLoader do if missing == [], do: :ok, else: {:error, {:missing_tools, missing}} end + # --- pure derivations (no I/O; the unit-tested core) ---------------------- + + # Validate `ref` and return the `skopeo` source `"docker://" <> ref`. A ref must + # be non-empty and contain no whitespace (refs never do; rejecting whitespace + # also closes the door on accidental arg-splitting surprises). + @doc false + @spec source(String.t()) :: {:ok, String.t()} | {:error, :invalid_ref} + def source(ref) when is_binary(ref) do + if ref != "" and not String.match?(ref, ~r/\s/), + do: {:ok, "docker://" <> ref}, + else: {:error, :invalid_ref} + end + + # Map a Hyper architecture to the Go/OCI arch name `skopeo --override-arch` wants. + @doc false + @spec goarch(Sys.Arch.t()) :: String.t() + def goarch(:x86_64), do: "amd64" + def goarch(:aarch64), do: "arm64" + + # ext4 image size (bytes) for a rootfs whose contents total `content_bytes`: + # content + scaled overhead (25% of content + 8 MiB base), rounded up to a whole + # MiB, never below 16 MiB. + @doc false + @spec ext4_bytes(non_neg_integer()) :: pos_integer() + def ext4_bytes(content_bytes) when is_integer(content_bytes) and content_bytes >= 0 do + raw = content_bytes + div(content_bytes, 4) + @base_overhead_bytes + rounded = div(raw + @mib - 1, @mib) * @mib + max(rounded, @floor_bytes) + end + # --- pull + flatten ------------------------------------------------------- # `skopeo copy` into a local OCI layout, then `umoci unpack` into a bundle. diff --git a/lib/hyper/img/oci_loader/params.ex b/lib/hyper/img/oci_loader/params.ex deleted file mode 100644 index 3a5bff3..0000000 --- a/lib/hyper/img/oci_loader/params.ex +++ /dev/null @@ -1,47 +0,0 @@ -defmodule Hyper.Img.OciLoader.Params do - @moduledoc """ - Pure derivations for `Hyper.Img.OciLoader`: registry-ref validation, the - Hyper-arch -> Go/OCI-arch name mapping `skopeo` expects, and the ext4 image - size for a rootfs of a given content size. No I/O — every function here is a - total function of its arguments, which is why this is the unit-tested core. - """ - - @mib 1024 * 1024 - # ext4 metadata (inode tables, journal, reserved blocks) plus slack so the - # rootfs always fits. Overhead scales with content -- a flat constant is far - # too small for large images -- as 25% of content plus an 8 MiB base, never - # below a 16 MiB floor. The base is a read-only dm-snapshot origin (guest - # writes land in the COW layer, never here), so generous slack is cheap. - @base_overhead_bytes 8 * @mib - @floor_bytes 16 * @mib - - @doc """ - Validate `ref` and return the `skopeo` source `"docker://" <> ref`. - - A ref must be non-empty and contain no whitespace (refs never do; rejecting - whitespace also closes the door on accidental arg-splitting surprises). - """ - @spec source(String.t()) :: {:ok, String.t()} | {:error, :invalid_ref} - def source(ref) when is_binary(ref) do - if ref != "" and not String.match?(ref, ~r/\s/), - do: {:ok, "docker://" <> ref}, - else: {:error, :invalid_ref} - end - - @doc "Map a Hyper architecture to the Go/OCI arch name `skopeo --override-arch` wants." - @spec goarch(Sys.Arch.t()) :: String.t() - def goarch(:x86_64), do: "amd64" - def goarch(:aarch64), do: "arm64" - - @doc """ - ext4 image size (bytes) for a rootfs whose contents total `content_bytes`: - content + scaled overhead (25% of content + 8 MiB base), rounded up to a whole - MiB, never below 16 MiB. - """ - @spec ext4_bytes(non_neg_integer()) :: pos_integer() - def ext4_bytes(content_bytes) when is_integer(content_bytes) and content_bytes >= 0 do - raw = content_bytes + div(content_bytes, 4) + @base_overhead_bytes - rounded = div(raw + @mib - 1, @mib) * @mib - max(rounded, @floor_bytes) - end -end diff --git a/test/hyper/img/oci_loader/params_test.exs b/test/hyper/img/oci_loader/params_test.exs deleted file mode 100644 index 3b6c9a8..0000000 --- a/test/hyper/img/oci_loader/params_test.exs +++ /dev/null @@ -1,61 +0,0 @@ -defmodule Hyper.Img.OciLoader.ParamsTest do - use ExUnit.Case, async: true - use ExUnitProperties - - alias Hyper.Img.OciLoader.Params - - @mib 1024 * 1024 - - describe "source/1" do - test "prefixes a valid ref with the docker transport" do - assert Params.source("docker.io/library/alpine:3.19") == - {:ok, "docker://docker.io/library/alpine:3.19"} - - assert Params.source("ghcr.io/foo/bar@sha256:abc") == - {:ok, "docker://ghcr.io/foo/bar@sha256:abc"} - end - - test "rejects empty, blank, or whitespace-bearing refs" do - assert Params.source("") == {:error, :invalid_ref} - assert Params.source(" ") == {:error, :invalid_ref} - assert Params.source("alpine 3.19") == {:error, :invalid_ref} - assert Params.source("alpine\n") == {:error, :invalid_ref} - end - end - - describe "goarch/1" do - test "maps Hyper arches to Go/OCI arch names" do - assert Params.goarch(:x86_64) == "amd64" - assert Params.goarch(:aarch64) == "arm64" - end - end - - describe "ext4_bytes/1" do - test "floors small inputs at 16 MiB" do - assert Params.ext4_bytes(0) == 16 * @mib - assert Params.ext4_bytes(1) == 16 * @mib - end - - test "leaves headroom above the content size, rounded up to a whole MiB" do - size = Params.ext4_bytes(100 * @mib) - assert size > 100 * @mib - assert rem(size, @mib) == 0 - end - - test "scales overhead with content above the floor crossover" do - # content small enough that 1.25x + 8 MiB stays under the 16 MiB floor - assert Params.ext4_bytes(4 * @mib) == 16 * @mib - # content past the crossover gets proportional slack, not the floor - assert Params.ext4_bytes(64 * @mib) == 88 * @mib - end - - property "always a whole-MiB size that fits the content and clears the floor" do - check all content <- integer(0..(8 * 1024 * @mib)) do - size = Params.ext4_bytes(content) - assert rem(size, @mib) == 0 - assert size >= content - assert size >= 16 * @mib - end - end - end -end diff --git a/test/hyper/img/oci_loader_test.exs b/test/hyper/img/oci_loader_test.exs index 48bfcfe..df1ff4a 100644 --- a/test/hyper/img/oci_loader_test.exs +++ b/test/hyper/img/oci_loader_test.exs @@ -1,14 +1,70 @@ defmodule Hyper.Img.OciLoaderTest do - # End-to-end: pulls a real (tiny) public image, builds the ext4 blob, and - # records it in the DB. Opt-in -- needs skopeo, umoci, mke2fs, network, and a - # running Postgres. Run with: mix test --include external use ExUnit.Case, async: false - @moduletag :external + use ExUnitProperties alias Hyper.Config alias Hyper.Img.Db.{Blob, Repo} alias Hyper.Img.OciLoader + @mib 1024 * 1024 + + describe "source/1" do + test "prefixes a valid ref with the docker transport" do + assert OciLoader.source("docker.io/library/alpine:3.19") == + {:ok, "docker://docker.io/library/alpine:3.19"} + + assert OciLoader.source("ghcr.io/foo/bar@sha256:abc") == + {:ok, "docker://ghcr.io/foo/bar@sha256:abc"} + end + + test "rejects empty, blank, or whitespace-bearing refs" do + assert OciLoader.source("") == {:error, :invalid_ref} + assert OciLoader.source(" ") == {:error, :invalid_ref} + assert OciLoader.source("alpine 3.19") == {:error, :invalid_ref} + assert OciLoader.source("alpine\n") == {:error, :invalid_ref} + end + end + + describe "goarch/1" do + test "maps Hyper arches to Go/OCI arch names" do + assert OciLoader.goarch(:x86_64) == "amd64" + assert OciLoader.goarch(:aarch64) == "arm64" + end + end + + describe "ext4_bytes/1" do + test "floors small inputs at 16 MiB" do + assert OciLoader.ext4_bytes(0) == 16 * @mib + assert OciLoader.ext4_bytes(1) == 16 * @mib + end + + test "leaves headroom above the content size, rounded up to a whole MiB" do + size = OciLoader.ext4_bytes(100 * @mib) + assert size > 100 * @mib + assert rem(size, @mib) == 0 + end + + test "scales overhead with content above the floor crossover" do + # content small enough that 1.25x + 8 MiB stays under the 16 MiB floor + assert OciLoader.ext4_bytes(4 * @mib) == 16 * @mib + # content past the crossover gets proportional slack, not the floor + assert OciLoader.ext4_bytes(64 * @mib) == 88 * @mib + end + + property "always a whole-MiB size that fits the content and clears the floor" do + check all content <- integer(0..(8 * 1024 * @mib)) do + size = OciLoader.ext4_bytes(content) + assert rem(size, @mib) == 0 + assert size >= content + assert size >= 16 * @mib + end + end + end + + # End-to-end: pulls a real (tiny) public image, builds the ext4 blob, and + # records it in the DB. Opt-in -- needs skopeo, umoci, mke2fs, network, and a + # running Postgres. Run with: mix test --include external + @tag :external test "load/1 publishes a busybox base image to the store and DB" do assert OciLoader.test_system() == :ok From b8471fb0c460bb888e084b84078ff9c65fcc4c4c Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 18:16:27 +0000 Subject: [PATCH 09/17] feat(grpc): LoadImage RPC -- load OCI images via gRPC --- lib/hyper/grpc/codec.ex | 29 +++++++++++++++++++ lib/hyper/grpc/server.ex | 12 ++++++++ proto/hyper/grpc/v0/hyper.proto | 29 +++++++++++++++++++ test/hyper/grpc/codec_test.exs | 50 +++++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 test/hyper/grpc/codec_test.exs diff --git a/lib/hyper/grpc/codec.ex b/lib/hyper/grpc/codec.ex index ca466be..650040a 100644 --- a/lib/hyper/grpc/codec.ex +++ b/lib/hyper/grpc/codec.ex @@ -15,6 +15,8 @@ defmodule Hyper.Grpc.Codec do CreateVmResponse, GetVmResponse, ListVmsResponse, + LoadImageRequest, + LoadImageResponse, Vm } @@ -72,6 +74,16 @@ defmodule Hyper.Grpc.Codec do end end + @spec from_grpc(LoadImageRequest.t()) :: + {:ok, {String.t(), keyword()}} | {:error, :missing_image_ref} + def from_grpc(%LoadImageRequest{image_ref: ref}) when ref in [nil, ""], + do: {:error, :missing_image_ref} + + def from_grpc(%LoadImageRequest{image_ref: ref, label: label}) do + opts = if label in [nil, ""], do: [], else: [label: label] + {:ok, {ref, opts}} + end + @doc "Convert a domain result to an outbound response message, or an error to `GRPC.RPCError`." @spec to_grpc({:created, Hyper.Vm.id(), node()}) :: CreateVmResponse.t() def to_grpc({:created, vm_id, node}) when is_binary(vm_id), @@ -85,6 +97,10 @@ defmodule Hyper.Grpc.Codec do def to_grpc({:vms, vms}), do: %ListVmsResponse{vms: Enum.map(vms, &vm/1)} + @spec to_grpc({:loaded, Hyper.Img.id()}) :: LoadImageResponse.t() + def to_grpc({:loaded, img_id}) when is_binary(img_id), + do: %LoadImageResponse{img_id: img_id} + @spec to_grpc(:stopped) :: Empty.t() def to_grpc(:stopped), do: %Empty{} @@ -124,6 +140,19 @@ defmodule Hyper.Grpc.Codec do defp rpc_error(reason) when reason in [:no_capacity, :exhausted], do: GRPC.RPCError.exception(:resource_exhausted, "no capacity") + defp rpc_error(:missing_image_ref), + do: GRPC.RPCError.exception(:invalid_argument, "image_ref is required") + + defp rpc_error(:invalid_ref), + do: GRPC.RPCError.exception(:invalid_argument, "image_ref is malformed") + + defp rpc_error({:missing_tools, tools}), + do: + GRPC.RPCError.exception( + :failed_precondition, + "node is missing required image tools: #{Enum.join(tools, ", ")}" + ) + defp rpc_error(reason), do: GRPC.RPCError.exception(:internal, "internal error: #{inspect(reason)}") end diff --git a/lib/hyper/grpc/server.ex b/lib/hyper/grpc/server.ex index 08d45bf..39c6482 100644 --- a/lib/hyper/grpc/server.ex +++ b/lib/hyper/grpc/server.ex @@ -17,9 +17,21 @@ defmodule Hyper.Grpc.Server do GetVmRequest, GetVmResponse, ListVmsResponse, + LoadImageRequest, + LoadImageResponse, StopVmRequest } + @spec load_image(LoadImageRequest.t(), GRPC.Server.Stream.t()) :: LoadImageResponse.t() + def load_image(%LoadImageRequest{} = req, _stream) do + with {:ok, {ref, opts}} <- Codec.from_grpc(req), + {:ok, img_id} <- Hyper.Img.OciLoader.load(ref, opts) do + Codec.to_grpc({:loaded, img_id}) + else + {:error, reason} -> raise Codec.to_grpc({:error, reason}) + end + end + @spec create_vm(CreateVmRequest.t(), GRPC.Server.Stream.t()) :: CreateVmResponse.t() def create_vm(%CreateVmRequest{} = req, _stream) do with {:ok, spec} <- Codec.from_grpc(req), diff --git a/proto/hyper/grpc/v0/hyper.proto b/proto/hyper/grpc/v0/hyper.proto index f43d0bf..f668ff8 100644 --- a/proto/hyper/grpc/v0/hyper.proto +++ b/proto/hyper/grpc/v0/hyper.proto @@ -52,6 +52,20 @@ service Hyper { // List every microVM currently known to the cluster, across all nodes. rpc ListVms(google.protobuf.Empty) returns (ListVmsResponse); + + // Load an OCI image into the cluster's shared media store and image database. + // + // Pulls the referenced image for this node's architecture, flattens it, builds + // an ext4 rootfs, content-addresses it, and records a base image. Runs on the + // node that receives the call; the result is visible cluster-wide. Blocks until + // the load completes -- this can take minutes -- so set a generous deadline. + // Returns the content-addressed `img_id` to pass to CreateVm. + // + // Errors: + // INVALID_ARGUMENT -- `image_ref` is empty or malformed. + // FAILED_PRECONDITION -- the node lacks the required tools (skopeo/umoci/mke2fs). + // INTERNAL -- the pull, unpack, build, publish, or DB record failed. + rpc LoadImage(LoadImageRequest) returns (LoadImageResponse); } // A fixed (vCPU, memory, disk) size, like a cloud instance class. Each step up @@ -142,3 +156,18 @@ message Vm { // The cluster node (Erlang node name) the VM runs on. string node = 2; } + +// Request to load an OCI image. +message LoadImageRequest { + // Required. An OCI image reference, e.g. "docker.io/library/alpine:3.19". + string image_ref = 1; + + // Optional. A human-readable label stored with the image (defaults to image_ref). + optional string label = 2; +} + +// Result of a successful LoadImage. +message LoadImageResponse { + // The content-addressed image id; pass it to CreateVm. + string img_id = 1; +} diff --git a/test/hyper/grpc/codec_test.exs b/test/hyper/grpc/codec_test.exs new file mode 100644 index 0000000..eab50f7 --- /dev/null +++ b/test/hyper/grpc/codec_test.exs @@ -0,0 +1,50 @@ +defmodule Hyper.Grpc.CodecTest do + use ExUnit.Case, async: true + + alias Hyper.Grpc.Codec + alias Hyper.Grpc.V0.{LoadImageRequest, LoadImageResponse} + + describe "from_grpc/1 (LoadImageRequest)" do + test "rejects a blank image_ref" do + assert Codec.from_grpc(%LoadImageRequest{image_ref: ""}) == {:error, :missing_image_ref} + assert Codec.from_grpc(%LoadImageRequest{image_ref: nil}) == {:error, :missing_image_ref} + end + + test "passes the ref through with no opts when label is unset" do + assert Codec.from_grpc(%LoadImageRequest{image_ref: "alpine:3.19"}) == + {:ok, {"alpine:3.19", []}} + + assert Codec.from_grpc(%LoadImageRequest{image_ref: "alpine:3.19", label: ""}) == + {:ok, {"alpine:3.19", []}} + end + + test "carries label as an opt when set" do + assert Codec.from_grpc(%LoadImageRequest{image_ref: "alpine:3.19", label: "base"}) == + {:ok, {"alpine:3.19", [label: "base"]}} + end + end + + describe "to_grpc/1 (loaded)" do + test "wraps the img_id in a LoadImageResponse" do + assert Codec.to_grpc({:loaded, "oci-abc"}) == %LoadImageResponse{img_id: "oci-abc"} + end + end + + describe "to_grpc/1 (LoadImage error mapping)" do + test "missing/invalid ref -> INVALID_ARGUMENT" do + assert %GRPC.RPCError{status: status} = Codec.to_grpc({:error, :missing_image_ref}) + assert status == GRPC.Status.invalid_argument() + assert Codec.to_grpc({:error, :invalid_ref}).status == GRPC.Status.invalid_argument() + end + + test "missing tools -> FAILED_PRECONDITION" do + err = Codec.to_grpc({:error, {:missing_tools, ["skopeo", "umoci"]}}) + assert err.status == GRPC.Status.failed_precondition() + end + + test "tool/build/db failures -> INTERNAL" do + assert Codec.to_grpc({:error, {:skopeo_failed, 1, "boom"}}).status == GRPC.Status.internal() + assert Codec.to_grpc({:error, {:record_failed, :blob, :db}}).status == GRPC.Status.internal() + end + end +end From ebe5d4276d2fa6d2c4ae832d4ad4ad86dcc92c7f Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 18:19:50 +0000 Subject: [PATCH 10/17] docs(grpc): document the LoadImage RPC --- docs/grpc.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/grpc.md b/docs/grpc.md index e1ed5cd..5b76276 100644 --- a/docs/grpc.md +++ b/docs/grpc.md @@ -51,6 +51,23 @@ from hyper.grpc.v0 import hyper_pb2, hyper_pb2_grpc client = hyper_pb2_grpc.HyperStub(grpc.aio.insecure_channel("localhost:50051")) ``` +### Loading Images + +Before you can create a VM you need an image in the cluster. `LoadImage` pulls an +OCI image, builds its rootfs, and records it -- returning the `img_id` you pass to +`CreateVm`. It blocks until the load finishes (this can take minutes), so set a +generous deadline. + +```python +loaded = await client.LoadImage( + hyper_pb2.LoadImageRequest( + image_ref="docker.io/library/alpine:3.19", + # label is optional; defaults to image_ref. + ) +) +print(loaded.img_id) # pass this to CreateVm +``` + ### Creating VMs You can create new VMs with the `CreateVm` RPC. @@ -58,7 +75,7 @@ You can create new VMs with the `CreateVm` RPC. ```python created = await client.CreateVm( hyper_pb2.CreateVmRequest( - img_id="img-abc", + img_id=loaded.img_id, instance_type=hyper_pb2.INSTANCE_TYPE_DECI, arch=hyper_pb2.ARCHITECTURE_X86_64, # boot_args is optional; omit it for the default kernel cmdline. From e62b0a785bc38f2e3367d3a32a2ccfdbe646e9bc Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 18:24:51 +0000 Subject: [PATCH 11/17] fix(img): fail fast in load/2 when required tools are missing --- lib/hyper/img/oci_loader.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index c29644e..474afe4 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -49,12 +49,17 @@ defmodule Hyper.Img.OciLoader do @doc """ Load `ref`. `opts[:label]` sets the human-readable `images.label` (defaults to `ref`). + + Returns `{:error, {:missing_tools, names}}` when the node lacks a required + external tool (`skopeo`/`umoci`/`mke2fs`); the check runs up front so the load + fails fast before the multi-minute pull. """ @spec load(String.t(), keyword()) :: {:ok, Hyper.Img.id()} | {:error, term()} def load(ref, opts) when is_binary(ref) and is_list(opts) do label = Keyword.get(opts, :label, ref) with {:ok, source} <- source(ref), + :ok <- test_system(), {:ok, arch} <- Sys.Arch.current() do Sys.Tmp.with_tempdir("hyper-oci", fn tmp -> with {:ok, rootfs} <- pull_and_unpack(source, goarch(arch), tmp), From 005799462e99c2df0e3bb258178fb6aed65603d4 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Wed, 24 Jun 2026 18:30:50 +0000 Subject: [PATCH 12/17] docs: fix skopeo link to containers/skopeo --- docs/cookbook/intro.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cookbook/intro.md b/docs/cookbook/intro.md index da1ea08..a0030d3 100644 --- a/docs/cookbook/intro.md +++ b/docs/cookbook/intro.md @@ -21,7 +21,7 @@ The absolute best way to get started with `Hyper` is to play with it. Hyper requires the following software be installed on each node running it: - - [`skopeo`](https://github.com/podman-container-tools/skopeo) + - [`skopeo`](https://github.com/containers/skopeo) - [`e2fsprogs`](https://github.com/tytso/e2fsprogs) Hyper has more runtime dependencies, but they are automatically redistributed From 1ae317fcb14a74f97fc2d2a559967e72d6ca45d4 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Thu, 25 Jun 2026 01:48:02 +0000 Subject: [PATCH 13/17] feat(img): auto-download a default umoci when unconfigured --- lib/hyper/config.ex | 11 ++- lib/hyper/img/oci_loader.ex | 6 +- lib/hyper/img/oci_loader/umoci.ex | 97 ++++++++++++++++++++++++ test/hyper/img/oci_loader/umoci_test.exs | 32 ++++++++ 4 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 lib/hyper/img/oci_loader/umoci.ex create mode 100644 test/hyper/img/oci_loader/umoci_test.exs diff --git a/lib/hyper/config.ex b/lib/hyper/config.ex index a510464..2b8a088 100644 --- a/lib/hyper/config.ex +++ b/lib/hyper/config.ex @@ -20,7 +20,7 @@ defmodule Hyper.Config do @dmsetup_path Application.compile_env(:hyper, :dmsetup_path, "dmsetup") @blockdev_path Application.compile_env(:hyper, :blockdev_path, "blockdev") @skopeo_path Application.compile_env(:hyper, :skopeo_path, "skopeo") - @umoci_path Application.compile_env(:hyper, :umoci_path, "umoci") + @umoci_path Application.compile_env(:hyper, :umoci_path, nil) @mke2fs_path Application.compile_env(:hyper, :mke2fs_path, "mke2fs") @vmlinux Application.compile_env(:hyper, :vmlinux, %{}) @@ -65,6 +65,10 @@ defmodule Hyper.Config do @spec vmlinux_install_dir :: Path.t() def vmlinux_install_dir, do: Path.join(redist_dir(), "vmlinux") + @doc "Directory where `Hyper.Img.OciLoader.Umoci` installs the default umoci binary." + @spec umoci_install_dir :: Path.t() + def umoci_install_dir, do: Path.join(redist_dir(), "umoci") + @doc """ Path to the directory where all VM chroot's are created (`/jails`). @@ -120,7 +124,10 @@ defmodule Hyper.Config do @doc "Path to the skopeo binary (used by `Hyper.Img.OciLoader` to pull OCI images)." def skopeo_path, do: @skopeo_path - @doc "Path to the umoci binary (used by `Hyper.Img.OciLoader` to flatten OCI layers)." + @doc """ + Operator-configured path to the umoci binary, or `nil` (the default) to let + `Hyper.Img.OciLoader.Umoci` download and manage a pinned default. + """ def umoci_path, do: @umoci_path @doc "Path to the mke2fs binary (used by `Hyper.Img.OciLoader` to build the ext4 rootfs)." diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index 474afe4..0fa5ac8 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -30,6 +30,7 @@ defmodule Hyper.Img.OciLoader do alias Hyper.Config alias Hyper.Img.Db.{Blob, Image, ImageLayer, Repo} + alias Hyper.Img.OciLoader.Umoci @mib 1024 * 1024 @hash_chunk 2 * @mib @@ -59,6 +60,7 @@ defmodule Hyper.Img.OciLoader do label = Keyword.get(opts, :label, ref) with {:ok, source} <- source(ref), + :ok <- Umoci.ensure_installed(), :ok <- test_system(), {:ok, arch} <- Sys.Arch.current() do Sys.Tmp.with_tempdir("hyper-oci", fn tmp -> @@ -81,7 +83,7 @@ defmodule Hyper.Img.OciLoader do @spec test_system() :: :ok | {:error, {:missing_tools, [String.t()]}} def test_system do missing = - [Config.skopeo_path(), Config.umoci_path(), Config.mke2fs_path()] + [Config.skopeo_path(), Umoci.bin(), Config.mke2fs_path()] |> Enum.reject(&System.find_executable/1) if missing == [], do: :ok, else: {:error, {:missing_tools, missing}} @@ -139,7 +141,7 @@ defmodule Hyper.Img.OciLoader do ]) with :ok <- tag(skopeo, :skopeo), - :ok <- tag(cmd(Config.umoci_path(), ["unpack", "--image", "#{oci}:img", bundle]), :umoci) do + :ok <- tag(cmd(Umoci.bin(), ["unpack", "--image", "#{oci}:img", bundle]), :umoci) do {:ok, Path.join(bundle, "rootfs")} end end diff --git a/lib/hyper/img/oci_loader/umoci.ex b/lib/hyper/img/oci_loader/umoci.ex new file mode 100644 index 0000000..d2f3c4f --- /dev/null +++ b/lib/hyper/img/oci_loader/umoci.ex @@ -0,0 +1,97 @@ +defmodule Hyper.Img.OciLoader.Umoci do + @moduledoc """ + Resolves and (when not operator-provided) installs the `umoci` binary that + `Hyper.Img.OciLoader` uses to flatten OCI image layers. + + Two sources, in priority order (mirrors `Hyper.Node.Vmlinux`): + + 1. An operator-configured path via `config :hyper, umoci_path: "/path/to/umoci"` + (`Hyper.Config.umoci_path/0`). If set, it wins and is never downloaded. + 2. Otherwise the pinned static binary downloaded by `ensure_installed/0` into + `Hyper.Config.umoci_install_dir/0` (`/redist/umoci`). + + umoci ships one raw static binary per architecture + (`umoci.linux.amd64` / `umoci.linux.arm64`); the version and per-arch SHA-256 + are pinned below and verified on download via `Hyper.Redist.File`. The download + is a plain file (not an archive) with no execute bit, so it is `chmod`'d after + install. + """ + + alias Hyper.Config + alias Hyper.Redist + + @version "v0.6.0" + @assets %{x86_64: "umoci.linux.amd64", aarch64: "umoci.linux.arm64"} + @sha256 %{ + x86_64: "b51c267ec394499e42c6fde47f240b7b7dba57ea49df0b5acd304378b82a3b71", + aarch64: "5cfd17f2e7a4bcf9ed67ea1b955ca893d200349b9ce6a3d3707dba415f458a1f" + } + + @doc """ + Ensure a usable `umoci` is available on this node. A no-op when the operator + configured `umoci_path` (they own it); otherwise downloads the pinned static + binary for this node's architecture into the redist cache if it is not already + present and executable, then marks it executable. Idempotent. + """ + @spec ensure_installed() :: :ok | {:error, term()} + def ensure_installed do + if Config.umoci_path() != nil do + :ok + else + with {:ok, arch} <- Sys.Arch.current() do + path = default_path(arch) + if Sys.Posix.executable?(path), do: :ok, else: install(arch, path) + end + end + end + + @doc """ + Absolute path to the `umoci` binary: the operator-configured path if set, + otherwise the downloaded default for this node's architecture. Raises if the + architecture is unsupported (boot's `test_system/0` is expected to catch that). + """ + @spec bin() :: Path.t() + def bin do + if Config.umoci_path() != nil do + Config.umoci_path() + else + {:ok, arch} = Sys.Arch.current() + default_path(arch) + end + end + + @doc """ + Verify the resolved `umoci` is present and executable. Returns + `{:error, {:missing_tools, ["umoci"]}}` otherwise -- the same shape + `Hyper.Img.OciLoader.test_system/0` uses, which the gRPC layer maps to + FAILED_PRECONDITION. + """ + @spec test_system() :: :ok | {:error, {:missing_tools, [String.t()]}} | {:error, term()} + def test_system do + with {:ok, _arch} <- Sys.Arch.current() do + if Sys.Posix.executable?(bin()), + do: :ok, + else: {:error, {:missing_tools, ["umoci"]}} + end + end + + @doc false + @spec asset_for(Sys.Arch.t()) :: String.t() + def asset_for(arch), do: Map.fetch!(@assets, arch) + + @doc false + @spec asset_url(Sys.Arch.t()) :: String.t() + def asset_url(arch) do + "https://github.com/opencontainers/umoci/releases/download/#{@version}/#{asset_for(arch)}" + end + + @spec default_path(Sys.Arch.t()) :: Path.t() + defp default_path(arch), do: Path.join(Config.umoci_install_dir(), asset_for(arch)) + + @spec install(Sys.Arch.t(), Path.t()) :: :ok | {:error, term()} + defp install(arch, path) do + with :ok <- Redist.File.install(asset_url(arch), Map.fetch!(@sha256, arch), path) do + File.chmod(path, 0o755) + end + end +end diff --git a/test/hyper/img/oci_loader/umoci_test.exs b/test/hyper/img/oci_loader/umoci_test.exs new file mode 100644 index 0000000..f8a8096 --- /dev/null +++ b/test/hyper/img/oci_loader/umoci_test.exs @@ -0,0 +1,32 @@ +defmodule Hyper.Img.OciLoader.UmociTest do + use ExUnit.Case, async: true + + alias Hyper.Img.OciLoader.Umoci + + describe "asset_for/1" do + test "maps each arch to its umoci release asset" do + assert Umoci.asset_for(:x86_64) == "umoci.linux.amd64" + assert Umoci.asset_for(:aarch64) == "umoci.linux.arm64" + end + end + + describe "asset_url/1" do + test "points at the pinned v0.6.0 release asset" do + assert Umoci.asset_url(:x86_64) == + "https://github.com/opencontainers/umoci/releases/download/v0.6.0/umoci.linux.amd64" + + assert Umoci.asset_url(:aarch64) == + "https://github.com/opencontainers/umoci/releases/download/v0.6.0/umoci.linux.arm64" + end + end + + describe "bin/0" do + test "defaults to the redist install path for this node's arch when unconfigured" do + # The test env sets no :umoci_path, so bin/0 resolves to the downloaded default. + {:ok, arch} = Sys.Arch.current() + + assert Umoci.bin() == + Path.join(Hyper.Config.umoci_install_dir(), Umoci.asset_for(arch)) + end + end +end From 0471d6f2094b4dbde666eb645cda61fa0f206f48 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Thu, 25 Jun 2026 01:52:18 +0000 Subject: [PATCH 14/17] refactor(img): bind umoci_path once in bin/0 --- lib/hyper/img/oci_loader/umoci.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/hyper/img/oci_loader/umoci.ex b/lib/hyper/img/oci_loader/umoci.ex index d2f3c4f..1e8d11c 100644 --- a/lib/hyper/img/oci_loader/umoci.ex +++ b/lib/hyper/img/oci_loader/umoci.ex @@ -52,8 +52,10 @@ defmodule Hyper.Img.OciLoader.Umoci do """ @spec bin() :: Path.t() def bin do - if Config.umoci_path() != nil do - Config.umoci_path() + configured = Config.umoci_path() + + if configured != nil do + configured else {:ok, arch} = Sys.Arch.current() default_path(arch) From 4ba105e3670b6f5d6001da3045792113a99c6f63 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Thu, 25 Jun 2026 01:53:17 +0000 Subject: [PATCH 15/17] feat(node): provision umoci at boot like firecracker/vmlinux --- lib/hyper/img/oci_loader.ex | 3 ++- lib/hyper/node.ex | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index 0fa5ac8..71c34e0 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -8,7 +8,8 @@ defmodule Hyper.Img.OciLoader do 1. **pulls** it with `skopeo`, selecting the manifest entry matching this node's architecture, into a local OCI layout; 2. **flattens** it with `umoci unpack`, which applies OCI whiteouts/opaque - dirs correctly, yielding a merged rootfs directory; + dirs correctly, yielding a merged rootfs directory (`umoci` is + auto-downloaded by `Hyper.Img.OciLoader.Umoci` when not operator-provided); 3. **builds** an ext4 image of that rootfs with `mke2fs -d` (no loopback, no privilege, no setuid helper); 4. **content-addresses** the image by the sha256 of its bytes -- that hash is diff --git a/lib/hyper/node.ex b/lib/hyper/node.ex index 1a4adac..51bb49a 100644 --- a/lib/hyper/node.ex +++ b/lib/hyper/node.ex @@ -147,6 +147,8 @@ defmodule Hyper.Node do :ok <- Hyper.Node.FireVMM.Provider.ensure_installed(), :ok <- Hyper.Node.FireVMM.VmLinux.Provider.ensure_installed(), :ok <- Hyper.Node.Vmlinux.test_system(), + :ok <- Hyper.Img.OciLoader.Umoci.ensure_installed(), + :ok <- Hyper.Img.OciLoader.Umoci.test_system(), :ok <- Hyper.Node.Users.test_system(), :ok <- Hyper.Node.Layer.Repo.test_system(), :ok <- Hyper.SuidHelper.test_system(), From 812865ee12e154020d55933ef5e1d602d05b9e5e Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Thu, 25 Jun 2026 01:58:13 +0000 Subject: [PATCH 16/17] fix(img): report friendly tool names in OciLoader.test_system --- lib/hyper/img/oci_loader.ex | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index 71c34e0..884797e 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -83,9 +83,13 @@ defmodule Hyper.Img.OciLoader do """ @spec test_system() :: :ok | {:error, {:missing_tools, [String.t()]}} def test_system do - missing = - [Config.skopeo_path(), Umoci.bin(), Config.mke2fs_path()] - |> Enum.reject(&System.find_executable/1) + tools = [ + {"skopeo", Config.skopeo_path()}, + {"umoci", Umoci.bin()}, + {"mke2fs", Config.mke2fs_path()} + ] + + missing = for {name, path} <- tools, System.find_executable(path) == nil, do: name if missing == [], do: :ok, else: {:error, {:missing_tools, missing}} end From 6d7cd5cb7342baf26dce2b6d77f99e8b46b0eb51 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Thu, 25 Jun 2026 03:42:02 +0000 Subject: [PATCH 17/17] fix(img): make mix check pass (format, credo, sha256 arg order, dialyzer) - format two test files (codec_test, oci_loader_test) - credo --strict: drop redundant with-clause result; cond -> if in publish_file - sha256_file: File.stream!/3 arg order was wrong (modes/bytes swapped) -- a real hashing bug; switch to the shared Hyper.Redist.Sha256.file/1 helper - dialyzer: silence the Ecto.Multi opacity false positive in record/3 --- lib/hyper/img/oci_loader.ex | 40 +++++++++++++----------------- test/hyper/grpc/codec_test.exs | 4 ++- test/hyper/img/oci_loader_test.exs | 2 +- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index 884797e..378f81a 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -33,8 +33,12 @@ defmodule Hyper.Img.OciLoader do alias Hyper.Img.Db.{Blob, Image, ImageLayer, Repo} alias Hyper.Img.OciLoader.Umoci + # `Ecto.Multi` is an opaque struct; building it through the pipe trips + # dialyzer's opacity check (a known Ecto false positive), so silence it for the + # one function that assembles a Multi. + @dialyzer {:no_opaque, record: 3} + @mib 1024 * 1024 - @hash_chunk 2 * @mib # ext4 metadata (inode tables, journal, reserved blocks) plus slack so the # rootfs always fits. Overhead scales with content -- a flat constant is far @@ -68,9 +72,8 @@ defmodule Hyper.Img.OciLoader do with {:ok, rootfs} <- pull_and_unpack(source, goarch(arch), tmp), {:ok, content} <- dir_bytes(rootfs), bytes = ext4_bytes(content), - {:ok, staged} <- build_ext4(rootfs, bytes), - {:ok, id} <- finalize(staged, bytes, label) do - {:ok, id} + {:ok, staged} <- build_ext4(rootfs, bytes) do + finalize(staged, bytes, label) end end) end @@ -210,17 +213,10 @@ defmodule Hyper.Img.OciLoader do # --- hash + publish ------------------------------------------------------- - # Streaming sha256 of `path`, lowercase hex. + # Streaming sha256 of `path`, lowercase hex (reuses the shared redist helper). @spec sha256_file(Path.t()) :: {:ok, String.t()} | {:error, term()} defp sha256_file(path) do - digest = - path - |> File.stream!([:read, :binary, :raw], @hash_chunk) - |> Enum.reduce(:crypto.hash_init(:sha256), &:crypto.hash_update(&2, &1)) - |> :crypto.hash_final() - |> Base.encode16(case: :lower) - - {:ok, digest} + {:ok, Hyper.Redist.Sha256.file(path)} rescue e -> {:error, {:hash_failed, Exception.message(e)}} end @@ -233,16 +229,14 @@ defmodule Hyper.Img.OciLoader do File.mkdir_p!(Config.layer_dir()) final = final_path(id) - cond do - File.exists?(final) -> - _ = File.rm(staged) - {:ok, final} - - true -> - case File.rename(staged, final) do - :ok -> {:ok, final} - {:error, reason} -> {:error, {:publish_failed, reason}} - end + if File.exists?(final) do + _ = File.rm(staged) + {:ok, final} + else + case File.rename(staged, final) do + :ok -> {:ok, final} + {:error, reason} -> {:error, {:publish_failed, reason}} + end end end diff --git a/test/hyper/grpc/codec_test.exs b/test/hyper/grpc/codec_test.exs index eab50f7..bdd0cc7 100644 --- a/test/hyper/grpc/codec_test.exs +++ b/test/hyper/grpc/codec_test.exs @@ -44,7 +44,9 @@ defmodule Hyper.Grpc.CodecTest do test "tool/build/db failures -> INTERNAL" do assert Codec.to_grpc({:error, {:skopeo_failed, 1, "boom"}}).status == GRPC.Status.internal() - assert Codec.to_grpc({:error, {:record_failed, :blob, :db}}).status == GRPC.Status.internal() + + assert Codec.to_grpc({:error, {:record_failed, :blob, :db}}).status == + GRPC.Status.internal() end end end diff --git a/test/hyper/img/oci_loader_test.exs b/test/hyper/img/oci_loader_test.exs index df1ff4a..3869d93 100644 --- a/test/hyper/img/oci_loader_test.exs +++ b/test/hyper/img/oci_loader_test.exs @@ -52,7 +52,7 @@ defmodule Hyper.Img.OciLoaderTest do end property "always a whole-MiB size that fits the content and clears the floor" do - check all content <- integer(0..(8 * 1024 * @mib)) do + check all(content <- integer(0..(8 * 1024 * @mib))) do size = OciLoader.ext4_bytes(content) assert rem(size, @mib) == 0 assert size >= content