feature: add ublk-based block device frontend (overlaybd-ublk)

.github/workflows/release/build.sh

@@ -73,11 +73,13 @@ elif [[ ${OS} =~ "centos" ]]; then
     fi
 
     yum install -y epel-release libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel
-    yum install -y rpm-build make git wget sudo autoconf automake libtool
+    # pkgconfig: ublksrv's configure uses PKG_CHECK_MODULES (needs pkg.m4 at autoreconf time)
+    yum install -y rpm-build make git wget sudo autoconf automake libtool pkgconfig
     yum install --skip-broken -y libzstd-static gcc gcc-c++ binutils libzstd-devel
 elif [[ ${OS} =~ "mariner" ]]; then
     yum install -y libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel glibc-devel libzstd-devel binutils ca-certificates-microsoft build-essential
-    yum install -y rpm-build make git wget sudo tar gcc gcc-c++ autoconf automake libtool
+    # pkg-config: ublksrv's configure uses PKG_CHECK_MODULES (needs pkg.m4 at autoreconf time)
+    yum install -y rpm-build make git wget sudo tar gcc gcc-c++ autoconf automake libtool pkg-config
 
     DISTRO=${OS/:/.}
     PACKAGE_RELEASE="-DPACKAGE_RELEASE=${RELEASE_NO}.${DISTRO}"

CMake/Findliburing.cmake

@@ -0,0 +1,72 @@
+# liburing (hard dependency of ublksrv).
+# Distro liburing packages are commonly missing or too old (>= 2.2 required)
+# and behavior differs across versions, so like every other dependency of
+# this project it is always fetched at a pinned version and statically
+# linked -- never taken from the system.
+# FetchContent only downloads; the build uses liburing's own configure/make.
+include(FetchContent)
+set(FETCHCONTENT_QUIET false)
+
+# CMake 3.30+ deprecates single-argument FetchContent_Populate (CMP0169);
+# explicitly keep the old behavior
+if(POLICY CMP0169)
+  cmake_policy(SET CMP0169 OLD)
+endif()
+
+FetchContent_Declare(
+  liburing
+  GIT_REPOSITORY https://github.com/axboe/liburing.git
+  GIT_TAG        liburing-2.8
+)
+
+# download only, no add_subdirectory (liburing is a plain Makefile project)
+FetchContent_GetProperties(liburing)
+if(NOT liburing_POPULATED)
+  FetchContent_Populate(liburing)
+endif()
+
+if(NOT TARGET liburing_build)
+  # OUTPUT points at the final artifact => incremental build, skipped once present
+  add_custom_command(
+    OUTPUT ${liburing_SOURCE_DIR}/src/liburing.a
+    WORKING_DIRECTORY ${liburing_SOURCE_DIR}
+    COMMAND ./configure
+    COMMAND make -C src -j
+    COMMENT "Building liburing with its own configure/make"
+    VERBATIM
+  )
+  add_custom_target(liburing_build DEPENDS ${liburing_SOURCE_DIR}/src/liburing.a)
+endif()
+
+set(LIBURING_LIBRARIES    ${liburing_SOURCE_DIR}/src/liburing.a)
+set(LIBURING_INCLUDE_DIRS ${liburing_SOURCE_DIR}/src/include)
+
+# Generate an in-build-tree liburing.pc for ublksrv's configure check
+# PKG_CHECK_MODULES([LIBURING], [liburing >= 2.2]) (the official
+# build_with_liburing_src flow relies on pkg-config metadata)
+set(LIBURING_PC_DIR ${CMAKE_BINARY_DIR}/liburing-pkgconfig)
+file(WRITE ${LIBURING_PC_DIR}/liburing.pc
+"prefix=${liburing_SOURCE_DIR}
+libdir=${liburing_SOURCE_DIR}/src
+includedir=${liburing_SOURCE_DIR}/src/include
+
+Name: liburing
+Version: 2.8
+Description: io_uring library
+Libs: -L\${libdir} -luring
+Cflags: -I\${includedir}
+")
+
+if(NOT TARGET liburing_static)
+  add_library(liburing_static STATIC IMPORTED GLOBAL)
+  set_target_properties(liburing_static PROPERTIES
+    IMPORTED_LOCATION             ${LIBURING_LIBRARIES}
+    INTERFACE_INCLUDE_DIRECTORIES ${LIBURING_INCLUDE_DIRS}
+  )
+endif()
+add_dependencies(liburing_static liburing_build)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(liburing DEFAULT_MSG LIBURING_LIBRARIES LIBURING_INCLUDE_DIRS)
+
+mark_as_advanced(LIBURING_LIBRARIES LIBURING_INCLUDE_DIRS)

CMake/Findublksrv.cmake

@@ -0,0 +1,78 @@
+# ublksrv (ublk userspace library; autotools project, no CMake).
+# Only the lib/ directory (pure-C libublksrv) is built:
+#   - avoids the C++20/-fcoroutines requirement of the ublk CLI tool;
+#   - libnfs/libiscsi/gnutls only serve its bundled targets, all disabled.
+# liburing is built from source by Findliburing.cmake and fed to configure
+# via pkg-config/CFLAGS/LDFLAGS (the official build_with_liburing_src flow).
+# License: lib/ + include/ublksrv.h are dual MIT/LGPL; linked under MIT.
+include(FetchContent)
+set(FETCHCONTENT_QUIET false)
+
+if(POLICY CMP0169)
+  cmake_policy(SET CMP0169 OLD)
+endif()
+
+# the autotools build needs extra host tools; fail early with a clear message
+find_program(AUTORECONF_EXECUTABLE autoreconf)
+find_program(LIBTOOLIZE_EXECUTABLE libtoolize)
+find_program(AUTOMAKE_EXECUTABLE automake)
+if(NOT AUTORECONF_EXECUTABLE OR NOT LIBTOOLIZE_EXECUTABLE OR NOT AUTOMAKE_EXECUTABLE)
+  message(FATAL_ERROR
+    "BUILD_UBLK_FRONTEND requires autotools (autoconf/automake/libtool) "
+    "to build ublksrv. Install them or configure with -DBUILD_UBLK_FRONTEND=off")
+endif()
+
+FetchContent_Declare(
+  ublksrv
+  GIT_REPOSITORY https://github.com/ublk-org/ublksrv.git
+  GIT_TAG        f6c643952d1cdc7f6460630638fe6b5454ca1c4d   # v1.7
+)
+
+FetchContent_GetProperties(ublksrv)
+if(NOT ublksrv_POPULATED)
+  FetchContent_Populate(ublksrv)
+endif()
+
+if(NOT TARGET libublksrv_build)
+  add_custom_command(
+    OUTPUT ${ublksrv_SOURCE_DIR}/lib/.libs/libublksrv.a
+    WORKING_DIRECTORY ${ublksrv_SOURCE_DIR}
+    COMMAND autoreconf -i
+    COMMAND ${CMAKE_COMMAND} -E env PKG_CONFIG_PATH=${LIBURING_PC_DIR}
+            ./configure
+            --without-libnfs --without-libiscsi --without-gnutls
+            # configure appends -fcoroutines whenever $CXX matches *g++*
+            # (GCC 10+ only, and only the ublk tool needs it, lib/ does not);
+            # use the name c++ to sidestep that match
+            CXX=c++
+            "CFLAGS=-I${LIBURING_INCLUDE_DIRS} -O2"
+            "CXXFLAGS=-I${LIBURING_INCLUDE_DIRS} -O2"
+            "LDFLAGS=-L${liburing_SOURCE_DIR}/src"
+    COMMAND make -C lib -j
+    DEPENDS ${LIBURING_LIBRARIES}
+    COMMENT "Building libublksrv (lib/ only) with autotools"
+    VERBATIM
+  )
+  add_custom_target(libublksrv_build DEPENDS ${ublksrv_SOURCE_DIR}/lib/.libs/libublksrv.a)
+  add_dependencies(libublksrv_build liburing_build)
+endif()
+
+set(UBLKSRV_LIBRARIES    ${ublksrv_SOURCE_DIR}/lib/.libs/libublksrv.a)
+set(UBLKSRV_INCLUDE_DIRS ${ublksrv_SOURCE_DIR}/include)
+
+if(NOT TARGET libublksrv_static)
+  add_library(libublksrv_static STATIC IMPORTED GLOBAL)
+  set_target_properties(libublksrv_static PROPERTIES
+    IMPORTED_LOCATION             ${UBLKSRV_LIBRARIES}
+    INTERFACE_INCLUDE_DIRECTORIES ${UBLKSRV_INCLUDE_DIRS}
+    # libublksrv.a references liburing symbols; propagate via INTERFACE so
+    # consumers get the correct link order automatically
+    INTERFACE_LINK_LIBRARIES      liburing_static
+  )
+endif()
+add_dependencies(libublksrv_static libublksrv_build)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ublksrv DEFAULT_MSG UBLKSRV_LIBRARIES UBLKSRV_INCLUDE_DIRS)
+
+mark_as_advanced(UBLKSRV_LIBRARIES UBLKSRV_INCLUDE_DIRS)

CMakeLists.txt

@@ -53,10 +53,17 @@ set(ENABLE_MIMIC_VDSO off)
 option(BUILD_CURL_FROM_SOURCE "Compile static libcurl" off)
 option(BUILD_STREAM_CONVERTOR "Build the stream convertor" on)
 option(ORIGIN_EXT2FS "Use original libext2fs" off)
+# Building it requires autotools on the build machine; disable with -DBUILD_UBLK_FRONTEND=off
+option(BUILD_UBLK_FRONTEND "Build the ublk frontend (overlaybd-ublk), requires autotools" on)
 
 find_package(photon REQUIRED)
 find_package(tcmu REQUIRED)
 
+if(BUILD_UBLK_FRONTEND)
+    find_package(liburing REQUIRED)
+    find_package(ublksrv REQUIRED)
+endif()
+
 if(BUILD_STREAM_CONVERTOR)
     find_package(yaml-cpp)
     if (NOT yaml-cpp_FOUND)

README.md

@@ -119,6 +119,73 @@ cmake -D ENABLE_QAT=1 ..
 
 For more information go to `overlaybd/src/overlaybd/zfile/README.md`.
 
+If you want to build the ublk frontend (`overlaybd-ublk`), which exposes an
+image as `/dev/ublkbN` without going through TCMU/SCSI. It is built by default
+(disable with `-D BUILD_UBLK_FRONTEND=off`); building it additionally requires
+`autoconf`, `automake` and `libtool` (liburing and libublksrv are fetched and
+built from source automatically).
+
+```bash
+cmake -D BUILD_UBLK_FRONTEND=on ..
+```
+
+Running it requires a kernel with the `ublk_drv` driver (mainline >= 6.0, or a
+distro kernel with ublk backported):
+
+```bash
+sudo modprobe ublk_drv
+sudo overlaybd-ublk add --config /path/to/config.v1.json   # prints /dev/ublkbN when ready
+sudo overlaybd-ublk list
+sudo overlaybd-ublk del -n 0
+```
+
+Unlike `overlaybd-tcmu`, one `overlaybd-ublk` process serves exactly one
+device; killing the daemon removes the device. Each device can write to its
+own log file via `add --log-path ...` (recommended when running multiple
+devices; daemons otherwise share the global log file and are distinguished
+by a `ublk-<pid>` tag).
+
+For hosting many devices in one process there is also `overlaybd-ublkd`, a
+centralized daemon sharing a single ImageService (and thus one cache tree,
+`<base>/daemon/` under `--cache-dir`, guarded by an exclusive lock). Its
+control API is HTTP over a root-only unix socket, debuggable with curl:
+
+```bash
+sudo overlaybd-ublkd &        # or: systemctl start overlaybd-ublkd
+SOCK=/var/run/overlaybd-ublk/ublkd.sock
+curl --unix-socket $SOCK -X POST -d '{"config":"/path/config.v1.json"}' http://d/v1/add
+curl --unix-socket $SOCK http://d/v1/list
+curl --unix-socket $SOCK -X POST -d '{"dev_id":0}' http://d/v1/del
+curl --unix-socket $SOCK -X POST -d '{"dev_id":0,"size_gb":50,"resize_fs":true}' http://d/v1/resize
+curl --unix-socket $SOCK -X POST http://d/v1/shutdown   # stops all devices
+```
+
+`add` replies once the device is usable; `del` replies after full teardown;
+online `resize` (grow only, writable images) additionally needs a kernel
+with `UBLK_F_UPDATE_SIZE` (mainline >= 6.11). Writable images are mounted
+exclusively across the daemon and CLI processes; devices owned by the
+daemon must be deleted through its API (the CLI `del` refuses them). A
+systemd unit template is installed at /opt/overlaybd/overlaybd-ublkd.service.
+
+Notes:
+
+* Always unmount filesystems on `/dev/ublkbN` before `del` (or before stopping
+  the daemon): the daemon serves the device's IO, so tearing it down with a
+  mounted filesystem aborts in-flight journal writes.
+* Caches are always isolated per device: each daemon uses
+  `/opt/overlaybd/ublk_cache/<image-key>/<instance>/` (override the base
+  directory with `add --cache-dir ...`), guarded by an exclusive lock -- the
+  file cache's locking is in-process only, so daemons must never share a
+  cache directory. Mounting the same read-only image multiple times gets
+  automatically numbered instances (or pin one with `--instance-id`);
+  remounting an image reuses its warm cache; mounting a writable image twice
+  is rejected to protect its upper layer. A per-device log file
+  (`add --log-path ...`) is recommended when running multiple devices.
+* Runtime verified on a 5.10 kernel with ublk backported (Alinux
+  5.10.134); a full regression on mainline 6.x kernels is still pending.
+  Known backport-kernel caveats: DISCARD is unavailable there, and `del`
+  takes ~2s instead of milliseconds.
+
 Finally, setup a systemd service for overlaybd-tcmu backstore.
 
 ```bash

src/CMakeLists.txt

@@ -75,6 +75,9 @@ if (NOT ORIGIN_EXT2FS)
 endif()
 
 add_subdirectory(tools)
+if (BUILD_UBLK_FRONTEND)
+    add_subdirectory(ublk)
+endif ()
 if (BUILD_TESTING)
     add_subdirectory(test)
 endif ()

src/example_config/overlaybd-ublkd.service

@@ -0,0 +1,30 @@
+[Unit]
+Description=overlaybd-ublkd service (centralized ublk device manager)
+After=network.target local-fs.target
+Before=shutdown.target
+DefaultDependencies=no
+Conflicts=shutdown.target
+
+[Service]
+LimitNOFILE=1048576
+LimitCORE=infinity
+Type=simple
+
+ExecStartPre=/sbin/modprobe ublk_drv
+
+ExecStart=/opt/overlaybd/bin/overlaybd-ublkd
+
+# Devices are stopped in parallel on shutdown, but each deletion may take
+# ~2s on kernels without DEL_DEV_ASYNC. Give teardown enough room: killing
+# a daemon that is still flushing dirty pages risks a writeback deadlock
+# (see the ublk notes in README).
+TimeoutStopSec=60
+
+GuessMainPID=no
+Restart=on-failure
+RestartSec=1s
+KillMode=process
+OOMScoreAdjust=-999
+
+[Install]
+WantedBy=multi-user.target

src/ublk/CMakeLists.txt

@@ -0,0 +1,53 @@
+# overlaybd-ublk: the ublk block device frontend (one process serves one device)
+# Entered only when BUILD_UBLK_FRONTEND=on
+
+add_library(ublk_frontend_lib
+    io_dispatch.cpp
+    ublk_device.cpp
+    cli.cpp
+    config_patch.cpp
+    ublkd_protocol.cpp
+)
+target_include_directories(ublk_frontend_lib PUBLIC
+    ${PHOTON_INCLUDE_DIR}
+    ${RAPIDJSON_INCLUDE_DIRS}
+)
+target_link_libraries(ublk_frontend_lib
+    photon_static
+    overlaybd_image_lib
+    libublksrv_static      # IMPORTED target; include dirs and liburing propagate via INTERFACE
+)
+
+add_executable(overlaybd-ublk main.cpp)
+target_link_libraries(overlaybd-ublk
+    ublk_frontend_lib
+    photon_static
+    overlaybd_image_lib
+    libublksrv_static
+    ${CURL_LIBRARIES}
+    ${OPENSSL_SSL_LIBRARY}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${AIO_LIBRARIES}
+)
+
+install(TARGETS overlaybd-ublk DESTINATION /opt/overlaybd/bin)
+
+# overlaybd-ublkd: daemon mode, all devices in one process (ADR-0006)
+add_executable(overlaybd-ublkd ublkd_main.cpp)
+target_link_libraries(overlaybd-ublkd
+    ublk_frontend_lib
+    photon_static
+    overlaybd_image_lib
+    libublksrv_static
+    ${CURL_LIBRARIES}
+    ${OPENSSL_SSL_LIBRARY}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${AIO_LIBRARIES}
+)
+install(TARGETS overlaybd-ublkd DESTINATION /opt/overlaybd/bin)
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../example_config/overlaybd-ublkd.service
+        DESTINATION /opt/overlaybd/)
+
+if (BUILD_TESTING)
+    add_subdirectory(test)
+endif ()