From 9fa9db0cff500c92c9f3fe280ca3bca2fc9d4583 Mon Sep 17 00:00:00 2001 From: wenkaifan0720 Date: Wed, 24 Jun 2026 16:25:44 -0700 Subject: [PATCH 1/2] feat(cef): expose frame IOSurface id to Dart for WebRTC streaming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add CefWebController.getFrameSurface() + an onSurface callback (and the CefSurfaceInfo model) that surface the existing global IOSurface id, so a host app can IOSurfaceLookup the live OSR CVPixelBuffer and feed it to a libwebrtc RTCVideoSource — with no webrtc dependency in flutter_cef. onSurface fires on each (re)allocation so consumers re-resolve after a resize realloc. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/src/cef_web_controller.dart | 34 +++++++++++++++++++ .../macos/Classes/CefWebSession.swift | 18 ++++++++-- .../macos/Classes/FlutterCefPlugin.swift | 24 +++++++++++++ .../lib/src/cef_events.dart | 27 +++++++++++++++ 4 files changed, 101 insertions(+), 2 deletions(-) diff --git a/lib/src/cef_web_controller.dart b/lib/src/cef_web_controller.dart index 9d2b5a6..8f1220a 100644 --- a/lib/src/cef_web_controller.dart +++ b/lib/src/cef_web_controller.dart @@ -120,6 +120,15 @@ class CefWebController { /// you generally don't set this yourself. void Function(Rect caretRect)? onImeCompositionBounds; + /// Called whenever the off-screen frame surface is (re)allocated — once at + /// session create and again on every resize (which frees the old IOSurface + /// and allocs a new one). Carries the new global IOSurface id and its + /// PHYSICAL (Retina) pixel dims. A consumer that mirrors the live page pixels + /// off-Flutter (e.g. a WebRTC capturer) resolves the surface by id and must + /// re-read on every fire — never cache the surface across frames. Pull the + /// current surface on demand with [getFrameSurface]. + void Function(CefSurfaceInfo info)? onSurface; + /// Handle a page `alert(...)`. Show your UI, then return to dismiss it. If /// unset, alerts are auto-dismissed. Future Function(CefJsDialogRequest request)? onJavaScriptAlertDialog; @@ -240,6 +249,13 @@ class CefWebController { (a['h'] as num? ?? 0).toDouble(), )); break; + case 'onSurface': + onSurface?.call(CefSurfaceInfo( + surfaceId: a['surfaceId'] as int? ?? 0, + width: a['width'] as int? ?? 0, + height: a['height'] as int? ?? 0, + )); + break; case 'processGone': // The native host dropped this session (crash, cache-lock loss, or a // create that failed — reason 'createFailed'). The texture is dead. Fail @@ -552,6 +568,24 @@ class CefWebController { Future disableAgentControl() => _channel.invokeMethod('disableAgentControl', {'sessionId': sessionId}); + /// Read the live off-screen frame surface: the global IOSurface id this + /// session's CVPixelBuffer is backed by, plus its PHYSICAL (Retina) pixel + /// dims. The on-demand pull counterpart to the [onSurface] event (which fires + /// on each (re)alloc). Returns null if the session doesn't exist yet, or a + /// [CefSurfaceInfo] with `surfaceId: 0` before the buffer is allocated. A + /// consumer mirroring the live pixels off-Flutter resolves the surface by id + /// and must re-read after any resize — the surface is freed and reallocated. + Future getFrameSurface() async { + final res = await _channel.invokeMapMethod( + 'getFrameSurface', {'sessionId': sessionId}); + if (res == null) return null; + return CefSurfaceInfo( + surfaceId: res['surfaceId'] as int? ?? 0, + width: res['width'] as int? ?? 0, + height: res['height'] as int? ?? 0, + ); + } + /// Load host-trusted content, bypassing the navigation scheme allowlist. /// Backs [loadHtmlString] (data:) and [loadFile] (file:): the host explicitly /// chose this content, so it isn't subject to `allowedSchemes` the way page diff --git a/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift b/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift index cbc42d5..4f296d2 100644 --- a/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift +++ b/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift @@ -90,6 +90,12 @@ final class CefWebSession: NSObject, FlutterTexture { var onDownload: ((String) -> Void)? // suggested name var onImeBounds: ((Int, Int, Int, Int) -> Void)? // caret rect x,y,w,h (DIP) var onCookies: ((Int, String) -> Void)? // request id, json array + // Fired when the backing IOSurface is (re)allocated — at create and on every + // resize() (which reallocs). Args are the live global surface id and the + // PHYSICAL (Retina) pixel dims. A consumer that mirrors the live frame + // (e.g. an off-Flutter capturer) reads the surface by id and must re-read on + // each fire, since resize() frees the old surface and allocs a new one. + var onSurface: ((UInt32, Int, Int) -> Void)? // surfaceId, physW, physH let sessionId: String private(set) var textureId: Int64 = 0 @@ -487,12 +493,20 @@ final class CefWebSession: NSObject, FlutterTexture { @discardableResult private func publishBuffers(_ surf: IOSurfaceRef, _ buffer: CVPixelBuffer, _ w: Int, _ h: Int) -> UInt32 { - bufferLock.lock(); defer { bufferLock.unlock() } + bufferLock.lock() ioSurface = surf pixelBuffer = buffer width = w height = h - return IOSurfaceGetID(surf) + let sid = IOSurfaceGetID(surf) + bufferLock.unlock() + // WebRTC frame export: notify any consumer that the surface (re)allocated — + // fires on init + each resize, both of which publish here. Outside + // bufferLock so the callback can read session accessors without + // self-deadlock. Physical (Retina) dims = logical * dpr. + onSurface?(sid, Int((Double(w) * Double(dpr)).rounded()), + Int((Double(h) * Double(dpr)).rounded())) + return sid } /// H4: read (w, h, dpr, surfaceId) as ONE consistent tuple under a single bufferLock diff --git a/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift b/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift index d2f6906..29b5bc2 100644 --- a/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift +++ b/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift @@ -100,6 +100,7 @@ public class FlutterCefPlugin: NSObject, FlutterPlugin { case "navigate": navigate(args, result) case "loadTrusted": loadTrusted(args, result) case "resize": resize(args, result) + case "getFrameSurface": getFrameSurface(args, result) case "dispose": destroy(args, result) case "pointer": pointer(args, result) case "key": key(args, result) @@ -403,6 +404,12 @@ public class FlutterCefPlugin: NSObject, FlutterPlugin { session.onCookies = { [weak self] id, json in self?.emit("cookies", ["sessionId": sessionId, "id": id, "json": json]) } + session.onSurface = { [weak self] surfaceId, width, height in + self?.emit("onSurface", [ + "sessionId": sessionId, "surfaceId": Int(surfaceId), + "width": width, "height": height, + ]) + } // Allocate the wire browserId + (when ready) issue opCreateBrowser. The // process arg --allowed-schemes is shared by every browser in the profile; // it's taken from the first browser that triggered the spawn. @@ -584,6 +591,23 @@ public class FlutterCefPlugin: NSObject, FlutterPlugin { } } + /// Read the session's live frame surface synchronously: the global IOSurface + /// id its CVPixelBuffer is backed by, plus the PHYSICAL (Retina) pixel dims + /// (logical w/h * dpr, matching allocateBuffers' rounding). A frame-export + /// consumer resolves the surface by id; the `onSurface` event fires it on each + /// (re)alloc, this verb is the on-demand pull. Returns nil for an unknown + /// session, or `surfaceId: 0` before the buffer is allocated. + private func getFrameSurface(_ a: [String: Any], _ result: @escaping FlutterResult) { + guard let id = a["sessionId"] as? String, let s = sessions[id] else { + result(nil) + return + } + let dpr = Double(s.scale) + let pw = max(1, Int((Double(s.w) * dpr).rounded())) + let ph = max(1, Int((Double(s.h) * dpr).rounded())) + result(["surfaceId": Int(s.surfaceId), "width": pw, "height": ph]) + } + private func destroy(_ a: [String: Any], _ result: @escaping FlutterResult) { if let id = a["sessionId"] as? String { disposeSession(id) } result(nil) diff --git a/packages/flutter_cef_platform_interface/lib/src/cef_events.dart b/packages/flutter_cef_platform_interface/lib/src/cef_events.dart index 3fb061e..c131235 100644 --- a/packages/flutter_cef_platform_interface/lib/src/cef_events.dart +++ b/packages/flutter_cef_platform_interface/lib/src/cef_events.dart @@ -72,6 +72,33 @@ class CefJsDialogRequest { String toString() => 'CefJsDialogRequest($message)'; } +/// The live frame surface backing a session: the global IOSurface id its +/// off-screen CVPixelBuffer is wrapped over, plus the surface's PHYSICAL +/// (Retina) pixel dimensions. Delivered by [CefWebController.onSurface] on each +/// (re)allocation (create + every resize) and pullable on demand via +/// [CefWebController.getFrameSurface]. A consumer resolves the surface by id +/// (e.g. `IOSurfaceLookup`) to mirror the live page pixels off-Flutter; it must +/// re-read on every change, since a resize frees the old surface. +class CefSurfaceInfo { + const CefSurfaceInfo({ + required this.surfaceId, + required this.width, + required this.height, + }); + + /// The global IOSurface id, resolvable cross-process. 0 before allocation. + final int surfaceId; + + /// Physical (Retina) pixel width of the surface. + final int width; + + /// Physical (Retina) pixel height of the surface. + final int height; + + @override + String toString() => 'CefSurfaceInfo($surfaceId, ${width}x$height)'; +} + /// A cookie returned by [CefWebController.getCookies]. class CefCookie { const CefCookie({ From 5aeec461d73e50df30548e348587e9f7092809c6 Mon Sep 17 00:00:00 2001 From: wenkaifan0720 Date: Thu, 25 Jun 2026 08:53:51 -0700 Subject: [PATCH 2/2] fix(cef): fire onSurface on resize realloc + initial; export CefSurfaceInfo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The onSurface callback was effectively dead: it fired only from the init publish (before the plugin wired the callback) and never on resize (sendResize swaps the surface inline; promotion happens in handleFrame(opPresent)/the resize watchdog, neither of which called publishBuffers). So a consumer's resize re-point (R2) never triggered — after any resize the freed/stale surface kept being sampled. - Route both resize-promotion paths (opPresent + watchdog force-promote) through a notifySurface() helper so onSurface fires on every realloc. - Add emitCurrentSurface(); the plugin calls it right after wiring onSurface so the initial surface (whose publish predated the callback) is delivered. - Export CefSurfaceInfo from the public barrel (was unreferenceable by consumers). Now matches the documented "fires on init + each resize" contract. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flutter_cef.dart | 3 +- .../macos/Classes/CefWebSession.swift | 43 ++++++++++++++++--- .../macos/Classes/FlutterCefPlugin.swift | 3 ++ 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/lib/flutter_cef.dart b/lib/flutter_cef.dart index d6f63e4..cc7c7b3 100644 --- a/lib/flutter_cef.dart +++ b/lib/flutter_cef.dart @@ -13,6 +13,7 @@ export 'package:flutter_cef_platform_interface/flutter_cef_platform_interface.da CefConsoleMessage, CefFindResult, CefJsDialogRequest, - CefLoadError; + CefLoadError, + CefSurfaceInfo; export 'src/cef_web_controller.dart' show CefWebController; export 'src/cef_web_view.dart' show CefWebView; diff --git a/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift b/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift index 4f296d2..77d86b7 100644 --- a/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift +++ b/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift @@ -269,9 +269,14 @@ final class CefWebSession: NSObject, FlutterTexture { let active = resizeInFlight && gen == resizeGen let givenUp = active && (nowNs() &- resizeSentAtNs) > 300_000_000 var promotedTid: Int64 = 0 + var promotedSid: UInt32 = 0 + var promotedW = 0, promotedH = 0 if givenUp { if let pending = pendingBuffer { pixelBuffer = pending + promotedSid = pendingSurfaceId // capture before clearing + promotedW = width + promotedH = height pendingBuffer = nil pendingSurfaceId = 0 promotedTid = textureId @@ -281,6 +286,8 @@ final class CefWebSession: NSObject, FlutterTexture { bufferLock.unlock() if givenUp { if promotedTid != 0 { registry?.textureFrameAvailable(promotedTid) } + // R2: force-promoted a resized surface — notify WebRTC consumers (same as opPresent). + if promotedSid != 0 { notifySurface(promotedSid, promotedW, promotedH) } maybeSendNextResize() return } @@ -500,15 +507,31 @@ final class CefWebSession: NSObject, FlutterTexture { height = h let sid = IOSurfaceGetID(surf) bufferLock.unlock() - // WebRTC frame export: notify any consumer that the surface (re)allocated — - // fires on init + each resize, both of which publish here. Outside - // bufferLock so the callback can read session accessors without - // self-deadlock. Physical (Retina) dims = logical * dpr. - onSurface?(sid, Int((Double(w) * Double(dpr)).rounded()), - Int((Double(h) * Double(dpr)).rounded())) + notifySurface(sid, w, h) return sid } + /// WebRTC frame export: notify any consumer that the live surface (re)allocated, so it + /// can IOSurfaceLookup the new id and re-point its capture (R2). Call OUTSIDE bufferLock + /// so the callback can read session accessors without self-deadlock. Reports PHYSICAL + /// (Retina) pixel dims = logical * dpr. + private func notifySurface(_ sid: UInt32, _ logicalW: Int, _ logicalH: Int) { + guard sid != 0 else { return } + onSurface?(sid, Int((Double(logicalW) * Double(dpr)).rounded()), + Int((Double(logicalH) * Double(dpr)).rounded())) + } + + /// Re-emit the current live surface to a just-attached onSurface consumer. The init + /// publish fires before the plugin wires onSurface, so the plugin calls this right + /// after assigning the callback to deliver the initial surface. + func emitCurrentSurface() { + bufferLock.lock() + let surf = ioSurface + let w = width, h = height + bufferLock.unlock() + if let surf = surf { notifySurface(IOSurfaceGetID(surf), w, h) } + } + /// H4: read (w, h, dpr, surfaceId) as ONE consistent tuple under a single bufferLock /// acquisition — the host builds opCreateBrowser from this so its payload can't /// capture a torn mix of stale dims + a freshly-reallocated surface id. @@ -532,6 +555,8 @@ final class CefWebSession: NSObject, FlutterTexture { // (BE u32). If it's our pending (resized) surface, promote it to live now — we // kept serving the old surface until this exact frame so Flutter never sampled the // blank new one. A present for the old/current surface just advances the frame. + var promotedSid: UInt32 = 0 + var promotedW = 0, promotedH = 0 if payload.count >= 4 { let psid = (UInt32(payload[0]) << 24) | (UInt32(payload[1]) << 16) | (UInt32(payload[2]) << 8) | UInt32(payload[3]) @@ -540,10 +565,16 @@ final class CefWebSession: NSObject, FlutterTexture { pendingBuffer = nil pendingSurfaceId = 0 resizeInFlight = false // its paint landed; free to send the next size + promotedSid = psid + promotedW = width + promotedH = height } } let tid = textureId bufferLock.unlock() + // R2: a resized surface just went live — tell WebRTC consumers to re-point their + // IOSurface capture at the new id (this is the "fires on each resize" half). + if promotedSid != 0 { notifySurface(promotedSid, promotedW, promotedH) } if tid != 0 { DispatchQueue.main.async { [weak self] in // Re-read on main (serialized with dispose()): the texture may have diff --git a/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift b/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift index 29b5bc2..bd27464 100644 --- a/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift +++ b/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift @@ -410,6 +410,9 @@ public class FlutterCefPlugin: NSObject, FlutterPlugin { "width": width, "height": height, ]) } + // The session's init publish fired onSurface before the callback above existed, so + // deliver the current surface now (no-op until the first surface is allocated). + session.emitCurrentSurface() // Allocate the wire browserId + (when ready) issue opCreateBrowser. The // process arg --allowed-schemes is shared by every browser in the profile; // it's taken from the first browser that triggered the spawn.