Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions packages/producer/src/services/renderOrchestrator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1763,12 +1763,28 @@ describe("shouldPreferParallelDrawElement (DE parallel router)", () => {
probeDeGated: false,
experimentalParallelDeOptIn: false,
routerEnabled: true,
totalMemoryMb: 32768,
minMemoryMb: 24576,
};

it("routes an auto-resolved multi-worker render for an eligible long comp", () => {
expect(shouldPreferParallelDrawElement(eligible)).toBe(true);
});

it("withholds the parallel bet below the RAM floor (16 GB black-slab report)", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, totalMemoryMb: 16384 })).toBe(false);
});

it("routes exactly at the RAM floor", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, totalMemoryMb: 24576 })).toBe(true);
});

it("minMemoryMb <= 0 disables the RAM guard", () => {
expect(
shouldPreferParallelDrawElement({ ...eligible, totalMemoryMb: 8192, minMemoryMb: 0 }),
).toBe(true);
});

it("is disabled by default (routerEnabled: false is the shipped default)", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, routerEnabled: false })).toBe(false);
});
Expand Down
27 changes: 26 additions & 1 deletion packages/producer/src/services/renderOrchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ import {
isDrawElementVerificationError,
} from "@hyperframes/engine";
import { join, dirname, resolve } from "path";
import { totalmem } from "node:os";
import { randomUUID } from "crypto";
import { fileURLToPath } from "url";
import {
Expand Down Expand Up @@ -1130,6 +1131,10 @@ export function shouldPreferParallelDrawElement(args: {
experimentalParallelDeOptIn: boolean;
/** HF_DE_PARALLEL_ROUTER === "true" — the router's own kill switch, default off. */
routerEnabled: boolean;
/** Machine RAM (os.totalmem, MB). */
totalMemoryMb: number;
/** RAM floor for routing; <=0 disables the guard. */
minMemoryMb: number;
}): boolean {
return (
args.routerEnabled &&
Expand All @@ -1144,7 +1149,14 @@ export function shouldPreferParallelDrawElement(args: {
!args.layeredOrEffectRoute &&
!args.supersampling &&
!args.probeDeGated &&
!args.experimentalParallelDeOptIn
!args.experimentalParallelDeOptIn &&
// RAM floor: routed parallel DE runs 3 concurrent hardware-GPU Chrome
// instances. On a 16 GB machine that produced vertical black slabs in the
// final MP4 (wild report, CLI 0.7.52) — compositor tiles evicted under
// GPU/memory pressure, and sampled self-verify can miss partial-frame
// damage. Single-worker DE (the inversion) stays available below the
// floor; only the parallel bet is withheld.
(args.minMemoryMb <= 0 || args.totalMemoryMb >= args.minMemoryMb)
);
}

Expand Down Expand Up @@ -1919,6 +1931,17 @@ export async function executeRenderJob(
const deParallelMinFrames = Number.isFinite(deParallelMinFramesNum)
? deParallelMinFramesNum
: 2000;
// RAM floor default 24 GB: the wild black-slab report was a 16 GB
// machine; every clean routed cohort in telemetry so far is >=24 GB.
// HF_DE_PARALLEL_MIN_MEM_MB overrides (0 disables the guard).
const deParallelMinMemRaw = process.env.HF_DE_PARALLEL_MIN_MEM_MB;
const deParallelMinMemNum =
deParallelMinMemRaw === undefined || deParallelMinMemRaw.trim() === ""
? 24576
: Number(deParallelMinMemRaw);
const deParallelMinMemoryMb = Number.isFinite(deParallelMinMemNum)
? deParallelMinMemNum
: 24576;
const deParallelRouterEligible = shouldPreferParallelDrawElement({
workerCount: WOULD_RESOLVE_MULTI_WORKER,
requestedWorkers: job.config.workers,
Expand All @@ -1938,6 +1961,8 @@ export async function executeRenderJob(
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE === "true" ||
process.env.HF_DE_PARALLEL_STREAM === "true",
routerEnabled: deParallelRouterEnabled,
totalMemoryMb: Math.round(totalmem() / (1024 * 1024)),
minMemoryMb: deParallelMinMemoryMb,
});
// Declared ahead of resolution (assigned below, after calibration) so
// captureStageObservationData can close over it for the calibration
Expand Down
Loading