Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/adapters/google-wire-compiler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,10 @@ function compileGenerationConfig(value: unknown): JsonObject | undefined {
: (["xhigh", "max", "ultra"].includes(raw) ? "high" : undefined);
if (thinkingLevel) out.thinkingConfig = { thinkingLevel };
}
if (Array.isArray(value.responseModalities)) {
const valid = value.responseModalities.filter((m): m is string => typeof m === "string" && ["TEXT", "IMAGE", "AUDIO"].includes(m));
if (valid.length > 0) out.responseModalities = valid;
}
return Object.keys(out).length > 0 ? out : undefined;
}

Expand Down
29 changes: 29 additions & 0 deletions src/adapters/google.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import type { AdapterFetchContext, AdapterRequest, ProviderAdapter } from "./base";
import { debugDroppedFrame } from "../lib/debug";
import { createHash } from "node:crypto";
import { createImageBudget, materializeInlineImage } from "../images/artifacts";
import type {
AdapterEvent,
OcxAssistantMessage,
Expand Down Expand Up @@ -194,6 +195,17 @@ function usageFromGemini(usage: Record<string, number> | undefined): OcxUsage |
};
}

const IMAGE_CAPABLE_MODELS = new Set([
"gemini-3.1-flash-image",
"gemini-2.0-flash-preview-image-generation",
"imagen-4.0-generate-001",
]);

function isImageCapableModel(modelId: string): boolean {
if (IMAGE_CAPABLE_MODELS.has(modelId)) return true;
return /image/.test(modelId) && /gemini/.test(modelId);
}

export function createGoogleAdapter(provider: OcxProviderConfig): ProviderAdapter {
// Per-request closure: resolveAdapter builds a fresh adapter per request (server.ts), so buildRequest
// can stash the CCA model/session for parseStream's reasoning-replay observation.
Expand Down Expand Up @@ -233,6 +245,9 @@ export function createGoogleAdapter(provider: OcxProviderConfig): ProviderAdapte
? mapReasoningEffort(provider, parsed.modelId, parsed.options.reasoning)
: undefined;
if (directFlashThinking) generationConfig.thinkingConfig = { thinkingLevel: directFlashThinking };
if (!generationConfig.thinkingConfig && isImageCapableModel(parsed.modelId)) {
generationConfig.responseModalities = ["TEXT", "IMAGE"];
}
if (Object.keys(generationConfig).length > 0) body.generationConfig = generationConfig;

const method = parsed.stream ? "streamGenerateContent" : "generateContent";
Expand Down Expand Up @@ -345,6 +360,7 @@ export function createGoogleAdapter(provider: OcxProviderConfig): ProviderAdapte
let pendingUsage: OcxUsage | undefined;
let toolCallsStarted = 0;
let lastFinishReason: string | undefined;
const imageBudget = createImageBudget();

try {
while (true) {
Expand Down Expand Up @@ -409,6 +425,12 @@ export function createGoogleAdapter(provider: OcxProviderConfig): ProviderAdapte
if (part.text) {
yield { type: "text_delta", text: part.text };
}
const inline = (part as { inlineData?: { mimeType?: string; data?: string } }).inlineData;
if (inline?.data && inline.mimeType) {
const filePath = await materializeInlineImage(inline.mimeType, inline.data, imageBudget);
const escapedPath = filePath.replace(/([() ])/g, "\\$1");
yield { type: "text_delta", text: `\n![image](${escapedPath})\n` };
}
if (part.functionCall) {
const id = `call_${crypto.randomUUID().slice(0, 8)}`;
toolCallsStarted++;
Expand Down Expand Up @@ -455,13 +477,20 @@ export function createGoogleAdapter(provider: OcxProviderConfig): ProviderAdapte
return [{ type: "error", message: "google response contained no candidates" }];
}
let toolCallsStarted = 0;
const imageBudget = createImageBudget();
if (candidates?.[0]?.content?.parts) {
// Non-streaming CCA: observe thoughtSignatures for the next turn, same as the stream path.
if (provider.googleMode === "cloud-code-assist" && antigravityModel && antigravitySession) {
observeAntigravityReplay(antigravityModel, antigravitySession, candidates[0].content.parts as unknown[]);
}
for (const part of candidates[0].content.parts) {
if (part.text) events.push({ type: "text_delta", text: part.text });
const inline = (part as { inlineData?: { mimeType?: string; data?: string } }).inlineData;
if (inline?.data && inline.mimeType) {
const filePath = await materializeInlineImage(inline.mimeType, inline.data, imageBudget);
const escapedPath = filePath.replace(/([() ])/g, "\\$1");
events.push({ type: "text_delta", text: `\n![image](${escapedPath})\n` });
}
if (part.functionCall) {
const id = `call_${crypto.randomUUID().slice(0, 8)}`;
toolCallsStarted++;
Expand Down
59 changes: 59 additions & 0 deletions src/images/artifacts.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { getConfigDir } from "../config";

const EXT_MAP: Record<string, string> = {
"image/png": "png",
"image/jpeg": "jpg",
"image/webp": "webp",
"image/gif": "gif",
};

const MAX_DECODED_BYTES_PER_IMAGE = 50 * 1024 * 1024;
const MAX_DECODED_BYTES_PER_RESPONSE = 100 * 1024 * 1024;

export interface ImageBudget {
spent: number;
}

export function createImageBudget(): ImageBudget {
return { spent: 0 };
}

export async function materializeInlineImage(
mimeType: string,
base64Data: string,
budget?: ImageBudget,
): Promise<string> {
const ext = EXT_MAP[mimeType] ?? "png";
const dir = join(getConfigDir(), "artifacts");
await mkdir(dir, { recursive: true, mode: 0o700 });

const buf = Buffer.from(base64Data, "base64");
if (buf.length === 0) throw new Error("inline image data is empty after base64 decode");
if (buf.length > MAX_DECODED_BYTES_PER_IMAGE) throw new Error(`inline image exceeds ${MAX_DECODED_BYTES_PER_IMAGE} byte per-image cap`);
if (budget) {
if (budget.spent + buf.length > MAX_DECODED_BYTES_PER_RESPONSE) {
throw new Error(`inline image response exceeds ${MAX_DECODED_BYTES_PER_RESPONSE} byte per-response cap`);
}
budget.spent += buf.length;
}

const now = new Date();
const ts = [
now.getFullYear(),
String(now.getMonth() + 1).padStart(2, "0"),
String(now.getDate()).padStart(2, "0"),
"-",
String(now.getHours()).padStart(2, "0"),
String(now.getMinutes()).padStart(2, "0"),
String(now.getSeconds()).padStart(2, "0"),
"-",
String(now.getMilliseconds()).padStart(3, "0"),
].join("");
const suffix = crypto.randomUUID();
const filePath = join(dir, `img-${ts}-${suffix}.${ext}`);

await writeFile(filePath, buf, { mode: 0o600 });
return filePath;
}
3 changes: 3 additions & 0 deletions src/providers/antigravity-models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const ANTIGRAVITY_WIRE_MODELS = [
"gemini-3.6-flash-high",
"gemini-3.1-pro-low",
"gemini-pro-agent",
"gemini-3.1-flash-image",
"claude-sonnet-4-6",
"claude-opus-4-6-thinking",
"gpt-oss-120b-medium",
Expand Down Expand Up @@ -85,6 +86,7 @@ export const ANTIGRAVITY_MODEL_ALIASES: Record<string, string> = {
export const ANTIGRAVITY_MODELS = [
"gemini-3.6-flash",
"gemini-3.1-pro",
"gemini-3.1-flash-image",
Comment thread
tizerluo marked this conversation as resolved.
"claude-sonnet-4-6",
"claude-opus-4-6-thinking",
"gpt-oss-120b-medium",
Expand All @@ -97,6 +99,7 @@ const ANTIGRAVITY_WIRE_MODEL_CONTEXT_WINDOWS: Record<string, number> = {
"gemini-3.6-flash-high": 1_048_576,
"gemini-3.1-pro-low": 1_048_576,
"gemini-pro-agent": 1_048_576,
"gemini-3.1-flash-image": 1_048_576,
"claude-sonnet-4-6": 200_000,
"claude-opus-4-6-thinking": 1_000_000,
"gpt-oss-120b-medium": 131_072,
Expand Down
107 changes: 104 additions & 3 deletions src/server/images.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import { readJsonRequestBody } from "./request-decompress";
import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "./auth-cors";
import type { RequestLogContext } from "./request-log";
import { codexLogAccountId, decodeRequestErrorResponse } from "./responses";
import { getValidAccessToken, getOAuthCredentialProjectId } from "../oauth/index";
import { safeAntigravityHttpErrorMessage } from "../adapters/google-errors";

export type ImagesEndpoint = "generations" | "edits";

Expand All @@ -40,6 +42,103 @@ const IMAGES_UPSTREAM_TIMEOUT_MS = 300_000;
*/
const IMAGES_RESPONSE_MAX_BYTES = 100 * 1024 * 1024;

const CCA_IMAGE_MODEL = "gemini-3.1-flash-image";

async function tryCcaImageGeneration(
body: unknown,
config: OcxConfig,
logCtx: RequestLogContext,
signal: AbortSignal,
endpoint: ImagesEndpoint,
): Promise<Response | undefined> {
if (endpoint !== "generations") return undefined;
const provider = config.providers?.["google-antigravity"];
if (!provider || provider.disabled) return undefined;

let token: string;
try {
token = await getValidAccessToken("google-antigravity");
} catch {
return undefined;
}
const project = getOAuthCredentialProjectId("google-antigravity");
if (!project) return undefined;

const prompt = (body as { prompt?: unknown })?.prompt;
if (typeof prompt !== "string" || !prompt) return undefined;

logCtx.provider = "google-antigravity";
logCtx.model = CCA_IMAGE_MODEL;

const baseUrl = provider.baseUrl?.trim() || "https://daily-cloudcode-pa.googleapis.com";
const envelope = {
model: CCA_IMAGE_MODEL,
userAgent: "antigravity",
requestType: "agent",
project,
requestId: `agent-${crypto.randomUUID()}`,
request: {
contents: [{ role: "user", parts: [{ text: prompt }] }],
generationConfig: { responseModalities: ["TEXT", "IMAGE"] },
sessionId: `ocx-img-${crypto.randomUUID().slice(0, 8)}`,
},
};

const timeoutMs = config.images?.timeoutMs ?? IMAGES_UPSTREAM_TIMEOUT_MS;
const linkedSignal = signalWithTimeout(timeoutMs, signal);
let upstream: Response;
try {
upstream = await fetch(`${baseUrl}/v1internal:generateContent`, {
method: "POST",
headers: {
"Content-Type": "application/json",
"Authorization": `Bearer ${token}`,
"User-Agent": "opencodex-images/1.0",
},
body: JSON.stringify(envelope),
signal: linkedSignal.signal,
});
} catch (err) {
if (signal.aborted) return formatErrorResponse(499, "client_closed_request", "CCA image request canceled by client");
if (err instanceof Error && err.name === "TimeoutError") {
return formatErrorResponse(504, "upstream_error", "CCA image generation timed out");
}
return formatErrorResponse(502, "upstream_error", `CCA image generation failed: ${err instanceof Error ? err.message : String(err)}`);
} finally {
linkedSignal.cleanup();
}

const payload = await upstream.arrayBuffer();
if (payload.byteLength > IMAGES_RESPONSE_MAX_BYTES) {
return formatErrorResponse(502, "upstream_error", `CCA image response too large (${payload.byteLength} bytes)`);
}

if (!upstream.ok) {
const text = new TextDecoder().decode(payload);
return formatErrorResponse(502, "upstream_error", safeAntigravityHttpErrorMessage(upstream.status, text));
}

let json: Record<string, unknown>;
try {
json = JSON.parse(new TextDecoder().decode(payload)) as Record<string, unknown>;
} catch {
return formatErrorResponse(502, "upstream_error", "CCA image response was not valid JSON");
}
const resp = (json.response ?? json) as { candidates?: { content?: { parts?: { inlineData?: { mimeType?: string; data?: string }; text?: string }[] } }[] };
const parts = resp.candidates?.[0]?.content?.parts ?? [];
const images: { b64_json: string }[] = [];
for (const part of parts) {
if (part.inlineData?.data) images.push({ b64_json: part.inlineData.data });
}
if (images.length === 0) {
return formatErrorResponse(502, "upstream_error", "CCA image model returned no image data");
}
return new Response(JSON.stringify({ created: Math.floor(Date.now() / 1000), data: images }), {
status: 200,
headers: { "content-type": "application/json" },
});
}

export async function handleImages(
req: Request,
config: OcxConfig,
Expand All @@ -62,14 +161,16 @@ export async function handleImages(

const candidates = selectOpenAiImagesProvider(config);
if (candidates.forwardCandidates.length === 0 && !candidates.keyed) {
const ccaResponse = await tryCcaImageGeneration(body, config, logCtx, req.signal, endpoint);
if (ccaResponse) return ccaResponse;
// 400, not 5xx: codex retries every 5xx up to 5 total attempts, and this is a permanent
// configuration state that must surface on the first attempt.
return formatErrorResponse(
400,
"invalid_request_error",
"Built-in image generation needs an OpenAI upstream (ChatGPT login or an OpenAI API-key provider), "
+ "but none is configured in opencodex. Routed providers cannot serve /v1/images/* — "
+ "add an OpenAI provider or disable the tool with `codex features disable image_generation`.",
"Built-in image generation needs an OpenAI upstream (ChatGPT login or an OpenAI API-key provider) "
+ "or a logged-in Google Antigravity (Cloud Code Assist) provider, "
+ "but none is configured in opencodex. Add a provider or disable the tool with `codex features disable image_generation`.",
);
}

Expand Down
1 change: 1 addition & 0 deletions tests/google-antigravity-wire.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ describe("antigravity CCA envelope", () => {
expect(ANTIGRAVITY_MODELS).toEqual([
"gemini-3.6-flash",
"gemini-3.1-pro",
"gemini-3.1-flash-image",
"claude-sonnet-4-6",
"claude-opus-4-6-thinking",
"gpt-oss-120b-medium",
Expand Down
Loading
Loading