mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-11 03:00:45 +08:00
重构主控编排与运行时预热链路,统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为(含 wiki worker)、更新 Admin 可观测性与相关测试,降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
248
openclaw/extensions/google/api.test.ts
Normal file
248
openclaw/extensions/google/api.test.ts
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
import type { ProviderRequestTransportOverrides } from "openclaw/plugin-sdk/provider-http";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
isGoogleGenerativeAiApi,
|
||||
normalizeGoogleApiBaseUrl,
|
||||
normalizeGoogleGenerativeAiBaseUrl,
|
||||
normalizeGoogleProviderConfig,
|
||||
parseGeminiAuth,
|
||||
resolveGoogleGenerativeAiHttpRequestConfig,
|
||||
resolveGoogleGenerativeAiApiOrigin,
|
||||
resolveGoogleGenerativeAiTransport,
|
||||
shouldNormalizeGoogleGenerativeAiProviderConfig,
|
||||
} from "./api.js";
|
||||
|
||||
describe("google generative ai helpers", () => {
|
||||
it("detects the Google Generative AI transport id", () => {
|
||||
expect(isGoogleGenerativeAiApi("google-generative-ai")).toBe(true);
|
||||
expect(isGoogleGenerativeAiApi("google-gemini-cli")).toBe(false);
|
||||
expect(isGoogleGenerativeAiApi(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it("normalizes only explicit Google Generative AI baseUrls", () => {
|
||||
expect(normalizeGoogleGenerativeAiBaseUrl("https://generativelanguage.googleapis.com")).toBe(
|
||||
"https://generativelanguage.googleapis.com/v1beta",
|
||||
);
|
||||
expect(normalizeGoogleGenerativeAiBaseUrl("https://proxy.example.com/google/v1beta")).toBe(
|
||||
"https://proxy.example.com/google/v1beta",
|
||||
);
|
||||
expect(normalizeGoogleGenerativeAiBaseUrl("https://aiplatform.googleapis.com")).toBe(
|
||||
"https://aiplatform.googleapis.com",
|
||||
);
|
||||
expect(normalizeGoogleGenerativeAiBaseUrl("proxy/generativelanguage.googleapis.com")).toBe(
|
||||
"proxy/generativelanguage.googleapis.com",
|
||||
);
|
||||
expect(normalizeGoogleGenerativeAiBaseUrl("generativelanguage.googleapis.com")).toBe(
|
||||
"generativelanguage.googleapis.com",
|
||||
);
|
||||
expect(normalizeGoogleGenerativeAiBaseUrl("https://xgenerativelanguage.googleapis.com")).toBe(
|
||||
"https://xgenerativelanguage.googleapis.com",
|
||||
);
|
||||
expect(normalizeGoogleGenerativeAiBaseUrl()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps /openai on generic Google base URL normalization and strips it only for native Gemini callers", () => {
|
||||
expect(
|
||||
normalizeGoogleApiBaseUrl("https://generativelanguage.googleapis.com/v1beta/openai"),
|
||||
).toBe("https://generativelanguage.googleapis.com/v1beta/openai");
|
||||
expect(
|
||||
normalizeGoogleGenerativeAiBaseUrl("https://generativelanguage.googleapis.com/v1beta/openai"),
|
||||
).toBe("https://generativelanguage.googleapis.com/v1beta");
|
||||
expect(
|
||||
normalizeGoogleGenerativeAiBaseUrl(
|
||||
"https://generativelanguage.googleapis.com/v1alpha/openai/",
|
||||
),
|
||||
).toBe("https://generativelanguage.googleapis.com/v1alpha");
|
||||
});
|
||||
|
||||
it("normalizes Google provider configs by provider key, provider api, or model api", () => {
|
||||
expect(
|
||||
shouldNormalizeGoogleGenerativeAiProviderConfig("google", {
|
||||
models: [{ api: "openai-completions" }],
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldNormalizeGoogleGenerativeAiProviderConfig("custom", {
|
||||
api: "google-generative-ai",
|
||||
models: [{ api: "openai-completions" }],
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldNormalizeGoogleGenerativeAiProviderConfig("custom", {
|
||||
models: [{ api: "google-generative-ai" }],
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldNormalizeGoogleGenerativeAiProviderConfig("custom", {
|
||||
api: "openai-completions",
|
||||
models: [{ api: "openai-completions" }],
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldNormalizeGoogleGenerativeAiProviderConfig("google", {
|
||||
api: "openai-completions",
|
||||
models: [{ api: "openai-completions" }],
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("normalizes transport baseUrls only for Google Generative AI", () => {
|
||||
expect(
|
||||
resolveGoogleGenerativeAiTransport({
|
||||
api: "google-generative-ai",
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
}),
|
||||
).toEqual({
|
||||
api: "google-generative-ai",
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
});
|
||||
expect(
|
||||
resolveGoogleGenerativeAiTransport({
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
}),
|
||||
).toEqual({
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes google-vertex model ids without rewriting the OpenAI-compatible baseUrl", () => {
|
||||
expect(
|
||||
normalizeGoogleProviderConfig("google-vertex", {
|
||||
api: "openai-completions",
|
||||
baseUrl:
|
||||
"https://aiplatform.googleapis.com/v1/projects/test/locations/us-central1/endpoints/openapi",
|
||||
models: [
|
||||
{
|
||||
id: "gemini-3.1-flash-lite",
|
||||
name: "Gemini Flash Lite",
|
||||
input: ["text"],
|
||||
reasoning: false,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1,
|
||||
maxTokens: 1,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toMatchObject({
|
||||
api: "openai-completions",
|
||||
baseUrl:
|
||||
"https://aiplatform.googleapis.com/v1/projects/test/locations/us-central1/endpoints/openapi",
|
||||
models: [
|
||||
expect.objectContaining({
|
||||
id: "gemini-3.1-flash-lite-preview",
|
||||
}),
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("derives the Gemini API origin without duplicating /v1beta", () => {
|
||||
expect(resolveGoogleGenerativeAiApiOrigin()).toBe("https://generativelanguage.googleapis.com");
|
||||
expect(resolveGoogleGenerativeAiApiOrigin("https://generativelanguage.googleapis.com")).toBe(
|
||||
"https://generativelanguage.googleapis.com",
|
||||
);
|
||||
expect(
|
||||
resolveGoogleGenerativeAiApiOrigin("https://generativelanguage.googleapis.com/v1beta"),
|
||||
).toBe("https://generativelanguage.googleapis.com");
|
||||
});
|
||||
|
||||
it("parses project-aware oauth auth payloads into bearer headers", () => {
|
||||
expect(
|
||||
parseGeminiAuth(JSON.stringify({ token: "oauth-token", projectId: "project-1" })),
|
||||
).toEqual({
|
||||
headers: {
|
||||
Authorization: "Bearer oauth-token",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to API key headers for raw tokens", () => {
|
||||
expect(parseGeminiAuth("api-key-123")).toEqual({
|
||||
headers: {
|
||||
"x-goog-api-key": "api-key-123",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("builds shared Google Generative AI HTTP request config", () => {
|
||||
const oauthConfig = resolveGoogleGenerativeAiHttpRequestConfig({
|
||||
apiKey: JSON.stringify({ token: "oauth-token" }),
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
capability: "audio",
|
||||
transport: "media-understanding",
|
||||
});
|
||||
expect(oauthConfig).toMatchObject({
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
allowPrivateNetwork: false,
|
||||
});
|
||||
expect(Object.fromEntries(new Headers(oauthConfig.headers).entries())).toEqual({
|
||||
authorization: "Bearer oauth-token",
|
||||
"content-type": "application/json",
|
||||
});
|
||||
|
||||
const apiKeyConfig = resolveGoogleGenerativeAiHttpRequestConfig({
|
||||
apiKey: "api-key-123",
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
});
|
||||
expect(apiKeyConfig).toMatchObject({
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
allowPrivateNetwork: false,
|
||||
});
|
||||
expect(Object.fromEntries(new Headers(apiKeyConfig.headers).entries())).toEqual({
|
||||
"content-type": "application/json",
|
||||
"x-goog-api-key": "api-key-123",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves explicit OpenAI-compatible Google endpoints during provider normalization", () => {
|
||||
expect(
|
||||
resolveGoogleGenerativeAiTransport({
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
}),
|
||||
).toEqual({
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
});
|
||||
});
|
||||
|
||||
it("strips URL credentials during Google base URL normalization", () => {
|
||||
const normalized = normalizeGoogleApiBaseUrl(
|
||||
"https://user:secret@generativelanguage.googleapis.com/v1beta/openai?x=1#frag",
|
||||
);
|
||||
expect(normalized).toBe("https://generativelanguage.googleapis.com/v1beta/openai");
|
||||
});
|
||||
|
||||
it("rejects non-Google Gemini base URLs and ignores smuggled private-network flags", () => {
|
||||
expect(() =>
|
||||
resolveGoogleGenerativeAiHttpRequestConfig({
|
||||
apiKey: "api-key-123",
|
||||
baseUrl: "https://proxy.example.com/v1beta",
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
}),
|
||||
).toThrow("Google Generative AI baseUrl must use https://generativelanguage.googleapis.com");
|
||||
|
||||
expect(() =>
|
||||
resolveGoogleGenerativeAiHttpRequestConfig({
|
||||
apiKey: "api-key-123",
|
||||
baseUrl: "http://generativelanguage.googleapis.com/v1beta",
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
}),
|
||||
).toThrow("Google Generative AI baseUrl must use https://generativelanguage.googleapis.com");
|
||||
|
||||
const config = resolveGoogleGenerativeAiHttpRequestConfig({
|
||||
apiKey: "api-key-123",
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
request: { allowPrivateNetwork: true } as unknown as ProviderRequestTransportOverrides,
|
||||
});
|
||||
expect(config.allowPrivateNetwork).toBe(false);
|
||||
});
|
||||
});
|
||||
88
openclaw/extensions/google/api.ts
Normal file
88
openclaw/extensions/google/api.ts
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import {
|
||||
resolveProviderHttpRequestConfig,
|
||||
type ProviderRequestTransportOverrides,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { parseGeminiAuth } from "./gemini-auth.js";
|
||||
export { parseGeminiAuth };
|
||||
export { applyGoogleGeminiModelDefault, GOOGLE_GEMINI_DEFAULT_MODEL } from "./onboard.js";
|
||||
import {
|
||||
DEFAULT_GOOGLE_API_BASE_URL,
|
||||
normalizeGoogleApiBaseUrl,
|
||||
normalizeGoogleGenerativeAiBaseUrl,
|
||||
} from "./provider-policy.js";
|
||||
export { normalizeAntigravityModelId, normalizeGoogleModelId } from "./model-id.js";
|
||||
export {
|
||||
createGoogleThinkingPayloadWrapper,
|
||||
createGoogleThinkingStreamWrapper,
|
||||
isGoogleGemini3FlashModel,
|
||||
isGoogleGemini3ProModel,
|
||||
isGoogleGemini3ThinkingLevelModel,
|
||||
isGoogleThinkingRequiredModel,
|
||||
resolveGoogleGemini3ThinkingLevel,
|
||||
sanitizeGoogleThinkingPayload,
|
||||
stripInvalidGoogleThinkingBudget,
|
||||
type GoogleThinkingInputLevel,
|
||||
type GoogleThinkingLevel,
|
||||
} from "./thinking-api.js";
|
||||
export {
|
||||
buildGoogleGenerativeAiParams,
|
||||
createGoogleGenerativeAiTransportStreamFn,
|
||||
} from "./transport-stream.js";
|
||||
export {
|
||||
DEFAULT_GOOGLE_API_BASE_URL,
|
||||
isGoogleGenerativeAiApi,
|
||||
normalizeGoogleApiBaseUrl,
|
||||
normalizeGoogleGenerativeAiBaseUrl,
|
||||
normalizeGoogleProviderConfig,
|
||||
resolveGoogleGenerativeAiApiOrigin,
|
||||
resolveGoogleGenerativeAiTransport,
|
||||
shouldNormalizeGoogleGenerativeAiProviderConfig,
|
||||
shouldNormalizeGoogleProviderConfig,
|
||||
} from "./provider-policy.js";
|
||||
export { buildGoogleGeminiCliProvider } from "./gemini-cli-provider.js";
|
||||
export { buildGoogleProvider } from "./provider-registration.js";
|
||||
|
||||
function resolveTrustedGoogleGenerativeAiBaseUrl(baseUrl?: string): string {
|
||||
const normalized =
|
||||
normalizeGoogleGenerativeAiBaseUrl(baseUrl ?? DEFAULT_GOOGLE_API_BASE_URL) ??
|
||||
DEFAULT_GOOGLE_API_BASE_URL;
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(normalized);
|
||||
} catch {
|
||||
throw new Error(
|
||||
"Google Generative AI baseUrl must be a valid https URL on generativelanguage.googleapis.com",
|
||||
);
|
||||
}
|
||||
if (
|
||||
url.protocol !== "https:" ||
|
||||
url.hostname.toLowerCase() !== "generativelanguage.googleapis.com"
|
||||
) {
|
||||
throw new Error(
|
||||
"Google Generative AI baseUrl must use https://generativelanguage.googleapis.com",
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function resolveGoogleGenerativeAiHttpRequestConfig(params: {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
headers?: Record<string, string>;
|
||||
request?: ProviderRequestTransportOverrides;
|
||||
capability: "image" | "audio" | "video";
|
||||
transport: "http" | "media-understanding";
|
||||
}) {
|
||||
return resolveProviderHttpRequestConfig({
|
||||
baseUrl: resolveTrustedGoogleGenerativeAiBaseUrl(params.baseUrl),
|
||||
defaultBaseUrl: DEFAULT_GOOGLE_API_BASE_URL,
|
||||
allowPrivateNetwork: false,
|
||||
headers: params.headers,
|
||||
request: params.request,
|
||||
defaultHeaders: parseGeminiAuth(params.apiKey).headers,
|
||||
provider: "google",
|
||||
api: "google-generative-ai",
|
||||
capability: params.capability,
|
||||
transport: params.transport,
|
||||
});
|
||||
}
|
||||
49
openclaw/extensions/google/cli-backend.ts
Normal file
49
openclaw/extensions/google/cli-backend.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import type { CliBackendPlugin } from "openclaw/plugin-sdk/cli-backend";
|
||||
import {
|
||||
CLI_FRESH_WATCHDOG_DEFAULTS,
|
||||
CLI_RESUME_WATCHDOG_DEFAULTS,
|
||||
} from "openclaw/plugin-sdk/cli-backend";
|
||||
|
||||
const GEMINI_MODEL_ALIASES: Record<string, string> = {
|
||||
pro: "gemini-3.1-pro-preview",
|
||||
flash: "gemini-3.1-flash-preview",
|
||||
"flash-lite": "gemini-3.1-flash-lite-preview",
|
||||
};
|
||||
const GEMINI_CLI_DEFAULT_MODEL_REF = "google-gemini-cli/gemini-3-flash-preview";
|
||||
|
||||
export function buildGoogleGeminiCliBackend(): CliBackendPlugin {
|
||||
return {
|
||||
id: "google-gemini-cli",
|
||||
liveTest: {
|
||||
defaultModelRef: GEMINI_CLI_DEFAULT_MODEL_REF,
|
||||
defaultImageProbe: true,
|
||||
defaultMcpProbe: true,
|
||||
docker: {
|
||||
npmPackage: "@google/gemini-cli",
|
||||
binaryName: "gemini",
|
||||
},
|
||||
},
|
||||
bundleMcp: true,
|
||||
bundleMcpMode: "gemini-system-settings",
|
||||
config: {
|
||||
command: "gemini",
|
||||
args: ["--output-format", "json", "--prompt", "{prompt}"],
|
||||
resumeArgs: ["--resume", "{sessionId}", "--output-format", "json", "--prompt", "{prompt}"],
|
||||
output: "json",
|
||||
input: "arg",
|
||||
imageArg: "@",
|
||||
imagePathScope: "workspace",
|
||||
modelArg: "--model",
|
||||
modelAliases: GEMINI_MODEL_ALIASES,
|
||||
sessionMode: "existing",
|
||||
sessionIdFields: ["session_id", "sessionId"],
|
||||
reliability: {
|
||||
watchdog: {
|
||||
fresh: { ...CLI_FRESH_WATCHDOG_DEFAULTS },
|
||||
resume: { ...CLI_RESUME_WATCHDOG_DEFAULTS },
|
||||
},
|
||||
},
|
||||
serialize: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
29
openclaw/extensions/google/default-model.test.ts
Normal file
29
openclaw/extensions/google/default-model.test.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-onboard";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyGoogleGeminiModelDefault, GOOGLE_GEMINI_DEFAULT_MODEL } from "./api.js";
|
||||
|
||||
describe("google default model", () => {
|
||||
it("sets defaults when model is unset", () => {
|
||||
const cfg: OpenClawConfig = { agents: { defaults: {} } };
|
||||
const applied = applyGoogleGeminiModelDefault(cfg);
|
||||
expect(applied.changed).toBe(true);
|
||||
expect(applied.next.agents?.defaults?.model).toEqual({ primary: GOOGLE_GEMINI_DEFAULT_MODEL });
|
||||
});
|
||||
|
||||
it("overrides existing models", () => {
|
||||
const applied = applyGoogleGeminiModelDefault({
|
||||
agents: { defaults: { model: { primary: "anthropic/claude-opus-4-6" } } },
|
||||
} as OpenClawConfig);
|
||||
expect(applied.changed).toBe(true);
|
||||
expect(applied.next.agents?.defaults?.model).toEqual({ primary: GOOGLE_GEMINI_DEFAULT_MODEL });
|
||||
});
|
||||
|
||||
it("no-ops when already on the target default", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { model: { primary: GOOGLE_GEMINI_DEFAULT_MODEL } } },
|
||||
} as OpenClawConfig;
|
||||
const applied = applyGoogleGeminiModelDefault(cfg);
|
||||
expect(applied.changed).toBe(false);
|
||||
expect(applied.next).toEqual(cfg);
|
||||
});
|
||||
});
|
||||
373
openclaw/extensions/google/embedding-batch.ts
Normal file
373
openclaw/extensions/google/embedding-batch.ts
Normal file
|
|
@ -0,0 +1,373 @@
|
|||
import crypto from "node:crypto";
|
||||
import {
|
||||
buildEmbeddingBatchGroupOptions,
|
||||
runEmbeddingBatchGroups,
|
||||
type EmbeddingBatchExecutionParams,
|
||||
buildBatchHeaders,
|
||||
debugEmbeddingsLog,
|
||||
normalizeBatchBaseUrl,
|
||||
sanitizeAndNormalizeEmbedding,
|
||||
withRemoteHttpResponse,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import type { GeminiEmbeddingClient, GeminiTextEmbeddingRequest } from "./embedding-provider.js";
|
||||
|
||||
export type GeminiBatchRequest = {
|
||||
custom_id: string;
|
||||
request: GeminiTextEmbeddingRequest;
|
||||
};
|
||||
|
||||
export type GeminiBatchStatus = {
|
||||
name?: string;
|
||||
state?: string;
|
||||
outputConfig?: { file?: string; fileId?: string };
|
||||
metadata?: {
|
||||
output?: {
|
||||
responsesFile?: string;
|
||||
};
|
||||
};
|
||||
error?: { message?: string };
|
||||
};
|
||||
|
||||
export type GeminiBatchOutputLine = {
|
||||
key?: string;
|
||||
custom_id?: string;
|
||||
request_id?: string;
|
||||
embedding?: { values?: number[] };
|
||||
response?: {
|
||||
embedding?: { values?: number[] };
|
||||
error?: { message?: string };
|
||||
};
|
||||
error?: { message?: string };
|
||||
};
|
||||
|
||||
const GEMINI_BATCH_MAX_REQUESTS = 50000;
|
||||
function hashText(text: string): string {
|
||||
return crypto.createHash("sha256").update(text).digest("hex");
|
||||
}
|
||||
|
||||
function getGeminiUploadUrl(baseUrl: string): string {
|
||||
if (baseUrl.includes("/v1beta")) {
|
||||
return baseUrl.replace(/\/v1beta\/?$/, "/upload/v1beta");
|
||||
}
|
||||
return `${baseUrl.replace(/\/$/, "")}/upload`;
|
||||
}
|
||||
|
||||
function buildGeminiUploadBody(params: { jsonl: string; displayName: string }): {
|
||||
body: Blob;
|
||||
contentType: string;
|
||||
} {
|
||||
const boundary = `openclaw-${hashText(params.displayName)}`;
|
||||
const jsonPart = JSON.stringify({
|
||||
file: {
|
||||
displayName: params.displayName,
|
||||
mimeType: "application/jsonl",
|
||||
},
|
||||
});
|
||||
const delimiter = `--${boundary}\r\n`;
|
||||
const closeDelimiter = `--${boundary}--\r\n`;
|
||||
const parts = [
|
||||
`${delimiter}Content-Type: application/json; charset=UTF-8\r\n\r\n${jsonPart}\r\n`,
|
||||
`${delimiter}Content-Type: application/jsonl; charset=UTF-8\r\n\r\n${params.jsonl}\r\n`,
|
||||
closeDelimiter,
|
||||
];
|
||||
const body = new Blob([parts.join("")], { type: "multipart/related" });
|
||||
return {
|
||||
body,
|
||||
contentType: `multipart/related; boundary=${boundary}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function submitGeminiBatch(params: {
|
||||
gemini: GeminiEmbeddingClient;
|
||||
requests: GeminiBatchRequest[];
|
||||
agentId: string;
|
||||
}): Promise<GeminiBatchStatus> {
|
||||
const baseUrl = normalizeBatchBaseUrl(params.gemini);
|
||||
const jsonl = params.requests
|
||||
.map((request) =>
|
||||
JSON.stringify({
|
||||
key: request.custom_id,
|
||||
request: request.request,
|
||||
}),
|
||||
)
|
||||
.join("\n");
|
||||
const displayName = `memory-embeddings-${hashText(String(Date.now()))}`;
|
||||
const uploadPayload = buildGeminiUploadBody({ jsonl, displayName });
|
||||
|
||||
const uploadUrl = `${getGeminiUploadUrl(baseUrl)}/files?uploadType=multipart`;
|
||||
debugEmbeddingsLog("memory embeddings: gemini batch upload", {
|
||||
uploadUrl,
|
||||
baseUrl,
|
||||
requests: params.requests.length,
|
||||
});
|
||||
const filePayload = await withRemoteHttpResponse({
|
||||
url: uploadUrl,
|
||||
ssrfPolicy: params.gemini.ssrfPolicy,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: {
|
||||
...buildBatchHeaders(params.gemini, { json: false }),
|
||||
"Content-Type": uploadPayload.contentType,
|
||||
},
|
||||
body: uploadPayload.body,
|
||||
},
|
||||
onResponse: async (fileRes) => {
|
||||
if (!fileRes.ok) {
|
||||
const text = await fileRes.text();
|
||||
throw new Error(`gemini batch file upload failed: ${fileRes.status} ${text}`);
|
||||
}
|
||||
return (await fileRes.json()) as { name?: string; file?: { name?: string } };
|
||||
},
|
||||
});
|
||||
const fileId = filePayload.name ?? filePayload.file?.name;
|
||||
if (!fileId) {
|
||||
throw new Error("gemini batch file upload failed: missing file id");
|
||||
}
|
||||
|
||||
const batchBody = {
|
||||
batch: {
|
||||
displayName: `memory-embeddings-${params.agentId}`,
|
||||
inputConfig: {
|
||||
file_name: fileId,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const batchEndpoint = `${baseUrl}/${params.gemini.modelPath}:asyncBatchEmbedContent`;
|
||||
debugEmbeddingsLog("memory embeddings: gemini batch create", {
|
||||
batchEndpoint,
|
||||
fileId,
|
||||
});
|
||||
return await withRemoteHttpResponse({
|
||||
url: batchEndpoint,
|
||||
ssrfPolicy: params.gemini.ssrfPolicy,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: buildBatchHeaders(params.gemini, { json: true }),
|
||||
body: JSON.stringify(batchBody),
|
||||
},
|
||||
onResponse: async (batchRes) => {
|
||||
if (batchRes.ok) {
|
||||
return (await batchRes.json()) as GeminiBatchStatus;
|
||||
}
|
||||
const text = await batchRes.text();
|
||||
if (batchRes.status === 404) {
|
||||
throw new Error(
|
||||
"gemini batch create failed: 404 (asyncBatchEmbedContent not available for this model/baseUrl). Disable remote.batch.enabled or switch providers.",
|
||||
);
|
||||
}
|
||||
throw new Error(`gemini batch create failed: ${batchRes.status} ${text}`);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchGeminiBatchStatus(params: {
|
||||
gemini: GeminiEmbeddingClient;
|
||||
batchName: string;
|
||||
}): Promise<GeminiBatchStatus> {
|
||||
const baseUrl = normalizeBatchBaseUrl(params.gemini);
|
||||
const name = params.batchName.startsWith("batches/")
|
||||
? params.batchName
|
||||
: `batches/${params.batchName}`;
|
||||
const statusUrl = `${baseUrl}/${name}`;
|
||||
debugEmbeddingsLog("memory embeddings: gemini batch status", { statusUrl });
|
||||
return await withRemoteHttpResponse({
|
||||
url: statusUrl,
|
||||
ssrfPolicy: params.gemini.ssrfPolicy,
|
||||
init: {
|
||||
headers: buildBatchHeaders(params.gemini, { json: true }),
|
||||
},
|
||||
onResponse: async (res) => {
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`gemini batch status failed: ${res.status} ${text}`);
|
||||
}
|
||||
return (await res.json()) as GeminiBatchStatus;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchGeminiFileContent(params: {
|
||||
gemini: GeminiEmbeddingClient;
|
||||
fileId: string;
|
||||
}): Promise<string> {
|
||||
const baseUrl = normalizeBatchBaseUrl(params.gemini);
|
||||
const file = params.fileId.startsWith("files/") ? params.fileId : `files/${params.fileId}`;
|
||||
const downloadUrl = `${baseUrl}/${file}:download`;
|
||||
debugEmbeddingsLog("memory embeddings: gemini batch download", { downloadUrl });
|
||||
return await withRemoteHttpResponse({
|
||||
url: downloadUrl,
|
||||
ssrfPolicy: params.gemini.ssrfPolicy,
|
||||
init: {
|
||||
headers: buildBatchHeaders(params.gemini, { json: true }),
|
||||
},
|
||||
onResponse: async (res) => {
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`gemini batch file content failed: ${res.status} ${text}`);
|
||||
}
|
||||
return await res.text();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function parseGeminiBatchOutput(text: string): GeminiBatchOutputLine[] {
|
||||
if (!text.trim()) {
|
||||
return [];
|
||||
}
|
||||
return text
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line) as GeminiBatchOutputLine);
|
||||
}
|
||||
|
||||
async function waitForGeminiBatch(params: {
|
||||
gemini: GeminiEmbeddingClient;
|
||||
batchName: string;
|
||||
wait: boolean;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
debug?: (message: string, data?: Record<string, unknown>) => void;
|
||||
initial?: GeminiBatchStatus;
|
||||
}): Promise<{ outputFileId: string }> {
|
||||
const start = Date.now();
|
||||
let current: GeminiBatchStatus | undefined = params.initial;
|
||||
while (true) {
|
||||
const status =
|
||||
current ??
|
||||
(await fetchGeminiBatchStatus({
|
||||
gemini: params.gemini,
|
||||
batchName: params.batchName,
|
||||
}));
|
||||
const state = status.state ?? "UNKNOWN";
|
||||
if (["SUCCEEDED", "COMPLETED", "DONE"].includes(state)) {
|
||||
const outputFileId =
|
||||
status.outputConfig?.file ??
|
||||
status.outputConfig?.fileId ??
|
||||
status.metadata?.output?.responsesFile;
|
||||
if (!outputFileId) {
|
||||
throw new Error(`gemini batch ${params.batchName} completed without output file`);
|
||||
}
|
||||
return { outputFileId };
|
||||
}
|
||||
if (["FAILED", "CANCELLED", "CANCELED", "EXPIRED"].includes(state)) {
|
||||
const message = status.error?.message ?? "unknown error";
|
||||
throw new Error(`gemini batch ${params.batchName} ${state}: ${message}`);
|
||||
}
|
||||
if (!params.wait) {
|
||||
throw new Error(`gemini batch ${params.batchName} still ${state}; wait disabled`);
|
||||
}
|
||||
if (Date.now() - start > params.timeoutMs) {
|
||||
throw new Error(`gemini batch ${params.batchName} timed out after ${params.timeoutMs}ms`);
|
||||
}
|
||||
params.debug?.(`gemini batch ${params.batchName} ${state}; waiting ${params.pollIntervalMs}ms`);
|
||||
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
|
||||
current = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runGeminiEmbeddingBatches(
|
||||
params: {
|
||||
gemini: GeminiEmbeddingClient;
|
||||
agentId: string;
|
||||
requests: GeminiBatchRequest[];
|
||||
} & EmbeddingBatchExecutionParams,
|
||||
): Promise<Map<string, number[]>> {
|
||||
return await runEmbeddingBatchGroups({
|
||||
...buildEmbeddingBatchGroupOptions(params, {
|
||||
maxRequests: GEMINI_BATCH_MAX_REQUESTS,
|
||||
debugLabel: "memory embeddings: gemini batch submit",
|
||||
}),
|
||||
runGroup: async ({ group, groupIndex, groups, byCustomId }) => {
|
||||
const batchInfo = await submitGeminiBatch({
|
||||
gemini: params.gemini,
|
||||
requests: group,
|
||||
agentId: params.agentId,
|
||||
});
|
||||
const batchName = batchInfo.name ?? "";
|
||||
if (!batchName) {
|
||||
throw new Error("gemini batch create failed: missing batch name");
|
||||
}
|
||||
|
||||
params.debug?.("memory embeddings: gemini batch created", {
|
||||
batchName,
|
||||
state: batchInfo.state,
|
||||
group: groupIndex + 1,
|
||||
groups,
|
||||
requests: group.length,
|
||||
});
|
||||
|
||||
if (
|
||||
!params.wait &&
|
||||
batchInfo.state &&
|
||||
!["SUCCEEDED", "COMPLETED", "DONE"].includes(batchInfo.state)
|
||||
) {
|
||||
throw new Error(
|
||||
`gemini batch ${batchName} submitted; enable remote.batch.wait to await completion`,
|
||||
);
|
||||
}
|
||||
|
||||
const completed =
|
||||
batchInfo.state && ["SUCCEEDED", "COMPLETED", "DONE"].includes(batchInfo.state)
|
||||
? {
|
||||
outputFileId:
|
||||
batchInfo.outputConfig?.file ??
|
||||
batchInfo.outputConfig?.fileId ??
|
||||
batchInfo.metadata?.output?.responsesFile ??
|
||||
"",
|
||||
}
|
||||
: await waitForGeminiBatch({
|
||||
gemini: params.gemini,
|
||||
batchName,
|
||||
wait: params.wait,
|
||||
pollIntervalMs: params.pollIntervalMs,
|
||||
timeoutMs: params.timeoutMs,
|
||||
debug: params.debug,
|
||||
initial: batchInfo,
|
||||
});
|
||||
if (!completed.outputFileId) {
|
||||
throw new Error(`gemini batch ${batchName} completed without output file`);
|
||||
}
|
||||
|
||||
const content = await fetchGeminiFileContent({
|
||||
gemini: params.gemini,
|
||||
fileId: completed.outputFileId,
|
||||
});
|
||||
const outputLines = parseGeminiBatchOutput(content);
|
||||
const errors: string[] = [];
|
||||
const remaining = new Set(group.map((request) => request.custom_id));
|
||||
|
||||
for (const line of outputLines) {
|
||||
const customId = line.key ?? line.custom_id ?? line.request_id;
|
||||
if (!customId) {
|
||||
continue;
|
||||
}
|
||||
remaining.delete(customId);
|
||||
if (line.error?.message) {
|
||||
errors.push(`${customId}: ${line.error.message}`);
|
||||
continue;
|
||||
}
|
||||
if (line.response?.error?.message) {
|
||||
errors.push(`${customId}: ${line.response.error.message}`);
|
||||
continue;
|
||||
}
|
||||
const embedding = sanitizeAndNormalizeEmbedding(
|
||||
line.embedding?.values ?? line.response?.embedding?.values ?? [],
|
||||
);
|
||||
if (embedding.length === 0) {
|
||||
errors.push(`${customId}: empty embedding`);
|
||||
continue;
|
||||
}
|
||||
byCustomId.set(customId, embedding);
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`gemini batch ${batchName} failed: ${errors.join("; ")}`);
|
||||
}
|
||||
if (remaining.size > 0) {
|
||||
throw new Error(`gemini batch ${batchName} missing ${remaining.size} embedding responses`);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
191
openclaw/extensions/google/embedding-provider.test.ts
Normal file
191
openclaw/extensions/google/embedding-provider.test.ts
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildGeminiEmbeddingRequest,
|
||||
buildGeminiTextEmbeddingRequest,
|
||||
createGeminiEmbeddingProvider,
|
||||
DEFAULT_GEMINI_EMBEDDING_MODEL,
|
||||
GEMINI_EMBEDDING_2_MODELS,
|
||||
isGeminiEmbedding2Model,
|
||||
normalizeGeminiModel,
|
||||
resolveGeminiOutputDimensionality,
|
||||
} from "./embedding-provider.js";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function installFetchMock(
|
||||
handler: (input: RequestInfo | URL, init?: RequestInit) => unknown,
|
||||
): ReturnType<typeof vi.fn> {
|
||||
const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
return new Response(JSON.stringify(handler(input, init)), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
return fetchMock;
|
||||
}
|
||||
|
||||
function fetchJsonBody(fetchMock: ReturnType<typeof vi.fn>, index: number): unknown {
|
||||
const init = fetchMock.mock.calls[index]?.[1] as RequestInit | undefined;
|
||||
const body = init?.body;
|
||||
if (typeof body !== "string") {
|
||||
throw new Error("Expected JSON string request body.");
|
||||
}
|
||||
return JSON.parse(body) as unknown;
|
||||
}
|
||||
|
||||
describe("Gemini embedding request helpers", () => {
|
||||
it("builds requests and resolves model settings", () => {
|
||||
expect(
|
||||
buildGeminiTextEmbeddingRequest({
|
||||
text: "hello",
|
||||
taskType: "RETRIEVAL_DOCUMENT",
|
||||
modelPath: "models/gemini-embedding-2-preview",
|
||||
outputDimensionality: 1536,
|
||||
}),
|
||||
).toEqual({
|
||||
model: "models/gemini-embedding-2-preview",
|
||||
content: { parts: [{ text: "hello" }] },
|
||||
taskType: "RETRIEVAL_DOCUMENT",
|
||||
outputDimensionality: 1536,
|
||||
});
|
||||
expect(
|
||||
buildGeminiEmbeddingRequest({
|
||||
input: {
|
||||
text: "Image file: diagram.png",
|
||||
parts: [
|
||||
{ type: "text", text: "Image file: diagram.png" },
|
||||
{ type: "inline-data", mimeType: "image/png", data: "abc123" },
|
||||
],
|
||||
},
|
||||
taskType: "RETRIEVAL_DOCUMENT",
|
||||
modelPath: "models/gemini-embedding-2-preview",
|
||||
outputDimensionality: 1536,
|
||||
}),
|
||||
).toEqual({
|
||||
model: "models/gemini-embedding-2-preview",
|
||||
content: {
|
||||
parts: [
|
||||
{ text: "Image file: diagram.png" },
|
||||
{ inlineData: { mimeType: "image/png", data: "abc123" } },
|
||||
],
|
||||
},
|
||||
taskType: "RETRIEVAL_DOCUMENT",
|
||||
outputDimensionality: 1536,
|
||||
});
|
||||
expect(GEMINI_EMBEDDING_2_MODELS.has("gemini-embedding-2-preview")).toBe(true);
|
||||
expect(isGeminiEmbedding2Model("gemini-embedding-2-preview")).toBe(true);
|
||||
expect(isGeminiEmbedding2Model("gemini-embedding-001")).toBe(false);
|
||||
expect(isGeminiEmbedding2Model("text-embedding-004")).toBe(false);
|
||||
expect(resolveGeminiOutputDimensionality("gemini-embedding-001")).toBeUndefined();
|
||||
expect(resolveGeminiOutputDimensionality("text-embedding-004")).toBeUndefined();
|
||||
expect(resolveGeminiOutputDimensionality("gemini-embedding-2-preview")).toBe(3072);
|
||||
expect(resolveGeminiOutputDimensionality("gemini-embedding-2-preview", 768)).toBe(768);
|
||||
expect(resolveGeminiOutputDimensionality("gemini-embedding-2-preview", 1536)).toBe(1536);
|
||||
expect(resolveGeminiOutputDimensionality("gemini-embedding-2-preview", 3072)).toBe(3072);
|
||||
expect(() => resolveGeminiOutputDimensionality("gemini-embedding-2-preview", 512)).toThrow(
|
||||
/Invalid outputDimensionality 512/,
|
||||
);
|
||||
expect(() => resolveGeminiOutputDimensionality("gemini-embedding-2-preview", 1024)).toThrow(
|
||||
/Valid values: 768, 1536, 3072/,
|
||||
);
|
||||
expect(normalizeGeminiModel("models/gemini-embedding-2-preview")).toBe(
|
||||
"gemini-embedding-2-preview",
|
||||
);
|
||||
expect(normalizeGeminiModel("gemini/gemini-embedding-2-preview")).toBe(
|
||||
"gemini-embedding-2-preview",
|
||||
);
|
||||
expect(normalizeGeminiModel("google/gemini-embedding-2-preview")).toBe(
|
||||
"gemini-embedding-2-preview",
|
||||
);
|
||||
expect(normalizeGeminiModel("")).toBe(DEFAULT_GEMINI_EMBEDDING_MODEL);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Gemini embedding provider", () => {
|
||||
it("handles legacy and v2 request/response behavior", async () => {
|
||||
const fetchMock = installFetchMock((input) => {
|
||||
const url = input instanceof URL ? input.href : typeof input === "string" ? input : input.url;
|
||||
return url.endsWith(":batchEmbedContents")
|
||||
? {
|
||||
embeddings: Array.from({ length: 2 }, () => ({
|
||||
values: [0, Number.POSITIVE_INFINITY, 5],
|
||||
})),
|
||||
}
|
||||
: { embedding: { values: [3, 4, Number.NaN] } };
|
||||
});
|
||||
|
||||
const { provider } = await createGeminiEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "gemini",
|
||||
remote: { apiKey: "test-key" },
|
||||
model: "gemini-embedding-2-preview",
|
||||
outputDimensionality: 768,
|
||||
taskType: "SEMANTIC_SIMILARITY",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
await expect(provider.embedQuery(" ")).resolves.toEqual([]);
|
||||
await expect(provider.embedBatch([])).resolves.toEqual([]);
|
||||
await expect(provider.embedQuery("test query")).resolves.toEqual([0.6, 0.8, 0]);
|
||||
|
||||
const structuredBatch = await provider.embedBatchInputs?.([
|
||||
{
|
||||
text: "Image file: diagram.png",
|
||||
parts: [
|
||||
{ type: "text", text: "Image file: diagram.png" },
|
||||
{ type: "inline-data", mimeType: "image/png", data: "img" },
|
||||
],
|
||||
},
|
||||
{
|
||||
text: "Audio file: note.wav",
|
||||
parts: [
|
||||
{ type: "text", text: "Audio file: note.wav" },
|
||||
{ type: "inline-data", mimeType: "audio/wav", data: "aud" },
|
||||
],
|
||||
},
|
||||
]);
|
||||
expect(structuredBatch).toEqual([
|
||||
[0, 0, 1],
|
||||
[0, 0, 1],
|
||||
]);
|
||||
|
||||
expect(fetchMock.mock.calls[0]?.[0]).toBe(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-2-preview:embedContent",
|
||||
);
|
||||
expect(fetchJsonBody(fetchMock, 0)).toMatchObject({
|
||||
outputDimensionality: 768,
|
||||
taskType: "SEMANTIC_SIMILARITY",
|
||||
content: { parts: [{ text: "test query" }] },
|
||||
});
|
||||
expect(fetchJsonBody(fetchMock, 1)).toMatchObject({
|
||||
requests: [
|
||||
{
|
||||
model: "models/gemini-embedding-2-preview",
|
||||
content: {
|
||||
parts: [
|
||||
{ text: "Image file: diagram.png" },
|
||||
{ inlineData: { mimeType: "image/png", data: "img" } },
|
||||
],
|
||||
},
|
||||
taskType: "SEMANTIC_SIMILARITY",
|
||||
outputDimensionality: 768,
|
||||
},
|
||||
{
|
||||
model: "models/gemini-embedding-2-preview",
|
||||
content: {
|
||||
parts: [
|
||||
{ text: "Audio file: note.wav" },
|
||||
{ inlineData: { mimeType: "audio/wav", data: "aud" } },
|
||||
],
|
||||
},
|
||||
taskType: "SEMANTIC_SIMILARITY",
|
||||
outputDimensionality: 768,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
357
openclaw/extensions/google/embedding-provider.ts
Normal file
357
openclaw/extensions/google/embedding-provider.ts
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
import { parseGeminiAuth } from "openclaw/plugin-sdk/image-generation-core";
|
||||
import {
|
||||
buildRemoteBaseUrlPolicy,
|
||||
debugEmbeddingsLog,
|
||||
sanitizeAndNormalizeEmbedding,
|
||||
withRemoteHttpResponse,
|
||||
type EmbeddingInput,
|
||||
type MemoryEmbeddingProvider,
|
||||
type MemoryEmbeddingProviderCreateOptions,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { resolveMemorySecretInputString } from "openclaw/plugin-sdk/memory-core-host-secret";
|
||||
import {
|
||||
collectProviderApiKeysForExecution,
|
||||
executeWithApiKeyRotation,
|
||||
requireApiKey,
|
||||
resolveApiKeyForProvider,
|
||||
} from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import type { SsrFPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
|
||||
|
||||
export type GeminiEmbeddingClient = {
|
||||
baseUrl: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
model: string;
|
||||
modelPath: string;
|
||||
apiKeys: string[];
|
||||
outputDimensionality?: number;
|
||||
};
|
||||
|
||||
export const DEFAULT_GEMINI_EMBEDDING_MODEL = "gemini-embedding-001";
|
||||
const DEFAULT_GOOGLE_API_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
|
||||
const GEMINI_MAX_INPUT_TOKENS: Record<string, number> = {
|
||||
"text-embedding-004": 2048,
|
||||
"gemini-embedding-001": 2048,
|
||||
"gemini-embedding-2-preview": 8192,
|
||||
};
|
||||
|
||||
export type GeminiTaskType = NonNullable<MemoryEmbeddingProviderCreateOptions["taskType"]>;
|
||||
|
||||
// --- gemini-embedding-2-preview support ---
|
||||
|
||||
export const GEMINI_EMBEDDING_2_MODELS = new Set([
|
||||
"gemini-embedding-2-preview",
|
||||
// Add the GA model name here once released.
|
||||
]);
|
||||
|
||||
const GEMINI_EMBEDDING_2_DEFAULT_DIMENSIONS = 3072;
|
||||
const GEMINI_EMBEDDING_2_VALID_DIMENSIONS = [768, 1536, 3072] as const;
|
||||
|
||||
export type GeminiTextPart = { text: string };
|
||||
export type GeminiInlinePart = {
|
||||
inlineData: { mimeType: string; data: string };
|
||||
};
|
||||
export type GeminiPart = GeminiTextPart | GeminiInlinePart;
|
||||
export type GeminiEmbeddingRequest = {
|
||||
content: { parts: GeminiPart[] };
|
||||
taskType: GeminiTaskType;
|
||||
outputDimensionality?: number;
|
||||
model?: string;
|
||||
};
|
||||
export type GeminiTextEmbeddingRequest = GeminiEmbeddingRequest;
|
||||
|
||||
/** Builds the text-only Gemini embedding request shape used across direct and batch APIs. */
|
||||
export function buildGeminiTextEmbeddingRequest(params: {
|
||||
text: string;
|
||||
taskType: GeminiTaskType;
|
||||
outputDimensionality?: number;
|
||||
modelPath?: string;
|
||||
}): GeminiTextEmbeddingRequest {
|
||||
return buildGeminiEmbeddingRequest({
|
||||
input: { text: params.text },
|
||||
taskType: params.taskType,
|
||||
outputDimensionality: params.outputDimensionality,
|
||||
modelPath: params.modelPath,
|
||||
});
|
||||
}
|
||||
|
||||
export function buildGeminiEmbeddingRequest(params: {
|
||||
input: EmbeddingInput;
|
||||
taskType: GeminiTaskType;
|
||||
outputDimensionality?: number;
|
||||
modelPath?: string;
|
||||
}): GeminiEmbeddingRequest {
|
||||
const request: GeminiEmbeddingRequest = {
|
||||
content: {
|
||||
parts: params.input.parts?.map((part) =>
|
||||
part.type === "text"
|
||||
? ({ text: part.text } satisfies GeminiTextPart)
|
||||
: ({
|
||||
inlineData: { mimeType: part.mimeType, data: part.data },
|
||||
} satisfies GeminiInlinePart),
|
||||
) ?? [{ text: params.input.text }],
|
||||
},
|
||||
taskType: params.taskType,
|
||||
};
|
||||
if (params.modelPath) {
|
||||
request.model = params.modelPath;
|
||||
}
|
||||
if (params.outputDimensionality != null) {
|
||||
request.outputDimensionality = params.outputDimensionality;
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the given model name is a gemini-embedding-2 variant that
|
||||
* supports `outputDimensionality` and extended task types.
|
||||
*/
|
||||
export function isGeminiEmbedding2Model(model: string): boolean {
|
||||
return GEMINI_EMBEDDING_2_MODELS.has(model);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and return the `outputDimensionality` for gemini-embedding-2 models.
|
||||
* Returns `undefined` for older models (they don't support the param).
|
||||
*/
|
||||
export function resolveGeminiOutputDimensionality(
|
||||
model: string,
|
||||
requested?: number,
|
||||
): number | undefined {
|
||||
if (!isGeminiEmbedding2Model(model)) {
|
||||
return undefined;
|
||||
}
|
||||
if (requested == null) {
|
||||
return GEMINI_EMBEDDING_2_DEFAULT_DIMENSIONS;
|
||||
}
|
||||
const valid: readonly number[] = GEMINI_EMBEDDING_2_VALID_DIMENSIONS;
|
||||
if (!valid.includes(requested)) {
|
||||
throw new Error(
|
||||
`Invalid outputDimensionality ${requested} for ${model}. Valid values: ${valid.join(", ")}`,
|
||||
);
|
||||
}
|
||||
return requested;
|
||||
}
|
||||
function resolveRemoteApiKey(remoteApiKey: unknown): string | undefined {
|
||||
const trimmed = resolveMemorySecretInputString({
|
||||
value: remoteApiKey,
|
||||
path: "agents.*.memorySearch.remote.apiKey",
|
||||
});
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
if (trimmed === "GOOGLE_API_KEY" || trimmed === "GEMINI_API_KEY") {
|
||||
return process.env[trimmed]?.trim();
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
export function normalizeGeminiModel(model: string): string {
|
||||
const trimmed = model.trim();
|
||||
if (!trimmed) {
|
||||
return DEFAULT_GEMINI_EMBEDDING_MODEL;
|
||||
}
|
||||
const withoutPrefix = trimmed.replace(/^models\//, "");
|
||||
if (withoutPrefix.startsWith("gemini/")) {
|
||||
return withoutPrefix.slice("gemini/".length);
|
||||
}
|
||||
if (withoutPrefix.startsWith("google/")) {
|
||||
return withoutPrefix.slice("google/".length);
|
||||
}
|
||||
return withoutPrefix;
|
||||
}
|
||||
|
||||
async function fetchGeminiEmbeddingPayload(params: {
|
||||
client: GeminiEmbeddingClient;
|
||||
endpoint: string;
|
||||
body: unknown;
|
||||
}): Promise<{
|
||||
embedding?: { values?: number[] };
|
||||
embeddings?: Array<{ values?: number[] }>;
|
||||
}> {
|
||||
return await executeWithApiKeyRotation({
|
||||
provider: "google",
|
||||
apiKeys: params.client.apiKeys,
|
||||
execute: async (apiKey) => {
|
||||
const authHeaders = parseGeminiAuth(apiKey);
|
||||
const headers = {
|
||||
...authHeaders.headers,
|
||||
...params.client.headers,
|
||||
};
|
||||
return await withRemoteHttpResponse({
|
||||
url: params.endpoint,
|
||||
ssrfPolicy: params.client.ssrfPolicy,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(params.body),
|
||||
},
|
||||
onResponse: async (res) => {
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`gemini embeddings failed: ${res.status} ${text}`);
|
||||
}
|
||||
return (await res.json()) as {
|
||||
embedding?: { values?: number[] };
|
||||
embeddings?: Array<{ values?: number[] }>;
|
||||
};
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeGeminiBaseUrl(raw: string): string {
|
||||
const trimmed = raw.replace(/\/+$/, "");
|
||||
const openAiIndex = trimmed.indexOf("/openai");
|
||||
if (openAiIndex > -1) {
|
||||
return normalizeGoogleApiBaseUrl(trimmed.slice(0, openAiIndex));
|
||||
}
|
||||
return normalizeGoogleApiBaseUrl(trimmed);
|
||||
}
|
||||
|
||||
function buildGeminiModelPath(model: string): string {
|
||||
return model.startsWith("models/") ? model : `models/${model}`;
|
||||
}
|
||||
|
||||
function normalizeGoogleApiBaseUrl(baseUrl: string): string {
|
||||
const trimmed = baseUrl.trim().replace(/\/+$/, "");
|
||||
if (!trimmed) {
|
||||
return DEFAULT_GOOGLE_API_BASE_URL;
|
||||
}
|
||||
try {
|
||||
const url = new URL(trimmed);
|
||||
url.hash = "";
|
||||
url.search = "";
|
||||
if (
|
||||
url.origin.toLowerCase() === "https://generativelanguage.googleapis.com" &&
|
||||
url.pathname.replace(/\/+$/, "") === ""
|
||||
) {
|
||||
url.pathname = "/v1beta";
|
||||
}
|
||||
return url.toString().replace(/\/+$/, "");
|
||||
} catch {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
export async function createGeminiEmbeddingProvider(
|
||||
options: MemoryEmbeddingProviderCreateOptions,
|
||||
): Promise<{ provider: MemoryEmbeddingProvider; client: GeminiEmbeddingClient }> {
|
||||
const client = await resolveGeminiEmbeddingClient(options);
|
||||
const baseUrl = client.baseUrl.replace(/\/$/, "");
|
||||
const embedUrl = `${baseUrl}/${client.modelPath}:embedContent`;
|
||||
const batchUrl = `${baseUrl}/${client.modelPath}:batchEmbedContents`;
|
||||
const isV2 = isGeminiEmbedding2Model(client.model);
|
||||
const outputDimensionality = client.outputDimensionality;
|
||||
|
||||
const embedQuery = async (text: string): Promise<number[]> => {
|
||||
if (!text.trim()) {
|
||||
return [];
|
||||
}
|
||||
const payload = await fetchGeminiEmbeddingPayload({
|
||||
client,
|
||||
endpoint: embedUrl,
|
||||
body: buildGeminiTextEmbeddingRequest({
|
||||
text,
|
||||
taskType: options.taskType ?? "RETRIEVAL_QUERY",
|
||||
outputDimensionality: isV2 ? outputDimensionality : undefined,
|
||||
}),
|
||||
});
|
||||
return sanitizeAndNormalizeEmbedding(payload.embedding?.values ?? []);
|
||||
};
|
||||
|
||||
const embedBatchInputs = async (inputs: EmbeddingInput[]): Promise<number[][]> => {
|
||||
if (inputs.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const payload = await fetchGeminiEmbeddingPayload({
|
||||
client,
|
||||
endpoint: batchUrl,
|
||||
body: {
|
||||
requests: inputs.map((input) =>
|
||||
buildGeminiEmbeddingRequest({
|
||||
input,
|
||||
modelPath: client.modelPath,
|
||||
taskType: options.taskType ?? "RETRIEVAL_DOCUMENT",
|
||||
outputDimensionality: isV2 ? outputDimensionality : undefined,
|
||||
}),
|
||||
),
|
||||
},
|
||||
});
|
||||
const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : [];
|
||||
return inputs.map((_, index) => sanitizeAndNormalizeEmbedding(embeddings[index]?.values ?? []));
|
||||
};
|
||||
|
||||
const embedBatch = async (texts: string[]): Promise<number[][]> => {
|
||||
return await embedBatchInputs(
|
||||
texts.map((text) => ({
|
||||
text,
|
||||
})),
|
||||
);
|
||||
};
|
||||
|
||||
return {
|
||||
provider: {
|
||||
id: "gemini",
|
||||
model: client.model,
|
||||
maxInputTokens: GEMINI_MAX_INPUT_TOKENS[client.model],
|
||||
embedQuery,
|
||||
embedBatch,
|
||||
embedBatchInputs,
|
||||
},
|
||||
client,
|
||||
};
|
||||
}
|
||||
|
||||
export async function resolveGeminiEmbeddingClient(
|
||||
options: MemoryEmbeddingProviderCreateOptions,
|
||||
): Promise<GeminiEmbeddingClient> {
|
||||
const remote = options.remote;
|
||||
const remoteApiKey = resolveRemoteApiKey(remote?.apiKey);
|
||||
const remoteBaseUrl = remote?.baseUrl?.trim();
|
||||
|
||||
const apiKey = remoteApiKey
|
||||
? remoteApiKey
|
||||
: requireApiKey(
|
||||
await resolveApiKeyForProvider({
|
||||
provider: "google",
|
||||
cfg: options.config,
|
||||
agentDir: options.agentDir,
|
||||
}),
|
||||
"google",
|
||||
);
|
||||
|
||||
const providerConfig = options.config.models?.providers?.google;
|
||||
const rawBaseUrl =
|
||||
remoteBaseUrl ||
|
||||
normalizeOptionalString(providerConfig?.baseUrl) ||
|
||||
DEFAULT_GOOGLE_API_BASE_URL;
|
||||
const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl);
|
||||
const ssrfPolicy = buildRemoteBaseUrlPolicy(baseUrl);
|
||||
const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
|
||||
const headers: Record<string, string> = {
|
||||
...headerOverrides,
|
||||
};
|
||||
const apiKeys = collectProviderApiKeysForExecution({
|
||||
provider: "google",
|
||||
primaryApiKey: apiKey,
|
||||
});
|
||||
const model = normalizeGeminiModel(options.model);
|
||||
const modelPath = buildGeminiModelPath(model);
|
||||
const outputDimensionality = resolveGeminiOutputDimensionality(
|
||||
model,
|
||||
options.outputDimensionality,
|
||||
);
|
||||
debugEmbeddingsLog("memory embeddings: gemini client", {
|
||||
rawBaseUrl,
|
||||
baseUrl,
|
||||
model,
|
||||
modelPath,
|
||||
outputDimensionality,
|
||||
embedEndpoint: `${baseUrl}/${modelPath}:embedContent`,
|
||||
batchEndpoint: `${baseUrl}/${modelPath}:batchEmbedContents`,
|
||||
});
|
||||
return { baseUrl, headers, ssrfPolicy, model, modelPath, apiKeys, outputDimensionality };
|
||||
}
|
||||
20
openclaw/extensions/google/gemini-auth.ts
Normal file
20
openclaw/extensions/google/gemini-auth.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { parseGoogleOauthApiKey } from "./oauth-token-shared.js";
|
||||
|
||||
export function parseGeminiAuth(apiKey: string): { headers: Record<string, string> } {
|
||||
const parsed = apiKey.startsWith("{") ? parseGoogleOauthApiKey(apiKey) : null;
|
||||
if (parsed?.token) {
|
||||
return {
|
||||
headers: {
|
||||
Authorization: `Bearer ${parsed.token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
headers: {
|
||||
"x-goog-api-key": apiKey,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
};
|
||||
}
|
||||
137
openclaw/extensions/google/gemini-cli-provider.ts
Normal file
137
openclaw/extensions/google/gemini-cli-provider.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import type {
|
||||
OpenClawPluginApi,
|
||||
ProviderAuthContext,
|
||||
ProviderFetchUsageSnapshotContext,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { buildOauthProviderAuthResult } from "openclaw/plugin-sdk/provider-auth-result";
|
||||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { buildProviderToolCompatFamilyHooks } from "openclaw/plugin-sdk/provider-tools";
|
||||
import { fetchGeminiUsage } from "openclaw/plugin-sdk/provider-usage";
|
||||
import { formatGoogleOauthApiKey, parseGoogleUsageToken } from "./oauth-token-shared.js";
|
||||
import { GOOGLE_GEMINI_PROVIDER_HOOKS } from "./provider-hooks.js";
|
||||
import { isModernGoogleModel, resolveGoogleGeminiForwardCompatModel } from "./provider-models.js";
|
||||
|
||||
const PROVIDER_ID = "google-gemini-cli";
|
||||
const PROVIDER_LABEL = "Gemini CLI OAuth";
|
||||
const DEFAULT_MODEL = "google-gemini-cli/gemini-3.1-pro-preview";
|
||||
const ENV_VARS = [
|
||||
"OPENCLAW_GEMINI_OAUTH_CLIENT_ID",
|
||||
"OPENCLAW_GEMINI_OAUTH_CLIENT_SECRET",
|
||||
"GEMINI_CLI_OAUTH_CLIENT_ID",
|
||||
"GEMINI_CLI_OAUTH_CLIENT_SECRET",
|
||||
] as const;
|
||||
|
||||
const GOOGLE_GEMINI_CLI_PROVIDER_HOOKS = {
|
||||
...GOOGLE_GEMINI_PROVIDER_HOOKS,
|
||||
...buildProviderToolCompatFamilyHooks("gemini"),
|
||||
};
|
||||
|
||||
async function fetchGeminiCliUsage(ctx: ProviderFetchUsageSnapshotContext) {
|
||||
return await fetchGeminiUsage(ctx.token, ctx.timeoutMs, ctx.fetchFn, PROVIDER_ID);
|
||||
}
|
||||
|
||||
export function buildGoogleGeminiCliProvider(): ProviderPlugin {
|
||||
return {
|
||||
id: PROVIDER_ID,
|
||||
label: PROVIDER_LABEL,
|
||||
docsPath: "/providers/models",
|
||||
aliases: ["gemini-cli"],
|
||||
envVars: [...ENV_VARS],
|
||||
auth: [
|
||||
{
|
||||
id: "oauth",
|
||||
label: "Google OAuth",
|
||||
hint: "PKCE + localhost callback",
|
||||
kind: "oauth",
|
||||
run: async (ctx: ProviderAuthContext) => {
|
||||
await ctx.prompter.note(
|
||||
[
|
||||
"This is an unofficial integration and is not endorsed by Google.",
|
||||
"Some users have reported account restrictions or suspensions after using third-party Gemini CLI and Antigravity OAuth clients.",
|
||||
"Proceed only if you understand and accept this risk.",
|
||||
].join("\n"),
|
||||
"Google Gemini CLI caution",
|
||||
);
|
||||
|
||||
const proceed = await ctx.prompter.confirm({
|
||||
message: "Continue with Google Gemini CLI OAuth?",
|
||||
initialValue: false,
|
||||
});
|
||||
if (!proceed) {
|
||||
await ctx.prompter.note("Skipped Google Gemini CLI OAuth setup.", "Setup skipped");
|
||||
return { profiles: [] };
|
||||
}
|
||||
|
||||
const spin = ctx.prompter.progress("Starting Gemini CLI OAuth…");
|
||||
try {
|
||||
const { loginGeminiCliOAuth } = await import("./oauth.runtime.js");
|
||||
const result = await loginGeminiCliOAuth({
|
||||
isRemote: ctx.isRemote,
|
||||
openUrl: ctx.openUrl,
|
||||
log: (msg) => ctx.runtime.log(msg),
|
||||
note: ctx.prompter.note,
|
||||
prompt: async (message) => ctx.prompter.text({ message }),
|
||||
progress: spin,
|
||||
});
|
||||
|
||||
spin.stop("Gemini CLI OAuth complete");
|
||||
return buildOauthProviderAuthResult({
|
||||
providerId: PROVIDER_ID,
|
||||
defaultModel: DEFAULT_MODEL,
|
||||
access: result.access,
|
||||
refresh: result.refresh,
|
||||
expires: result.expires,
|
||||
email: result.email,
|
||||
...(result.projectId ? { credentialExtra: { projectId: result.projectId } } : {}),
|
||||
...(result.projectId
|
||||
? {
|
||||
notes: [
|
||||
"If requests fail, set GOOGLE_CLOUD_PROJECT or GOOGLE_CLOUD_PROJECT_ID.",
|
||||
],
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
} catch (err) {
|
||||
spin.stop("Gemini CLI OAuth failed");
|
||||
await ctx.prompter.note(
|
||||
"Trouble with OAuth? Ensure your Google account has Gemini CLI access.",
|
||||
"OAuth help",
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
},
|
||||
],
|
||||
wizard: {
|
||||
setup: {
|
||||
choiceId: "google-gemini-cli",
|
||||
choiceLabel: "Gemini CLI OAuth",
|
||||
choiceHint: "Google OAuth with project-aware token payload",
|
||||
methodId: "oauth",
|
||||
},
|
||||
},
|
||||
resolveDynamicModel: (ctx) =>
|
||||
resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: PROVIDER_ID,
|
||||
ctx,
|
||||
}),
|
||||
...GOOGLE_GEMINI_CLI_PROVIDER_HOOKS,
|
||||
isModernModelRef: ({ modelId }) => isModernGoogleModel(modelId),
|
||||
formatApiKey: (cred) => formatGoogleOauthApiKey(cred),
|
||||
resolveUsageAuth: async (ctx) => {
|
||||
const auth = await ctx.resolveOAuthToken();
|
||||
if (!auth) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
...auth,
|
||||
token: parseGoogleUsageToken(auth.token),
|
||||
};
|
||||
},
|
||||
fetchUsageSnapshot: async (ctx) => await fetchGeminiCliUsage(ctx),
|
||||
};
|
||||
}
|
||||
|
||||
export function registerGoogleGeminiCliProvider(api: OpenClawPluginApi) {
|
||||
api.registerProvider(buildGoogleGeminiCliProvider());
|
||||
}
|
||||
121
openclaw/extensions/google/generation-provider-metadata.ts
Normal file
121
openclaw/extensions/google/generation-provider-metadata.ts
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
import type { MusicGenerationProvider } from "openclaw/plugin-sdk/music-generation";
|
||||
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import type {
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationProviderConfiguredContext,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
|
||||
export const DEFAULT_GOOGLE_MUSIC_MODEL = "lyria-3-clip-preview";
|
||||
export const GOOGLE_PRO_MUSIC_MODEL = "lyria-3-pro-preview";
|
||||
export const GOOGLE_MAX_INPUT_IMAGES = 10;
|
||||
|
||||
export const DEFAULT_GOOGLE_VIDEO_MODEL = "veo-3.1-fast-generate-preview";
|
||||
export const GOOGLE_VIDEO_ALLOWED_DURATION_SECONDS = [4, 6, 8] as const;
|
||||
export const GOOGLE_VIDEO_MIN_DURATION_SECONDS = GOOGLE_VIDEO_ALLOWED_DURATION_SECONDS[0];
|
||||
export const GOOGLE_VIDEO_MAX_DURATION_SECONDS =
|
||||
GOOGLE_VIDEO_ALLOWED_DURATION_SECONDS[GOOGLE_VIDEO_ALLOWED_DURATION_SECONDS.length - 1];
|
||||
|
||||
function isGoogleProviderConfigured(
|
||||
ctx: { agentDir?: string } | VideoGenerationProviderConfiguredContext,
|
||||
): boolean {
|
||||
return isProviderApiKeyConfigured({
|
||||
provider: "google",
|
||||
agentDir: ctx.agentDir,
|
||||
});
|
||||
}
|
||||
|
||||
export function createGoogleMusicGenerationProviderMetadata(): Omit<
|
||||
MusicGenerationProvider,
|
||||
"generateMusic"
|
||||
> {
|
||||
return {
|
||||
id: "google",
|
||||
label: "Google",
|
||||
defaultModel: DEFAULT_GOOGLE_MUSIC_MODEL,
|
||||
models: [DEFAULT_GOOGLE_MUSIC_MODEL, GOOGLE_PRO_MUSIC_MODEL],
|
||||
isConfigured: isGoogleProviderConfigured,
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxTracks: 1,
|
||||
supportsLyrics: true,
|
||||
supportsInstrumental: true,
|
||||
supportsFormat: true,
|
||||
supportedFormatsByModel: {
|
||||
[DEFAULT_GOOGLE_MUSIC_MODEL]: ["mp3"],
|
||||
[GOOGLE_PRO_MUSIC_MODEL]: ["mp3", "wav"],
|
||||
},
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxTracks: 1,
|
||||
maxInputImages: GOOGLE_MAX_INPUT_IMAGES,
|
||||
supportsLyrics: true,
|
||||
supportsInstrumental: true,
|
||||
supportsFormat: true,
|
||||
supportedFormatsByModel: {
|
||||
[DEFAULT_GOOGLE_MUSIC_MODEL]: ["mp3"],
|
||||
[GOOGLE_PRO_MUSIC_MODEL]: ["mp3", "wav"],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createGoogleVideoGenerationProviderMetadata(): Omit<
|
||||
VideoGenerationProvider,
|
||||
"generateVideo"
|
||||
> {
|
||||
return {
|
||||
id: "google",
|
||||
label: "Google",
|
||||
defaultModel: DEFAULT_GOOGLE_VIDEO_MODEL,
|
||||
models: [
|
||||
DEFAULT_GOOGLE_VIDEO_MODEL,
|
||||
"veo-3.1-generate-preview",
|
||||
"veo-3.1-lite-generate-preview",
|
||||
"veo-3.0-fast-generate-001",
|
||||
"veo-3.0-generate-001",
|
||||
"veo-2.0-generate-001",
|
||||
],
|
||||
isConfigured: isGoogleProviderConfigured,
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxVideos: 1,
|
||||
maxDurationSeconds: GOOGLE_VIDEO_MAX_DURATION_SECONDS,
|
||||
supportedDurationSeconds: [...GOOGLE_VIDEO_ALLOWED_DURATION_SECONDS],
|
||||
aspectRatios: ["16:9", "9:16"],
|
||||
resolutions: ["720P", "1080P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: true,
|
||||
supportsAudio: true,
|
||||
},
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
maxDurationSeconds: GOOGLE_VIDEO_MAX_DURATION_SECONDS,
|
||||
supportedDurationSeconds: [...GOOGLE_VIDEO_ALLOWED_DURATION_SECONDS],
|
||||
aspectRatios: ["16:9", "9:16"],
|
||||
resolutions: ["720P", "1080P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: true,
|
||||
supportsAudio: true,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputVideos: 1,
|
||||
maxDurationSeconds: GOOGLE_VIDEO_MAX_DURATION_SECONDS,
|
||||
supportedDurationSeconds: [...GOOGLE_VIDEO_ALLOWED_DURATION_SECONDS],
|
||||
aspectRatios: ["16:9", "9:16"],
|
||||
resolutions: ["720P", "1080P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: true,
|
||||
supportsAudio: true,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
99
openclaw/extensions/google/google-shared.test-helpers.ts
Normal file
99
openclaw/extensions/google/google-shared.test-helpers.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import type { Model } from "@mariozechner/pi-ai";
|
||||
import { expect } from "vitest";
|
||||
|
||||
function makeZeroUsageSnapshot() {
|
||||
return {
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cacheCreationInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
total: 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const asRecord = (value: unknown): Record<string, unknown> => {
|
||||
expect(value).toBeTruthy();
|
||||
expect(typeof value).toBe("object");
|
||||
expect(Array.isArray(value)).toBe(false);
|
||||
return value as Record<string, unknown>;
|
||||
};
|
||||
|
||||
type ConvertedTools = ReadonlyArray<{
|
||||
functionDeclarations?: ReadonlyArray<{
|
||||
parametersJsonSchema?: unknown;
|
||||
parameters?: unknown;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export const getFirstToolParameters = (converted: ConvertedTools): Record<string, unknown> => {
|
||||
const functionDeclaration = asRecord(converted?.[0]?.functionDeclarations?.[0]);
|
||||
return asRecord(functionDeclaration.parametersJsonSchema ?? functionDeclaration.parameters);
|
||||
};
|
||||
|
||||
export const makeModel = (id: string): Model<"google-generative-ai"> =>
|
||||
({
|
||||
id,
|
||||
name: id,
|
||||
api: "google-generative-ai",
|
||||
provider: "google",
|
||||
baseUrl: "https://example.invalid",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1,
|
||||
maxTokens: 1,
|
||||
}) as Model<"google-generative-ai">;
|
||||
|
||||
export const makeGeminiCliModel = (id: string): Model<"google-gemini-cli"> =>
|
||||
({
|
||||
id,
|
||||
name: id,
|
||||
api: "google-gemini-cli",
|
||||
provider: "google-gemini-cli",
|
||||
baseUrl: "https://example.invalid",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1,
|
||||
maxTokens: 1,
|
||||
}) as Model<"google-gemini-cli">;
|
||||
|
||||
export function makeGoogleAssistantMessage(model: string, content: unknown) {
|
||||
return {
|
||||
role: "assistant",
|
||||
content,
|
||||
api: "google-generative-ai",
|
||||
provider: "google",
|
||||
model,
|
||||
usage: makeZeroUsageSnapshot(),
|
||||
stopReason: "stop",
|
||||
timestamp: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function makeGeminiCliAssistantMessage(model: string, content: unknown) {
|
||||
return {
|
||||
role: "assistant",
|
||||
content,
|
||||
api: "google-gemini-cli",
|
||||
provider: "google-gemini-cli",
|
||||
model,
|
||||
usage: makeZeroUsageSnapshot(),
|
||||
stopReason: "stop",
|
||||
timestamp: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function expectConvertedRoles(contents: Array<{ role?: string }>, expectedRoles: string[]) {
|
||||
expect(contents).toHaveLength(expectedRoles.length);
|
||||
for (const [index, role] of expectedRoles.entries()) {
|
||||
expect(contents[index]?.role).toBe(role);
|
||||
}
|
||||
}
|
||||
363
openclaw/extensions/google/google-shared.test.ts
Normal file
363
openclaw/extensions/google/google-shared.test.ts
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
import type { Context, Tool } from "@mariozechner/pi-ai";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
convertMessages,
|
||||
convertTools,
|
||||
} from "../../node_modules/@mariozechner/pi-ai/dist/providers/google-shared.js";
|
||||
import {
|
||||
asRecord,
|
||||
expectConvertedRoles,
|
||||
getFirstToolParameters,
|
||||
makeGeminiCliAssistantMessage,
|
||||
makeGeminiCliModel,
|
||||
makeGoogleAssistantMessage,
|
||||
makeModel,
|
||||
} from "./google-shared.test-helpers.js";
|
||||
|
||||
describe("google-shared convertTools", () => {
|
||||
it("preserves parameters when type is missing", () => {
|
||||
const tools = [
|
||||
{
|
||||
name: "noType",
|
||||
description: "Tool with properties but no type",
|
||||
parameters: {
|
||||
properties: {
|
||||
action: { type: "string" },
|
||||
},
|
||||
required: ["action"],
|
||||
},
|
||||
},
|
||||
] as unknown as Tool[];
|
||||
|
||||
const converted = convertTools(tools);
|
||||
const params = getFirstToolParameters(
|
||||
converted as Parameters<typeof getFirstToolParameters>[0],
|
||||
);
|
||||
|
||||
expect(params.type).toBeUndefined();
|
||||
expect(params.properties).toBeDefined();
|
||||
expect(params.required).toEqual(["action"]);
|
||||
});
|
||||
|
||||
it("keeps unsupported JSON Schema keywords intact", () => {
|
||||
const tools = [
|
||||
{
|
||||
name: "example",
|
||||
description: "Example tool",
|
||||
parameters: {
|
||||
type: "object",
|
||||
patternProperties: {
|
||||
"^x-": { type: "string" },
|
||||
},
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
mode: {
|
||||
type: "string",
|
||||
const: "fast",
|
||||
},
|
||||
options: {
|
||||
anyOf: [{ type: "string" }, { type: "number" }],
|
||||
},
|
||||
list: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "string",
|
||||
const: "item",
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ["mode"],
|
||||
},
|
||||
},
|
||||
] as unknown as Tool[];
|
||||
|
||||
const converted = convertTools(tools);
|
||||
const params = getFirstToolParameters(
|
||||
converted as Parameters<typeof getFirstToolParameters>[0],
|
||||
);
|
||||
const properties = asRecord(params.properties);
|
||||
const mode = asRecord(properties.mode);
|
||||
const options = asRecord(properties.options);
|
||||
const list = asRecord(properties.list);
|
||||
const items = asRecord(list.items);
|
||||
|
||||
expect(params.patternProperties).toEqual({ "^x-": { type: "string" } });
|
||||
expect(params.additionalProperties).toBe(false);
|
||||
expect(mode.const).toBe("fast");
|
||||
expect(options.anyOf).toEqual([{ type: "string" }, { type: "number" }]);
|
||||
expect(items.const).toBe("item");
|
||||
expect(params.required).toEqual(["mode"]);
|
||||
});
|
||||
|
||||
it("keeps supported schema fields", () => {
|
||||
const tools = [
|
||||
{
|
||||
name: "settings",
|
||||
description: "Settings tool",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
config: {
|
||||
type: "object",
|
||||
properties: {
|
||||
retries: { type: "number", minimum: 1 },
|
||||
tags: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
},
|
||||
},
|
||||
required: ["retries"],
|
||||
},
|
||||
},
|
||||
required: ["config"],
|
||||
},
|
||||
},
|
||||
] as unknown as Tool[];
|
||||
|
||||
const converted = convertTools(tools);
|
||||
const params = getFirstToolParameters(
|
||||
converted as Parameters<typeof getFirstToolParameters>[0],
|
||||
);
|
||||
const config = asRecord(asRecord(params.properties).config);
|
||||
const configProps = asRecord(config.properties);
|
||||
const retries = asRecord(configProps.retries);
|
||||
const tags = asRecord(configProps.tags);
|
||||
const items = asRecord(tags.items);
|
||||
|
||||
expect(params.type).toBe("object");
|
||||
expect(config.type).toBe("object");
|
||||
expect(retries.minimum).toBe(1);
|
||||
expect(tags.type).toBe("array");
|
||||
expect(items.type).toBe("string");
|
||||
expect(config.required).toEqual(["retries"]);
|
||||
expect(params.required).toEqual(["config"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("google-shared convertMessages", () => {
|
||||
function expectConsecutiveMessagesNotMerged(params: {
|
||||
modelId: string;
|
||||
first: string;
|
||||
second: string;
|
||||
}) {
|
||||
const model = makeModel(params.modelId);
|
||||
const context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: params.first,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: params.second,
|
||||
},
|
||||
],
|
||||
} as unknown as Context;
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
expect(contents).toHaveLength(2);
|
||||
expect(contents[0].role).toBe("user");
|
||||
expect(contents[1].role).toBe("user");
|
||||
expect(contents[0].parts).toHaveLength(1);
|
||||
expect(contents[1].parts).toHaveLength(1);
|
||||
}
|
||||
|
||||
it("keeps thinking blocks when provider/model match", () => {
|
||||
const model = makeModel("gemini-1.5-pro");
|
||||
const context = {
|
||||
messages: [
|
||||
makeGoogleAssistantMessage(model.id, [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "hidden",
|
||||
thinkingSignature: "c2ln",
|
||||
},
|
||||
]),
|
||||
],
|
||||
} as unknown as Context;
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("model");
|
||||
expect(contents[0].parts?.[0]).toMatchObject({
|
||||
thought: true,
|
||||
thoughtSignature: "c2ln",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps thought signatures for Claude models", () => {
|
||||
const model = makeModel("claude-3-opus");
|
||||
const context = {
|
||||
messages: [
|
||||
makeGoogleAssistantMessage(model.id, [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "structured",
|
||||
thinkingSignature: "c2ln",
|
||||
},
|
||||
]),
|
||||
],
|
||||
} as unknown as Context;
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
const parts = contents?.[0]?.parts ?? [];
|
||||
expect(parts).toHaveLength(1);
|
||||
expect(parts[0]).toMatchObject({
|
||||
thought: true,
|
||||
thoughtSignature: "c2ln",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not merge consecutive user messages for Gemini", () => {
|
||||
expectConsecutiveMessagesNotMerged({
|
||||
modelId: "gemini-1.5-pro",
|
||||
first: "Hello",
|
||||
second: "How are you?",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not merge consecutive user messages for non-Gemini Google models", () => {
|
||||
expectConsecutiveMessagesNotMerged({
|
||||
modelId: "claude-3-opus",
|
||||
first: "First",
|
||||
second: "Second",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not merge consecutive model messages for Gemini", () => {
|
||||
const model = makeModel("gemini-1.5-pro");
|
||||
const context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Hello",
|
||||
},
|
||||
makeGoogleAssistantMessage(model.id, [{ type: "text", text: "Hi there!" }]),
|
||||
makeGoogleAssistantMessage(model.id, [{ type: "text", text: "How can I help?" }]),
|
||||
],
|
||||
} as unknown as Context;
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
expectConvertedRoles(contents, ["user", "model", "model"]);
|
||||
expect(contents[1].parts).toHaveLength(1);
|
||||
expect(contents[2].parts).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("handles user message after tool result without model response in between", () => {
|
||||
const model = makeModel("gemini-1.5-pro");
|
||||
const context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Use a tool",
|
||||
},
|
||||
makeGoogleAssistantMessage(model.id, [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "myTool",
|
||||
arguments: { arg: "value" },
|
||||
},
|
||||
]),
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_1",
|
||||
toolName: "myTool",
|
||||
content: [{ type: "text", text: "Tool result" }],
|
||||
isError: false,
|
||||
timestamp: 0,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "Now do something else",
|
||||
},
|
||||
],
|
||||
} as unknown as Context;
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
expect(contents).toHaveLength(4);
|
||||
expect(contents[0].role).toBe("user");
|
||||
expect(contents[1].role).toBe("model");
|
||||
expect(contents[2].role).toBe("user");
|
||||
expect(contents[3].role).toBe("user");
|
||||
const toolResponsePart = contents[2].parts?.find(
|
||||
(part) => typeof part === "object" && part !== null && "functionResponse" in part,
|
||||
);
|
||||
const toolResponse = asRecord(toolResponsePart);
|
||||
expect(toolResponse.functionResponse).toBeTruthy();
|
||||
expect(contents[3].role).toBe("user");
|
||||
});
|
||||
|
||||
it("ensures function call comes after user turn, not after model turn", () => {
|
||||
const model = makeModel("gemini-1.5-pro");
|
||||
const context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Hello",
|
||||
},
|
||||
makeGoogleAssistantMessage(model.id, [{ type: "text", text: "Hi!" }]),
|
||||
makeGoogleAssistantMessage(model.id, [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "myTool",
|
||||
arguments: {},
|
||||
},
|
||||
]),
|
||||
],
|
||||
} as unknown as Context;
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
expectConvertedRoles(contents, ["user", "model", "model"]);
|
||||
const toolCallPart = contents[2].parts?.find(
|
||||
(part) => typeof part === "object" && part !== null && "functionCall" in part,
|
||||
);
|
||||
const toolCall = asRecord(toolCallPart);
|
||||
expect(toolCall.functionCall).toBeTruthy();
|
||||
});
|
||||
|
||||
it("strips tool call and response ids for google-gemini-cli", () => {
|
||||
const model = makeGeminiCliModel("gemini-3-flash");
|
||||
const context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Use a tool",
|
||||
},
|
||||
makeGeminiCliAssistantMessage(model.id, [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "myTool",
|
||||
arguments: { arg: "value" },
|
||||
thoughtSignature: "dGVzdA==",
|
||||
},
|
||||
]),
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_1",
|
||||
toolName: "myTool",
|
||||
content: [{ type: "text", text: "Tool result" }],
|
||||
isError: false,
|
||||
timestamp: 0,
|
||||
},
|
||||
],
|
||||
} as unknown as Context;
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
const parts = contents.flatMap((content) => content.parts ?? []);
|
||||
const toolCallPart = parts.find(
|
||||
(part) => typeof part === "object" && part !== null && "functionCall" in part,
|
||||
);
|
||||
const toolResponsePart = parts.find(
|
||||
(part) => typeof part === "object" && part !== null && "functionResponse" in part,
|
||||
);
|
||||
|
||||
const toolCall = asRecord(toolCallPart);
|
||||
const toolResponse = asRecord(toolResponsePart);
|
||||
|
||||
expect(asRecord(toolCall.functionCall).id).toBeUndefined();
|
||||
expect(asRecord(toolResponse.functionResponse).id).toBeUndefined();
|
||||
});
|
||||
});
|
||||
350
openclaw/extensions/google/image-generation-provider.test.ts
Normal file
350
openclaw/extensions/google/image-generation-provider.test.ts
Normal file
|
|
@ -0,0 +1,350 @@
|
|||
import * as providerAuthRuntime from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import * as providerHttp from "openclaw/plugin-sdk/provider-http";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildGoogleImageGenerationProvider } from "./image-generation-provider.js";
|
||||
import { __testing as geminiWebSearchTesting } from "./src/gemini-web-search-provider.js";
|
||||
|
||||
function mockGoogleApiKeyAuth() {
|
||||
vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "google-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
}
|
||||
|
||||
function installGoogleFetchMock(params?: {
|
||||
data?: string;
|
||||
mimeType?: string;
|
||||
inlineDataKey?: "inlineData" | "inline_data";
|
||||
}) {
|
||||
const mimeType = params?.mimeType ?? "image/png";
|
||||
const data = params?.data ?? "png-data";
|
||||
const inlineDataKey = params?.inlineDataKey ?? "inlineData";
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{
|
||||
[inlineDataKey]: {
|
||||
[inlineDataKey === "inlineData" ? "mimeType" : "mime_type"]: mimeType,
|
||||
data: Buffer.from(data).toString("base64"),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
return fetchMock;
|
||||
}
|
||||
|
||||
describe("Google image-generation provider", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("generates image buffers from the Gemini generateContent API", async () => {
|
||||
vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "google-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{ text: "generated" },
|
||||
{
|
||||
inlineData: {
|
||||
mimeType: "image/png",
|
||||
data: Buffer.from("png-data").toString("base64"),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const provider = buildGoogleImageGenerationProvider();
|
||||
const result = await provider.generateImage({
|
||||
provider: "google",
|
||||
model: "gemini-3.1-flash-image-preview",
|
||||
prompt: "draw a cat",
|
||||
cfg: {},
|
||||
size: "1536x1024",
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ text: "draw a cat" }],
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseModalities: ["TEXT", "IMAGE"],
|
||||
imageConfig: {
|
||||
aspectRatio: "3:2",
|
||||
imageSize: "2K",
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(result).toEqual({
|
||||
images: [
|
||||
{
|
||||
buffer: Buffer.from("png-data"),
|
||||
mimeType: "image/png",
|
||||
fileName: "image-1.png",
|
||||
},
|
||||
],
|
||||
model: "gemini-3.1-flash-image-preview",
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts OAuth JSON auth and inline_data responses", async () => {
|
||||
vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: JSON.stringify({ token: "oauth-token" }),
|
||||
source: "profile",
|
||||
mode: "token",
|
||||
});
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{
|
||||
inline_data: {
|
||||
mime_type: "image/jpeg",
|
||||
data: Buffer.from("jpg-data").toString("base64"),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const provider = buildGoogleImageGenerationProvider();
|
||||
const result = await provider.generateImage({
|
||||
provider: "google",
|
||||
model: "gemini-3.1-flash-image-preview",
|
||||
prompt: "draw a dog",
|
||||
cfg: {},
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
headers: expect.any(Headers),
|
||||
}),
|
||||
);
|
||||
const [, init] = fetchMock.mock.calls[0];
|
||||
expect(new Headers(init.headers).get("authorization")).toBe("Bearer oauth-token");
|
||||
expect(result).toEqual({
|
||||
images: [
|
||||
{
|
||||
buffer: Buffer.from("jpg-data"),
|
||||
mimeType: "image/jpeg",
|
||||
fileName: "image-1.jpg",
|
||||
},
|
||||
],
|
||||
model: "gemini-3.1-flash-image-preview",
|
||||
});
|
||||
});
|
||||
|
||||
it("sends reference images and explicit resolution for edit flows", async () => {
|
||||
mockGoogleApiKeyAuth();
|
||||
const fetchMock = installGoogleFetchMock();
|
||||
|
||||
const provider = buildGoogleImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "google",
|
||||
model: "gemini-3-pro-image-preview",
|
||||
prompt: "Change only the sky to a sunset.",
|
||||
cfg: {},
|
||||
resolution: "4K",
|
||||
inputImages: [
|
||||
{
|
||||
buffer: Buffer.from("reference-bytes"),
|
||||
mimeType: "image/png",
|
||||
fileName: "reference.png",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3-pro-image-preview:generateContent",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{
|
||||
inlineData: {
|
||||
mimeType: "image/png",
|
||||
data: Buffer.from("reference-bytes").toString("base64"),
|
||||
},
|
||||
},
|
||||
{ text: "Change only the sky to a sunset." },
|
||||
],
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseModalities: ["TEXT", "IMAGE"],
|
||||
imageConfig: {
|
||||
imageSize: "4K",
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("forwards explicit aspect ratio without forcing a default when size is omitted", async () => {
|
||||
mockGoogleApiKeyAuth();
|
||||
const fetchMock = installGoogleFetchMock();
|
||||
|
||||
const provider = buildGoogleImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "google",
|
||||
model: "gemini-3-pro-image-preview",
|
||||
prompt: "portrait photo",
|
||||
cfg: {},
|
||||
aspectRatio: "9:16",
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3-pro-image-preview:generateContent",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ text: "portrait photo" }],
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseModalities: ["TEXT", "IMAGE"],
|
||||
imageConfig: {
|
||||
aspectRatio: "9:16",
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("disables DNS pinning for Google image generation requests", async () => {
|
||||
mockGoogleApiKeyAuth();
|
||||
installGoogleFetchMock();
|
||||
const postJsonRequestSpy = vi.spyOn(providerHttp, "postJsonRequest");
|
||||
|
||||
const provider = buildGoogleImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "google",
|
||||
model: "gemini-3.1-flash-image-preview",
|
||||
prompt: "draw a fox",
|
||||
cfg: {},
|
||||
});
|
||||
|
||||
expect(postJsonRequestSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
pinDns: false,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes a configured bare Google host to the v1beta API root", async () => {
|
||||
mockGoogleApiKeyAuth();
|
||||
const fetchMock = installGoogleFetchMock();
|
||||
|
||||
const provider = buildGoogleImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "google",
|
||||
model: "gemini-3-pro-image-preview",
|
||||
prompt: "draw a cat",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
google: {
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3-pro-image-preview:generateContent",
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("strips a configured /openai suffix before calling the native Gemini image API", async () => {
|
||||
mockGoogleApiKeyAuth();
|
||||
const fetchMock = installGoogleFetchMock();
|
||||
|
||||
const provider = buildGoogleImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "google",
|
||||
model: "gemini-3-pro-image-preview",
|
||||
prompt: "draw a fox",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
google: {
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3-pro-image-preview:generateContent",
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("prefers scoped configured Gemini API keys over environment fallbacks", () => {
|
||||
expect(
|
||||
geminiWebSearchTesting.resolveGeminiApiKey({
|
||||
apiKey: "gemini-secret",
|
||||
}),
|
||||
).toBe("gemini-secret");
|
||||
});
|
||||
|
||||
it("falls back to the default Gemini model when unset or blank", () => {
|
||||
expect(geminiWebSearchTesting.resolveGeminiModel()).toBe("gemini-2.5-flash");
|
||||
expect(geminiWebSearchTesting.resolveGeminiModel({ model: " " })).toBe("gemini-2.5-flash");
|
||||
expect(geminiWebSearchTesting.resolveGeminiModel({ model: "gemini-2.5-pro" })).toBe(
|
||||
"gemini-2.5-pro",
|
||||
);
|
||||
});
|
||||
});
|
||||
212
openclaw/extensions/google/image-generation-provider.ts
Normal file
212
openclaw/extensions/google/image-generation-provider.ts
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation";
|
||||
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import { assertOkOrThrowHttpError, postJsonRequest } from "openclaw/plugin-sdk/provider-http";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { normalizeGoogleModelId, resolveGoogleGenerativeAiHttpRequestConfig } from "./api.js";
|
||||
|
||||
const DEFAULT_GOOGLE_IMAGE_MODEL = "gemini-3.1-flash-image-preview";
|
||||
const DEFAULT_OUTPUT_MIME = "image/png";
|
||||
const GOOGLE_SUPPORTED_SIZES = [
|
||||
"1024x1024",
|
||||
"1024x1536",
|
||||
"1536x1024",
|
||||
"1024x1792",
|
||||
"1792x1024",
|
||||
] as const;
|
||||
const GOOGLE_SUPPORTED_ASPECT_RATIOS = [
|
||||
"1:1",
|
||||
"2:3",
|
||||
"3:2",
|
||||
"3:4",
|
||||
"4:3",
|
||||
"4:5",
|
||||
"5:4",
|
||||
"9:16",
|
||||
"16:9",
|
||||
"21:9",
|
||||
] as const;
|
||||
|
||||
type GoogleInlineDataPart = {
|
||||
mimeType?: string;
|
||||
mime_type?: string;
|
||||
data?: string;
|
||||
};
|
||||
|
||||
type GoogleGenerateImageResponse = {
|
||||
candidates?: Array<{
|
||||
content?: {
|
||||
parts?: Array<{
|
||||
text?: string;
|
||||
inlineData?: GoogleInlineDataPart;
|
||||
inline_data?: GoogleInlineDataPart;
|
||||
}>;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
|
||||
function normalizeGoogleImageModel(model: string | undefined): string {
|
||||
const trimmed = model?.trim();
|
||||
return normalizeGoogleModelId(trimmed || DEFAULT_GOOGLE_IMAGE_MODEL);
|
||||
}
|
||||
|
||||
function mapSizeToImageConfig(
|
||||
size: string | undefined,
|
||||
): { aspectRatio?: string; imageSize?: "2K" | "4K" } | undefined {
|
||||
const trimmed = size?.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const normalized = normalizeLowercaseStringOrEmpty(trimmed);
|
||||
const mapping = new Map<string, string>([
|
||||
["1024x1024", "1:1"],
|
||||
["1024x1536", "2:3"],
|
||||
["1536x1024", "3:2"],
|
||||
["1024x1792", "9:16"],
|
||||
["1792x1024", "16:9"],
|
||||
]);
|
||||
const aspectRatio = mapping.get(normalized);
|
||||
|
||||
const [widthRaw, heightRaw] = normalized.split("x");
|
||||
const width = Number.parseInt(widthRaw ?? "", 10);
|
||||
const height = Number.parseInt(heightRaw ?? "", 10);
|
||||
const longestEdge = Math.max(width, height);
|
||||
const imageSize = longestEdge >= 3072 ? "4K" : longestEdge >= 1536 ? "2K" : undefined;
|
||||
|
||||
if (!aspectRatio && !imageSize) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
...(aspectRatio ? { aspectRatio } : {}),
|
||||
...(imageSize ? { imageSize } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildGoogleImageGenerationProvider(): ImageGenerationProvider {
|
||||
return {
|
||||
id: "google",
|
||||
label: "Google",
|
||||
defaultModel: DEFAULT_GOOGLE_IMAGE_MODEL,
|
||||
models: [DEFAULT_GOOGLE_IMAGE_MODEL, "gemini-3-pro-image-preview"],
|
||||
isConfigured: ({ agentDir }) =>
|
||||
isProviderApiKeyConfigured({
|
||||
provider: "google",
|
||||
agentDir,
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxCount: 4,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxCount: 4,
|
||||
maxInputImages: 5,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
geometry: {
|
||||
sizes: [...GOOGLE_SUPPORTED_SIZES],
|
||||
aspectRatios: [...GOOGLE_SUPPORTED_ASPECT_RATIOS],
|
||||
resolutions: ["1K", "2K", "4K"],
|
||||
},
|
||||
},
|
||||
async generateImage(req) {
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "google",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
});
|
||||
if (!auth.apiKey) {
|
||||
throw new Error("Google API key missing");
|
||||
}
|
||||
|
||||
const model = normalizeGoogleImageModel(req.model);
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveGoogleGenerativeAiHttpRequestConfig({
|
||||
apiKey: auth.apiKey,
|
||||
baseUrl: req.cfg?.models?.providers?.google?.baseUrl,
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
});
|
||||
const imageConfig = mapSizeToImageConfig(req.size);
|
||||
const inputParts = (req.inputImages ?? []).map((image) => ({
|
||||
inlineData: {
|
||||
mimeType: image.mimeType,
|
||||
data: image.buffer.toString("base64"),
|
||||
},
|
||||
}));
|
||||
const resolvedImageConfig = {
|
||||
...imageConfig,
|
||||
...(req.aspectRatio?.trim() ? { aspectRatio: req.aspectRatio.trim() } : {}),
|
||||
...(req.resolution ? { imageSize: req.resolution } : {}),
|
||||
};
|
||||
|
||||
const { response: res, release } = await postJsonRequest({
|
||||
url: `${baseUrl}/models/${model}:generateContent`,
|
||||
headers,
|
||||
body: {
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [...inputParts, { text: req.prompt }],
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseModalities: ["TEXT", "IMAGE"],
|
||||
...(Object.keys(resolvedImageConfig).length > 0
|
||||
? { imageConfig: resolvedImageConfig }
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
timeoutMs: 60_000,
|
||||
fetchFn: fetch,
|
||||
pinDns: false,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
|
||||
try {
|
||||
await assertOkOrThrowHttpError(res, "Google image generation failed");
|
||||
|
||||
const payload = (await res.json()) as GoogleGenerateImageResponse;
|
||||
let imageIndex = 0;
|
||||
const images = (payload.candidates ?? [])
|
||||
.flatMap((candidate) => candidate.content?.parts ?? [])
|
||||
.map((part) => {
|
||||
const inline = part.inlineData ?? part.inline_data;
|
||||
const data = inline?.data?.trim();
|
||||
if (!data) {
|
||||
return null;
|
||||
}
|
||||
const mimeType = inline?.mimeType ?? inline?.mime_type ?? DEFAULT_OUTPUT_MIME;
|
||||
const extension = mimeType.includes("jpeg") ? "jpg" : (mimeType.split("/")[1] ?? "png");
|
||||
imageIndex += 1;
|
||||
return {
|
||||
buffer: Buffer.from(data, "base64"),
|
||||
mimeType,
|
||||
fileName: `image-${imageIndex}.${extension}`,
|
||||
};
|
||||
})
|
||||
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
|
||||
|
||||
if (images.length === 0) {
|
||||
throw new Error("Google image generation response missing image data");
|
||||
}
|
||||
|
||||
return {
|
||||
images,
|
||||
model,
|
||||
};
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
204
openclaw/extensions/google/index.test.ts
Normal file
204
openclaw/extensions/google/index.test.ts
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
import type { StreamFn } from "@mariozechner/pi-agent-core";
|
||||
import type { Context, Model } from "@mariozechner/pi-ai";
|
||||
import type {
|
||||
ProviderReplaySessionEntry,
|
||||
ProviderSanitizeReplayHistoryContext,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
registerProviderPlugin,
|
||||
requireRegisteredProvider,
|
||||
} from "../../test/helpers/plugins/provider-registration.js";
|
||||
import { registerGoogleGeminiCliProvider } from "./gemini-cli-provider.js";
|
||||
import { registerGoogleProvider } from "./provider-registration.js";
|
||||
|
||||
const googleProviderPlugin = {
|
||||
register(api: Parameters<typeof registerGoogleProvider>[0]) {
|
||||
registerGoogleProvider(api);
|
||||
registerGoogleGeminiCliProvider(api);
|
||||
},
|
||||
};
|
||||
|
||||
describe("google provider plugin hooks", () => {
|
||||
it("owns replay policy and reasoning mode for the direct Gemini provider", async () => {
|
||||
const { providers } = await registerProviderPlugin({
|
||||
plugin: googleProviderPlugin,
|
||||
id: "google",
|
||||
name: "Google Provider",
|
||||
});
|
||||
const provider = requireRegisteredProvider(providers, "google");
|
||||
const customEntries: ProviderReplaySessionEntry[] = [];
|
||||
|
||||
expect(
|
||||
provider.buildReplayPolicy?.({
|
||||
provider: "google",
|
||||
modelApi: "google-generative-ai",
|
||||
modelId: "gemini-3.1-pro-preview",
|
||||
} as never),
|
||||
).toEqual({
|
||||
sanitizeMode: "full",
|
||||
sanitizeToolCallIds: true,
|
||||
toolCallIdMode: "strict",
|
||||
sanitizeThoughtSignatures: {
|
||||
allowBase64Only: true,
|
||||
includeCamelCase: true,
|
||||
},
|
||||
repairToolUseResultPairing: true,
|
||||
applyAssistantFirstOrderingFix: true,
|
||||
validateGeminiTurns: true,
|
||||
validateAnthropicTurns: false,
|
||||
allowSyntheticToolResults: true,
|
||||
});
|
||||
|
||||
expect(
|
||||
provider.resolveReasoningOutputMode?.({
|
||||
provider: "google",
|
||||
modelApi: "google-generative-ai",
|
||||
modelId: "gemini-3.1-pro-preview",
|
||||
} as never),
|
||||
).toBe("tagged");
|
||||
|
||||
const sanitized = await Promise.resolve(
|
||||
provider.sanitizeReplayHistory?.({
|
||||
provider: "google",
|
||||
modelApi: "google-generative-ai",
|
||||
modelId: "gemini-3.1-pro-preview",
|
||||
sessionId: "session-1",
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "hello" }],
|
||||
},
|
||||
],
|
||||
sessionState: {
|
||||
getCustomEntries: () => customEntries,
|
||||
appendCustomEntry: (customType: string, data: unknown) => {
|
||||
customEntries.push({ customType, data });
|
||||
},
|
||||
},
|
||||
} as ProviderSanitizeReplayHistoryContext),
|
||||
);
|
||||
|
||||
expect(sanitized).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
role: "user",
|
||||
content: "(session bootstrap)",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(customEntries).toHaveLength(1);
|
||||
expect(customEntries[0]?.customType).toBe("google-turn-ordering-bootstrap");
|
||||
});
|
||||
|
||||
it("owns Gemini CLI tool schema normalization", async () => {
|
||||
const { providers } = await registerProviderPlugin({
|
||||
plugin: googleProviderPlugin,
|
||||
id: "google",
|
||||
name: "Google Provider",
|
||||
});
|
||||
const provider = requireRegisteredProvider(providers, "google-gemini-cli");
|
||||
|
||||
const [tool] =
|
||||
provider.normalizeToolSchemas?.({
|
||||
provider: "google-gemini-cli",
|
||||
tools: [
|
||||
{
|
||||
name: "write_file",
|
||||
description: "Write a file",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
path: { type: "string", pattern: "^src/" },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
} as never) ?? [];
|
||||
|
||||
expect(tool).toMatchObject({
|
||||
name: "write_file",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
path: { type: "string" },
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(tool?.parameters).not.toHaveProperty("additionalProperties");
|
||||
expect(
|
||||
(tool?.parameters as { properties?: { path?: Record<string, unknown> } })?.properties?.path,
|
||||
).not.toHaveProperty("pattern");
|
||||
expect(
|
||||
provider.inspectToolSchemas?.({
|
||||
provider: "google-gemini-cli",
|
||||
tools: [tool],
|
||||
} as never),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("wires google-thinking stream hooks for direct and Gemini CLI providers", async () => {
|
||||
const { providers } = await registerProviderPlugin({
|
||||
plugin: googleProviderPlugin,
|
||||
id: "google",
|
||||
name: "Google Provider",
|
||||
});
|
||||
const googleProvider = requireRegisteredProvider(providers, "google");
|
||||
const cliProvider = requireRegisteredProvider(providers, "google-gemini-cli");
|
||||
let capturedPayload: Record<string, unknown> | undefined;
|
||||
|
||||
const baseStreamFn: StreamFn = (model, _context, options) => {
|
||||
const payload = { config: { thinkingConfig: { thinkingBudget: -1 } } } as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
options?.onPayload?.(payload as never, model as never);
|
||||
capturedPayload = payload;
|
||||
return {} as never;
|
||||
};
|
||||
|
||||
const runCase = (provider: typeof googleProvider, providerId: string) => {
|
||||
const wrapped = provider.wrapStreamFn?.({
|
||||
provider: providerId,
|
||||
modelId: "gemini-3.1-pro-preview",
|
||||
thinkingLevel: "high",
|
||||
streamFn: baseStreamFn,
|
||||
} as never);
|
||||
|
||||
void wrapped?.(
|
||||
{
|
||||
api: "google-generative-ai",
|
||||
provider: providerId,
|
||||
id: "gemini-3.1-pro-preview",
|
||||
} as Model<"google-generative-ai">,
|
||||
{ messages: [] } as Context,
|
||||
{},
|
||||
);
|
||||
|
||||
expect(capturedPayload).toMatchObject({
|
||||
config: { thinkingConfig: { thinkingLevel: "HIGH" } },
|
||||
});
|
||||
const thinkingConfig = (
|
||||
(capturedPayload as Record<string, unknown>).config as Record<string, unknown>
|
||||
).thinkingConfig as Record<string, unknown>;
|
||||
expect(thinkingConfig).not.toHaveProperty("thinkingBudget");
|
||||
};
|
||||
|
||||
runCase(googleProvider, "google");
|
||||
runCase(cliProvider, "google-gemini-cli");
|
||||
});
|
||||
|
||||
it("shares Gemini replay and stream hooks across Google provider variants", async () => {
|
||||
const { providers } = await registerProviderPlugin({
|
||||
plugin: googleProviderPlugin,
|
||||
id: "google",
|
||||
name: "Google Provider",
|
||||
});
|
||||
const googleProvider = requireRegisteredProvider(providers, "google");
|
||||
const cliProvider = requireRegisteredProvider(providers, "google-gemini-cli");
|
||||
|
||||
expect(googleProvider.buildReplayPolicy).toBe(cliProvider.buildReplayPolicy);
|
||||
expect(googleProvider.wrapStreamFn).toBe(cliProvider.wrapStreamFn);
|
||||
});
|
||||
});
|
||||
163
openclaw/extensions/google/index.ts
Normal file
163
openclaw/extensions/google/index.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation";
|
||||
import type { MediaUnderstandingProvider } from "openclaw/plugin-sdk/media-understanding";
|
||||
import type { MusicGenerationProvider } from "openclaw/plugin-sdk/music-generation";
|
||||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { VideoGenerationProvider } from "openclaw/plugin-sdk/video-generation";
|
||||
import { buildGoogleGeminiCliBackend } from "./cli-backend.js";
|
||||
import { registerGoogleGeminiCliProvider } from "./gemini-cli-provider.js";
|
||||
import {
|
||||
createGoogleMusicGenerationProviderMetadata,
|
||||
createGoogleVideoGenerationProviderMetadata,
|
||||
} from "./generation-provider-metadata.js";
|
||||
import { geminiMemoryEmbeddingProviderAdapter } from "./memory-embedding-adapter.js";
|
||||
import { registerGoogleProvider } from "./provider-registration.js";
|
||||
import { buildGoogleSpeechProvider } from "./speech-provider.js";
|
||||
import { createGeminiWebSearchProvider } from "./src/gemini-web-search-provider.js";
|
||||
|
||||
let googleImageGenerationProviderPromise: Promise<ImageGenerationProvider> | null = null;
|
||||
let googleMediaUnderstandingProviderPromise: Promise<MediaUnderstandingProvider> | null = null;
|
||||
let googleMusicGenerationProviderPromise: Promise<MusicGenerationProvider> | null = null;
|
||||
let googleVideoGenerationProviderPromise: Promise<VideoGenerationProvider> | null = null;
|
||||
|
||||
type GoogleMediaUnderstandingProvider = Required<
|
||||
Pick<
|
||||
MediaUnderstandingProvider,
|
||||
"describeImage" | "describeImages" | "transcribeAudio" | "describeVideo"
|
||||
>
|
||||
>;
|
||||
|
||||
async function loadGoogleImageGenerationProvider(): Promise<ImageGenerationProvider> {
|
||||
if (!googleImageGenerationProviderPromise) {
|
||||
googleImageGenerationProviderPromise = import("./image-generation-provider.js").then((mod) =>
|
||||
mod.buildGoogleImageGenerationProvider(),
|
||||
);
|
||||
}
|
||||
return await googleImageGenerationProviderPromise;
|
||||
}
|
||||
|
||||
async function loadGoogleMediaUnderstandingProvider(): Promise<MediaUnderstandingProvider> {
|
||||
if (!googleMediaUnderstandingProviderPromise) {
|
||||
googleMediaUnderstandingProviderPromise = import("./media-understanding-provider.js").then(
|
||||
(mod) => mod.googleMediaUnderstandingProvider,
|
||||
);
|
||||
}
|
||||
return await googleMediaUnderstandingProviderPromise;
|
||||
}
|
||||
|
||||
async function loadGoogleMusicGenerationProvider(): Promise<MusicGenerationProvider> {
|
||||
if (!googleMusicGenerationProviderPromise) {
|
||||
googleMusicGenerationProviderPromise = import("./music-generation-provider.js").then((mod) =>
|
||||
mod.buildGoogleMusicGenerationProvider(),
|
||||
);
|
||||
}
|
||||
return await googleMusicGenerationProviderPromise;
|
||||
}
|
||||
|
||||
async function loadGoogleVideoGenerationProvider(): Promise<VideoGenerationProvider> {
|
||||
if (!googleVideoGenerationProviderPromise) {
|
||||
googleVideoGenerationProviderPromise = import("./video-generation-provider.js").then((mod) =>
|
||||
mod.buildGoogleVideoGenerationProvider(),
|
||||
);
|
||||
}
|
||||
return await googleVideoGenerationProviderPromise;
|
||||
}
|
||||
|
||||
async function loadGoogleRequiredMediaUnderstandingProvider(): Promise<GoogleMediaUnderstandingProvider> {
|
||||
const provider = await loadGoogleMediaUnderstandingProvider();
|
||||
if (
|
||||
!provider.describeImage ||
|
||||
!provider.describeImages ||
|
||||
!provider.transcribeAudio ||
|
||||
!provider.describeVideo
|
||||
) {
|
||||
throw new Error("google media understanding provider missing required handlers");
|
||||
}
|
||||
return provider as GoogleMediaUnderstandingProvider;
|
||||
}
|
||||
|
||||
function createLazyGoogleImageGenerationProvider(): ImageGenerationProvider {
|
||||
return {
|
||||
id: "google",
|
||||
label: "Google",
|
||||
defaultModel: "gemini-3.1-flash-image-preview",
|
||||
models: ["gemini-3.1-flash-image-preview", "gemini-3-pro-image-preview"],
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxCount: 4,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxCount: 4,
|
||||
maxInputImages: 5,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
geometry: {
|
||||
sizes: ["1024x1024", "1024x1536", "1536x1024", "1024x1792", "1792x1024"],
|
||||
aspectRatios: ["1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"],
|
||||
resolutions: ["1K", "2K", "4K"],
|
||||
},
|
||||
},
|
||||
generateImage: async (req) => (await loadGoogleImageGenerationProvider()).generateImage(req),
|
||||
};
|
||||
}
|
||||
|
||||
function createLazyGoogleMediaUnderstandingProvider(): MediaUnderstandingProvider {
|
||||
return {
|
||||
id: "google",
|
||||
capabilities: ["image", "audio", "video"],
|
||||
defaultModels: {
|
||||
image: "gemini-3-flash-preview",
|
||||
audio: "gemini-3-flash-preview",
|
||||
video: "gemini-3-flash-preview",
|
||||
},
|
||||
autoPriority: { image: 30, audio: 40, video: 10 },
|
||||
nativeDocumentInputs: ["pdf"],
|
||||
describeImage: async (...args) =>
|
||||
await (await loadGoogleRequiredMediaUnderstandingProvider()).describeImage(...args),
|
||||
describeImages: async (...args) =>
|
||||
await (await loadGoogleRequiredMediaUnderstandingProvider()).describeImages(...args),
|
||||
transcribeAudio: async (...args) =>
|
||||
await (await loadGoogleRequiredMediaUnderstandingProvider()).transcribeAudio(...args),
|
||||
describeVideo: async (...args) =>
|
||||
await (await loadGoogleRequiredMediaUnderstandingProvider()).describeVideo(...args),
|
||||
};
|
||||
}
|
||||
|
||||
function createLazyGoogleMusicGenerationProvider(): MusicGenerationProvider {
|
||||
return {
|
||||
...createGoogleMusicGenerationProviderMetadata(),
|
||||
generateMusic: async (...args) =>
|
||||
await (await loadGoogleMusicGenerationProvider()).generateMusic(...args),
|
||||
};
|
||||
}
|
||||
|
||||
function createLazyGoogleVideoGenerationProvider(): VideoGenerationProvider {
|
||||
return {
|
||||
...createGoogleVideoGenerationProviderMetadata(),
|
||||
generateVideo: async (...args) =>
|
||||
await (await loadGoogleVideoGenerationProvider()).generateVideo(...args),
|
||||
};
|
||||
}
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "google",
|
||||
name: "Google Plugin",
|
||||
description: "Bundled Google plugin",
|
||||
register(api) {
|
||||
api.registerCliBackend(buildGoogleGeminiCliBackend());
|
||||
registerGoogleGeminiCliProvider(api);
|
||||
registerGoogleProvider(api);
|
||||
api.registerMemoryEmbeddingProvider(geminiMemoryEmbeddingProviderAdapter);
|
||||
api.registerImageGenerationProvider(createLazyGoogleImageGenerationProvider());
|
||||
api.registerMediaUnderstandingProvider(createLazyGoogleMediaUnderstandingProvider());
|
||||
api.registerMusicGenerationProvider(createLazyGoogleMusicGenerationProvider());
|
||||
api.registerSpeechProvider(buildGoogleSpeechProvider());
|
||||
api.registerVideoGenerationProvider(createLazyGoogleVideoGenerationProvider());
|
||||
api.registerWebSearchProvider(createGeminiWebSearchProvider());
|
||||
},
|
||||
});
|
||||
164
openclaw/extensions/google/media-understanding-provider.ts
Normal file
164
openclaw/extensions/google/media-understanding-provider.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
import {
|
||||
describeImageWithModel,
|
||||
describeImagesWithModel,
|
||||
type AudioTranscriptionRequest,
|
||||
type AudioTranscriptionResult,
|
||||
type MediaUnderstandingProvider,
|
||||
type VideoDescriptionRequest,
|
||||
type VideoDescriptionResult,
|
||||
} from "openclaw/plugin-sdk/media-understanding";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
postJsonRequest,
|
||||
type ProviderRequestTransportOverrides,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
DEFAULT_GOOGLE_API_BASE_URL,
|
||||
normalizeGoogleModelId,
|
||||
resolveGoogleGenerativeAiHttpRequestConfig,
|
||||
} from "./runtime-api.js";
|
||||
|
||||
export const DEFAULT_GOOGLE_AUDIO_BASE_URL = DEFAULT_GOOGLE_API_BASE_URL;
|
||||
export const DEFAULT_GOOGLE_VIDEO_BASE_URL = DEFAULT_GOOGLE_API_BASE_URL;
|
||||
const DEFAULT_GOOGLE_AUDIO_MODEL = "gemini-3-flash-preview";
|
||||
const DEFAULT_GOOGLE_VIDEO_MODEL = "gemini-3-flash-preview";
|
||||
const DEFAULT_GOOGLE_AUDIO_PROMPT = "Transcribe the audio.";
|
||||
const DEFAULT_GOOGLE_VIDEO_PROMPT = "Describe the video.";
|
||||
|
||||
async function generateGeminiInlineDataText(params: {
|
||||
buffer: Buffer;
|
||||
mime?: string;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
headers?: Record<string, string>;
|
||||
request?: ProviderRequestTransportOverrides;
|
||||
model?: string;
|
||||
prompt?: string;
|
||||
timeoutMs: number;
|
||||
fetchFn?: typeof fetch;
|
||||
defaultBaseUrl: string;
|
||||
defaultModel: string;
|
||||
defaultPrompt: string;
|
||||
defaultMime: string;
|
||||
httpErrorLabel: string;
|
||||
missingTextError: string;
|
||||
}): Promise<{ text: string; model: string }> {
|
||||
const fetchFn = params.fetchFn ?? fetch;
|
||||
const model = (() => {
|
||||
const trimmed = params.model?.trim();
|
||||
if (!trimmed) {
|
||||
return params.defaultModel;
|
||||
}
|
||||
return normalizeGoogleModelId(trimmed);
|
||||
})();
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveGoogleGenerativeAiHttpRequestConfig({
|
||||
apiKey: params.apiKey,
|
||||
baseUrl: params.baseUrl,
|
||||
headers: params.headers,
|
||||
request: params.request,
|
||||
capability: params.defaultMime.startsWith("audio/") ? "audio" : "video",
|
||||
transport: "media-understanding",
|
||||
});
|
||||
const resolvedBaseUrl = baseUrl ?? params.defaultBaseUrl;
|
||||
const url = `${resolvedBaseUrl}/models/${model}:generateContent`;
|
||||
|
||||
const prompt = (() => {
|
||||
const trimmed = params.prompt?.trim();
|
||||
return trimmed || params.defaultPrompt;
|
||||
})();
|
||||
|
||||
const body = {
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{ text: prompt },
|
||||
{
|
||||
inline_data: {
|
||||
mime_type: params.mime ?? params.defaultMime,
|
||||
data: params.buffer.toString("base64"),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { response: res, release } = await postJsonRequest({
|
||||
url,
|
||||
headers,
|
||||
body,
|
||||
timeoutMs: params.timeoutMs,
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
|
||||
try {
|
||||
await assertOkOrThrowHttpError(res, params.httpErrorLabel);
|
||||
|
||||
const payload = (await res.json()) as {
|
||||
candidates?: Array<{
|
||||
content?: { parts?: Array<{ text?: string }> };
|
||||
}>;
|
||||
};
|
||||
const parts = payload.candidates?.[0]?.content?.parts ?? [];
|
||||
const text = parts
|
||||
.map((part) => part?.text?.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
if (!text) {
|
||||
throw new Error(params.missingTextError);
|
||||
}
|
||||
return { text, model };
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function transcribeGeminiAudio(
|
||||
params: AudioTranscriptionRequest,
|
||||
): Promise<AudioTranscriptionResult> {
|
||||
const { text, model } = await generateGeminiInlineDataText({
|
||||
...params,
|
||||
defaultBaseUrl: DEFAULT_GOOGLE_AUDIO_BASE_URL,
|
||||
defaultModel: DEFAULT_GOOGLE_AUDIO_MODEL,
|
||||
defaultPrompt: DEFAULT_GOOGLE_AUDIO_PROMPT,
|
||||
defaultMime: "audio/wav",
|
||||
httpErrorLabel: "Audio transcription failed",
|
||||
missingTextError: "Audio transcription response missing text",
|
||||
});
|
||||
return { text, model };
|
||||
}
|
||||
|
||||
export async function describeGeminiVideo(
|
||||
params: VideoDescriptionRequest,
|
||||
): Promise<VideoDescriptionResult> {
|
||||
const { text, model } = await generateGeminiInlineDataText({
|
||||
...params,
|
||||
defaultBaseUrl: DEFAULT_GOOGLE_VIDEO_BASE_URL,
|
||||
defaultModel: DEFAULT_GOOGLE_VIDEO_MODEL,
|
||||
defaultPrompt: DEFAULT_GOOGLE_VIDEO_PROMPT,
|
||||
defaultMime: "video/mp4",
|
||||
httpErrorLabel: "Video description failed",
|
||||
missingTextError: "Video description response missing text",
|
||||
});
|
||||
return { text, model };
|
||||
}
|
||||
|
||||
export const googleMediaUnderstandingProvider: MediaUnderstandingProvider = {
|
||||
id: "google",
|
||||
capabilities: ["image", "audio", "video"],
|
||||
defaultModels: {
|
||||
image: DEFAULT_GOOGLE_VIDEO_MODEL,
|
||||
audio: DEFAULT_GOOGLE_AUDIO_MODEL,
|
||||
video: DEFAULT_GOOGLE_VIDEO_MODEL,
|
||||
},
|
||||
autoPriority: { image: 30, audio: 40, video: 10 },
|
||||
nativeDocumentInputs: ["pdf"],
|
||||
describeImage: describeImageWithModel,
|
||||
describeImages: describeImagesWithModel,
|
||||
transcribeAudio: transcribeGeminiAudio,
|
||||
describeVideo: describeGeminiVideo,
|
||||
};
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
import { withFetchPreconnect } from "openclaw/plugin-sdk/testing";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
createRequestCaptureJsonFetch,
|
||||
installPinnedHostnameTestHooks,
|
||||
} from "../../src/media-understanding/audio.test-helpers.js";
|
||||
import { describeGeminiVideo } from "./media-understanding-provider.js";
|
||||
import { resolveGoogleGenerativeAiHttpRequestConfig } from "./runtime-api.js";
|
||||
|
||||
installPinnedHostnameTestHooks();
|
||||
|
||||
describe("describeGeminiVideo", () => {
|
||||
it("respects case-insensitive x-goog-api-key overrides", async () => {
|
||||
let seenKey: string | null = null;
|
||||
const fetchFn = withFetchPreconnect(async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers);
|
||||
seenKey = headers.get("x-goog-api-key");
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
candidates: [{ content: { parts: [{ text: "video ok" }] } }],
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
});
|
||||
|
||||
const result = await describeGeminiVideo({
|
||||
buffer: Buffer.from("video"),
|
||||
fileName: "clip.mp4",
|
||||
apiKey: "test-key",
|
||||
timeoutMs: 1000,
|
||||
headers: { "X-Goog-Api-Key": "override" },
|
||||
fetchFn,
|
||||
});
|
||||
|
||||
expect(seenKey).toBe("override");
|
||||
expect(result.text).toBe("video ok");
|
||||
});
|
||||
|
||||
it("keeps private-network disabled for the default Google media endpoint", async () => {
|
||||
expect(
|
||||
resolveGoogleGenerativeAiHttpRequestConfig({
|
||||
apiKey: "test-key",
|
||||
capability: "video",
|
||||
transport: "media-understanding",
|
||||
}).allowPrivateNetwork,
|
||||
).toBe(false);
|
||||
|
||||
const fetchFn = withFetchPreconnect(async () => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
candidates: [{ content: { parts: [{ text: "video ok" }] } }],
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
});
|
||||
|
||||
await describeGeminiVideo({
|
||||
buffer: Buffer.from("video"),
|
||||
fileName: "clip.mp4",
|
||||
apiKey: "test-key",
|
||||
timeoutMs: 1000,
|
||||
fetchFn,
|
||||
});
|
||||
});
|
||||
|
||||
it("builds the expected request payload", async () => {
|
||||
const { fetchFn, getRequest } = createRequestCaptureJsonFetch({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [{ text: "first" }, { text: " second " }, { text: "" }],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const result = await describeGeminiVideo({
|
||||
buffer: Buffer.from("video-bytes"),
|
||||
fileName: "clip.mp4",
|
||||
apiKey: "test-key",
|
||||
timeoutMs: 1500,
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta/",
|
||||
model: "gemini-3-pro",
|
||||
headers: { "X-Other": "1" },
|
||||
fetchFn,
|
||||
});
|
||||
const { url: seenUrl, init: seenInit } = getRequest();
|
||||
|
||||
expect(result.model).toBe("gemini-3-pro-preview");
|
||||
expect(result.text).toBe("first\nsecond");
|
||||
expect(seenUrl).toBe(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3-pro-preview:generateContent",
|
||||
);
|
||||
expect(seenInit?.method).toBe("POST");
|
||||
expect(seenInit?.signal).toBeInstanceOf(AbortSignal);
|
||||
|
||||
const headers = new Headers(seenInit?.headers);
|
||||
expect(headers.get("x-goog-api-key")).toBe("test-key");
|
||||
expect(headers.get("content-type")).toBe("application/json");
|
||||
expect(headers.get("x-other")).toBe("1");
|
||||
|
||||
const bodyText =
|
||||
typeof seenInit?.body === "string"
|
||||
? seenInit.body
|
||||
: Buffer.isBuffer(seenInit?.body)
|
||||
? seenInit.body.toString("utf8")
|
||||
: "";
|
||||
const body = JSON.parse(bodyText);
|
||||
expect(body.contents?.[0]?.parts?.[0]?.text).toBe("Describe the video.");
|
||||
expect(body.contents?.[0]?.parts?.[1]?.inline_data?.mime_type).toBe("video/mp4");
|
||||
expect(body.contents?.[0]?.parts?.[1]?.inline_data?.data).toBe(
|
||||
Buffer.from("video-bytes").toString("base64"),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects non-Google video base URLs before sending authenticated requests", async () => {
|
||||
await expect(
|
||||
describeGeminiVideo({
|
||||
buffer: Buffer.from("video-bytes"),
|
||||
fileName: "clip.mp4",
|
||||
apiKey: "test-key",
|
||||
timeoutMs: 1500,
|
||||
baseUrl: "https://example.com/v1beta/",
|
||||
fetchFn: async () => {
|
||||
throw new Error("fetch should not run");
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"Google Generative AI baseUrl must use https://generativelanguage.googleapis.com",
|
||||
);
|
||||
});
|
||||
});
|
||||
79
openclaw/extensions/google/memory-embedding-adapter.ts
Normal file
79
openclaw/extensions/google/memory-embedding-adapter.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import {
|
||||
hasNonTextEmbeddingParts,
|
||||
isMissingEmbeddingApiKeyError,
|
||||
mapBatchEmbeddingsByIndex,
|
||||
sanitizeEmbeddingCacheHeaders,
|
||||
type MemoryEmbeddingProviderAdapter,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { runGeminiEmbeddingBatches } from "./embedding-batch.js";
|
||||
import {
|
||||
buildGeminiEmbeddingRequest,
|
||||
createGeminiEmbeddingProvider,
|
||||
DEFAULT_GEMINI_EMBEDDING_MODEL,
|
||||
} from "./embedding-provider.js";
|
||||
|
||||
function supportsGeminiMultimodalEmbeddings(model: string): boolean {
|
||||
const normalized = model
|
||||
.trim()
|
||||
.replace(/^models\//, "")
|
||||
.replace(/^(gemini|google)\//, "");
|
||||
return normalized === "gemini-embedding-2-preview";
|
||||
}
|
||||
|
||||
export const geminiMemoryEmbeddingProviderAdapter: MemoryEmbeddingProviderAdapter = {
|
||||
id: "gemini",
|
||||
defaultModel: DEFAULT_GEMINI_EMBEDDING_MODEL,
|
||||
transport: "remote",
|
||||
authProviderId: "google",
|
||||
autoSelectPriority: 30,
|
||||
allowExplicitWhenConfiguredAuto: true,
|
||||
supportsMultimodalEmbeddings: ({ model }) => supportsGeminiMultimodalEmbeddings(model),
|
||||
shouldContinueAutoSelection: isMissingEmbeddingApiKeyError,
|
||||
create: async (options) => {
|
||||
const { provider, client } = await createGeminiEmbeddingProvider({
|
||||
...options,
|
||||
provider: "gemini",
|
||||
fallback: "none",
|
||||
});
|
||||
return {
|
||||
provider,
|
||||
runtime: {
|
||||
id: "gemini",
|
||||
cacheKeyData: {
|
||||
provider: "gemini",
|
||||
baseUrl: client.baseUrl,
|
||||
model: client.model,
|
||||
outputDimensionality: client.outputDimensionality,
|
||||
headers: sanitizeEmbeddingCacheHeaders(client.headers, [
|
||||
"authorization",
|
||||
"x-goog-api-key",
|
||||
]),
|
||||
},
|
||||
batchEmbed: async (batch) => {
|
||||
if (batch.chunks.some((chunk) => hasNonTextEmbeddingParts(chunk.embeddingInput))) {
|
||||
return null;
|
||||
}
|
||||
const byCustomId = await runGeminiEmbeddingBatches({
|
||||
gemini: client,
|
||||
agentId: batch.agentId,
|
||||
requests: batch.chunks.map((chunk, index) => ({
|
||||
custom_id: String(index),
|
||||
request: buildGeminiEmbeddingRequest({
|
||||
input: chunk.embeddingInput ?? { text: chunk.text },
|
||||
taskType: "RETRIEVAL_DOCUMENT",
|
||||
modelPath: client.modelPath,
|
||||
outputDimensionality: client.outputDimensionality,
|
||||
}),
|
||||
})),
|
||||
wait: batch.wait,
|
||||
concurrency: batch.concurrency,
|
||||
pollIntervalMs: batch.pollIntervalMs,
|
||||
timeoutMs: batch.timeoutMs,
|
||||
debug: batch.debug,
|
||||
});
|
||||
return mapBatchEmbeddingsByIndex(byCustomId, batch.chunks.length);
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
29
openclaw/extensions/google/model-id.test.ts
Normal file
29
openclaw/extensions/google/model-id.test.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeAntigravityModelId, normalizeGoogleModelId } from "./api.js";
|
||||
|
||||
describe("google model id helpers", () => {
|
||||
it.each(["gemini-3-pro", "gemini-3.1-pro", "gemini-3-1-pro"])(
|
||||
"adds default -low suffix to bare antigravity pro id: %s",
|
||||
(id) => {
|
||||
expect(normalizeAntigravityModelId(id)).toBe(`${id}-low`);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
"gemini-3-pro-low",
|
||||
"gemini-3-pro-high",
|
||||
"gemini-3.1-flash",
|
||||
"claude-opus-4-6-thinking",
|
||||
])("keeps already-tiered and non-pro ids unchanged: %s", (id) => {
|
||||
expect(normalizeAntigravityModelId(id)).toBe(id);
|
||||
});
|
||||
|
||||
it("maps the deprecated 3.1 flash alias to the real preview model", () => {
|
||||
expect(normalizeGoogleModelId("gemini-3.1-flash")).toBe("gemini-3-flash-preview");
|
||||
expect(normalizeGoogleModelId("gemini-3.1-flash-preview")).toBe("gemini-3-flash-preview");
|
||||
});
|
||||
|
||||
it("adds the preview suffix for gemini 3.1 flash-lite", () => {
|
||||
expect(normalizeGoogleModelId("gemini-3.1-flash-lite")).toBe("gemini-3.1-flash-lite-preview");
|
||||
});
|
||||
});
|
||||
27
openclaw/extensions/google/model-id.ts
Normal file
27
openclaw/extensions/google/model-id.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
const ANTIGRAVITY_BARE_PRO_IDS = new Set(["gemini-3-pro", "gemini-3.1-pro", "gemini-3-1-pro"]);
|
||||
|
||||
export function normalizeGoogleModelId(id: string): string {
|
||||
if (id === "gemini-3-pro") {
|
||||
return "gemini-3-pro-preview";
|
||||
}
|
||||
if (id === "gemini-3-flash") {
|
||||
return "gemini-3-flash-preview";
|
||||
}
|
||||
if (id === "gemini-3.1-pro") {
|
||||
return "gemini-3.1-pro-preview";
|
||||
}
|
||||
if (id === "gemini-3.1-flash-lite") {
|
||||
return "gemini-3.1-flash-lite-preview";
|
||||
}
|
||||
if (id === "gemini-3.1-flash" || id === "gemini-3.1-flash-preview") {
|
||||
return "gemini-3-flash-preview";
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
export function normalizeAntigravityModelId(id: string): string {
|
||||
if (ANTIGRAVITY_BARE_PRO_IDS.has(id)) {
|
||||
return `${id}-low`;
|
||||
}
|
||||
return id;
|
||||
}
|
||||
103
openclaw/extensions/google/music-generation-provider.test.ts
Normal file
103
openclaw/extensions/google/music-generation-provider.test.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { GoogleGenAIMock, generateContentMock } = vi.hoisted(() => {
|
||||
const generateContentMock = vi.fn();
|
||||
const GoogleGenAIMock = vi.fn(function GoogleGenAI() {
|
||||
return {
|
||||
models: {
|
||||
generateContent: generateContentMock,
|
||||
},
|
||||
};
|
||||
});
|
||||
return { GoogleGenAIMock, generateContentMock };
|
||||
});
|
||||
|
||||
vi.mock("@google/genai", () => ({
|
||||
GoogleGenAI: GoogleGenAIMock,
|
||||
}));
|
||||
|
||||
import * as providerAuthRuntime from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import { expectExplicitMusicGenerationCapabilities } from "../../test/helpers/media-generation/provider-capability-assertions.js";
|
||||
import { buildGoogleMusicGenerationProvider } from "./music-generation-provider.js";
|
||||
|
||||
describe("google music generation provider", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
generateContentMock.mockReset();
|
||||
GoogleGenAIMock.mockClear();
|
||||
});
|
||||
|
||||
it("declares explicit mode capabilities", () => {
|
||||
expectExplicitMusicGenerationCapabilities(buildGoogleMusicGenerationProvider());
|
||||
});
|
||||
|
||||
it("submits generation and returns inline audio bytes plus lyrics", async () => {
|
||||
vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "google-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
generateContentMock.mockResolvedValue({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{ text: "wake the city up" },
|
||||
{
|
||||
inlineData: {
|
||||
data: Buffer.from("mp3-bytes").toString("base64"),
|
||||
mimeType: "audio/mpeg",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const provider = buildGoogleMusicGenerationProvider();
|
||||
const result = await provider.generateMusic({
|
||||
provider: "google",
|
||||
model: "lyria-3-clip-preview",
|
||||
prompt: "upbeat synthpop anthem",
|
||||
cfg: {},
|
||||
instrumental: true,
|
||||
});
|
||||
|
||||
expect(generateContentMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
model: "lyria-3-clip-preview",
|
||||
config: {
|
||||
responseModalities: ["AUDIO", "TEXT"],
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(result.tracks).toHaveLength(1);
|
||||
expect(result.tracks[0]?.mimeType).toBe("audio/mpeg");
|
||||
expect(result.lyrics).toEqual(["wake the city up"]);
|
||||
expect(GoogleGenAIMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
apiKey: "google-key",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects unsupported wav output on clip model", async () => {
|
||||
vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "google-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
const provider = buildGoogleMusicGenerationProvider();
|
||||
|
||||
await expect(
|
||||
provider.generateMusic({
|
||||
provider: "google",
|
||||
model: "lyria-3-clip-preview",
|
||||
prompt: "ambient ocean",
|
||||
cfg: {},
|
||||
format: "wav",
|
||||
}),
|
||||
).rejects.toThrow("supports mp3 output");
|
||||
});
|
||||
});
|
||||
176
openclaw/extensions/google/music-generation-provider.ts
Normal file
176
openclaw/extensions/google/music-generation-provider.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
import { GoogleGenAI } from "@google/genai";
|
||||
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
|
||||
import type {
|
||||
GeneratedMusicAsset,
|
||||
MusicGenerationProvider,
|
||||
MusicGenerationRequest,
|
||||
} from "openclaw/plugin-sdk/music-generation";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { normalizeGoogleApiBaseUrl } from "./api.js";
|
||||
import {
|
||||
createGoogleMusicGenerationProviderMetadata,
|
||||
DEFAULT_GOOGLE_MUSIC_MODEL,
|
||||
GOOGLE_MAX_INPUT_IMAGES,
|
||||
GOOGLE_PRO_MUSIC_MODEL,
|
||||
} from "./generation-provider-metadata.js";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 180_000;
|
||||
|
||||
type GoogleInlineDataPart = {
|
||||
mimeType?: string;
|
||||
mime_type?: string;
|
||||
data?: string;
|
||||
};
|
||||
|
||||
type GoogleGenerateMusicResponse = {
|
||||
candidates?: Array<{
|
||||
content?: {
|
||||
parts?: Array<{
|
||||
text?: string;
|
||||
inlineData?: GoogleInlineDataPart;
|
||||
inline_data?: GoogleInlineDataPart;
|
||||
}>;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
|
||||
function resolveConfiguredGoogleMusicBaseUrl(req: MusicGenerationRequest): string | undefined {
|
||||
const configured = normalizeOptionalString(req.cfg?.models?.providers?.google?.baseUrl);
|
||||
return configured ? normalizeGoogleApiBaseUrl(configured) : undefined;
|
||||
}
|
||||
|
||||
function buildMusicPrompt(req: MusicGenerationRequest): string {
|
||||
const parts = [req.prompt.trim()];
|
||||
const lyrics = normalizeOptionalString(req.lyrics);
|
||||
if (req.instrumental === true) {
|
||||
parts.push("Instrumental only. No vocals, no sung lyrics, no spoken word.");
|
||||
}
|
||||
if (lyrics) {
|
||||
parts.push(`Lyrics:\n${lyrics}`);
|
||||
}
|
||||
return parts.join("\n\n");
|
||||
}
|
||||
|
||||
function resolveSupportedFormats(model: string): readonly string[] {
|
||||
return model === GOOGLE_PRO_MUSIC_MODEL ? ["mp3", "wav"] : ["mp3"];
|
||||
}
|
||||
|
||||
function resolveTrackFileName(params: { index: number; mimeType: string; model: string }): string {
|
||||
const ext =
|
||||
extensionForMime(params.mimeType)?.replace(/^\./u, "") ||
|
||||
(params.model === GOOGLE_PRO_MUSIC_MODEL ? "wav" : "mp3");
|
||||
return `track-${params.index + 1}.${ext}`;
|
||||
}
|
||||
|
||||
function extractTracks(params: { payload: GoogleGenerateMusicResponse; model: string }): {
|
||||
tracks: GeneratedMusicAsset[];
|
||||
lyrics: string[];
|
||||
} {
|
||||
const lyrics: string[] = [];
|
||||
const tracks: GeneratedMusicAsset[] = [];
|
||||
for (const candidate of params.payload.candidates ?? []) {
|
||||
for (const part of candidate.content?.parts ?? []) {
|
||||
const text = normalizeOptionalString(part.text);
|
||||
if (text) {
|
||||
lyrics.push(text);
|
||||
continue;
|
||||
}
|
||||
const inline = part.inlineData ?? part.inline_data;
|
||||
const data = normalizeOptionalString(inline?.data);
|
||||
if (!data) {
|
||||
continue;
|
||||
}
|
||||
const mimeType =
|
||||
normalizeOptionalString(inline?.mimeType) ||
|
||||
normalizeOptionalString(inline?.mime_type) ||
|
||||
"audio/mpeg";
|
||||
tracks.push({
|
||||
buffer: Buffer.from(data, "base64"),
|
||||
mimeType,
|
||||
fileName: resolveTrackFileName({
|
||||
index: tracks.length,
|
||||
mimeType,
|
||||
model: params.model,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
return { tracks, lyrics };
|
||||
}
|
||||
|
||||
export function buildGoogleMusicGenerationProvider(): MusicGenerationProvider {
|
||||
return {
|
||||
...createGoogleMusicGenerationProviderMetadata(),
|
||||
async generateMusic(req) {
|
||||
if ((req.inputImages?.length ?? 0) > GOOGLE_MAX_INPUT_IMAGES) {
|
||||
throw new Error(
|
||||
`Google music generation supports at most ${GOOGLE_MAX_INPUT_IMAGES} reference images.`,
|
||||
);
|
||||
}
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "google",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
});
|
||||
if (!auth.apiKey) {
|
||||
throw new Error("Google API key missing");
|
||||
}
|
||||
|
||||
const model = normalizeOptionalString(req.model) || DEFAULT_GOOGLE_MUSIC_MODEL;
|
||||
if (req.format) {
|
||||
const supportedFormats = resolveSupportedFormats(model);
|
||||
if (!supportedFormats.includes(req.format)) {
|
||||
throw new Error(
|
||||
`Google music generation model ${model} supports ${supportedFormats.join(", ")} output.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const client = new GoogleGenAI({
|
||||
apiKey: auth.apiKey,
|
||||
httpOptions: {
|
||||
...(resolveConfiguredGoogleMusicBaseUrl(req)
|
||||
? { baseUrl: resolveConfiguredGoogleMusicBaseUrl(req) }
|
||||
: {}),
|
||||
timeout: req.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||
},
|
||||
});
|
||||
const response = (await client.models.generateContent({
|
||||
model,
|
||||
contents: [
|
||||
{ text: buildMusicPrompt(req) },
|
||||
...(req.inputImages ?? []).map((image) => ({
|
||||
inlineData: {
|
||||
mimeType: normalizeOptionalString(image.mimeType) || "image/png",
|
||||
data: image.buffer?.toString("base64") ?? "",
|
||||
},
|
||||
})),
|
||||
],
|
||||
config: {
|
||||
responseModalities: ["AUDIO", "TEXT"],
|
||||
},
|
||||
})) as GoogleGenerateMusicResponse;
|
||||
|
||||
const { tracks, lyrics } = extractTracks({
|
||||
payload: response,
|
||||
model,
|
||||
});
|
||||
if (tracks.length === 0) {
|
||||
throw new Error("Google music generation response missing audio data");
|
||||
}
|
||||
return {
|
||||
tracks,
|
||||
...(lyrics.length > 0 ? { lyrics } : {}),
|
||||
model,
|
||||
metadata: {
|
||||
inputImageCount: req.inputImages?.length ?? 0,
|
||||
instrumental: req.instrumental === true,
|
||||
...(normalizeOptionalString(req.lyrics) ? { requestedLyrics: true } : {}),
|
||||
...(req.format ? { requestedFormat: req.format } : {}),
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
39
openclaw/extensions/google/oauth-token-shared.test.ts
Normal file
39
openclaw/extensions/google/oauth-token-shared.test.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formatGoogleOauthApiKey,
|
||||
parseGoogleOauthApiKey,
|
||||
parseGoogleUsageToken,
|
||||
} from "./oauth-token-shared.js";
|
||||
|
||||
describe("google oauth token helpers", () => {
|
||||
it("formats oauth credentials with project-aware payloads", () => {
|
||||
expect(
|
||||
formatGoogleOauthApiKey({
|
||||
type: "oauth",
|
||||
access: "token-123",
|
||||
projectId: "project-abc",
|
||||
}),
|
||||
).toBe(JSON.stringify({ token: "token-123", projectId: "project-abc" }));
|
||||
});
|
||||
|
||||
it("returns an empty string for non-oauth credentials", () => {
|
||||
expect(formatGoogleOauthApiKey({ type: "token", access: "token-123" })).toBe("");
|
||||
});
|
||||
|
||||
it("parses project-aware oauth payloads for usage auth", () => {
|
||||
expect(parseGoogleUsageToken(JSON.stringify({ token: "usage-token" }))).toBe("usage-token");
|
||||
});
|
||||
|
||||
it("parses structured oauth payload fields", () => {
|
||||
expect(
|
||||
parseGoogleOauthApiKey(JSON.stringify({ token: "usage-token", projectId: "proj-1" })),
|
||||
).toEqual({
|
||||
token: "usage-token",
|
||||
projectId: "proj-1",
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to the raw token when the payload is not JSON", () => {
|
||||
expect(parseGoogleUsageToken("raw-token")).toBe("raw-token");
|
||||
});
|
||||
});
|
||||
42
openclaw/extensions/google/oauth-token-shared.ts
Normal file
42
openclaw/extensions/google/oauth-token-shared.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import { readStringValue } from "openclaw/plugin-sdk/text-runtime";
|
||||
|
||||
type GoogleOauthApiKeyCredential = {
|
||||
type?: string;
|
||||
access?: string;
|
||||
projectId?: string;
|
||||
};
|
||||
|
||||
export function parseGoogleOauthApiKey(apiKey: string): {
|
||||
token?: string;
|
||||
projectId?: string;
|
||||
} | null {
|
||||
try {
|
||||
const parsed = JSON.parse(apiKey) as { token?: unknown; projectId?: unknown };
|
||||
return {
|
||||
token: readStringValue(parsed.token),
|
||||
projectId: readStringValue(parsed.projectId),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function formatGoogleOauthApiKey(cred: GoogleOauthApiKeyCredential): string {
|
||||
if (cred.type !== "oauth" || typeof cred.access !== "string" || !cred.access.trim()) {
|
||||
return "";
|
||||
}
|
||||
return JSON.stringify({
|
||||
token: cred.access,
|
||||
projectId: cred.projectId,
|
||||
});
|
||||
}
|
||||
|
||||
export function parseGoogleUsageToken(apiKey: string): string {
|
||||
const parsed = parseGoogleOauthApiKey(apiKey);
|
||||
if (parsed?.token) {
|
||||
return parsed.token;
|
||||
}
|
||||
|
||||
// Keep the raw token when the stored credential is not a project-aware JSON payload.
|
||||
return apiKey;
|
||||
}
|
||||
273
openclaw/extensions/google/oauth.credentials.ts
Normal file
273
openclaw/extensions/google/oauth.credentials.ts
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
|
||||
import type { Dirent } from "node:fs";
|
||||
import { delimiter, dirname, join } from "node:path";
|
||||
import { lowercasePreservingWhitespace } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { CLIENT_ID_KEYS, CLIENT_SECRET_KEYS } from "./oauth.shared.js";
|
||||
|
||||
type CredentialFs = {
|
||||
existsSync: (path: Parameters<typeof existsSync>[0]) => ReturnType<typeof existsSync>;
|
||||
readFileSync: (path: Parameters<typeof readFileSync>[0], encoding: "utf8") => string;
|
||||
realpathSync: (path: Parameters<typeof realpathSync>[0]) => string;
|
||||
readdirSync: (
|
||||
path: Parameters<typeof readdirSync>[0],
|
||||
options: { withFileTypes: true },
|
||||
) => Dirent[];
|
||||
};
|
||||
|
||||
const defaultFs: CredentialFs = {
|
||||
existsSync,
|
||||
readFileSync,
|
||||
realpathSync,
|
||||
readdirSync,
|
||||
};
|
||||
|
||||
let credentialFs: CredentialFs = defaultFs;
|
||||
|
||||
function resolveEnv(keys: string[]): string | undefined {
|
||||
for (const key of keys) {
|
||||
const value = process.env[key]?.trim();
|
||||
if (value) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
let cachedGeminiCliCredentials: { clientId: string; clientSecret: string } | null = null;
|
||||
|
||||
export function clearCredentialsCache(): void {
|
||||
cachedGeminiCliCredentials = null;
|
||||
}
|
||||
|
||||
export function setOAuthCredentialsFsForTest(overrides?: Partial<CredentialFs>): void {
|
||||
credentialFs = overrides ? { ...defaultFs, ...overrides } : defaultFs;
|
||||
}
|
||||
|
||||
export function extractGeminiCliCredentials(): { clientId: string; clientSecret: string } | null {
|
||||
if (cachedGeminiCliCredentials) {
|
||||
return cachedGeminiCliCredentials;
|
||||
}
|
||||
|
||||
try {
|
||||
const geminiPath = findInPath("gemini");
|
||||
if (!geminiPath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const resolvedPath = credentialFs.realpathSync(geminiPath);
|
||||
const geminiCliDirs = resolveGeminiCliDirs(geminiPath, resolvedPath);
|
||||
|
||||
for (const geminiCliDir of geminiCliDirs) {
|
||||
const directCredentials = readGeminiCliCredentialsFromKnownPaths(geminiCliDir);
|
||||
if (directCredentials) {
|
||||
cachedGeminiCliCredentials = directCredentials;
|
||||
return directCredentials;
|
||||
}
|
||||
|
||||
const bundledCredentials = readGeminiCliCredentialsFromBundle(geminiCliDir);
|
||||
if (bundledCredentials) {
|
||||
cachedGeminiCliCredentials = bundledCredentials;
|
||||
return bundledCredentials;
|
||||
}
|
||||
|
||||
const discoveredCredentials = findGeminiCliCredentialsInTree(geminiCliDir, 10);
|
||||
if (discoveredCredentials) {
|
||||
cachedGeminiCliCredentials = discoveredCredentials;
|
||||
return discoveredCredentials;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Gemini CLI not installed or extraction failed
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveGeminiCliDirs(geminiPath: string, resolvedPath: string): string[] {
|
||||
const binDir = dirname(geminiPath);
|
||||
const candidates = [
|
||||
dirname(dirname(resolvedPath)),
|
||||
join(dirname(resolvedPath), "node_modules", "@google", "gemini-cli"),
|
||||
join(binDir, "node_modules", "@google", "gemini-cli"),
|
||||
join(dirname(binDir), "node_modules", "@google", "gemini-cli"),
|
||||
join(dirname(binDir), "lib", "node_modules", "@google", "gemini-cli"),
|
||||
];
|
||||
|
||||
const deduped: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const candidate of candidates) {
|
||||
for (const searchDir of resolveGeminiCliSearchDirs(candidate)) {
|
||||
const key =
|
||||
process.platform === "win32"
|
||||
? lowercasePreservingWhitespace(searchDir.replace(/\\/g, "/"))
|
||||
: searchDir;
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
deduped.push(searchDir);
|
||||
}
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
function resolveGeminiCliSearchDirs(candidate: string): string[] {
|
||||
const searchDirs = [
|
||||
candidate,
|
||||
join(candidate, "node_modules", "@google", "gemini-cli"),
|
||||
join(candidate, "lib", "node_modules", "@google", "gemini-cli"),
|
||||
];
|
||||
return searchDirs.filter(looksLikeGeminiCliDir);
|
||||
}
|
||||
|
||||
function looksLikeGeminiCliDir(candidate: string): boolean {
|
||||
return (
|
||||
credentialFs.existsSync(join(candidate, "package.json")) ||
|
||||
credentialFs.existsSync(join(candidate, "node_modules", "@google", "gemini-cli-core"))
|
||||
);
|
||||
}
|
||||
|
||||
function findInPath(name: string): string | null {
|
||||
const exts = process.platform === "win32" ? [".cmd", ".bat", ".exe", ""] : [""];
|
||||
for (const dir of (process.env.PATH ?? "").split(delimiter)) {
|
||||
for (const ext of exts) {
|
||||
const path = join(dir, name + ext);
|
||||
if (credentialFs.existsSync(path)) {
|
||||
return path;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function readGeminiCliCredentialsFile(
|
||||
path: string,
|
||||
): { clientId: string; clientSecret: string } | null {
|
||||
try {
|
||||
return parseGeminiCliCredentials(credentialFs.readFileSync(path, "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function parseGeminiCliCredentials(
|
||||
content: string,
|
||||
): { clientId: string; clientSecret: string } | null {
|
||||
const clientId =
|
||||
content.match(/OAUTH_CLIENT_ID\s*=\s*["']([^"']+)["']/)?.[1] ??
|
||||
content.match(/(\d+-[a-z0-9]+\.apps\.googleusercontent\.com)/)?.[1];
|
||||
const clientSecret =
|
||||
content.match(/OAUTH_CLIENT_SECRET\s*=\s*["']([^"']+)["']/)?.[1] ??
|
||||
content.match(/(GOCSPX-[A-Za-z0-9_-]+)/)?.[1];
|
||||
if (!clientId || !clientSecret) {
|
||||
return null;
|
||||
}
|
||||
return { clientId, clientSecret };
|
||||
}
|
||||
|
||||
function readGeminiCliCredentialsFromKnownPaths(
|
||||
geminiCliDir: string,
|
||||
): { clientId: string; clientSecret: string } | null {
|
||||
const searchPaths = [
|
||||
join(
|
||||
geminiCliDir,
|
||||
"node_modules",
|
||||
"@google",
|
||||
"gemini-cli-core",
|
||||
"dist",
|
||||
"src",
|
||||
"code_assist",
|
||||
"oauth2.js",
|
||||
),
|
||||
join(
|
||||
geminiCliDir,
|
||||
"node_modules",
|
||||
"@google",
|
||||
"gemini-cli-core",
|
||||
"dist",
|
||||
"code_assist",
|
||||
"oauth2.js",
|
||||
),
|
||||
];
|
||||
|
||||
for (const path of searchPaths) {
|
||||
if (!credentialFs.existsSync(path)) {
|
||||
continue;
|
||||
}
|
||||
const credentials = readGeminiCliCredentialsFile(path);
|
||||
if (credentials) {
|
||||
return credentials;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function readGeminiCliCredentialsFromBundle(
|
||||
geminiCliDir: string,
|
||||
): { clientId: string; clientSecret: string } | null {
|
||||
const bundleDir = join(geminiCliDir, "bundle");
|
||||
if (!credentialFs.existsSync(bundleDir)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
for (const entry of credentialFs.readdirSync(bundleDir, { withFileTypes: true })) {
|
||||
if (!entry.isFile() || !entry.name.endsWith(".js")) {
|
||||
continue;
|
||||
}
|
||||
const credentials = readGeminiCliCredentialsFile(join(bundleDir, entry.name));
|
||||
if (credentials) {
|
||||
return credentials;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore bundle traversal failures and fall back to the recursive search.
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function findGeminiCliCredentialsInTree(
|
||||
dir: string,
|
||||
depth: number,
|
||||
): { clientId: string; clientSecret: string } | null {
|
||||
if (depth <= 0) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
for (const entry of credentialFs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isFile() && entry.name === "oauth2.js") {
|
||||
const credentials = readGeminiCliCredentialsFile(path);
|
||||
if (credentials) {
|
||||
return credentials;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (entry.isDirectory() && !entry.name.startsWith(".")) {
|
||||
const found = findGeminiCliCredentialsInTree(path, depth - 1);
|
||||
if (found) {
|
||||
return found;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function resolveOAuthClientConfig(): { clientId: string; clientSecret?: string } {
|
||||
const envClientId = resolveEnv(CLIENT_ID_KEYS);
|
||||
const envClientSecret = resolveEnv(CLIENT_SECRET_KEYS);
|
||||
if (envClientId) {
|
||||
return { clientId: envClientId, clientSecret: envClientSecret };
|
||||
}
|
||||
|
||||
const extracted = extractGeminiCliCredentials();
|
||||
if (extracted) {
|
||||
return extracted;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"Gemini CLI not found. Install it first: brew install gemini-cli (or npm install -g @google/gemini-cli), or set GEMINI_CLI_OAUTH_CLIENT_ID.",
|
||||
);
|
||||
}
|
||||
152
openclaw/extensions/google/oauth.flow.ts
Normal file
152
openclaw/extensions/google/oauth.flow.ts
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { createServer } from "node:http";
|
||||
import { isWSL2Sync } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { resolveOAuthClientConfig } from "./oauth.credentials.js";
|
||||
import { AUTH_URL, REDIRECT_URI, SCOPES } from "./oauth.shared.js";
|
||||
|
||||
export function shouldUseManualOAuthFlow(isRemote: boolean): boolean {
|
||||
return isRemote || isWSL2Sync();
|
||||
}
|
||||
|
||||
export function generatePkce(): { verifier: string; challenge: string } {
|
||||
const verifier = randomBytes(32).toString("hex");
|
||||
const challenge = createHash("sha256").update(verifier).digest("base64url");
|
||||
return { verifier, challenge };
|
||||
}
|
||||
|
||||
export function generateOAuthState(): string {
|
||||
return randomBytes(32).toString("hex");
|
||||
}
|
||||
|
||||
export function buildAuthUrl(challenge: string, state: string): string {
|
||||
const { clientId } = resolveOAuthClientConfig();
|
||||
const params = new URLSearchParams({
|
||||
client_id: clientId,
|
||||
response_type: "code",
|
||||
redirect_uri: REDIRECT_URI,
|
||||
scope: SCOPES.join(" "),
|
||||
code_challenge: challenge,
|
||||
code_challenge_method: "S256",
|
||||
state,
|
||||
access_type: "offline",
|
||||
prompt: "consent",
|
||||
});
|
||||
return `${AUTH_URL}?${params.toString()}`;
|
||||
}
|
||||
|
||||
export function parseCallbackInput(
|
||||
input: string,
|
||||
): { code: string; state: string } | { error: string } {
|
||||
const trimmed = input.trim();
|
||||
if (!trimmed) {
|
||||
return { error: "No input provided" };
|
||||
}
|
||||
|
||||
try {
|
||||
const url = new URL(trimmed);
|
||||
const code = url.searchParams.get("code");
|
||||
const state = url.searchParams.get("state");
|
||||
if (!code) {
|
||||
return { error: "Missing 'code' parameter in URL" };
|
||||
}
|
||||
if (!state) {
|
||||
return { error: "Missing 'state' parameter. Paste the full URL." };
|
||||
}
|
||||
return { code, state };
|
||||
} catch {
|
||||
return { error: "Paste the full redirect URL, not just the code." };
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForLocalCallback(params: {
|
||||
expectedState: string;
|
||||
timeoutMs: number;
|
||||
onProgress?: (message: string) => void;
|
||||
}): Promise<{ code: string; state: string }> {
|
||||
const port = 8085;
|
||||
const hostname = "localhost";
|
||||
const expectedPath = "/oauth2callback";
|
||||
|
||||
return new Promise<{ code: string; state: string }>((resolve, reject) => {
|
||||
let timeout: NodeJS.Timeout | null = null;
|
||||
const server = createServer((req, res) => {
|
||||
try {
|
||||
const requestUrl = new URL(req.url ?? "/", `http://${hostname}:${port}`);
|
||||
if (requestUrl.pathname !== expectedPath) {
|
||||
res.statusCode = 404;
|
||||
res.setHeader("Content-Type", "text/plain");
|
||||
res.end("Not found");
|
||||
return;
|
||||
}
|
||||
|
||||
const error = requestUrl.searchParams.get("error");
|
||||
const code = requestUrl.searchParams.get("code")?.trim();
|
||||
const state = requestUrl.searchParams.get("state")?.trim();
|
||||
|
||||
if (error) {
|
||||
res.statusCode = 400;
|
||||
res.setHeader("Content-Type", "text/plain");
|
||||
res.end(`Authentication failed: ${error}`);
|
||||
finish(new Error(`OAuth error: ${error}`));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!code || !state) {
|
||||
res.statusCode = 400;
|
||||
res.setHeader("Content-Type", "text/plain");
|
||||
res.end("Missing code or state");
|
||||
finish(new Error("Missing OAuth code or state"));
|
||||
return;
|
||||
}
|
||||
|
||||
if (state !== params.expectedState) {
|
||||
res.statusCode = 400;
|
||||
res.setHeader("Content-Type", "text/plain");
|
||||
res.end("Invalid state");
|
||||
finish(new Error("OAuth state mismatch"));
|
||||
return;
|
||||
}
|
||||
|
||||
res.statusCode = 200;
|
||||
res.setHeader("Content-Type", "text/html; charset=utf-8");
|
||||
res.end(
|
||||
"<!doctype html><html><head><meta charset='utf-8'/></head>" +
|
||||
"<body><h2>Gemini CLI OAuth complete</h2>" +
|
||||
"<p>You can close this window and return to OpenClaw.</p></body></html>",
|
||||
);
|
||||
|
||||
finish(undefined, { code, state });
|
||||
} catch (err) {
|
||||
finish(err instanceof Error ? err : new Error("OAuth callback failed"));
|
||||
}
|
||||
});
|
||||
|
||||
const finish = (err?: Error, result?: { code: string; state: string }) => {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
try {
|
||||
server.close();
|
||||
} catch {
|
||||
// ignore close errors
|
||||
}
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else if (result) {
|
||||
resolve(result);
|
||||
}
|
||||
};
|
||||
|
||||
server.once("error", (err) => {
|
||||
finish(err instanceof Error ? err : new Error("OAuth callback server error"));
|
||||
});
|
||||
|
||||
server.listen(port, hostname, () => {
|
||||
params.onProgress?.(`Waiting for OAuth callback on ${REDIRECT_URI}…`);
|
||||
});
|
||||
|
||||
timeout = setTimeout(() => {
|
||||
finish(new Error("OAuth callback timeout"));
|
||||
}, params.timeoutMs);
|
||||
});
|
||||
}
|
||||
24
openclaw/extensions/google/oauth.http.ts
Normal file
24
openclaw/extensions/google/oauth.http.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { DEFAULT_FETCH_TIMEOUT_MS } from "./oauth.shared.js";
|
||||
|
||||
export async function fetchWithTimeout(
|
||||
url: string,
|
||||
init: RequestInit,
|
||||
timeoutMs = DEFAULT_FETCH_TIMEOUT_MS,
|
||||
): Promise<Response> {
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
url,
|
||||
init,
|
||||
timeoutMs,
|
||||
});
|
||||
try {
|
||||
const body = await response.arrayBuffer();
|
||||
return new Response(body, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
232
openclaw/extensions/google/oauth.project.ts
Normal file
232
openclaw/extensions/google/oauth.project.ts
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
import { fetchWithTimeout } from "./oauth.http.js";
|
||||
import {
|
||||
CODE_ASSIST_ENDPOINT_PROD,
|
||||
LOAD_CODE_ASSIST_ENDPOINTS,
|
||||
TIER_FREE,
|
||||
TIER_LEGACY,
|
||||
TIER_STANDARD,
|
||||
USERINFO_URL,
|
||||
} from "./oauth.shared.js";
|
||||
|
||||
const LOAD_CODE_ASSIST_METADATA = {
|
||||
ideType: "IDE_UNSPECIFIED",
|
||||
platform: "PLATFORM_UNSPECIFIED",
|
||||
pluginType: "GEMINI",
|
||||
} as const;
|
||||
|
||||
async function getUserEmail(accessToken: string): Promise<string | undefined> {
|
||||
try {
|
||||
const response = await fetchWithTimeout(USERINFO_URL, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
});
|
||||
if (response.ok) {
|
||||
const data = (await response.json()) as { email?: string };
|
||||
return data.email;
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isVpcScAffected(payload: unknown): boolean {
|
||||
if (!payload || typeof payload !== "object") {
|
||||
return false;
|
||||
}
|
||||
const error = (payload as { error?: unknown }).error;
|
||||
if (!error || typeof error !== "object") {
|
||||
return false;
|
||||
}
|
||||
const details = (error as { details?: unknown[] }).details;
|
||||
if (!Array.isArray(details)) {
|
||||
return false;
|
||||
}
|
||||
return details.some(
|
||||
(item) =>
|
||||
typeof item === "object" &&
|
||||
item &&
|
||||
(item as { reason?: string }).reason === "SECURITY_POLICY_VIOLATED",
|
||||
);
|
||||
}
|
||||
|
||||
function getDefaultTier(
|
||||
allowedTiers?: Array<{ id?: string; isDefault?: boolean }>,
|
||||
): { id?: string } | undefined {
|
||||
if (!allowedTiers?.length) {
|
||||
return { id: TIER_LEGACY };
|
||||
}
|
||||
return allowedTiers.find((tier) => tier.isDefault) ?? { id: TIER_LEGACY };
|
||||
}
|
||||
|
||||
async function pollOperation(
|
||||
endpoint: string,
|
||||
operationName: string,
|
||||
headers: Record<string, string>,
|
||||
): Promise<{ done?: boolean; response?: { cloudaicompanionProject?: { id?: string } } }> {
|
||||
for (let attempt = 0; attempt < 24; attempt += 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
const response = await fetchWithTimeout(`${endpoint}/v1internal/${operationName}`, {
|
||||
headers,
|
||||
});
|
||||
if (!response.ok) {
|
||||
continue;
|
||||
}
|
||||
const data = (await response.json()) as {
|
||||
done?: boolean;
|
||||
response?: { cloudaicompanionProject?: { id?: string } };
|
||||
};
|
||||
if (data.done) {
|
||||
return data;
|
||||
}
|
||||
}
|
||||
throw new Error("Operation polling timeout");
|
||||
}
|
||||
|
||||
export async function resolveGoogleOAuthIdentity(accessToken: string): Promise<{
|
||||
email?: string;
|
||||
projectId?: string;
|
||||
}> {
|
||||
const email = await getUserEmail(accessToken);
|
||||
const projectId = await discoverProject(accessToken);
|
||||
return { email, projectId };
|
||||
}
|
||||
|
||||
export async function resolveGooglePersonalOAuthIdentity(accessToken: string): Promise<{
|
||||
email?: string;
|
||||
projectId?: string;
|
||||
}> {
|
||||
return { email: await getUserEmail(accessToken) };
|
||||
}
|
||||
|
||||
async function discoverProject(accessToken: string): Promise<string> {
|
||||
const envProject = process.env.GOOGLE_CLOUD_PROJECT || process.env.GOOGLE_CLOUD_PROJECT_ID;
|
||||
const headers = {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "google-api-nodejs-client/9.15.1",
|
||||
"X-Goog-Api-Client": `gl-node/${process.versions.node}`,
|
||||
"Client-Metadata": JSON.stringify(LOAD_CODE_ASSIST_METADATA),
|
||||
};
|
||||
|
||||
const loadBody = {
|
||||
...(envProject ? { cloudaicompanionProject: envProject } : {}),
|
||||
metadata: {
|
||||
...LOAD_CODE_ASSIST_METADATA,
|
||||
...(envProject ? { duetProject: envProject } : {}),
|
||||
},
|
||||
};
|
||||
|
||||
let data: {
|
||||
currentTier?: { id?: string };
|
||||
cloudaicompanionProject?: string | { id?: string };
|
||||
allowedTiers?: Array<{ id?: string; isDefault?: boolean }>;
|
||||
} = {};
|
||||
let activeEndpoint = CODE_ASSIST_ENDPOINT_PROD;
|
||||
let loadError: Error | undefined;
|
||||
for (const endpoint of LOAD_CODE_ASSIST_ENDPOINTS) {
|
||||
try {
|
||||
const response = await fetchWithTimeout(`${endpoint}/v1internal:loadCodeAssist`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(loadBody),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorPayload = await response.json().catch(() => null);
|
||||
if (isVpcScAffected(errorPayload)) {
|
||||
data = { currentTier: { id: TIER_STANDARD } };
|
||||
activeEndpoint = endpoint;
|
||||
loadError = undefined;
|
||||
break;
|
||||
}
|
||||
loadError = new Error(`loadCodeAssist failed: ${response.status} ${response.statusText}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
data = (await response.json()) as typeof data;
|
||||
activeEndpoint = endpoint;
|
||||
loadError = undefined;
|
||||
break;
|
||||
} catch (err) {
|
||||
loadError = err instanceof Error ? err : new Error("loadCodeAssist failed", { cause: err });
|
||||
}
|
||||
}
|
||||
|
||||
const hasLoadCodeAssistData =
|
||||
Boolean(data.currentTier) ||
|
||||
Boolean(data.cloudaicompanionProject) ||
|
||||
Boolean(data.allowedTiers?.length);
|
||||
if (!hasLoadCodeAssistData && loadError) {
|
||||
if (envProject) {
|
||||
return envProject;
|
||||
}
|
||||
throw loadError;
|
||||
}
|
||||
|
||||
if (data.currentTier) {
|
||||
const project = data.cloudaicompanionProject;
|
||||
if (typeof project === "string" && project) {
|
||||
return project;
|
||||
}
|
||||
if (typeof project === "object" && project?.id) {
|
||||
return project.id;
|
||||
}
|
||||
if (envProject) {
|
||||
return envProject;
|
||||
}
|
||||
throw new Error(
|
||||
"This account requires GOOGLE_CLOUD_PROJECT or GOOGLE_CLOUD_PROJECT_ID to be set.",
|
||||
);
|
||||
}
|
||||
|
||||
const tier = getDefaultTier(data.allowedTiers);
|
||||
const tierId = tier?.id || TIER_FREE;
|
||||
if (tierId !== TIER_FREE && !envProject) {
|
||||
throw new Error(
|
||||
"This account requires GOOGLE_CLOUD_PROJECT or GOOGLE_CLOUD_PROJECT_ID to be set.",
|
||||
);
|
||||
}
|
||||
|
||||
const onboardBody: Record<string, unknown> = {
|
||||
tierId,
|
||||
metadata: {
|
||||
...LOAD_CODE_ASSIST_METADATA,
|
||||
},
|
||||
};
|
||||
if (tierId !== TIER_FREE && envProject) {
|
||||
onboardBody.cloudaicompanionProject = envProject;
|
||||
(onboardBody.metadata as Record<string, unknown>).duetProject = envProject;
|
||||
}
|
||||
|
||||
const onboardResponse = await fetchWithTimeout(`${activeEndpoint}/v1internal:onboardUser`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(onboardBody),
|
||||
});
|
||||
|
||||
if (!onboardResponse.ok) {
|
||||
throw new Error(`onboardUser failed: ${onboardResponse.status} ${onboardResponse.statusText}`);
|
||||
}
|
||||
|
||||
let lro = (await onboardResponse.json()) as {
|
||||
done?: boolean;
|
||||
name?: string;
|
||||
response?: { cloudaicompanionProject?: { id?: string } };
|
||||
};
|
||||
|
||||
if (!lro.done && lro.name) {
|
||||
lro = await pollOperation(activeEndpoint, lro.name, headers);
|
||||
}
|
||||
|
||||
const projectId = lro.response?.cloudaicompanionProject?.id;
|
||||
if (projectId) {
|
||||
return projectId;
|
||||
}
|
||||
if (envProject) {
|
||||
return envProject;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"Could not discover or provision a Google Cloud project. Set GOOGLE_CLOUD_PROJECT or GOOGLE_CLOUD_PROJECT_ID.",
|
||||
);
|
||||
}
|
||||
1
openclaw/extensions/google/oauth.runtime.ts
Normal file
1
openclaw/extensions/google/oauth.runtime.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export { loginGeminiCliOAuth } from "./oauth.js";
|
||||
72
openclaw/extensions/google/oauth.settings.ts
Normal file
72
openclaw/extensions/google/oauth.settings.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
|
||||
|
||||
type OAuthSettingsFs = {
|
||||
existsSync: (path: Parameters<typeof existsSync>[0]) => ReturnType<typeof existsSync>;
|
||||
readFileSync: (path: Parameters<typeof readFileSync>[0], encoding: "utf8") => string;
|
||||
homedir: typeof homedir;
|
||||
};
|
||||
|
||||
const defaultFs: OAuthSettingsFs = {
|
||||
existsSync,
|
||||
readFileSync,
|
||||
homedir,
|
||||
};
|
||||
|
||||
let oauthSettingsFs: OAuthSettingsFs = defaultFs;
|
||||
|
||||
type GeminiCliAuthSettings = {
|
||||
security?: {
|
||||
auth?: {
|
||||
selectedType?: unknown;
|
||||
enforcedType?: unknown;
|
||||
};
|
||||
};
|
||||
selectedAuthType?: unknown;
|
||||
enforcedAuthType?: unknown;
|
||||
};
|
||||
|
||||
function readSettingsFile(): GeminiCliAuthSettings | null {
|
||||
const settingsPath = join(oauthSettingsFs.homedir(), ".gemini", "settings.json");
|
||||
if (!oauthSettingsFs.existsSync(settingsPath)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(oauthSettingsFs.readFileSync(settingsPath, "utf8")) as unknown;
|
||||
return isRecord(parsed) ? (parsed as GeminiCliAuthSettings) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function setOAuthSettingsFsForTest(overrides?: Partial<OAuthSettingsFs>): void {
|
||||
oauthSettingsFs = overrides ? { ...defaultFs, ...overrides } : defaultFs;
|
||||
}
|
||||
|
||||
export function resolveGeminiCliSelectedAuthType(): string | undefined {
|
||||
const settings = readSettingsFile();
|
||||
if (settings) {
|
||||
const security = isRecord(settings.security) ? settings.security : undefined;
|
||||
const auth = isRecord(security?.auth) ? security.auth : undefined;
|
||||
const selectedAuthType =
|
||||
normalizeOptionalString(auth?.selectedType) ??
|
||||
normalizeOptionalString(auth?.enforcedType) ??
|
||||
normalizeOptionalString(settings.selectedAuthType) ??
|
||||
normalizeOptionalString(settings.enforcedAuthType);
|
||||
if (selectedAuthType) {
|
||||
return selectedAuthType;
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env.GOOGLE_GENAI_USE_GCA === "true") {
|
||||
return "oauth-personal";
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function isGeminiCliPersonalOAuth(): boolean {
|
||||
return resolveGeminiCliSelectedAuthType() === "oauth-personal";
|
||||
}
|
||||
44
openclaw/extensions/google/oauth.shared.ts
Normal file
44
openclaw/extensions/google/oauth.shared.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
export const CLIENT_ID_KEYS = ["OPENCLAW_GEMINI_OAUTH_CLIENT_ID", "GEMINI_CLI_OAUTH_CLIENT_ID"];
|
||||
export const CLIENT_SECRET_KEYS = [
|
||||
"OPENCLAW_GEMINI_OAUTH_CLIENT_SECRET",
|
||||
"GEMINI_CLI_OAUTH_CLIENT_SECRET",
|
||||
];
|
||||
export const REDIRECT_URI = "http://localhost:8085/oauth2callback";
|
||||
export const AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth";
|
||||
export const TOKEN_URL = "https://oauth2.googleapis.com/token";
|
||||
export const USERINFO_URL = "https://www.googleapis.com/oauth2/v1/userinfo?alt=json";
|
||||
export const CODE_ASSIST_ENDPOINT_PROD = "https://cloudcode-pa.googleapis.com";
|
||||
export const CODE_ASSIST_ENDPOINT_DAILY = "https://daily-cloudcode-pa.sandbox.googleapis.com";
|
||||
export const CODE_ASSIST_ENDPOINT_AUTOPUSH = "https://autopush-cloudcode-pa.sandbox.googleapis.com";
|
||||
export const LOAD_CODE_ASSIST_ENDPOINTS = [
|
||||
CODE_ASSIST_ENDPOINT_PROD,
|
||||
CODE_ASSIST_ENDPOINT_DAILY,
|
||||
CODE_ASSIST_ENDPOINT_AUTOPUSH,
|
||||
];
|
||||
export const DEFAULT_FETCH_TIMEOUT_MS = 10_000;
|
||||
export const SCOPES = [
|
||||
"https://www.googleapis.com/auth/cloud-platform",
|
||||
"https://www.googleapis.com/auth/userinfo.email",
|
||||
"https://www.googleapis.com/auth/userinfo.profile",
|
||||
];
|
||||
|
||||
export const TIER_FREE = "free-tier";
|
||||
export const TIER_LEGACY = "legacy-tier";
|
||||
export const TIER_STANDARD = "standard-tier";
|
||||
|
||||
export type GeminiCliOAuthCredentials = {
|
||||
access: string;
|
||||
refresh: string;
|
||||
expires: number;
|
||||
email?: string;
|
||||
projectId?: string;
|
||||
};
|
||||
|
||||
export type GeminiCliOAuthContext = {
|
||||
isRemote: boolean;
|
||||
openUrl: (url: string) => Promise<void>;
|
||||
log: (msg: string) => void;
|
||||
note: (message: string, title?: string) => Promise<void>;
|
||||
prompt: (message: string) => Promise<string>;
|
||||
progress: { update: (msg: string) => void; stop: (msg?: string) => void };
|
||||
};
|
||||
854
openclaw/extensions/google/oauth.test.ts
Normal file
854
openclaw/extensions/google/oauth.test.ts
Normal file
|
|
@ -0,0 +1,854 @@
|
|||
import { join, parse } from "node:path";
|
||||
import { describe, expect, it, vi, beforeAll, beforeEach, afterEach } from "vitest";
|
||||
|
||||
vi.mock("../../src/infra/wsl.js", () => ({
|
||||
isWSL2Sync: () => false,
|
||||
}));
|
||||
|
||||
vi.mock("../../src/infra/net/fetch-guard.js", () => ({
|
||||
fetchWithSsrFGuard: async (params: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
fetchImpl?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
}) => {
|
||||
const fetchImpl = params.fetchImpl ?? globalThis.fetch;
|
||||
const response = await fetchImpl(params.url, params.init);
|
||||
return {
|
||||
response,
|
||||
finalUrl: params.url,
|
||||
release: async () => {},
|
||||
};
|
||||
},
|
||||
}));
|
||||
|
||||
const mockExistsSync = vi.fn();
|
||||
const mockReadFileSync = vi.fn();
|
||||
const mockRealpathSync = vi.fn();
|
||||
const mockReaddirSync = vi.fn();
|
||||
const mockSettingsExistsSync = vi.fn();
|
||||
const mockSettingsReadFileSync = vi.fn();
|
||||
|
||||
describe("resolveGeminiCliSelectedAuthType", () => {
|
||||
const ENV_KEYS = ["GOOGLE_GENAI_USE_GCA"] as const;
|
||||
|
||||
let envSnapshot: Partial<Record<(typeof ENV_KEYS)[number], string>>;
|
||||
let resolveGeminiCliSelectedAuthType: typeof import("./oauth.settings.js").resolveGeminiCliSelectedAuthType;
|
||||
let setOAuthSettingsFsForTest: typeof import("./oauth.settings.js").setOAuthSettingsFsForTest;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ resolveGeminiCliSelectedAuthType, setOAuthSettingsFsForTest } =
|
||||
await import("./oauth.settings.js"));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
envSnapshot = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]]));
|
||||
delete process.env.GOOGLE_GENAI_USE_GCA;
|
||||
mockSettingsExistsSync.mockReset();
|
||||
mockSettingsReadFileSync.mockReset();
|
||||
setOAuthSettingsFsForTest({
|
||||
existsSync: (...args) => mockSettingsExistsSync(...args),
|
||||
readFileSync: (...args) => mockSettingsReadFileSync(...args),
|
||||
homedir: () => "/mock/home",
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const key of ENV_KEYS) {
|
||||
const value = envSnapshot[key];
|
||||
if (value === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = value;
|
||||
}
|
||||
}
|
||||
setOAuthSettingsFsForTest();
|
||||
});
|
||||
|
||||
it("uses GOOGLE_GENAI_USE_GCA as an oauth-personal fallback when settings are absent", () => {
|
||||
process.env.GOOGLE_GENAI_USE_GCA = "true";
|
||||
mockSettingsExistsSync.mockReturnValue(false);
|
||||
|
||||
expect(resolveGeminiCliSelectedAuthType()).toBe("oauth-personal");
|
||||
});
|
||||
|
||||
it("prefers settings auth selection over the GOOGLE_GENAI_USE_GCA fallback", () => {
|
||||
process.env.GOOGLE_GENAI_USE_GCA = "true";
|
||||
mockSettingsExistsSync.mockReturnValue(true);
|
||||
mockSettingsReadFileSync.mockReturnValue(
|
||||
JSON.stringify({
|
||||
security: {
|
||||
auth: {
|
||||
selectedType: "oauth-code-assist",
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(resolveGeminiCliSelectedAuthType()).toBe("oauth-code-assist");
|
||||
});
|
||||
|
||||
it("reads the nested security auth selection from ~/.gemini/settings.json", () => {
|
||||
mockSettingsExistsSync.mockReturnValue(true);
|
||||
mockSettingsReadFileSync.mockReturnValue(
|
||||
JSON.stringify({
|
||||
security: {
|
||||
auth: {
|
||||
selectedType: "oauth-personal",
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(resolveGeminiCliSelectedAuthType()).toBe("oauth-personal");
|
||||
});
|
||||
|
||||
it("falls back to legacy top-level selectedAuthType keys", () => {
|
||||
mockSettingsExistsSync.mockReturnValue(true);
|
||||
mockSettingsReadFileSync.mockReturnValue(
|
||||
JSON.stringify({ selectedAuthType: "oauth-personal" }),
|
||||
);
|
||||
|
||||
expect(resolveGeminiCliSelectedAuthType()).toBe("oauth-personal");
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractGeminiCliCredentials", () => {
|
||||
const normalizePath = (value: string) =>
|
||||
value.replace(/\\/g, "/").replace(/\/+$/, "").toLowerCase();
|
||||
const rootDir = parse(process.cwd()).root || "/";
|
||||
const FAKE_CLIENT_ID = "123456789-abcdef.apps.googleusercontent.com";
|
||||
const FAKE_CLIENT_SECRET = "GOCSPX-FakeSecretValue123";
|
||||
const FAKE_OAUTH2_CONTENT = `
|
||||
const clientId = "${FAKE_CLIENT_ID}";
|
||||
const clientSecret = "${FAKE_CLIENT_SECRET}";
|
||||
`;
|
||||
|
||||
let originalPath: string | undefined;
|
||||
let extractGeminiCliCredentials: typeof import("./oauth.credentials.js").extractGeminiCliCredentials;
|
||||
let clearCredentialsCache: typeof import("./oauth.credentials.js").clearCredentialsCache;
|
||||
let setOAuthCredentialsFsForTest: typeof import("./oauth.credentials.js").setOAuthCredentialsFsForTest;
|
||||
|
||||
async function installMockFs() {
|
||||
setOAuthCredentialsFsForTest({
|
||||
existsSync: (...args) => mockExistsSync(...args),
|
||||
readFileSync: (...args) => mockReadFileSync(...args),
|
||||
realpathSync: (...args) => mockRealpathSync(...args),
|
||||
readdirSync: (...args) => mockReaddirSync(...args),
|
||||
});
|
||||
}
|
||||
|
||||
function makeFakeLayout() {
|
||||
const binDir = join(rootDir, "fake", "bin");
|
||||
const geminiPath = join(binDir, "gemini");
|
||||
const resolvedPath = join(
|
||||
rootDir,
|
||||
"fake",
|
||||
"lib",
|
||||
"node_modules",
|
||||
"@google",
|
||||
"gemini-cli",
|
||||
"dist",
|
||||
"index.js",
|
||||
);
|
||||
const oauth2Path = join(
|
||||
rootDir,
|
||||
"fake",
|
||||
"lib",
|
||||
"node_modules",
|
||||
"@google",
|
||||
"gemini-cli",
|
||||
"node_modules",
|
||||
"@google",
|
||||
"gemini-cli-core",
|
||||
"dist",
|
||||
"src",
|
||||
"code_assist",
|
||||
"oauth2.js",
|
||||
);
|
||||
|
||||
return { binDir, geminiPath, resolvedPath, oauth2Path };
|
||||
}
|
||||
|
||||
function installGeminiLayout(params: {
|
||||
oauth2Exists?: boolean;
|
||||
oauth2Content?: string;
|
||||
readdir?: string[];
|
||||
}) {
|
||||
const layout = makeFakeLayout();
|
||||
process.env.PATH = layout.binDir;
|
||||
|
||||
// resolveGeminiCliDirs checks package.json to validate candidate directories
|
||||
const geminiCliDir = join(rootDir, "fake", "lib", "node_modules", "@google", "gemini-cli");
|
||||
const packageJsonPath = normalizePath(join(geminiCliDir, "package.json"));
|
||||
|
||||
mockExistsSync.mockImplementation((p: string) => {
|
||||
const normalized = normalizePath(p);
|
||||
if (normalized === normalizePath(layout.geminiPath)) {
|
||||
return true;
|
||||
}
|
||||
if (normalized === packageJsonPath) {
|
||||
return true;
|
||||
}
|
||||
if (params.oauth2Exists && normalized === normalizePath(layout.oauth2Path)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
mockRealpathSync.mockReturnValue(layout.resolvedPath);
|
||||
if (params.oauth2Content !== undefined) {
|
||||
mockReadFileSync.mockReturnValue(params.oauth2Content);
|
||||
}
|
||||
if (params.readdir) {
|
||||
mockReaddirSync.mockReturnValue(params.readdir);
|
||||
}
|
||||
|
||||
return layout;
|
||||
}
|
||||
|
||||
function installNpmShimLayout(params: { oauth2Exists?: boolean; oauth2Content?: string }) {
|
||||
const binDir = join(rootDir, "fake", "npm-bin");
|
||||
const geminiPath = join(binDir, "gemini");
|
||||
const resolvedPath = geminiPath;
|
||||
const geminiCliDir = join(binDir, "node_modules", "@google", "gemini-cli");
|
||||
const oauth2Path = join(
|
||||
geminiCliDir,
|
||||
"node_modules",
|
||||
"@google",
|
||||
"gemini-cli-core",
|
||||
"dist",
|
||||
"src",
|
||||
"code_assist",
|
||||
"oauth2.js",
|
||||
);
|
||||
const packageJsonPath = normalizePath(join(geminiCliDir, "package.json"));
|
||||
process.env.PATH = binDir;
|
||||
|
||||
mockExistsSync.mockImplementation((p: string) => {
|
||||
const normalized = normalizePath(p);
|
||||
if (normalized === normalizePath(geminiPath)) {
|
||||
return true;
|
||||
}
|
||||
if (normalized === packageJsonPath) {
|
||||
return true;
|
||||
}
|
||||
if (params.oauth2Exists && normalized === normalizePath(oauth2Path)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
mockRealpathSync.mockReturnValue(resolvedPath);
|
||||
if (params.oauth2Content !== undefined) {
|
||||
mockReadFileSync.mockReturnValue(params.oauth2Content);
|
||||
}
|
||||
}
|
||||
|
||||
function installBundledNpmLayout(params: { bundleContent: string }) {
|
||||
const binDir = join(rootDir, "fake", "npm-bundle-bin");
|
||||
const geminiPath = join(binDir, "gemini");
|
||||
const resolvedPath = geminiPath;
|
||||
const geminiCliDir = join(binDir, "node_modules", "@google", "gemini-cli");
|
||||
const packageJsonPath = normalizePath(join(geminiCliDir, "package.json"));
|
||||
const bundleDir = join(geminiCliDir, "bundle");
|
||||
const chunkPath = join(bundleDir, "chunk-ABC123.js");
|
||||
|
||||
process.env.PATH = binDir;
|
||||
mockExistsSync.mockImplementation((p: string) => {
|
||||
const normalized = normalizePath(p);
|
||||
return (
|
||||
normalized === normalizePath(geminiPath) ||
|
||||
normalized === packageJsonPath ||
|
||||
normalized === normalizePath(bundleDir)
|
||||
);
|
||||
});
|
||||
mockRealpathSync.mockReturnValue(resolvedPath);
|
||||
mockReaddirSync.mockImplementation((p: string) => {
|
||||
if (normalizePath(p) === normalizePath(bundleDir)) {
|
||||
return [dirent("chunk-ABC123.js", false)];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
mockReadFileSync.mockImplementation((p: string) => {
|
||||
if (normalizePath(p) === normalizePath(chunkPath)) {
|
||||
return params.bundleContent;
|
||||
}
|
||||
throw new Error(`Unexpected read for ${p}`);
|
||||
});
|
||||
}
|
||||
|
||||
function installHomebrewLibexecLayout(params: { oauth2Content: string }) {
|
||||
const brewPrefix = join(rootDir, "opt", "homebrew");
|
||||
const cellarRoot = join(brewPrefix, "Cellar", "gemini-cli", "1.2.3");
|
||||
const binDir = join(brewPrefix, "bin");
|
||||
const geminiPath = join(binDir, "gemini");
|
||||
const resolvedPath = join(cellarRoot, "libexec", "bin", "gemini");
|
||||
const geminiCliDir = join(
|
||||
cellarRoot,
|
||||
"libexec",
|
||||
"lib",
|
||||
"node_modules",
|
||||
"@google",
|
||||
"gemini-cli",
|
||||
);
|
||||
const packageJsonPath = normalizePath(join(geminiCliDir, "package.json"));
|
||||
const oauth2Path = join(
|
||||
geminiCliDir,
|
||||
"node_modules",
|
||||
"@google",
|
||||
"gemini-cli-core",
|
||||
"dist",
|
||||
"src",
|
||||
"code_assist",
|
||||
"oauth2.js",
|
||||
);
|
||||
|
||||
process.env.PATH = binDir;
|
||||
mockExistsSync.mockImplementation((p: string) => {
|
||||
const normalized = normalizePath(p);
|
||||
return (
|
||||
normalized === normalizePath(geminiPath) ||
|
||||
normalized === packageJsonPath ||
|
||||
normalized === normalizePath(oauth2Path)
|
||||
);
|
||||
});
|
||||
mockRealpathSync.mockReturnValue(resolvedPath);
|
||||
mockReadFileSync.mockImplementation((p: string) => {
|
||||
if (normalizePath(p) === normalizePath(oauth2Path)) {
|
||||
return params.oauth2Content;
|
||||
}
|
||||
throw new Error(`Unexpected read for ${p}`);
|
||||
});
|
||||
}
|
||||
|
||||
function installWindowsNvmLayoutWithUnrelatedOauth(params: {
|
||||
oauth2Content: string;
|
||||
unrelatedOauth2Content: string;
|
||||
}) {
|
||||
const nvmRoot = join(rootDir, "fake", "Users", "lobster", "AppData", "Local", "nvm");
|
||||
const versionDir = join(nvmRoot, "v24.1.0");
|
||||
const geminiPath = join(versionDir, process.platform === "win32" ? "gemini.cmd" : "gemini");
|
||||
const resolvedPath = geminiPath;
|
||||
const geminiCliDir = join(versionDir, "node_modules", "@google", "gemini-cli");
|
||||
const packageJsonPath = normalizePath(join(geminiCliDir, "package.json"));
|
||||
const oauth2Path = join(
|
||||
geminiCliDir,
|
||||
"node_modules",
|
||||
"@google",
|
||||
"gemini-cli-core",
|
||||
"dist",
|
||||
"src",
|
||||
"code_assist",
|
||||
"oauth2.js",
|
||||
);
|
||||
const unrelatedOauth2Path = join(
|
||||
nvmRoot,
|
||||
"node_modules",
|
||||
"discord-api-types",
|
||||
"payloads",
|
||||
"v10",
|
||||
"oauth2.js",
|
||||
);
|
||||
|
||||
process.env.PATH = versionDir;
|
||||
mockExistsSync.mockImplementation((p: string) => {
|
||||
const normalized = normalizePath(p);
|
||||
return (
|
||||
normalized === normalizePath(geminiPath) ||
|
||||
normalized === packageJsonPath ||
|
||||
normalized === normalizePath(oauth2Path)
|
||||
);
|
||||
});
|
||||
mockRealpathSync.mockReturnValue(resolvedPath);
|
||||
mockReadFileSync.mockImplementation((p: string) => {
|
||||
const normalized = normalizePath(p);
|
||||
if (normalized === normalizePath(oauth2Path)) {
|
||||
return params.oauth2Content;
|
||||
}
|
||||
if (normalized === normalizePath(unrelatedOauth2Path)) {
|
||||
return params.unrelatedOauth2Content;
|
||||
}
|
||||
throw new Error(`Unexpected read for ${p}`);
|
||||
});
|
||||
mockReaddirSync.mockImplementation((p: string) => {
|
||||
const normalized = normalizePath(p);
|
||||
if (normalized === normalizePath(nvmRoot)) {
|
||||
return [dirent("node_modules", true)];
|
||||
}
|
||||
if (normalized === normalizePath(join(nvmRoot, "node_modules"))) {
|
||||
return [dirent("discord-api-types", true)];
|
||||
}
|
||||
if (normalized === normalizePath(join(nvmRoot, "node_modules", "discord-api-types"))) {
|
||||
return [dirent("payloads", true)];
|
||||
}
|
||||
if (
|
||||
normalized === normalizePath(join(nvmRoot, "node_modules", "discord-api-types", "payloads"))
|
||||
) {
|
||||
return [dirent("v10", true)];
|
||||
}
|
||||
if (
|
||||
normalized ===
|
||||
normalizePath(join(nvmRoot, "node_modules", "discord-api-types", "payloads", "v10"))
|
||||
) {
|
||||
return [dirent("oauth2.js", false)];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
|
||||
return { unrelatedOauth2Path };
|
||||
}
|
||||
|
||||
function dirent(name: string, isDirectory: boolean) {
|
||||
return {
|
||||
name,
|
||||
isBlockDevice: () => false,
|
||||
isCharacterDevice: () => false,
|
||||
isDirectory: () => isDirectory,
|
||||
isFIFO: () => false,
|
||||
isFile: () => !isDirectory,
|
||||
isSocket: () => false,
|
||||
isSymbolicLink: () => false,
|
||||
};
|
||||
}
|
||||
|
||||
function expectFakeCliCredentials(result: unknown) {
|
||||
expect(result).toEqual({
|
||||
clientId: FAKE_CLIENT_ID,
|
||||
clientSecret: FAKE_CLIENT_SECRET,
|
||||
});
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
({ extractGeminiCliCredentials, clearCredentialsCache, setOAuthCredentialsFsForTest } =
|
||||
await import("./oauth.credentials.js"));
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
originalPath = process.env.PATH;
|
||||
await installMockFs();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
process.env.PATH = originalPath;
|
||||
setOAuthCredentialsFsForTest();
|
||||
});
|
||||
|
||||
it("returns null when gemini binary is not in PATH", async () => {
|
||||
process.env.PATH = "/nonexistent";
|
||||
mockExistsSync.mockReturnValue(false);
|
||||
|
||||
clearCredentialsCache();
|
||||
expect(extractGeminiCliCredentials()).toBeNull();
|
||||
});
|
||||
|
||||
it("extracts credentials from oauth2.js in known path", async () => {
|
||||
installGeminiLayout({ oauth2Exists: true, oauth2Content: FAKE_OAUTH2_CONTENT });
|
||||
|
||||
clearCredentialsCache();
|
||||
const result = extractGeminiCliCredentials();
|
||||
|
||||
expectFakeCliCredentials(result);
|
||||
});
|
||||
|
||||
it("extracts credentials when PATH entry is an npm global shim", async () => {
|
||||
installNpmShimLayout({ oauth2Exists: true, oauth2Content: FAKE_OAUTH2_CONTENT });
|
||||
|
||||
clearCredentialsCache();
|
||||
const result = extractGeminiCliCredentials();
|
||||
|
||||
expectFakeCliCredentials(result);
|
||||
});
|
||||
|
||||
it("extracts credentials from bundled npm installs", async () => {
|
||||
installBundledNpmLayout({
|
||||
bundleContent: `
|
||||
const OAUTH_CLIENT_ID = "${FAKE_CLIENT_ID}";
|
||||
const OAUTH_CLIENT_SECRET = "${FAKE_CLIENT_SECRET}";
|
||||
`,
|
||||
});
|
||||
|
||||
clearCredentialsCache();
|
||||
const result = extractGeminiCliCredentials();
|
||||
|
||||
expectFakeCliCredentials(result);
|
||||
});
|
||||
|
||||
it("extracts credentials from Homebrew libexec installs", async () => {
|
||||
installHomebrewLibexecLayout({ oauth2Content: FAKE_OAUTH2_CONTENT });
|
||||
|
||||
clearCredentialsCache();
|
||||
const result = extractGeminiCliCredentials();
|
||||
|
||||
expectFakeCliCredentials(result);
|
||||
});
|
||||
|
||||
it("returns null when oauth2.js cannot be found", async () => {
|
||||
installGeminiLayout({ oauth2Exists: false, readdir: [] });
|
||||
|
||||
clearCredentialsCache();
|
||||
expect(extractGeminiCliCredentials()).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when oauth2.js lacks credentials", async () => {
|
||||
installGeminiLayout({ oauth2Exists: true, oauth2Content: "// no credentials here" });
|
||||
|
||||
clearCredentialsCache();
|
||||
expect(extractGeminiCliCredentials()).toBeNull();
|
||||
});
|
||||
|
||||
it("caches credentials after first extraction", async () => {
|
||||
installGeminiLayout({ oauth2Exists: true, oauth2Content: FAKE_OAUTH2_CONTENT });
|
||||
|
||||
clearCredentialsCache();
|
||||
|
||||
// First call
|
||||
const result1 = extractGeminiCliCredentials();
|
||||
expect(result1).not.toBeNull();
|
||||
|
||||
// Second call should use cache (readFileSync not called again)
|
||||
const readCount = mockReadFileSync.mock.calls.length;
|
||||
const result2 = extractGeminiCliCredentials();
|
||||
expect(result2).toEqual(result1);
|
||||
expect(mockReadFileSync.mock.calls.length).toBe(readCount);
|
||||
});
|
||||
|
||||
it("skips unrelated oauth2.js files when gemini resolves inside a Windows nvm root", async () => {
|
||||
const { unrelatedOauth2Path } = installWindowsNvmLayoutWithUnrelatedOauth({
|
||||
oauth2Content: FAKE_OAUTH2_CONTENT,
|
||||
unrelatedOauth2Content: "// unrelated oauth file",
|
||||
});
|
||||
|
||||
clearCredentialsCache();
|
||||
const result = extractGeminiCliCredentials();
|
||||
|
||||
expectFakeCliCredentials(result);
|
||||
expect(
|
||||
mockReadFileSync.mock.calls.some(
|
||||
([path]) => normalizePath(String(path)) === normalizePath(unrelatedOauth2Path),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("loginGeminiCliOAuth", () => {
|
||||
const TOKEN_URL = "https://oauth2.googleapis.com/token";
|
||||
const USERINFO_URL = "https://www.googleapis.com/oauth2/v1/userinfo?alt=json";
|
||||
const LOAD_PROD = "https://cloudcode-pa.googleapis.com/v1internal:loadCodeAssist";
|
||||
const LOAD_DAILY = "https://daily-cloudcode-pa.sandbox.googleapis.com/v1internal:loadCodeAssist";
|
||||
const LOAD_AUTOPUSH =
|
||||
"https://autopush-cloudcode-pa.sandbox.googleapis.com/v1internal:loadCodeAssist";
|
||||
|
||||
const ENV_KEYS = [
|
||||
"OPENCLAW_GEMINI_OAUTH_CLIENT_ID",
|
||||
"OPENCLAW_GEMINI_OAUTH_CLIENT_SECRET",
|
||||
"GEMINI_CLI_OAUTH_CLIENT_ID",
|
||||
"GEMINI_CLI_OAUTH_CLIENT_SECRET",
|
||||
"GOOGLE_CLOUD_PROJECT",
|
||||
"GOOGLE_CLOUD_PROJECT_ID",
|
||||
"GOOGLE_GENAI_USE_GCA",
|
||||
] as const;
|
||||
|
||||
const EXPECTED_LOAD_CODE_ASSIST_METADATA = {
|
||||
ideType: "IDE_UNSPECIFIED",
|
||||
platform: "PLATFORM_UNSPECIFIED",
|
||||
pluginType: "GEMINI",
|
||||
} as const;
|
||||
|
||||
function getRequestUrl(input: string | URL | Request): string {
|
||||
if (typeof input === "string") {
|
||||
return input;
|
||||
}
|
||||
if (input instanceof URL) {
|
||||
return input.toString();
|
||||
}
|
||||
return input.url;
|
||||
}
|
||||
|
||||
function getHeaderValue(headers: HeadersInit | undefined, name: string): string | undefined {
|
||||
if (!headers) {
|
||||
return undefined;
|
||||
}
|
||||
if (headers instanceof Headers) {
|
||||
return headers.get(name) ?? undefined;
|
||||
}
|
||||
if (Array.isArray(headers)) {
|
||||
return headers.find(([key]) => key.toLowerCase() === name.toLowerCase())?.[1];
|
||||
}
|
||||
return headers[name];
|
||||
}
|
||||
|
||||
function responseJson(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function getFormField(body: RequestInit["body"], name: string): string | null {
|
||||
if (!(body instanceof URLSearchParams)) {
|
||||
throw new Error("Expected URLSearchParams body");
|
||||
}
|
||||
return body.get(name);
|
||||
}
|
||||
|
||||
function parseJsonString(value: unknown, label: string): unknown {
|
||||
if (typeof value !== "string") {
|
||||
throw new Error(`Expected ${label} JSON string`);
|
||||
}
|
||||
return JSON.parse(value);
|
||||
}
|
||||
|
||||
type LoginGeminiCliOAuthFn = (options: {
|
||||
isRemote: boolean;
|
||||
openUrl: () => Promise<void>;
|
||||
log: (msg: string) => void;
|
||||
note: () => Promise<void>;
|
||||
prompt: () => Promise<string>;
|
||||
progress: { update: () => void; stop: () => void };
|
||||
}) => Promise<{ projectId?: string }>;
|
||||
|
||||
async function runRemoteLoginWithCapturedAuthUrl(loginGeminiCliOAuth: LoginGeminiCliOAuthFn) {
|
||||
let authUrl = "";
|
||||
const result = await loginGeminiCliOAuth({
|
||||
isRemote: true,
|
||||
openUrl: async () => {},
|
||||
log: (msg) => {
|
||||
const found = msg.match(/https:\/\/accounts\.google\.com\/o\/oauth2\/v2\/auth\?[^\s]+/);
|
||||
if (found?.[0]) {
|
||||
authUrl = found[0];
|
||||
}
|
||||
},
|
||||
note: async () => {},
|
||||
prompt: async () => {
|
||||
const state = new URL(authUrl).searchParams.get("state");
|
||||
return `http://localhost:8085/oauth2callback?code=oauth-code&state=${state}`;
|
||||
},
|
||||
progress: { update: () => {}, stop: () => {} },
|
||||
});
|
||||
return { result, authUrl };
|
||||
}
|
||||
|
||||
async function runRemoteLoginExpectingProjectId(
|
||||
loginGeminiCliOAuth: LoginGeminiCliOAuthFn,
|
||||
projectId: string,
|
||||
) {
|
||||
const { result } = await runRemoteLoginWithCapturedAuthUrl(loginGeminiCliOAuth);
|
||||
expect(result.projectId).toBe(projectId);
|
||||
}
|
||||
|
||||
let envSnapshot: Partial<Record<(typeof ENV_KEYS)[number], string>>;
|
||||
let setOAuthSettingsFsForTest: typeof import("./oauth.settings.js").setOAuthSettingsFsForTest;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ setOAuthSettingsFsForTest } = await import("./oauth.settings.js"));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
envSnapshot = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]]));
|
||||
process.env.OPENCLAW_GEMINI_OAUTH_CLIENT_ID = "test-client-id.apps.googleusercontent.com";
|
||||
process.env.OPENCLAW_GEMINI_OAUTH_CLIENT_SECRET = "GOCSPX-test-client-secret"; // pragma: allowlist secret
|
||||
delete process.env.GEMINI_CLI_OAUTH_CLIENT_ID;
|
||||
delete process.env.GEMINI_CLI_OAUTH_CLIENT_SECRET;
|
||||
delete process.env.GOOGLE_CLOUD_PROJECT;
|
||||
delete process.env.GOOGLE_CLOUD_PROJECT_ID;
|
||||
delete process.env.GOOGLE_GENAI_USE_GCA;
|
||||
mockSettingsExistsSync.mockReset();
|
||||
mockSettingsReadFileSync.mockReset();
|
||||
setOAuthSettingsFsForTest({
|
||||
existsSync: (...args) => mockSettingsExistsSync(...args),
|
||||
readFileSync: (...args) => mockSettingsReadFileSync(...args),
|
||||
homedir: () => "/mock/home",
|
||||
});
|
||||
mockSettingsExistsSync.mockReturnValue(false);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const key of ENV_KEYS) {
|
||||
const value = envSnapshot[key];
|
||||
if (value === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = value;
|
||||
}
|
||||
}
|
||||
setOAuthSettingsFsForTest();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("falls back across loadCodeAssist endpoints with aligned headers and metadata", async () => {
|
||||
const requests: Array<{ url: string; init?: RequestInit }> = [];
|
||||
const fetchMock = vi.fn(async (input: string | URL | Request, init?: RequestInit) => {
|
||||
const url = getRequestUrl(input);
|
||||
requests.push({ url, init });
|
||||
|
||||
if (url === TOKEN_URL) {
|
||||
return responseJson({
|
||||
access_token: "access-token",
|
||||
refresh_token: "refresh-token",
|
||||
expires_in: 3600,
|
||||
});
|
||||
}
|
||||
if (url === USERINFO_URL) {
|
||||
return responseJson({ email: "lobster@openclaw.ai" });
|
||||
}
|
||||
if (url === LOAD_PROD) {
|
||||
return responseJson({ error: { message: "temporary failure" } }, 503);
|
||||
}
|
||||
if (url === LOAD_DAILY) {
|
||||
return responseJson({
|
||||
currentTier: { id: "standard-tier" },
|
||||
cloudaicompanionProject: { id: "daily-project" },
|
||||
});
|
||||
}
|
||||
throw new Error(`Unexpected request: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const { loginGeminiCliOAuth } = await import("./oauth.js");
|
||||
await runRemoteLoginExpectingProjectId(loginGeminiCliOAuth, "daily-project");
|
||||
const loadRequests = requests.filter((request) =>
|
||||
request.url.includes("v1internal:loadCodeAssist"),
|
||||
);
|
||||
expect(loadRequests.map((request) => request.url)).toEqual([LOAD_PROD, LOAD_DAILY]);
|
||||
|
||||
const firstHeaders = loadRequests[0]?.init?.headers;
|
||||
expect(getHeaderValue(firstHeaders, "X-Goog-Api-Client")).toBe(
|
||||
`gl-node/${process.versions.node}`,
|
||||
);
|
||||
|
||||
const clientMetadata = getHeaderValue(firstHeaders, "Client-Metadata");
|
||||
expect(clientMetadata).toBeDefined();
|
||||
expect(parseJsonString(clientMetadata, "Client-Metadata")).toEqual(
|
||||
EXPECTED_LOAD_CODE_ASSIST_METADATA,
|
||||
);
|
||||
|
||||
const loadBody = loadRequests[0]?.init?.body;
|
||||
const body = parseJsonString(loadBody, "loadCodeAssist body");
|
||||
expect(body).toEqual({
|
||||
metadata: EXPECTED_LOAD_CODE_ASSIST_METADATA,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps OAuth state separate from the PKCE verifier during manual login", async () => {
|
||||
const requests: Array<{ url: string; init?: RequestInit }> = [];
|
||||
const fetchMock = vi.fn(async (input: string | URL | Request, init?: RequestInit) => {
|
||||
const url = getRequestUrl(input);
|
||||
requests.push({ url, init });
|
||||
|
||||
if (url === TOKEN_URL) {
|
||||
return responseJson({
|
||||
access_token: "access-token",
|
||||
refresh_token: "refresh-token",
|
||||
expires_in: 3600,
|
||||
});
|
||||
}
|
||||
if (url === USERINFO_URL) {
|
||||
return responseJson({ email: "lobster@openclaw.ai" });
|
||||
}
|
||||
if (url === LOAD_PROD) {
|
||||
return responseJson({
|
||||
currentTier: { id: "standard-tier" },
|
||||
cloudaicompanionProject: { id: "prod-project" },
|
||||
});
|
||||
}
|
||||
throw new Error(`Unexpected request: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const { loginGeminiCliOAuth } = await import("./oauth.js");
|
||||
const { authUrl } = await runRemoteLoginWithCapturedAuthUrl(loginGeminiCliOAuth);
|
||||
|
||||
const authState = new URL(authUrl).searchParams.get("state");
|
||||
expect(authState).toBeTruthy();
|
||||
|
||||
const tokenRequest = requests.find((request) => request.url === TOKEN_URL);
|
||||
expect(tokenRequest).toBeDefined();
|
||||
const codeVerifier = getFormField(tokenRequest?.init?.body, "code_verifier");
|
||||
expect(codeVerifier).toBeTruthy();
|
||||
expect(codeVerifier).not.toBe(authState);
|
||||
});
|
||||
|
||||
it("rejects manual callback input when the returned state does not match", async () => {
|
||||
const { loginGeminiCliOAuth } = await import("./oauth.js");
|
||||
|
||||
await expect(
|
||||
loginGeminiCliOAuth({
|
||||
isRemote: true,
|
||||
openUrl: async () => {},
|
||||
log: () => {},
|
||||
note: async () => {},
|
||||
prompt: async () =>
|
||||
"http://localhost:8085/oauth2callback?code=oauth-code&state=wrong-state",
|
||||
progress: { update: () => {}, stop: () => {} },
|
||||
}),
|
||||
).rejects.toThrow("OAuth state mismatch - please try again");
|
||||
});
|
||||
|
||||
it("falls back to GOOGLE_CLOUD_PROJECT when all loadCodeAssist endpoints fail", async () => {
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "env-project";
|
||||
|
||||
const requests: string[] = [];
|
||||
const fetchMock = vi.fn(async (input: string | URL | Request) => {
|
||||
const url = getRequestUrl(input);
|
||||
requests.push(url);
|
||||
|
||||
if (url === TOKEN_URL) {
|
||||
return responseJson({
|
||||
access_token: "access-token",
|
||||
refresh_token: "refresh-token",
|
||||
expires_in: 3600,
|
||||
});
|
||||
}
|
||||
if (url === USERINFO_URL) {
|
||||
return responseJson({ email: "lobster@openclaw.ai" });
|
||||
}
|
||||
if ([LOAD_PROD, LOAD_DAILY, LOAD_AUTOPUSH].includes(url)) {
|
||||
return responseJson({ error: { message: "unavailable" } }, 503);
|
||||
}
|
||||
throw new Error(`Unexpected request: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const { loginGeminiCliOAuth } = await import("./oauth.js");
|
||||
await runRemoteLoginExpectingProjectId(loginGeminiCliOAuth, "env-project");
|
||||
expect(requests.filter((url) => url.includes("v1internal:loadCodeAssist"))).toHaveLength(3);
|
||||
expect(requests.some((url) => url.includes("v1internal:onboardUser"))).toBe(false);
|
||||
});
|
||||
|
||||
it("skips loadCodeAssist entirely when Gemini CLI is configured for personal OAuth", async () => {
|
||||
mockSettingsExistsSync.mockReturnValue(true);
|
||||
mockSettingsReadFileSync.mockReturnValue(
|
||||
JSON.stringify({
|
||||
security: {
|
||||
auth: {
|
||||
selectedType: "oauth-personal",
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const requests: string[] = [];
|
||||
const fetchMock = vi.fn(async (input: string | URL | Request) => {
|
||||
const url = getRequestUrl(input);
|
||||
requests.push(url);
|
||||
|
||||
if (url === TOKEN_URL) {
|
||||
return responseJson({
|
||||
access_token: "access-token",
|
||||
refresh_token: "refresh-token",
|
||||
expires_in: 3600,
|
||||
});
|
||||
}
|
||||
if (url === USERINFO_URL) {
|
||||
return responseJson({ email: "lobster@openclaw.ai" });
|
||||
}
|
||||
throw new Error(`Unexpected request: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const { loginGeminiCliOAuth } = await import("./oauth.js");
|
||||
const { result } = await runRemoteLoginWithCapturedAuthUrl(loginGeminiCliOAuth);
|
||||
|
||||
expect(result.projectId).toBeUndefined();
|
||||
expect(requests).toEqual([TOKEN_URL, USERINFO_URL]);
|
||||
});
|
||||
});
|
||||
60
openclaw/extensions/google/oauth.token.ts
Normal file
60
openclaw/extensions/google/oauth.token.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import { resolveOAuthClientConfig } from "./oauth.credentials.js";
|
||||
import { fetchWithTimeout } from "./oauth.http.js";
|
||||
import { resolveGoogleOAuthIdentity, resolveGooglePersonalOAuthIdentity } from "./oauth.project.js";
|
||||
import { isGeminiCliPersonalOAuth } from "./oauth.settings.js";
|
||||
import { REDIRECT_URI, TOKEN_URL, type GeminiCliOAuthCredentials } from "./oauth.shared.js";
|
||||
|
||||
export async function exchangeCodeForTokens(
|
||||
code: string,
|
||||
verifier: string,
|
||||
): Promise<GeminiCliOAuthCredentials> {
|
||||
const { clientId, clientSecret } = resolveOAuthClientConfig();
|
||||
const body = new URLSearchParams({
|
||||
client_id: clientId,
|
||||
code,
|
||||
grant_type: "authorization_code",
|
||||
redirect_uri: REDIRECT_URI,
|
||||
code_verifier: verifier,
|
||||
});
|
||||
if (clientSecret) {
|
||||
body.set("client_secret", clientSecret);
|
||||
}
|
||||
|
||||
const response = await fetchWithTimeout(TOKEN_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
|
||||
Accept: "*/*",
|
||||
"User-Agent": "google-api-nodejs-client/9.15.1",
|
||||
},
|
||||
body,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`Token exchange failed: ${errorText}`);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
access_token: string;
|
||||
refresh_token: string;
|
||||
expires_in: number;
|
||||
};
|
||||
|
||||
if (!data.refresh_token) {
|
||||
throw new Error("No refresh token received. Please try again.");
|
||||
}
|
||||
|
||||
const identity = isGeminiCliPersonalOAuth()
|
||||
? await resolveGooglePersonalOAuthIdentity(data.access_token)
|
||||
: await resolveGoogleOAuthIdentity(data.access_token);
|
||||
const expiresAt = Date.now() + data.expires_in * 1000 - 5 * 60 * 1000;
|
||||
|
||||
return {
|
||||
refresh: data.refresh_token,
|
||||
access: data.access_token,
|
||||
expires: expiresAt,
|
||||
projectId: identity.projectId,
|
||||
email: identity.email,
|
||||
};
|
||||
}
|
||||
92
openclaw/extensions/google/oauth.ts
Normal file
92
openclaw/extensions/google/oauth.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import { clearCredentialsCache, extractGeminiCliCredentials } from "./oauth.credentials.js";
|
||||
import {
|
||||
buildAuthUrl,
|
||||
generateOAuthState,
|
||||
generatePkce,
|
||||
parseCallbackInput,
|
||||
shouldUseManualOAuthFlow,
|
||||
waitForLocalCallback,
|
||||
} from "./oauth.flow.js";
|
||||
import type { GeminiCliOAuthContext, GeminiCliOAuthCredentials } from "./oauth.shared.js";
|
||||
import { exchangeCodeForTokens } from "./oauth.token.js";
|
||||
|
||||
export { clearCredentialsCache, extractGeminiCliCredentials };
|
||||
export type { GeminiCliOAuthContext, GeminiCliOAuthCredentials };
|
||||
|
||||
export async function loginGeminiCliOAuth(
|
||||
ctx: GeminiCliOAuthContext,
|
||||
): Promise<GeminiCliOAuthCredentials> {
|
||||
const needsManual = shouldUseManualOAuthFlow(ctx.isRemote);
|
||||
await ctx.note(
|
||||
needsManual
|
||||
? [
|
||||
"You are running in a remote/VPS environment.",
|
||||
"A URL will be shown for you to open in your LOCAL browser.",
|
||||
"After signing in, copy the redirect URL and paste it back here.",
|
||||
].join("\n")
|
||||
: [
|
||||
"Browser will open for Google authentication.",
|
||||
"Sign in with your Google account for Gemini CLI access.",
|
||||
"The callback will be captured automatically on localhost:8085.",
|
||||
].join("\n"),
|
||||
"Gemini CLI OAuth",
|
||||
);
|
||||
|
||||
const { verifier, challenge } = generatePkce();
|
||||
const state = generateOAuthState();
|
||||
const authUrl = buildAuthUrl(challenge, state);
|
||||
|
||||
if (needsManual) {
|
||||
ctx.progress.update("OAuth URL ready");
|
||||
ctx.log(`\nOpen this URL in your LOCAL browser:\n\n${authUrl}\n`);
|
||||
ctx.progress.update("Waiting for you to paste the callback URL...");
|
||||
const callbackInput = await ctx.prompt("Paste the redirect URL here: ");
|
||||
const parsed = parseCallbackInput(callbackInput);
|
||||
if ("error" in parsed) {
|
||||
throw new Error(parsed.error);
|
||||
}
|
||||
if (parsed.state !== state) {
|
||||
throw new Error("OAuth state mismatch - please try again");
|
||||
}
|
||||
ctx.progress.update("Exchanging authorization code for tokens...");
|
||||
return exchangeCodeForTokens(parsed.code, verifier);
|
||||
}
|
||||
|
||||
ctx.progress.update("Complete sign-in in browser...");
|
||||
try {
|
||||
await ctx.openUrl(authUrl);
|
||||
} catch {
|
||||
ctx.log(`\nOpen this URL in your browser:\n\n${authUrl}\n`);
|
||||
}
|
||||
|
||||
try {
|
||||
const { code } = await waitForLocalCallback({
|
||||
expectedState: state,
|
||||
timeoutMs: 5 * 60 * 1000,
|
||||
onProgress: (msg) => ctx.progress.update(msg),
|
||||
});
|
||||
ctx.progress.update("Exchanging authorization code for tokens...");
|
||||
return await exchangeCodeForTokens(code, verifier);
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof Error &&
|
||||
(err.message.includes("EADDRINUSE") ||
|
||||
err.message.includes("port") ||
|
||||
err.message.includes("listen"))
|
||||
) {
|
||||
ctx.progress.update("Local callback server failed. Switching to manual mode...");
|
||||
ctx.log(`\nOpen this URL in your LOCAL browser:\n\n${authUrl}\n`);
|
||||
const callbackInput = await ctx.prompt("Paste the redirect URL here: ");
|
||||
const parsed = parseCallbackInput(callbackInput);
|
||||
if ("error" in parsed) {
|
||||
throw new Error(parsed.error, { cause: err });
|
||||
}
|
||||
if (parsed.state !== state) {
|
||||
throw new Error("OAuth state mismatch - please try again", { cause: err });
|
||||
}
|
||||
ctx.progress.update("Exchanging authorization code for tokens...");
|
||||
return exchangeCodeForTokens(parsed.code, verifier);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
28
openclaw/extensions/google/onboard.ts
Normal file
28
openclaw/extensions/google/onboard.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import {
|
||||
applyAgentDefaultModelPrimary,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/provider-onboard";
|
||||
|
||||
export const GOOGLE_GEMINI_DEFAULT_MODEL = "google/gemini-3.1-pro-preview";
|
||||
|
||||
export function applyGoogleGeminiModelDefault(cfg: OpenClawConfig): {
|
||||
next: OpenClawConfig;
|
||||
changed: boolean;
|
||||
} {
|
||||
const current = cfg.agents?.defaults?.model as unknown;
|
||||
const currentPrimary =
|
||||
typeof current === "string"
|
||||
? current.trim() || undefined
|
||||
: current &&
|
||||
typeof current === "object" &&
|
||||
typeof (current as { primary?: unknown }).primary === "string"
|
||||
? ((current as { primary: string }).primary || "").trim() || undefined
|
||||
: undefined;
|
||||
if (currentPrimary === GOOGLE_GEMINI_DEFAULT_MODEL) {
|
||||
return { next: cfg, changed: false };
|
||||
}
|
||||
return {
|
||||
next: applyAgentDefaultModelPrimary(cfg, GOOGLE_GEMINI_DEFAULT_MODEL),
|
||||
changed: true,
|
||||
};
|
||||
}
|
||||
74
openclaw/extensions/google/openclaw.plugin.json
Normal file
74
openclaw/extensions/google/openclaw.plugin.json
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
{
|
||||
"id": "google",
|
||||
"enabledByDefault": true,
|
||||
"providers": ["google", "google-gemini-cli"],
|
||||
"autoEnableWhenConfiguredProviders": ["google-gemini-cli"],
|
||||
"cliBackends": ["google-gemini-cli"],
|
||||
"providerAuthEnvVars": {
|
||||
"google": ["GEMINI_API_KEY", "GOOGLE_API_KEY"]
|
||||
},
|
||||
"providerAuthChoices": [
|
||||
{
|
||||
"provider": "google",
|
||||
"method": "api-key",
|
||||
"choiceId": "gemini-api-key",
|
||||
"choiceLabel": "Google Gemini API key",
|
||||
"groupId": "google",
|
||||
"groupLabel": "Google",
|
||||
"groupHint": "Gemini API key + OAuth",
|
||||
"optionKey": "geminiApiKey",
|
||||
"cliFlag": "--gemini-api-key",
|
||||
"cliOption": "--gemini-api-key <key>",
|
||||
"cliDescription": "Gemini API key"
|
||||
},
|
||||
{
|
||||
"provider": "google-gemini-cli",
|
||||
"method": "oauth",
|
||||
"choiceId": "google-gemini-cli",
|
||||
"choiceLabel": "Gemini CLI OAuth",
|
||||
"choiceHint": "Google OAuth with project-aware token payload",
|
||||
"groupId": "google",
|
||||
"groupLabel": "Google",
|
||||
"groupHint": "Gemini API key + OAuth"
|
||||
}
|
||||
],
|
||||
"uiHints": {
|
||||
"webSearch.apiKey": {
|
||||
"label": "Gemini Search API Key",
|
||||
"help": "Gemini API key for Google Search grounding (fallback: GEMINI_API_KEY env var).",
|
||||
"sensitive": true,
|
||||
"placeholder": "AIza..."
|
||||
},
|
||||
"webSearch.model": {
|
||||
"label": "Gemini Search Model",
|
||||
"help": "Gemini model override for web search grounding."
|
||||
}
|
||||
},
|
||||
"contracts": {
|
||||
"mediaUnderstandingProviders": ["google"],
|
||||
"memoryEmbeddingProviders": ["gemini"],
|
||||
"imageGenerationProviders": ["google"],
|
||||
"musicGenerationProviders": ["google"],
|
||||
"speechProviders": ["google"],
|
||||
"videoGenerationProviders": ["google"],
|
||||
"webSearchProviders": ["gemini"]
|
||||
},
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"webSearch": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"apiKey": {
|
||||
"type": ["string", "object"]
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
21
openclaw/extensions/google/package.json
Normal file
21
openclaw/extensions/google/package.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"name": "@openclaw/google-plugin",
|
||||
"version": "2026.4.20",
|
||||
"private": true,
|
||||
"description": "OpenClaw Google plugin",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@google/genai": "^1.50.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openclaw/plugin-sdk": "workspace:*"
|
||||
},
|
||||
"openclaw": {
|
||||
"bundle": {
|
||||
"stageRuntimeDependencies": true
|
||||
},
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
import { pluginRegistrationContractCases } from "../../test/helpers/plugins/plugin-registration-contract-cases.js";
|
||||
import { describePluginRegistrationContract } from "../../test/helpers/plugins/plugin-registration-contract.js";
|
||||
|
||||
describePluginRegistrationContract({
|
||||
...pluginRegistrationContractCases.google,
|
||||
speechProviderIds: ["google"],
|
||||
videoGenerationProviderIds: ["google"],
|
||||
webSearchProviderIds: ["gemini"],
|
||||
requireDescribeImages: true,
|
||||
requireGenerateImage: true,
|
||||
requireGenerateVideo: true,
|
||||
});
|
||||
61
openclaw/extensions/google/provider-contract-api.ts
Normal file
61
openclaw/extensions/google/provider-contract-api.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
|
||||
const noopAuth = async () => ({ profiles: [] });
|
||||
|
||||
export function createGoogleProvider(): ProviderPlugin {
|
||||
return {
|
||||
id: "google",
|
||||
label: "Google AI Studio",
|
||||
docsPath: "/providers/models",
|
||||
hookAliases: ["google-antigravity", "google-vertex"],
|
||||
envVars: ["GEMINI_API_KEY", "GOOGLE_API_KEY"],
|
||||
auth: [
|
||||
{
|
||||
id: "api-key",
|
||||
kind: "api_key",
|
||||
label: "Google Gemini API key",
|
||||
hint: "AI Studio / Gemini API key",
|
||||
run: noopAuth,
|
||||
wizard: {
|
||||
choiceId: "gemini-api-key",
|
||||
choiceLabel: "Google Gemini API key",
|
||||
groupId: "google",
|
||||
groupLabel: "Google",
|
||||
groupHint: "Gemini API key + OAuth",
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function createGoogleGeminiCliProvider(): ProviderPlugin {
|
||||
return {
|
||||
id: "google-gemini-cli",
|
||||
label: "Gemini CLI OAuth",
|
||||
docsPath: "/providers/models",
|
||||
aliases: ["gemini-cli"],
|
||||
envVars: [
|
||||
"OPENCLAW_GEMINI_OAUTH_CLIENT_ID",
|
||||
"OPENCLAW_GEMINI_OAUTH_CLIENT_SECRET",
|
||||
"GEMINI_CLI_OAUTH_CLIENT_ID",
|
||||
"GEMINI_CLI_OAUTH_CLIENT_SECRET",
|
||||
],
|
||||
auth: [
|
||||
{
|
||||
id: "oauth",
|
||||
kind: "oauth",
|
||||
label: "Google OAuth",
|
||||
hint: "PKCE + localhost callback",
|
||||
run: noopAuth,
|
||||
},
|
||||
],
|
||||
wizard: {
|
||||
setup: {
|
||||
choiceId: "google-gemini-cli",
|
||||
choiceLabel: "Gemini CLI OAuth",
|
||||
choiceHint: "Google OAuth with project-aware token payload",
|
||||
methodId: "oauth",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
9
openclaw/extensions/google/provider-hooks.ts
Normal file
9
openclaw/extensions/google/provider-hooks.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import { buildProviderReplayFamilyHooks } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { createGoogleThinkingStreamWrapper } from "./thinking-api.js";
|
||||
|
||||
export const GOOGLE_GEMINI_PROVIDER_HOOKS = {
|
||||
...buildProviderReplayFamilyHooks({
|
||||
family: "google-gemini",
|
||||
}),
|
||||
wrapStreamFn: createGoogleThinkingStreamWrapper,
|
||||
};
|
||||
412
openclaw/extensions/google/provider-models.test.ts
Normal file
412
openclaw/extensions/google/provider-models.test.ts
Normal file
|
|
@ -0,0 +1,412 @@
|
|||
import type { ModelRegistry } from "@mariozechner/pi-coding-agent";
|
||||
import type {
|
||||
ProviderResolveDynamicModelContext,
|
||||
ProviderRuntimeModel,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isModernGoogleModel, resolveGoogleGeminiForwardCompatModel } from "./provider-models.js";
|
||||
|
||||
function createTemplateModel(
|
||||
provider: string,
|
||||
id: string,
|
||||
overrides: Partial<ProviderRuntimeModel> = {},
|
||||
): ProviderRuntimeModel {
|
||||
return {
|
||||
id,
|
||||
name: id,
|
||||
provider,
|
||||
api: provider === "google-gemini-cli" ? "google-gemini-cli" : "google-generative-ai",
|
||||
baseUrl:
|
||||
provider === "google-gemini-cli"
|
||||
? "https://cloudcode-pa.googleapis.com"
|
||||
: "https://generativelanguage.googleapis.com/v1beta",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 64_000,
|
||||
...overrides,
|
||||
} as ProviderRuntimeModel;
|
||||
}
|
||||
|
||||
function createContext(params: {
|
||||
provider: string;
|
||||
modelId: string;
|
||||
models: ProviderRuntimeModel[];
|
||||
}): ProviderResolveDynamicModelContext {
|
||||
return {
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
modelRegistry: {
|
||||
find(providerId: string, modelId: string) {
|
||||
return (
|
||||
params.models.find(
|
||||
(model) =>
|
||||
model.provider === providerId && model.id.toLowerCase() === modelId.toLowerCase(),
|
||||
) ?? null
|
||||
);
|
||||
},
|
||||
} as ModelRegistry,
|
||||
};
|
||||
}
|
||||
|
||||
describe("resolveGoogleGeminiForwardCompatModel", () => {
|
||||
it("resolves stable gemini 2.5 flash-lite from direct google templates for Gemini CLI when available", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-gemini-cli",
|
||||
ctx: createContext({
|
||||
provider: "google-gemini-cli",
|
||||
modelId: "gemini-2.5-flash-lite",
|
||||
models: [createTemplateModel("google", "gemini-2.5-flash-lite")],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google-gemini-cli",
|
||||
id: "gemini-2.5-flash-lite",
|
||||
api: "google-generative-ai",
|
||||
reasoning: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves stable gemini 2.5 flash-lite from Gemini CLI templates when direct google templates are unavailable", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-gemini-cli",
|
||||
ctx: createContext({
|
||||
provider: "google-gemini-cli",
|
||||
modelId: "gemini-2.5-flash-lite",
|
||||
models: [
|
||||
createTemplateModel("google-gemini-cli", "gemini-3.1-flash-lite-preview", {
|
||||
contextWindow: 1_048_576,
|
||||
api: "google-gemini-cli",
|
||||
baseUrl: "https://cloudcode-pa.googleapis.com",
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google-gemini-cli",
|
||||
id: "gemini-2.5-flash-lite",
|
||||
api: "google-gemini-cli",
|
||||
contextWindow: 1_048_576,
|
||||
reasoning: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves gemini 3.1 pro for google aliases via an alternate template provider", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-vertex",
|
||||
ctx: createContext({
|
||||
provider: "google-vertex",
|
||||
modelId: "gemini-3.1-pro-preview",
|
||||
models: [createTemplateModel("google-gemini-cli", "gemini-3-pro-preview")],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google-vertex",
|
||||
id: "gemini-3.1-pro-preview",
|
||||
api: "google-gemini-cli",
|
||||
reasoning: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps Gemini CLI 3.1 clones sourced from CLI templates when both catalogs exist", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-gemini-cli",
|
||||
ctx: createContext({
|
||||
provider: "google-gemini-cli",
|
||||
modelId: "gemini-3.1-pro-preview",
|
||||
models: [
|
||||
createTemplateModel("google-gemini-cli", "gemini-3-pro-preview", {
|
||||
api: "google-gemini-cli",
|
||||
baseUrl: "https://cloudcode-pa.googleapis.com",
|
||||
contextWindow: 1_048_576,
|
||||
}),
|
||||
createTemplateModel("google", "gemini-3-pro-preview", {
|
||||
api: "google-generative-ai",
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
contextWindow: 200_000,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google-gemini-cli",
|
||||
id: "gemini-3.1-pro-preview",
|
||||
api: "google-gemini-cli",
|
||||
baseUrl: "https://cloudcode-pa.googleapis.com",
|
||||
contextWindow: 1_048_576,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves template reasoning metadata instead of forcing it on forward-compat clones", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google",
|
||||
ctx: createContext({
|
||||
provider: "google",
|
||||
modelId: "gemini-3.1-flash-preview",
|
||||
models: [
|
||||
createTemplateModel("google-gemini-cli", "gemini-3-flash-preview", {
|
||||
reasoning: true,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google",
|
||||
id: "gemini-3.1-flash-preview",
|
||||
api: "google-gemini-cli",
|
||||
reasoning: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves gemini 3.1 flash from direct google templates", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google",
|
||||
ctx: createContext({
|
||||
provider: "google",
|
||||
modelId: "gemini-3.1-flash-preview",
|
||||
models: [
|
||||
createTemplateModel("google", "gemini-3-flash-preview", {
|
||||
reasoning: false,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google",
|
||||
id: "gemini-3.1-flash-preview",
|
||||
api: "google-generative-ai",
|
||||
reasoning: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves Gemini latest aliases from current Google templates", () => {
|
||||
const models = [
|
||||
createTemplateModel("google", "gemini-3-pro-preview", { reasoning: true }),
|
||||
createTemplateModel("google", "gemini-3-flash-preview", { reasoning: true }),
|
||||
createTemplateModel("google", "gemini-3.1-flash-lite-preview", { reasoning: true }),
|
||||
];
|
||||
|
||||
expect(
|
||||
resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google",
|
||||
ctx: createContext({ provider: "google", modelId: "gemini-pro-latest", models }),
|
||||
}),
|
||||
).toMatchObject({
|
||||
provider: "google",
|
||||
id: "gemini-pro-latest",
|
||||
api: "google-generative-ai",
|
||||
reasoning: true,
|
||||
});
|
||||
expect(
|
||||
resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google",
|
||||
ctx: createContext({ provider: "google", modelId: "gemini-flash-latest", models }),
|
||||
}),
|
||||
).toMatchObject({
|
||||
provider: "google",
|
||||
id: "gemini-flash-latest",
|
||||
api: "google-generative-ai",
|
||||
reasoning: true,
|
||||
});
|
||||
expect(
|
||||
resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google",
|
||||
ctx: createContext({ provider: "google", modelId: "gemini-flash-lite-latest", models }),
|
||||
}),
|
||||
).toMatchObject({
|
||||
provider: "google",
|
||||
id: "gemini-flash-lite-latest",
|
||||
api: "google-generative-ai",
|
||||
reasoning: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves Antigravity Gemini 3.1 pro customtools from the low template", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-antigravity",
|
||||
ctx: createContext({
|
||||
provider: "google-antigravity",
|
||||
modelId: "gemini-3.1-pro-preview-customtools",
|
||||
models: [
|
||||
createTemplateModel("google-antigravity", "gemini-3-pro-low", {
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://antigravity.example/v1",
|
||||
contextWindow: 1_048_576,
|
||||
reasoning: true,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google-antigravity",
|
||||
id: "gemini-3.1-pro-preview-customtools",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://antigravity.example/v1",
|
||||
contextWindow: 1_048_576,
|
||||
reasoning: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to the Antigravity high template when the low template is unavailable", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-antigravity",
|
||||
ctx: createContext({
|
||||
provider: "google-antigravity",
|
||||
modelId: "gemini-3.1-pro-preview",
|
||||
models: [
|
||||
createTemplateModel("google-antigravity", "gemini-3-pro-high", {
|
||||
api: "openai-completions",
|
||||
maxTokens: 65_536,
|
||||
reasoning: true,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google-antigravity",
|
||||
id: "gemini-3.1-pro-preview",
|
||||
api: "openai-completions",
|
||||
maxTokens: 65_536,
|
||||
reasoning: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves Antigravity Gemini 3.1 flash variants from the flash template", () => {
|
||||
const models = [
|
||||
createTemplateModel("google-antigravity", "gemini-3-flash", {
|
||||
api: "openai-completions",
|
||||
contextWindow: 1_048_576,
|
||||
}),
|
||||
];
|
||||
|
||||
expect(
|
||||
resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-antigravity",
|
||||
ctx: createContext({
|
||||
provider: "google-antigravity",
|
||||
modelId: "gemini-3.1-flash-preview",
|
||||
models,
|
||||
}),
|
||||
}),
|
||||
).toMatchObject({
|
||||
provider: "google-antigravity",
|
||||
id: "gemini-3.1-flash-preview",
|
||||
api: "openai-completions",
|
||||
contextWindow: 1_048_576,
|
||||
});
|
||||
|
||||
expect(
|
||||
resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-antigravity",
|
||||
ctx: createContext({
|
||||
provider: "google-antigravity",
|
||||
modelId: "gemini-3.1-flash-lite-preview",
|
||||
models,
|
||||
}),
|
||||
}),
|
||||
).toMatchObject({
|
||||
provider: "google-antigravity",
|
||||
id: "gemini-3.1-flash-lite-preview",
|
||||
api: "openai-completions",
|
||||
contextWindow: 1_048_576,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns undefined for Antigravity Gemini 3.1 models without a matching template", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-antigravity",
|
||||
ctx: createContext({
|
||||
provider: "google-antigravity",
|
||||
modelId: "gemini-3.1-pro-preview-customtools",
|
||||
models: [createTemplateModel("google-antigravity", "claude-opus-4-6-thinking")],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toBeUndefined();
|
||||
});
|
||||
|
||||
it("prefers the flash-lite template before the broader flash prefix", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google-vertex",
|
||||
ctx: createContext({
|
||||
provider: "google-vertex",
|
||||
modelId: "gemini-3.1-flash-lite-preview",
|
||||
models: [
|
||||
createTemplateModel("google-gemini-cli", "gemini-3-flash-preview", {
|
||||
contextWindow: 128_000,
|
||||
}),
|
||||
createTemplateModel("google-gemini-cli", "gemini-3.1-flash-lite-preview", {
|
||||
contextWindow: 1_048_576,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google-vertex",
|
||||
id: "gemini-3.1-flash-lite-preview",
|
||||
contextWindow: 1_048_576,
|
||||
reasoning: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("treats gemini 2.5 ids as modern google models", () => {
|
||||
expect(isModernGoogleModel("gemini-2.5-pro")).toBe(true);
|
||||
expect(isModernGoogleModel("gemini-2.5-flash-lite")).toBe(true);
|
||||
expect(isModernGoogleModel("gemini-1.5-pro")).toBe(false);
|
||||
});
|
||||
|
||||
it("treats Gemini latest aliases as modern google models", () => {
|
||||
expect(isModernGoogleModel("gemini-pro-latest")).toBe(true);
|
||||
expect(isModernGoogleModel("gemini-flash-latest")).toBe(true);
|
||||
expect(isModernGoogleModel("gemini-flash-lite-latest")).toBe(true);
|
||||
});
|
||||
|
||||
it("treats gemma models as modern google models", () => {
|
||||
expect(isModernGoogleModel("gemma-4-26b-a4b-it")).toBe(true);
|
||||
expect(isModernGoogleModel("gemma-3-4b-it")).toBe(true);
|
||||
});
|
||||
|
||||
it("resolves Gemma 4 models with reasoning enabled regardless of template", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google",
|
||||
ctx: createContext({
|
||||
provider: "google",
|
||||
modelId: "gemma-4-26b-a4b-it",
|
||||
models: [createTemplateModel("google", "gemini-3-flash-preview", { reasoning: false })],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google",
|
||||
id: "gemma-4-26b-a4b-it",
|
||||
reasoning: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves template reasoning for non-Gemma 4 gemma models", () => {
|
||||
const model = resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: "google",
|
||||
ctx: createContext({
|
||||
provider: "google",
|
||||
modelId: "gemma-3-4b-it",
|
||||
models: [createTemplateModel("google", "gemini-3-flash-preview", { reasoning: false })],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "google",
|
||||
id: "gemma-3-4b-it",
|
||||
reasoning: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
213
openclaw/extensions/google/provider-models.ts
Normal file
213
openclaw/extensions/google/provider-models.ts
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
import type {
|
||||
ProviderResolveDynamicModelContext,
|
||||
ProviderRuntimeModel,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { cloneFirstTemplateModel } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { normalizeOptionalLowercaseString } from "openclaw/plugin-sdk/text-runtime";
|
||||
|
||||
const GOOGLE_GEMINI_CLI_PROVIDER_ID = "google-gemini-cli";
|
||||
const GOOGLE_ANTIGRAVITY_PROVIDER_ID = "google-antigravity";
|
||||
const GEMINI_2_5_PRO_PREFIX = "gemini-2.5-pro";
|
||||
const GEMINI_2_5_FLASH_LITE_PREFIX = "gemini-2.5-flash-lite";
|
||||
const GEMINI_2_5_FLASH_PREFIX = "gemini-2.5-flash";
|
||||
const GEMINI_3_1_PRO_PREFIX = "gemini-3.1-pro";
|
||||
const GEMINI_3_1_FLASH_LITE_PREFIX = "gemini-3.1-flash-lite";
|
||||
const GEMINI_3_1_FLASH_PREFIX = "gemini-3.1-flash";
|
||||
const GEMINI_PRO_LATEST_ID = "gemini-pro-latest";
|
||||
const GEMINI_FLASH_LATEST_ID = "gemini-flash-latest";
|
||||
const GEMINI_FLASH_LITE_LATEST_ID = "gemini-flash-lite-latest";
|
||||
const GEMMA_PREFIX = "gemma-";
|
||||
const GEMINI_2_5_PRO_TEMPLATE_IDS = ["gemini-2.5-pro"] as const;
|
||||
const GEMINI_2_5_FLASH_LITE_TEMPLATE_IDS = ["gemini-2.5-flash-lite"] as const;
|
||||
const GEMINI_2_5_FLASH_TEMPLATE_IDS = ["gemini-2.5-flash"] as const;
|
||||
const GEMINI_3_1_PRO_TEMPLATE_IDS = ["gemini-3-pro-preview"] as const;
|
||||
const GEMINI_3_1_FLASH_LITE_TEMPLATE_IDS = ["gemini-3.1-flash-lite-preview"] as const;
|
||||
const GEMINI_3_1_FLASH_TEMPLATE_IDS = ["gemini-3-flash-preview"] as const;
|
||||
const GEMINI_3_PRO_ANTIGRAVITY_TEMPLATE_IDS = ["gemini-3-pro-low", "gemini-3-pro-high"] as const;
|
||||
const GEMINI_3_FLASH_ANTIGRAVITY_TEMPLATE_IDS = ["gemini-3-flash"] as const;
|
||||
// Gemma uses the Gemini flash template as a forward-compat approximation
|
||||
// until a dedicated Gemma template is registered in the catalog.
|
||||
const GEMMA_TEMPLATE_IDS = GEMINI_3_1_FLASH_TEMPLATE_IDS;
|
||||
|
||||
type GoogleForwardCompatFamily = {
|
||||
googleTemplateIds: readonly string[];
|
||||
cliTemplateIds: readonly string[];
|
||||
antigravityTemplateIds?: readonly string[];
|
||||
preferExternalFirstForCli?: boolean;
|
||||
};
|
||||
|
||||
type GoogleTemplateSource = {
|
||||
templateProviderId: string;
|
||||
templateIds: readonly string[];
|
||||
};
|
||||
|
||||
function cloneGoogleTemplateModel(params: {
|
||||
providerId: string;
|
||||
modelId: string;
|
||||
templateProviderId: string;
|
||||
templateIds: readonly string[];
|
||||
ctx: ProviderResolveDynamicModelContext;
|
||||
patch?: Partial<ProviderRuntimeModel>;
|
||||
}): ProviderRuntimeModel | undefined {
|
||||
return cloneFirstTemplateModel({
|
||||
providerId: params.templateProviderId,
|
||||
modelId: params.modelId,
|
||||
templateIds: params.templateIds,
|
||||
ctx: params.ctx,
|
||||
patch: {
|
||||
...params.patch,
|
||||
provider: params.providerId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function isGoogleGeminiCliProvider(providerId: string): boolean {
|
||||
return normalizeOptionalLowercaseString(providerId) === GOOGLE_GEMINI_CLI_PROVIDER_ID;
|
||||
}
|
||||
|
||||
function isGoogleAntigravityProvider(providerId: string): boolean {
|
||||
return normalizeOptionalLowercaseString(providerId) === GOOGLE_ANTIGRAVITY_PROVIDER_ID;
|
||||
}
|
||||
|
||||
function templateIdsForProvider(
|
||||
templateProviderId: string,
|
||||
family: GoogleForwardCompatFamily,
|
||||
): readonly string[] {
|
||||
if (isGoogleGeminiCliProvider(templateProviderId)) {
|
||||
return family.cliTemplateIds;
|
||||
}
|
||||
if (isGoogleAntigravityProvider(templateProviderId)) {
|
||||
return family.antigravityTemplateIds ?? family.googleTemplateIds;
|
||||
}
|
||||
return family.googleTemplateIds;
|
||||
}
|
||||
|
||||
function buildGoogleTemplateSources(params: {
|
||||
providerId: string;
|
||||
templateProviderId?: string;
|
||||
family: GoogleForwardCompatFamily;
|
||||
}): GoogleTemplateSource[] {
|
||||
const defaultTemplateProviderId = params.templateProviderId?.trim()
|
||||
? params.templateProviderId
|
||||
: isGoogleGeminiCliProvider(params.providerId)
|
||||
? "google"
|
||||
: GOOGLE_GEMINI_CLI_PROVIDER_ID;
|
||||
const preferredExternalFirst =
|
||||
isGoogleGeminiCliProvider(params.providerId) &&
|
||||
params.family.preferExternalFirstForCli === true;
|
||||
const orderedTemplateProviderIds = preferredExternalFirst
|
||||
? [defaultTemplateProviderId, params.providerId]
|
||||
: [params.providerId, defaultTemplateProviderId];
|
||||
|
||||
const seen = new Set<string>();
|
||||
const sources: GoogleTemplateSource[] = [];
|
||||
for (const providerId of orderedTemplateProviderIds) {
|
||||
const trimmed = providerId?.trim();
|
||||
if (!trimmed || seen.has(trimmed)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(trimmed);
|
||||
sources.push({
|
||||
templateProviderId: trimmed,
|
||||
templateIds: templateIdsForProvider(trimmed, params.family),
|
||||
});
|
||||
}
|
||||
return sources;
|
||||
}
|
||||
|
||||
export function resolveGoogleGeminiForwardCompatModel(params: {
|
||||
providerId: string;
|
||||
templateProviderId?: string;
|
||||
ctx: ProviderResolveDynamicModelContext;
|
||||
}): ProviderRuntimeModel | undefined {
|
||||
const trimmed = params.ctx.modelId.trim();
|
||||
const lower = normalizeOptionalLowercaseString(trimmed) ?? "";
|
||||
|
||||
let family: GoogleForwardCompatFamily;
|
||||
let patch: Partial<ProviderRuntimeModel> | undefined;
|
||||
if (lower.startsWith(GEMINI_2_5_PRO_PREFIX)) {
|
||||
family = {
|
||||
googleTemplateIds: GEMINI_2_5_PRO_TEMPLATE_IDS,
|
||||
cliTemplateIds: GEMINI_3_1_PRO_TEMPLATE_IDS,
|
||||
preferExternalFirstForCli: true,
|
||||
};
|
||||
} else if (lower.startsWith(GEMINI_2_5_FLASH_LITE_PREFIX)) {
|
||||
family = {
|
||||
googleTemplateIds: GEMINI_2_5_FLASH_LITE_TEMPLATE_IDS,
|
||||
cliTemplateIds: GEMINI_3_1_FLASH_LITE_TEMPLATE_IDS,
|
||||
preferExternalFirstForCli: true,
|
||||
};
|
||||
} else if (lower.startsWith(GEMINI_2_5_FLASH_PREFIX)) {
|
||||
family = {
|
||||
googleTemplateIds: GEMINI_2_5_FLASH_TEMPLATE_IDS,
|
||||
cliTemplateIds: GEMINI_3_1_FLASH_TEMPLATE_IDS,
|
||||
preferExternalFirstForCli: true,
|
||||
};
|
||||
} else if (lower.startsWith(GEMINI_3_1_PRO_PREFIX) || lower === GEMINI_PRO_LATEST_ID) {
|
||||
family = {
|
||||
googleTemplateIds: GEMINI_3_1_PRO_TEMPLATE_IDS,
|
||||
cliTemplateIds: GEMINI_3_1_PRO_TEMPLATE_IDS,
|
||||
antigravityTemplateIds: GEMINI_3_PRO_ANTIGRAVITY_TEMPLATE_IDS,
|
||||
};
|
||||
if (params.providerId === "google" || params.providerId === GOOGLE_GEMINI_CLI_PROVIDER_ID) {
|
||||
patch = { reasoning: true };
|
||||
}
|
||||
} else if (
|
||||
lower.startsWith(GEMINI_3_1_FLASH_LITE_PREFIX) ||
|
||||
lower === GEMINI_FLASH_LITE_LATEST_ID
|
||||
) {
|
||||
family = {
|
||||
googleTemplateIds: GEMINI_3_1_FLASH_LITE_TEMPLATE_IDS,
|
||||
cliTemplateIds: GEMINI_3_1_FLASH_LITE_TEMPLATE_IDS,
|
||||
antigravityTemplateIds: GEMINI_3_FLASH_ANTIGRAVITY_TEMPLATE_IDS,
|
||||
};
|
||||
} else if (lower.startsWith(GEMINI_3_1_FLASH_PREFIX) || lower === GEMINI_FLASH_LATEST_ID) {
|
||||
family = {
|
||||
googleTemplateIds: GEMINI_3_1_FLASH_TEMPLATE_IDS,
|
||||
cliTemplateIds: GEMINI_3_1_FLASH_TEMPLATE_IDS,
|
||||
antigravityTemplateIds: GEMINI_3_FLASH_ANTIGRAVITY_TEMPLATE_IDS,
|
||||
};
|
||||
} else if (lower.startsWith(GEMMA_PREFIX)) {
|
||||
family = {
|
||||
googleTemplateIds: GEMMA_TEMPLATE_IDS,
|
||||
cliTemplateIds: GEMMA_TEMPLATE_IDS,
|
||||
};
|
||||
if (lower.startsWith("gemma-4")) {
|
||||
patch = { reasoning: true };
|
||||
}
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
for (const source of buildGoogleTemplateSources({
|
||||
providerId: params.providerId,
|
||||
templateProviderId: params.templateProviderId,
|
||||
family,
|
||||
})) {
|
||||
const model = cloneGoogleTemplateModel({
|
||||
providerId: params.providerId,
|
||||
modelId: trimmed,
|
||||
templateProviderId: source.templateProviderId,
|
||||
templateIds: source.templateIds,
|
||||
ctx: params.ctx,
|
||||
patch,
|
||||
});
|
||||
if (model) {
|
||||
return model;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function isModernGoogleModel(modelId: string): boolean {
|
||||
const lower = normalizeOptionalLowercaseString(modelId) ?? "";
|
||||
return (
|
||||
lower.startsWith("gemini-2.5") ||
|
||||
lower.startsWith("gemini-3") ||
|
||||
lower === GEMINI_PRO_LATEST_ID ||
|
||||
lower === GEMINI_FLASH_LATEST_ID ||
|
||||
lower === GEMINI_FLASH_LITE_LATEST_ID ||
|
||||
lower.startsWith(GEMMA_PREFIX)
|
||||
);
|
||||
}
|
||||
47
openclaw/extensions/google/provider-policy-api.test.ts
Normal file
47
openclaw/extensions/google/provider-policy-api.test.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeConfig } from "./provider-policy-api.js";
|
||||
|
||||
describe("google provider policy public artifact", () => {
|
||||
it("normalizes Google provider config without loading the full provider plugin", () => {
|
||||
expect(
|
||||
normalizeConfig({
|
||||
provider: "google",
|
||||
providerConfig: {
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
api: "google-generative-ai",
|
||||
apiKey: "GEMINI_API_KEY",
|
||||
models: [
|
||||
{
|
||||
id: "gemini-3-pro",
|
||||
name: "Gemini 3 Pro",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1_048_576,
|
||||
maxTokens: 65_536,
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
models: [{ id: "gemini-3-pro-preview" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves explicit OpenAI-compatible Google endpoints during normalization", () => {
|
||||
expect(
|
||||
normalizeConfig({
|
||||
provider: "google",
|
||||
providerConfig: {
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
api: "openai-completions",
|
||||
models: [],
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
api: "openai-completions",
|
||||
});
|
||||
});
|
||||
});
|
||||
6
openclaw/extensions/google/provider-policy-api.ts
Normal file
6
openclaw/extensions/google/provider-policy-api.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-types";
|
||||
import { normalizeGoogleProviderConfig } from "./provider-policy.js";
|
||||
|
||||
export function normalizeConfig(params: { provider: string; providerConfig: ModelProviderConfig }) {
|
||||
return normalizeGoogleProviderConfig(params.provider, params.providerConfig);
|
||||
}
|
||||
177
openclaw/extensions/google/provider-policy.ts
Normal file
177
openclaw/extensions/google/provider-policy.ts
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-types";
|
||||
import { normalizeAntigravityModelId, normalizeGoogleModelId } from "./model-id.js";
|
||||
|
||||
type GoogleApiCarrier = {
|
||||
api?: string | null;
|
||||
};
|
||||
|
||||
type GoogleProviderConfigLike = GoogleApiCarrier & {
|
||||
models?: ReadonlyArray<GoogleApiCarrier | null | undefined> | null;
|
||||
};
|
||||
|
||||
export const DEFAULT_GOOGLE_API_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
|
||||
|
||||
function normalizeOptionalString(value: unknown): string | undefined {
|
||||
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function trimTrailingSlashes(value: string): string {
|
||||
return value.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function isCanonicalGoogleApiOriginShorthand(value: string): boolean {
|
||||
return /^https:\/\/generativelanguage\.googleapis\.com\/?$/i.test(value);
|
||||
}
|
||||
|
||||
function isGoogleGenerativeAiUrl(url: URL): boolean {
|
||||
return (
|
||||
url.protocol === "https:" && url.hostname.toLowerCase() === "generativelanguage.googleapis.com"
|
||||
);
|
||||
}
|
||||
|
||||
function stripUrlUserInfo(url: URL): void {
|
||||
url.username = "";
|
||||
url.password = "";
|
||||
}
|
||||
|
||||
export function normalizeGoogleApiBaseUrl(baseUrl?: string): string {
|
||||
const raw = trimTrailingSlashes(normalizeOptionalString(baseUrl) || DEFAULT_GOOGLE_API_BASE_URL);
|
||||
try {
|
||||
const url = new URL(raw);
|
||||
url.hash = "";
|
||||
url.search = "";
|
||||
stripUrlUserInfo(url);
|
||||
if (isGoogleGenerativeAiUrl(url)) {
|
||||
const normalizedPath = trimTrailingSlashes(url.pathname || "");
|
||||
url.pathname = normalizedPath || "/v1beta";
|
||||
}
|
||||
return trimTrailingSlashes(url.toString());
|
||||
} catch {
|
||||
if (isCanonicalGoogleApiOriginShorthand(raw)) {
|
||||
return DEFAULT_GOOGLE_API_BASE_URL;
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
}
|
||||
|
||||
export function isGoogleGenerativeAiApi(api?: string | null): boolean {
|
||||
return api === "google-generative-ai";
|
||||
}
|
||||
|
||||
export function normalizeGoogleGenerativeAiBaseUrl(baseUrl?: string): string | undefined {
|
||||
if (!baseUrl) {
|
||||
return baseUrl;
|
||||
}
|
||||
|
||||
const normalized = normalizeGoogleApiBaseUrl(baseUrl);
|
||||
try {
|
||||
const url = new URL(normalized);
|
||||
stripUrlUserInfo(url);
|
||||
if (isGoogleGenerativeAiUrl(url)) {
|
||||
url.pathname = trimTrailingSlashes(url.pathname || "").replace(/\/openai$/i, "") || "/v1beta";
|
||||
return trimTrailingSlashes(url.toString());
|
||||
}
|
||||
} catch {
|
||||
// `normalizeGoogleApiBaseUrl` already returned the best-effort input form.
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function resolveGoogleGenerativeAiTransport<TApi extends string | null | undefined>(params: {
|
||||
api: TApi;
|
||||
baseUrl?: string;
|
||||
}): { api: TApi; baseUrl?: string } {
|
||||
return {
|
||||
api: params.api,
|
||||
baseUrl: isGoogleGenerativeAiApi(params.api)
|
||||
? normalizeGoogleGenerativeAiBaseUrl(params.baseUrl)
|
||||
: params.baseUrl,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveGoogleGenerativeAiApiOrigin(baseUrl?: string): string {
|
||||
return (
|
||||
normalizeGoogleGenerativeAiBaseUrl(baseUrl) ?? normalizeGoogleApiBaseUrl(baseUrl)
|
||||
).replace(/\/v1beta$/i, "");
|
||||
}
|
||||
|
||||
export function shouldNormalizeGoogleGenerativeAiProviderConfig(
|
||||
providerKey: string,
|
||||
provider: GoogleProviderConfigLike,
|
||||
): boolean {
|
||||
if (isGoogleGenerativeAiApi(provider.api)) {
|
||||
return true;
|
||||
}
|
||||
const hasGoogleGenerativeAiModelApi =
|
||||
provider.models?.some((model) => isGoogleGenerativeAiApi(model?.api)) ?? false;
|
||||
if (hasGoogleGenerativeAiModelApi) {
|
||||
return true;
|
||||
}
|
||||
if (providerKey !== "google" && providerKey !== "google-vertex") {
|
||||
return false;
|
||||
}
|
||||
const hasExplicitNonGoogleApi = normalizeOptionalString(provider.api) !== undefined;
|
||||
return !hasExplicitNonGoogleApi;
|
||||
}
|
||||
|
||||
export function shouldNormalizeGoogleProviderConfig(
|
||||
providerKey: string,
|
||||
provider: GoogleProviderConfigLike,
|
||||
): boolean {
|
||||
return (
|
||||
providerKey === "google-antigravity" ||
|
||||
shouldNormalizeGoogleGenerativeAiProviderConfig(providerKey, provider)
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeProviderModels(
|
||||
provider: ModelProviderConfig,
|
||||
normalizeId: (id: string) => string,
|
||||
): ModelProviderConfig {
|
||||
const models = provider.models;
|
||||
if (!Array.isArray(models) || models.length === 0) {
|
||||
return provider;
|
||||
}
|
||||
|
||||
let mutated = false;
|
||||
const nextModels = models.map((model) => {
|
||||
const nextId = normalizeId(model.id);
|
||||
if (nextId === model.id) {
|
||||
return model;
|
||||
}
|
||||
mutated = true;
|
||||
return Object.assign({}, model, { id: nextId });
|
||||
});
|
||||
|
||||
return mutated ? { ...provider, models: nextModels } : provider;
|
||||
}
|
||||
|
||||
export function normalizeGoogleProviderConfig(
|
||||
providerKey: string,
|
||||
provider: ModelProviderConfig,
|
||||
): ModelProviderConfig {
|
||||
let nextProvider = provider;
|
||||
const shouldNormalizeModelIds =
|
||||
providerKey === "google-vertex" ||
|
||||
shouldNormalizeGoogleGenerativeAiProviderConfig(providerKey, nextProvider);
|
||||
|
||||
if (shouldNormalizeModelIds) {
|
||||
const modelNormalized = normalizeProviderModels(nextProvider, normalizeGoogleModelId);
|
||||
if (shouldNormalizeGoogleGenerativeAiProviderConfig(providerKey, modelNormalized)) {
|
||||
const normalizedBaseUrl = normalizeGoogleGenerativeAiBaseUrl(modelNormalized.baseUrl);
|
||||
nextProvider =
|
||||
normalizedBaseUrl !== modelNormalized.baseUrl
|
||||
? { ...modelNormalized, baseUrl: normalizedBaseUrl ?? modelNormalized.baseUrl }
|
||||
: modelNormalized;
|
||||
} else {
|
||||
nextProvider = modelNormalized;
|
||||
}
|
||||
}
|
||||
|
||||
if (providerKey === "google-antigravity") {
|
||||
nextProvider = normalizeProviderModels(nextProvider, normalizeAntigravityModelId);
|
||||
}
|
||||
|
||||
return nextProvider;
|
||||
}
|
||||
63
openclaw/extensions/google/provider-registration.ts
Normal file
63
openclaw/extensions/google/provider-registration.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth-api-key";
|
||||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { normalizeGoogleModelId } from "./model-id.js";
|
||||
import { GOOGLE_GEMINI_DEFAULT_MODEL, applyGoogleGeminiModelDefault } from "./onboard.js";
|
||||
import { GOOGLE_GEMINI_PROVIDER_HOOKS } from "./provider-hooks.js";
|
||||
import { isModernGoogleModel, resolveGoogleGeminiForwardCompatModel } from "./provider-models.js";
|
||||
import {
|
||||
normalizeGoogleProviderConfig,
|
||||
resolveGoogleGenerativeAiTransport,
|
||||
} from "./provider-policy.js";
|
||||
import { createGoogleGenerativeAiTransportStreamFn } from "./transport-stream.js";
|
||||
|
||||
export function buildGoogleProvider(): ProviderPlugin {
|
||||
return {
|
||||
id: "google",
|
||||
label: "Google AI Studio",
|
||||
docsPath: "/providers/models",
|
||||
hookAliases: ["google-antigravity", "google-vertex"],
|
||||
envVars: ["GEMINI_API_KEY", "GOOGLE_API_KEY"],
|
||||
auth: [
|
||||
createProviderApiKeyAuthMethod({
|
||||
providerId: "google",
|
||||
methodId: "api-key",
|
||||
label: "Google Gemini API key",
|
||||
hint: "AI Studio / Gemini API key",
|
||||
optionKey: "geminiApiKey",
|
||||
flagName: "--gemini-api-key",
|
||||
envVar: "GEMINI_API_KEY",
|
||||
promptMessage: "Enter Gemini API key",
|
||||
defaultModel: GOOGLE_GEMINI_DEFAULT_MODEL,
|
||||
expectedProviders: ["google"],
|
||||
applyConfig: (cfg) => applyGoogleGeminiModelDefault(cfg).next,
|
||||
wizard: {
|
||||
choiceId: "gemini-api-key",
|
||||
choiceLabel: "Google Gemini API key",
|
||||
groupId: "google",
|
||||
groupLabel: "Google",
|
||||
groupHint: "Gemini API key + OAuth",
|
||||
},
|
||||
}),
|
||||
],
|
||||
normalizeTransport: ({ api, baseUrl }) => resolveGoogleGenerativeAiTransport({ api, baseUrl }),
|
||||
normalizeConfig: ({ provider, providerConfig }) =>
|
||||
normalizeGoogleProviderConfig(provider, providerConfig),
|
||||
normalizeModelId: ({ modelId }) => normalizeGoogleModelId(modelId),
|
||||
resolveDynamicModel: (ctx) =>
|
||||
resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: ctx.provider,
|
||||
ctx,
|
||||
}),
|
||||
createStreamFn: ({ model }) =>
|
||||
model.api === "google-generative-ai"
|
||||
? createGoogleGenerativeAiTransportStreamFn()
|
||||
: undefined,
|
||||
...GOOGLE_GEMINI_PROVIDER_HOOKS,
|
||||
isModernModelRef: ({ modelId }) => isModernGoogleModel(modelId),
|
||||
};
|
||||
}
|
||||
|
||||
export function registerGoogleProvider(api: OpenClawPluginApi) {
|
||||
api.registerProvider(buildGoogleProvider());
|
||||
}
|
||||
19
openclaw/extensions/google/runtime-api.ts
Normal file
19
openclaw/extensions/google/runtime-api.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
export {
|
||||
DEFAULT_GOOGLE_API_BASE_URL,
|
||||
createGoogleThinkingPayloadWrapper,
|
||||
createGoogleThinkingStreamWrapper,
|
||||
isGoogleGemini3FlashModel,
|
||||
isGoogleGemini3ProModel,
|
||||
isGoogleGemini3ThinkingLevelModel,
|
||||
isGoogleThinkingRequiredModel,
|
||||
normalizeGoogleApiBaseUrl,
|
||||
normalizeGoogleModelId,
|
||||
parseGeminiAuth,
|
||||
buildGoogleGenerativeAiParams,
|
||||
createGoogleGenerativeAiTransportStreamFn,
|
||||
resolveGoogleGemini3ThinkingLevel,
|
||||
resolveGoogleGenerativeAiHttpRequestConfig,
|
||||
sanitizeGoogleThinkingPayload,
|
||||
stripInvalidGoogleThinkingBudget,
|
||||
} from "./api.js";
|
||||
export type { GoogleThinkingInputLevel, GoogleThinkingLevel } from "./api.js";
|
||||
11
openclaw/extensions/google/setup-api.ts
Normal file
11
openclaw/extensions/google/setup-api.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { buildGoogleGeminiCliBackend } from "./cli-backend.js";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "google",
|
||||
name: "Google Setup",
|
||||
description: "Lightweight Google setup hooks",
|
||||
register(api) {
|
||||
api.registerCliBackend(buildGoogleGeminiCliBackend());
|
||||
},
|
||||
});
|
||||
248
openclaw/extensions/google/speech-provider.test.ts
Normal file
248
openclaw/extensions/google/speech-provider.test.ts
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildGoogleSpeechProvider, __testing } from "./speech-provider.js";
|
||||
|
||||
function installGoogleTtsFetchMock(pcm = Buffer.from([1, 0, 2, 0])) {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{
|
||||
inlineData: {
|
||||
mimeType: "audio/L16;codec=pcm;rate=24000",
|
||||
data: pcm.toString("base64"),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
return fetchMock;
|
||||
}
|
||||
|
||||
describe("Google speech provider", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("synthesizes Gemini PCM as WAV and preserves audio tags in the request text", async () => {
|
||||
const fetchMock = installGoogleTtsFetchMock();
|
||||
const provider = buildGoogleSpeechProvider();
|
||||
|
||||
const result = await provider.synthesize({
|
||||
text: "[whispers] The door is open.",
|
||||
cfg: {},
|
||||
providerConfig: {
|
||||
apiKey: "google-test-key",
|
||||
model: "google/gemini-3.1-flash-tts",
|
||||
voiceName: "Puck",
|
||||
},
|
||||
target: "audio-file",
|
||||
timeoutMs: 12_345,
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-tts-preview:generateContent",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ text: "[whispers] The door is open." }],
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseModalities: ["AUDIO"],
|
||||
speechConfig: {
|
||||
voiceConfig: {
|
||||
prebuiltVoiceConfig: {
|
||||
voiceName: "Puck",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const [, init] = fetchMock.mock.calls[0];
|
||||
expect(new Headers(init.headers).get("x-goog-api-key")).toBe("google-test-key");
|
||||
expect(result.outputFormat).toBe("wav");
|
||||
expect(result.fileExtension).toBe(".wav");
|
||||
expect(result.voiceCompatible).toBe(false);
|
||||
expect(result.audioBuffer.subarray(0, 4).toString("ascii")).toBe("RIFF");
|
||||
expect(result.audioBuffer.subarray(8, 12).toString("ascii")).toBe("WAVE");
|
||||
expect(result.audioBuffer.readUInt32LE(24)).toBe(__testing.GOOGLE_TTS_SAMPLE_RATE);
|
||||
expect(result.audioBuffer.subarray(44)).toEqual(Buffer.from([1, 0, 2, 0]));
|
||||
});
|
||||
|
||||
it("falls back to GEMINI_API_KEY and configured Google API base URL", async () => {
|
||||
vi.stubEnv("GEMINI_API_KEY", "env-google-key");
|
||||
const fetchMock = installGoogleTtsFetchMock();
|
||||
const provider = buildGoogleSpeechProvider();
|
||||
|
||||
expect(provider.isConfigured({ providerConfig: {}, timeoutMs: 1 })).toBe(true);
|
||||
|
||||
await provider.synthesize({
|
||||
text: "Read this plainly.",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
google: {
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
providerConfig: {},
|
||||
target: "voice-note",
|
||||
timeoutMs: 10_000,
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-tts-preview:generateContent",
|
||||
expect.any(Object),
|
||||
);
|
||||
const [, init] = fetchMock.mock.calls[0];
|
||||
expect(new Headers(init.headers).get("x-goog-api-key")).toBe("env-google-key");
|
||||
});
|
||||
|
||||
it("can reuse a configured Google model-provider API key without auth profiles", async () => {
|
||||
const fetchMock = installGoogleTtsFetchMock();
|
||||
const provider = buildGoogleSpeechProvider();
|
||||
const cfg = {
|
||||
models: {
|
||||
providers: {
|
||||
google: {
|
||||
apiKey: "model-provider-google-key",
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(provider.isConfigured({ cfg, providerConfig: {}, timeoutMs: 1 })).toBe(true);
|
||||
|
||||
await provider.synthesize({
|
||||
text: "Use the configured model provider key.",
|
||||
cfg,
|
||||
providerConfig: {},
|
||||
target: "audio-file",
|
||||
timeoutMs: 10_000,
|
||||
});
|
||||
|
||||
const [, init] = fetchMock.mock.calls[0];
|
||||
expect(new Headers(init.headers).get("x-goog-api-key")).toBe("model-provider-google-key");
|
||||
});
|
||||
|
||||
it("returns Gemini PCM directly for telephony synthesis", async () => {
|
||||
const pcm = Buffer.from([3, 0, 4, 0]);
|
||||
installGoogleTtsFetchMock(pcm);
|
||||
const provider = buildGoogleSpeechProvider();
|
||||
|
||||
const result = await provider.synthesizeTelephony?.({
|
||||
text: "Phone call audio.",
|
||||
cfg: {},
|
||||
providerConfig: {
|
||||
apiKey: "google-test-key",
|
||||
voice: "Kore",
|
||||
},
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
audioBuffer: pcm,
|
||||
outputFormat: "pcm",
|
||||
sampleRate: 24_000,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves provider config and directive overrides", () => {
|
||||
const provider = buildGoogleSpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.resolveConfig?.({
|
||||
cfg: {},
|
||||
rawConfig: {
|
||||
providers: {
|
||||
google: {
|
||||
apiKey: "configured-key",
|
||||
model: "google/gemini-3.1-flash-tts-preview",
|
||||
voice: "Leda",
|
||||
},
|
||||
},
|
||||
},
|
||||
timeoutMs: 1,
|
||||
}),
|
||||
).toEqual({
|
||||
apiKey: "configured-key",
|
||||
baseUrl: undefined,
|
||||
model: "gemini-3.1-flash-tts-preview",
|
||||
voiceName: "Leda",
|
||||
});
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "google_voice",
|
||||
value: "Aoede",
|
||||
policy: {
|
||||
enabled: true,
|
||||
allowText: true,
|
||||
allowProvider: true,
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
allowNormalization: true,
|
||||
allowSeed: true,
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
overrides: {
|
||||
voiceName: "Aoede",
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "google_model",
|
||||
value: "gemini-3.1-flash-tts-preview",
|
||||
policy: {
|
||||
enabled: true,
|
||||
allowText: true,
|
||||
allowProvider: true,
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
allowNormalization: true,
|
||||
allowSeed: true,
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
overrides: {
|
||||
model: "gemini-3.1-flash-tts-preview",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("lists Gemini prebuilt TTS voices", async () => {
|
||||
const provider = buildGoogleSpeechProvider();
|
||||
|
||||
await expect(provider.listVoices?.({ providerConfig: {} })).resolves.toEqual(
|
||||
expect.arrayContaining([
|
||||
{ id: "Kore", name: "Kore" },
|
||||
{ id: "Puck", name: "Puck" },
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
391
openclaw/extensions/google/speech-provider.ts
Normal file
391
openclaw/extensions/google/speech-provider.ts
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
import { assertOkOrThrowHttpError, postJsonRequest } from "openclaw/plugin-sdk/provider-http";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-onboard";
|
||||
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
|
||||
import type {
|
||||
SpeechDirectiveTokenParseContext,
|
||||
SpeechProviderConfig,
|
||||
SpeechProviderOverrides,
|
||||
SpeechProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/speech-core";
|
||||
import { asObject, trimToUndefined } from "openclaw/plugin-sdk/speech-core";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { resolveGoogleGenerativeAiHttpRequestConfig } from "./api.js";
|
||||
|
||||
const DEFAULT_GOOGLE_TTS_MODEL = "gemini-3.1-flash-tts-preview";
|
||||
const DEFAULT_GOOGLE_TTS_VOICE = "Kore";
|
||||
const GOOGLE_TTS_SAMPLE_RATE = 24_000;
|
||||
const GOOGLE_TTS_CHANNELS = 1;
|
||||
const GOOGLE_TTS_BITS_PER_SAMPLE = 16;
|
||||
|
||||
const GOOGLE_TTS_VOICES = [
|
||||
"Zephyr",
|
||||
"Puck",
|
||||
"Charon",
|
||||
"Kore",
|
||||
"Fenrir",
|
||||
"Leda",
|
||||
"Orus",
|
||||
"Aoede",
|
||||
"Callirrhoe",
|
||||
"Autonoe",
|
||||
"Enceladus",
|
||||
"Iapetus",
|
||||
"Umbriel",
|
||||
"Algieba",
|
||||
"Despina",
|
||||
"Erinome",
|
||||
"Algenib",
|
||||
"Rasalgethi",
|
||||
"Laomedeia",
|
||||
"Achernar",
|
||||
"Alnilam",
|
||||
"Schedar",
|
||||
"Gacrux",
|
||||
"Pulcherrima",
|
||||
"Achird",
|
||||
"Zubenelgenubi",
|
||||
"Vindemiatrix",
|
||||
"Sadachbia",
|
||||
"Sadaltager",
|
||||
"Sulafat",
|
||||
] as const;
|
||||
|
||||
type GoogleTtsProviderConfig = {
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
model: string;
|
||||
voiceName: string;
|
||||
};
|
||||
|
||||
type GoogleTtsProviderOverrides = {
|
||||
model?: string;
|
||||
voiceName?: string;
|
||||
};
|
||||
|
||||
type Maybe<T> = T | undefined;
|
||||
|
||||
type GoogleInlineDataPart = {
|
||||
mimeType?: string;
|
||||
mime_type?: string;
|
||||
data?: string;
|
||||
};
|
||||
|
||||
type GoogleGenerateSpeechResponse = {
|
||||
candidates?: Array<{
|
||||
content?: {
|
||||
parts?: Array<{
|
||||
text?: string;
|
||||
inlineData?: GoogleInlineDataPart;
|
||||
inline_data?: GoogleInlineDataPart;
|
||||
}>;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
|
||||
function normalizeGoogleTtsModel(model: unknown): string {
|
||||
const trimmed = normalizeOptionalString(model);
|
||||
if (!trimmed) {
|
||||
return DEFAULT_GOOGLE_TTS_MODEL;
|
||||
}
|
||||
const withoutProvider = trimmed.startsWith("google/") ? trimmed.slice("google/".length) : trimmed;
|
||||
return withoutProvider === "gemini-3.1-flash-tts" ? DEFAULT_GOOGLE_TTS_MODEL : withoutProvider;
|
||||
}
|
||||
|
||||
function normalizeGoogleTtsVoiceName(voiceName: unknown): string {
|
||||
return normalizeOptionalString(voiceName) ?? DEFAULT_GOOGLE_TTS_VOICE;
|
||||
}
|
||||
|
||||
function resolveGoogleTtsEnvApiKey(): string | undefined {
|
||||
return (
|
||||
normalizeOptionalString(process.env.GEMINI_API_KEY) ??
|
||||
normalizeOptionalString(process.env.GOOGLE_API_KEY)
|
||||
);
|
||||
}
|
||||
|
||||
function resolveGoogleTtsModelProviderApiKey(cfg?: OpenClawConfig): string | undefined {
|
||||
return normalizeResolvedSecretInputString({
|
||||
value: cfg?.models?.providers?.google?.apiKey,
|
||||
path: "models.providers.google.apiKey",
|
||||
});
|
||||
}
|
||||
|
||||
function resolveGoogleTtsApiKey(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
providerConfig: SpeechProviderConfig;
|
||||
}): string | undefined {
|
||||
return (
|
||||
readGoogleTtsProviderConfig(params.providerConfig).apiKey ??
|
||||
resolveGoogleTtsModelProviderApiKey(params.cfg) ??
|
||||
resolveGoogleTtsEnvApiKey()
|
||||
);
|
||||
}
|
||||
|
||||
function resolveGoogleTtsBaseUrl(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
providerConfig: GoogleTtsProviderConfig;
|
||||
}): string | undefined {
|
||||
return (
|
||||
params.providerConfig.baseUrl ?? trimToUndefined(params.cfg?.models?.providers?.google?.baseUrl)
|
||||
);
|
||||
}
|
||||
|
||||
function resolveGoogleTtsConfigRecord(
|
||||
rawConfig: Record<string, unknown>,
|
||||
): Record<string, unknown> | undefined {
|
||||
const providers = asObject(rawConfig.providers);
|
||||
return asObject(providers?.google) ?? asObject(rawConfig.google);
|
||||
}
|
||||
|
||||
function normalizeGoogleTtsProviderConfig(
|
||||
rawConfig: Record<string, unknown>,
|
||||
): GoogleTtsProviderConfig {
|
||||
const raw = resolveGoogleTtsConfigRecord(rawConfig);
|
||||
return {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: raw?.apiKey,
|
||||
path: "messages.tts.providers.google.apiKey",
|
||||
}),
|
||||
baseUrl: trimToUndefined(raw?.baseUrl),
|
||||
model: normalizeGoogleTtsModel(raw?.model),
|
||||
voiceName: normalizeGoogleTtsVoiceName(raw?.voiceName ?? raw?.voice),
|
||||
};
|
||||
}
|
||||
|
||||
function readGoogleTtsProviderConfig(config: SpeechProviderConfig): GoogleTtsProviderConfig {
|
||||
const normalized = normalizeGoogleTtsProviderConfig({});
|
||||
return {
|
||||
apiKey: trimToUndefined(config.apiKey) ?? normalized.apiKey,
|
||||
baseUrl: trimToUndefined(config.baseUrl) ?? normalized.baseUrl,
|
||||
model: normalizeGoogleTtsModel(config.model ?? normalized.model),
|
||||
voiceName: normalizeGoogleTtsVoiceName(
|
||||
config.voiceName ?? config.voice ?? normalized.voiceName,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function readGoogleTtsOverrides(
|
||||
overrides: Maybe<SpeechProviderOverrides>,
|
||||
): GoogleTtsProviderOverrides {
|
||||
if (!overrides) {
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
model: normalizeOptionalString(overrides.model),
|
||||
voiceName: normalizeOptionalString(overrides.voiceName ?? overrides.voice),
|
||||
};
|
||||
}
|
||||
|
||||
function parseDirectiveToken(ctx: SpeechDirectiveTokenParseContext): {
|
||||
handled: boolean;
|
||||
overrides?: SpeechProviderOverrides;
|
||||
warnings?: string[];
|
||||
} {
|
||||
switch (ctx.key) {
|
||||
case "voicename":
|
||||
case "voice_name":
|
||||
case "google_voice":
|
||||
case "googlevoice":
|
||||
if (!ctx.policy.allowVoice) {
|
||||
return { handled: true };
|
||||
}
|
||||
return { handled: true, overrides: { voiceName: ctx.value } };
|
||||
case "google_model":
|
||||
case "googlemodel":
|
||||
if (!ctx.policy.allowModelId) {
|
||||
return { handled: true };
|
||||
}
|
||||
return { handled: true, overrides: { model: ctx.value } };
|
||||
default:
|
||||
return { handled: false };
|
||||
}
|
||||
}
|
||||
|
||||
function extractGoogleSpeechPcm(payload: GoogleGenerateSpeechResponse): Buffer {
|
||||
for (const candidate of payload.candidates ?? []) {
|
||||
for (const part of candidate.content?.parts ?? []) {
|
||||
const inline = part.inlineData ?? part.inline_data;
|
||||
const data = normalizeOptionalString(inline?.data);
|
||||
if (!data) {
|
||||
continue;
|
||||
}
|
||||
return Buffer.from(data, "base64");
|
||||
}
|
||||
}
|
||||
throw new Error("Google TTS response missing audio data");
|
||||
}
|
||||
|
||||
function wrapPcm16MonoToWav(pcm: Buffer, sampleRate = GOOGLE_TTS_SAMPLE_RATE): Buffer {
|
||||
const byteRate = sampleRate * GOOGLE_TTS_CHANNELS * (GOOGLE_TTS_BITS_PER_SAMPLE / 8);
|
||||
const blockAlign = GOOGLE_TTS_CHANNELS * (GOOGLE_TTS_BITS_PER_SAMPLE / 8);
|
||||
const header = Buffer.alloc(44);
|
||||
|
||||
header.write("RIFF", 0, "ascii");
|
||||
header.writeUInt32LE(36 + pcm.length, 4);
|
||||
header.write("WAVE", 8, "ascii");
|
||||
header.write("fmt ", 12, "ascii");
|
||||
header.writeUInt32LE(16, 16);
|
||||
header.writeUInt16LE(1, 20);
|
||||
header.writeUInt16LE(GOOGLE_TTS_CHANNELS, 22);
|
||||
header.writeUInt32LE(sampleRate, 24);
|
||||
header.writeUInt32LE(byteRate, 28);
|
||||
header.writeUInt16LE(blockAlign, 32);
|
||||
header.writeUInt16LE(GOOGLE_TTS_BITS_PER_SAMPLE, 34);
|
||||
header.write("data", 36, "ascii");
|
||||
header.writeUInt32LE(pcm.length, 40);
|
||||
|
||||
return Buffer.concat([header, pcm]);
|
||||
}
|
||||
|
||||
async function synthesizeGoogleTtsPcm(params: {
|
||||
text: string;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
model: string;
|
||||
voiceName: string;
|
||||
timeoutMs: number;
|
||||
}): Promise<Buffer> {
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveGoogleGenerativeAiHttpRequestConfig({
|
||||
apiKey: params.apiKey,
|
||||
baseUrl: params.baseUrl,
|
||||
capability: "audio",
|
||||
transport: "http",
|
||||
});
|
||||
|
||||
const { response: res, release } = await postJsonRequest({
|
||||
url: `${baseUrl}/models/${params.model}:generateContent`,
|
||||
headers,
|
||||
body: {
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ text: params.text }],
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseModalities: ["AUDIO"],
|
||||
speechConfig: {
|
||||
voiceConfig: {
|
||||
prebuiltVoiceConfig: {
|
||||
voiceName: params.voiceName,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
fetchFn: fetch,
|
||||
pinDns: false,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
|
||||
try {
|
||||
await assertOkOrThrowHttpError(res, "Google TTS failed");
|
||||
return extractGoogleSpeechPcm((await res.json()) as GoogleGenerateSpeechResponse);
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
export function buildGoogleSpeechProvider(): SpeechProviderPlugin {
|
||||
return {
|
||||
id: "google",
|
||||
label: "Google",
|
||||
autoSelectOrder: 50,
|
||||
models: [DEFAULT_GOOGLE_TTS_MODEL],
|
||||
voices: GOOGLE_TTS_VOICES,
|
||||
resolveConfig: ({ rawConfig }) => normalizeGoogleTtsProviderConfig(rawConfig),
|
||||
parseDirectiveToken,
|
||||
resolveTalkConfig: ({ baseTtsConfig, talkProviderConfig }) => {
|
||||
const base = normalizeGoogleTtsProviderConfig(baseTtsConfig);
|
||||
return {
|
||||
...base,
|
||||
...(talkProviderConfig.apiKey === undefined
|
||||
? {}
|
||||
: {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: talkProviderConfig.apiKey,
|
||||
path: "talk.providers.google.apiKey",
|
||||
}),
|
||||
}),
|
||||
...(trimToUndefined(talkProviderConfig.baseUrl) == null
|
||||
? {}
|
||||
: { baseUrl: trimToUndefined(talkProviderConfig.baseUrl) }),
|
||||
...(trimToUndefined(talkProviderConfig.modelId) == null
|
||||
? {}
|
||||
: { model: normalizeGoogleTtsModel(talkProviderConfig.modelId) }),
|
||||
...(trimToUndefined(talkProviderConfig.voiceId) == null
|
||||
? {}
|
||||
: { voiceName: normalizeGoogleTtsVoiceName(talkProviderConfig.voiceId) }),
|
||||
};
|
||||
},
|
||||
resolveTalkOverrides: ({ params }) => ({
|
||||
...(trimToUndefined(params.voiceId) == null
|
||||
? {}
|
||||
: { voiceName: normalizeGoogleTtsVoiceName(params.voiceId) }),
|
||||
...(trimToUndefined(params.modelId) == null
|
||||
? {}
|
||||
: { model: normalizeGoogleTtsModel(params.modelId) }),
|
||||
}),
|
||||
listVoices: async () => GOOGLE_TTS_VOICES.map((voice) => ({ id: voice, name: voice })),
|
||||
isConfigured: ({ cfg, providerConfig }) =>
|
||||
Boolean(resolveGoogleTtsApiKey({ cfg, providerConfig })),
|
||||
synthesize: async (req) => {
|
||||
const config = readGoogleTtsProviderConfig(req.providerConfig);
|
||||
const overrides = readGoogleTtsOverrides(req.providerOverrides);
|
||||
const apiKey = resolveGoogleTtsApiKey({
|
||||
cfg: req.cfg,
|
||||
providerConfig: req.providerConfig,
|
||||
});
|
||||
if (!apiKey) {
|
||||
throw new Error("Google API key missing");
|
||||
}
|
||||
const pcm = await synthesizeGoogleTtsPcm({
|
||||
text: req.text,
|
||||
apiKey,
|
||||
baseUrl: resolveGoogleTtsBaseUrl({ cfg: req.cfg, providerConfig: config }),
|
||||
model: normalizeGoogleTtsModel(overrides.model ?? config.model),
|
||||
voiceName: normalizeGoogleTtsVoiceName(overrides.voiceName ?? config.voiceName),
|
||||
timeoutMs: req.timeoutMs,
|
||||
});
|
||||
return {
|
||||
audioBuffer: wrapPcm16MonoToWav(pcm),
|
||||
outputFormat: "wav",
|
||||
fileExtension: ".wav",
|
||||
voiceCompatible: false,
|
||||
};
|
||||
},
|
||||
synthesizeTelephony: async (req) => {
|
||||
const config = readGoogleTtsProviderConfig(req.providerConfig);
|
||||
const apiKey = resolveGoogleTtsApiKey({
|
||||
cfg: req.cfg,
|
||||
providerConfig: req.providerConfig,
|
||||
});
|
||||
if (!apiKey) {
|
||||
throw new Error("Google API key missing");
|
||||
}
|
||||
const pcm = await synthesizeGoogleTtsPcm({
|
||||
text: req.text,
|
||||
apiKey,
|
||||
baseUrl: resolveGoogleTtsBaseUrl({ cfg: req.cfg, providerConfig: config }),
|
||||
model: config.model,
|
||||
voiceName: config.voiceName,
|
||||
timeoutMs: req.timeoutMs,
|
||||
});
|
||||
return {
|
||||
audioBuffer: pcm,
|
||||
outputFormat: "pcm",
|
||||
sampleRate: GOOGLE_TTS_SAMPLE_RATE,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
DEFAULT_GOOGLE_TTS_MODEL,
|
||||
DEFAULT_GOOGLE_TTS_VOICE,
|
||||
GOOGLE_TTS_SAMPLE_RATE,
|
||||
normalizeGoogleTtsModel,
|
||||
wrapPcm16MonoToWav,
|
||||
};
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
import {
|
||||
buildSearchCacheKey,
|
||||
buildUnsupportedSearchFilterResponse,
|
||||
DEFAULT_SEARCH_COUNT,
|
||||
readCachedSearchPayload,
|
||||
readConfiguredSecretString,
|
||||
readNumberParam,
|
||||
readProviderEnvValue,
|
||||
readStringParam,
|
||||
resolveCitationRedirectUrl,
|
||||
resolveSearchCacheTtlMs,
|
||||
resolveSearchCount,
|
||||
resolveSearchTimeoutSeconds,
|
||||
type SearchConfigRecord,
|
||||
withTrustedWebSearchEndpoint,
|
||||
wrapWebContent,
|
||||
writeCachedSearchPayload,
|
||||
} from "openclaw/plugin-sdk/provider-web-search";
|
||||
import { DEFAULT_GOOGLE_API_BASE_URL } from "../api.js";
|
||||
import {
|
||||
resolveGeminiConfig,
|
||||
resolveGeminiModel,
|
||||
type GeminiConfig,
|
||||
} from "./gemini-web-search-provider.shared.js";
|
||||
|
||||
const GEMINI_API_BASE = DEFAULT_GOOGLE_API_BASE_URL;
|
||||
|
||||
type GeminiGroundingResponse = {
|
||||
candidates?: Array<{
|
||||
content?: {
|
||||
parts?: Array<{
|
||||
text?: string;
|
||||
}>;
|
||||
};
|
||||
groundingMetadata?: {
|
||||
groundingChunks?: Array<{
|
||||
web?: {
|
||||
uri?: string;
|
||||
title?: string;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
}>;
|
||||
error?: {
|
||||
code?: number;
|
||||
message?: string;
|
||||
status?: string;
|
||||
};
|
||||
};
|
||||
|
||||
export function resolveGeminiRuntimeApiKey(gemini?: GeminiConfig): string | undefined {
|
||||
return (
|
||||
readConfiguredSecretString(gemini?.apiKey, "tools.web.search.gemini.apiKey") ??
|
||||
readProviderEnvValue(["GEMINI_API_KEY"])
|
||||
);
|
||||
}
|
||||
|
||||
async function runGeminiSearch(params: {
|
||||
query: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
timeoutSeconds: number;
|
||||
}): Promise<{ content: string; citations: Array<{ url: string; title?: string }> }> {
|
||||
const endpoint = `${GEMINI_API_BASE}/models/${params.model}:generateContent`;
|
||||
|
||||
return withTrustedWebSearchEndpoint(
|
||||
{
|
||||
url: endpoint,
|
||||
timeoutSeconds: params.timeoutSeconds,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"x-goog-api-key": params.apiKey,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
contents: [{ parts: [{ text: params.query }] }],
|
||||
tools: [{ google_search: {} }],
|
||||
}),
|
||||
},
|
||||
},
|
||||
async (res) => {
|
||||
if (!res.ok) {
|
||||
const safeDetail = ((await res.text()) || res.statusText).replace(
|
||||
/key=[^&\s]+/giu,
|
||||
"key=***",
|
||||
);
|
||||
throw new Error(`Gemini API error (${res.status}): ${safeDetail}`);
|
||||
}
|
||||
|
||||
let data: GeminiGroundingResponse;
|
||||
try {
|
||||
data = (await res.json()) as GeminiGroundingResponse;
|
||||
} catch (error) {
|
||||
const safeError = String(error).replace(/key=[^&\s]+/giu, "key=***");
|
||||
throw new Error(`Gemini API returned invalid JSON: ${safeError}`, { cause: error });
|
||||
}
|
||||
|
||||
if (data.error) {
|
||||
const rawMessage = data.error.message || data.error.status || "unknown";
|
||||
throw new Error(
|
||||
`Gemini API error (${data.error.code}): ${rawMessage.replace(/key=[^&\s]+/giu, "key=***")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const candidate = data.candidates?.[0];
|
||||
const content =
|
||||
candidate?.content?.parts
|
||||
?.map((part) => part.text)
|
||||
.filter(Boolean)
|
||||
.join("\n") ?? "No response";
|
||||
const rawCitations = (candidate?.groundingMetadata?.groundingChunks ?? [])
|
||||
.filter((chunk) => chunk.web?.uri)
|
||||
.map((chunk) => ({
|
||||
url: chunk.web!.uri!,
|
||||
title: chunk.web?.title || undefined,
|
||||
}));
|
||||
|
||||
const citations: Array<{ url: string; title?: string }> = [];
|
||||
for (let index = 0; index < rawCitations.length; index += 10) {
|
||||
const batch = rawCitations.slice(index, index + 10);
|
||||
const resolved = await Promise.all(
|
||||
batch.map(async (citation) =>
|
||||
Object.assign({}, citation, { url: await resolveCitationRedirectUrl(citation.url) }),
|
||||
),
|
||||
);
|
||||
citations.push(...resolved);
|
||||
}
|
||||
|
||||
return { content, citations };
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export async function executeGeminiSearch(
|
||||
args: Record<string, unknown>,
|
||||
searchConfig?: SearchConfigRecord,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const unsupportedResponse = buildUnsupportedSearchFilterResponse(args, "gemini");
|
||||
if (unsupportedResponse) {
|
||||
return unsupportedResponse;
|
||||
}
|
||||
|
||||
const geminiConfig = resolveGeminiConfig(searchConfig);
|
||||
const apiKey = resolveGeminiRuntimeApiKey(geminiConfig);
|
||||
if (!apiKey) {
|
||||
return {
|
||||
error: "missing_gemini_api_key",
|
||||
message:
|
||||
"web_search (gemini) needs an API key. Set GEMINI_API_KEY in the Gateway environment, or configure tools.web.search.gemini.apiKey.",
|
||||
docs: "https://docs.openclaw.ai/tools/web",
|
||||
};
|
||||
}
|
||||
|
||||
const query = readStringParam(args, "query", { required: true });
|
||||
const count =
|
||||
readNumberParam(args, "count", { integer: true }) ?? searchConfig?.maxResults ?? undefined;
|
||||
const model = resolveGeminiModel(geminiConfig);
|
||||
const cacheKey = buildSearchCacheKey([
|
||||
"gemini",
|
||||
query,
|
||||
resolveSearchCount(count, DEFAULT_SEARCH_COUNT),
|
||||
model,
|
||||
]);
|
||||
const cached = readCachedSearchPayload(cacheKey);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
const result = await runGeminiSearch({
|
||||
query,
|
||||
apiKey,
|
||||
model,
|
||||
timeoutSeconds: resolveSearchTimeoutSeconds(searchConfig),
|
||||
});
|
||||
const payload = {
|
||||
query,
|
||||
provider: "gemini",
|
||||
model,
|
||||
tookMs: Date.now() - start,
|
||||
externalContent: {
|
||||
untrusted: true,
|
||||
source: "web_search",
|
||||
provider: "gemini",
|
||||
wrapped: true,
|
||||
},
|
||||
content: wrapWebContent(result.content),
|
||||
citations: result.citations,
|
||||
};
|
||||
writeCachedSearchPayload(cacheKey, payload, resolveSearchCacheTtlMs(searchConfig));
|
||||
return payload;
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
export const DEFAULT_GEMINI_WEB_SEARCH_MODEL = "gemini-2.5-flash";
|
||||
|
||||
export type GeminiConfig = {
|
||||
apiKey?: unknown;
|
||||
model?: unknown;
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function trimToUndefined(value: unknown): string | undefined {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
export function resolveGeminiConfig(searchConfig?: Record<string, unknown>): GeminiConfig {
|
||||
const gemini = searchConfig?.gemini;
|
||||
return isRecord(gemini) ? gemini : {};
|
||||
}
|
||||
|
||||
export function resolveGeminiApiKey(
|
||||
gemini?: GeminiConfig,
|
||||
env: Record<string, string | undefined> = process.env,
|
||||
): string | undefined {
|
||||
return trimToUndefined(gemini?.apiKey) ?? trimToUndefined(env.GEMINI_API_KEY);
|
||||
}
|
||||
|
||||
export function resolveGeminiModel(gemini?: GeminiConfig): string {
|
||||
return trimToUndefined(gemini?.model) ?? DEFAULT_GEMINI_WEB_SEARCH_MODEL;
|
||||
}
|
||||
86
openclaw/extensions/google/src/gemini-web-search-provider.ts
Normal file
86
openclaw/extensions/google/src/gemini-web-search-provider.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import {
|
||||
createWebSearchProviderContractFields,
|
||||
mergeScopedSearchConfig,
|
||||
resolveProviderWebSearchPluginConfig,
|
||||
type WebSearchProviderPlugin,
|
||||
type WebSearchProviderToolDefinition,
|
||||
} from "openclaw/plugin-sdk/provider-web-search-config-contract";
|
||||
import { resolveGeminiApiKey, resolveGeminiModel } from "./gemini-web-search-provider.shared.js";
|
||||
|
||||
const GEMINI_CREDENTIAL_PATH = "plugins.entries.google.config.webSearch.apiKey";
|
||||
|
||||
type GeminiWebSearchRuntime = typeof import("./gemini-web-search-provider.runtime.js");
|
||||
|
||||
let geminiWebSearchRuntimePromise: Promise<GeminiWebSearchRuntime> | undefined;
|
||||
|
||||
function loadGeminiWebSearchRuntime(): Promise<GeminiWebSearchRuntime> {
|
||||
geminiWebSearchRuntimePromise ??= import("./gemini-web-search-provider.runtime.js");
|
||||
return geminiWebSearchRuntimePromise;
|
||||
}
|
||||
|
||||
const GEMINI_TOOL_PARAMETERS = {
|
||||
type: "object",
|
||||
properties: {
|
||||
query: { type: "string", description: "Search query string." },
|
||||
count: {
|
||||
type: "number",
|
||||
description: "Number of results to return (1-10).",
|
||||
minimum: 1,
|
||||
maximum: 10,
|
||||
},
|
||||
country: { type: "string", description: "Not supported by Gemini." },
|
||||
language: { type: "string", description: "Not supported by Gemini." },
|
||||
freshness: { type: "string", description: "Not supported by Gemini." },
|
||||
date_after: { type: "string", description: "Not supported by Gemini." },
|
||||
date_before: { type: "string", description: "Not supported by Gemini." },
|
||||
},
|
||||
required: ["query"],
|
||||
} satisfies Record<string, unknown>;
|
||||
|
||||
function createGeminiToolDefinition(
|
||||
searchConfig?: Record<string, unknown>,
|
||||
): WebSearchProviderToolDefinition {
|
||||
return {
|
||||
description:
|
||||
"Search the web using Gemini with Google Search grounding. Returns AI-synthesized answers with citations from Google Search.",
|
||||
parameters: GEMINI_TOOL_PARAMETERS,
|
||||
execute: async (args) => {
|
||||
const { executeGeminiSearch } = await loadGeminiWebSearchRuntime();
|
||||
return await executeGeminiSearch(args, searchConfig);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createGeminiWebSearchProvider(): WebSearchProviderPlugin {
|
||||
return {
|
||||
id: "gemini",
|
||||
label: "Gemini (Google Search)",
|
||||
hint: "Requires Google Gemini API key · Google Search grounding",
|
||||
onboardingScopes: ["text-inference"],
|
||||
credentialLabel: "Google Gemini API key",
|
||||
envVars: ["GEMINI_API_KEY"],
|
||||
placeholder: "AIza...",
|
||||
signupUrl: "https://aistudio.google.com/apikey",
|
||||
docsUrl: "https://docs.openclaw.ai/tools/web",
|
||||
autoDetectOrder: 20,
|
||||
credentialPath: GEMINI_CREDENTIAL_PATH,
|
||||
...createWebSearchProviderContractFields({
|
||||
credentialPath: GEMINI_CREDENTIAL_PATH,
|
||||
searchCredential: { type: "scoped", scopeId: "gemini" },
|
||||
configuredCredential: { pluginId: "google" },
|
||||
}),
|
||||
createTool: (ctx) =>
|
||||
createGeminiToolDefinition(
|
||||
mergeScopedSearchConfig(
|
||||
ctx.searchConfig,
|
||||
"gemini",
|
||||
resolveProviderWebSearchPluginConfig(ctx.config, "google"),
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
resolveGeminiApiKey,
|
||||
resolveGeminiModel,
|
||||
} as const;
|
||||
6
openclaw/extensions/google/test-api.ts
Normal file
6
openclaw/extensions/google/test-api.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
export { buildGoogleGeminiCliBackend } from "./cli-backend.js";
|
||||
export { buildGoogleImageGenerationProvider } from "./image-generation-provider.js";
|
||||
export { buildGoogleMusicGenerationProvider } from "./music-generation-provider.js";
|
||||
export { buildGoogleSpeechProvider } from "./speech-provider.js";
|
||||
export { googleMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
export { buildGoogleVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
13
openclaw/extensions/google/thinking-api.ts
Normal file
13
openclaw/extensions/google/thinking-api.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
export {
|
||||
createGoogleThinkingPayloadWrapper,
|
||||
createGoogleThinkingStreamWrapper,
|
||||
isGoogleGemini3FlashModel,
|
||||
isGoogleGemini3ProModel,
|
||||
isGoogleGemini3ThinkingLevelModel,
|
||||
isGoogleThinkingRequiredModel,
|
||||
resolveGoogleGemini3ThinkingLevel,
|
||||
sanitizeGoogleThinkingPayload,
|
||||
stripInvalidGoogleThinkingBudget,
|
||||
type GoogleThinkingInputLevel,
|
||||
type GoogleThinkingLevel,
|
||||
} from "./thinking.js";
|
||||
116
openclaw/extensions/google/thinking.test.ts
Normal file
116
openclaw/extensions/google/thinking.test.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
resolveGoogleGemini3ThinkingLevel,
|
||||
sanitizeGoogleThinkingPayload,
|
||||
} from "./thinking-api.js";
|
||||
|
||||
describe("google thinking policy", () => {
|
||||
it.each([
|
||||
["off", "LOW"],
|
||||
["minimal", "LOW"],
|
||||
["low", "LOW"],
|
||||
["medium", "HIGH"],
|
||||
["adaptive", "HIGH"],
|
||||
["high", "HIGH"],
|
||||
["xhigh", "HIGH"],
|
||||
] as const)("maps Gemini 3 Pro thinking level %s to %s", (thinkingLevel, expected) => {
|
||||
expect(
|
||||
resolveGoogleGemini3ThinkingLevel({
|
||||
modelId: "gemini-3.1-pro-preview",
|
||||
thinkingLevel,
|
||||
}),
|
||||
).toBe(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[0, "LOW"],
|
||||
[2048, "LOW"],
|
||||
[2049, "HIGH"],
|
||||
] as const)("maps Gemini 3 Pro budget %s to %s", (thinkingBudget, expected) => {
|
||||
expect(
|
||||
resolveGoogleGemini3ThinkingLevel({
|
||||
modelId: "gemini-pro-latest",
|
||||
thinkingBudget,
|
||||
}),
|
||||
).toBe(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["off", "MINIMAL"],
|
||||
["minimal", "MINIMAL"],
|
||||
["low", "LOW"],
|
||||
["medium", "MEDIUM"],
|
||||
["adaptive", "MEDIUM"],
|
||||
["high", "HIGH"],
|
||||
["xhigh", "HIGH"],
|
||||
] as const)("maps Gemini 3 Flash thinking level %s to %s", (thinkingLevel, expected) => {
|
||||
expect(
|
||||
resolveGoogleGemini3ThinkingLevel({
|
||||
modelId: "gemini-flash-latest",
|
||||
thinkingLevel,
|
||||
}),
|
||||
).toBe(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[-1, "MINIMAL"],
|
||||
[0, "MINIMAL"],
|
||||
[2048, "LOW"],
|
||||
[8192, "MEDIUM"],
|
||||
[8193, "HIGH"],
|
||||
] as const)("maps Gemini 3 Flash budget %s to %s", (thinkingBudget, expected) => {
|
||||
expect(
|
||||
resolveGoogleGemini3ThinkingLevel({
|
||||
modelId: "gemini-3.1-flash-lite-preview",
|
||||
thinkingBudget,
|
||||
}),
|
||||
).toBe(expected);
|
||||
});
|
||||
|
||||
it("removes thinkingBudget=0 for Gemini 2.5 Pro", () => {
|
||||
const payload = {
|
||||
config: {
|
||||
thinkingConfig: { thinkingBudget: 0 },
|
||||
},
|
||||
};
|
||||
|
||||
sanitizeGoogleThinkingPayload({ payload, modelId: "google/gemini-2.5-pro-preview" });
|
||||
|
||||
expect(payload.config).not.toHaveProperty("thinkingConfig");
|
||||
});
|
||||
|
||||
it("rewrites Gemini 3 thinking budgets to thinkingLevel", () => {
|
||||
const payload = {
|
||||
generationConfig: {
|
||||
thinkingConfig: { thinkingBudget: 8193, includeThoughts: true },
|
||||
},
|
||||
};
|
||||
|
||||
sanitizeGoogleThinkingPayload({
|
||||
payload,
|
||||
modelId: "gemini-3-flash-preview",
|
||||
thinkingLevel: "medium",
|
||||
});
|
||||
|
||||
expect(payload.generationConfig.thinkingConfig).toEqual({
|
||||
includeThoughts: true,
|
||||
thinkingLevel: "MEDIUM",
|
||||
});
|
||||
});
|
||||
|
||||
it("maps Gemma 4 thinking mode without sending thinkingBudget", () => {
|
||||
const payload = {
|
||||
config: {
|
||||
thinkingConfig: { thinkingBudget: 4096 },
|
||||
},
|
||||
};
|
||||
|
||||
sanitizeGoogleThinkingPayload({
|
||||
payload,
|
||||
modelId: "gemma-4-26b-a4b-it",
|
||||
thinkingLevel: "high",
|
||||
});
|
||||
|
||||
expect(payload.config.thinkingConfig).toEqual({ thinkingLevel: "HIGH" });
|
||||
});
|
||||
});
|
||||
13
openclaw/extensions/google/thinking.ts
Normal file
13
openclaw/extensions/google/thinking.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
export {
|
||||
createGoogleThinkingPayloadWrapper,
|
||||
createGoogleThinkingStreamWrapper,
|
||||
isGoogleGemini3FlashModel,
|
||||
isGoogleGemini3ProModel,
|
||||
isGoogleGemini3ThinkingLevelModel,
|
||||
isGoogleThinkingRequiredModel,
|
||||
resolveGoogleGemini3ThinkingLevel,
|
||||
sanitizeGoogleThinkingPayload,
|
||||
stripInvalidGoogleThinkingBudget,
|
||||
type GoogleThinkingInputLevel,
|
||||
type GoogleThinkingLevel,
|
||||
} from "openclaw/plugin-sdk/provider-stream-shared";
|
||||
479
openclaw/extensions/google/transport-stream.test.ts
Normal file
479
openclaw/extensions/google/transport-stream.test.ts
Normal file
|
|
@ -0,0 +1,479 @@
|
|||
import type { Model } from "@mariozechner/pi-ai";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { buildGuardedModelFetchMock, guardedFetchMock } = vi.hoisted(() => ({
|
||||
buildGuardedModelFetchMock: vi.fn(),
|
||||
guardedFetchMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/provider-transport-runtime", async (importOriginal) => ({
|
||||
...(await importOriginal()),
|
||||
buildGuardedModelFetch: buildGuardedModelFetchMock,
|
||||
}));
|
||||
|
||||
let buildGoogleGenerativeAiParams: typeof import("./transport-stream.js").buildGoogleGenerativeAiParams;
|
||||
let createGoogleGenerativeAiTransportStreamFn: typeof import("./transport-stream.js").createGoogleGenerativeAiTransportStreamFn;
|
||||
|
||||
const MODEL_PROVIDER_REQUEST_TRANSPORT_SYMBOL = Symbol.for(
|
||||
"openclaw.modelProviderRequestTransport",
|
||||
);
|
||||
|
||||
function attachModelProviderRequestTransport<TModel extends object>(
|
||||
model: TModel,
|
||||
request: unknown,
|
||||
): TModel {
|
||||
return {
|
||||
...model,
|
||||
[MODEL_PROVIDER_REQUEST_TRANSPORT_SYMBOL]: request,
|
||||
};
|
||||
}
|
||||
|
||||
function buildGeminiModel(
|
||||
overrides: Partial<Model<"google-generative-ai">> = {},
|
||||
): Model<"google-generative-ai"> {
|
||||
return {
|
||||
id: "gemini-2.5-pro",
|
||||
name: "Gemini 2.5 Pro",
|
||||
api: "google-generative-ai",
|
||||
provider: "google",
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSseResponse(events: unknown[]): Response {
|
||||
const sse = `${events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("")}data: [DONE]\n\n`;
|
||||
const encoder = new TextEncoder();
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(sse));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
return new Response(body, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
}
|
||||
|
||||
describe("google transport stream", () => {
|
||||
beforeAll(async () => {
|
||||
({ buildGoogleGenerativeAiParams, createGoogleGenerativeAiTransportStreamFn } =
|
||||
await import("./transport-stream.js"));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
buildGuardedModelFetchMock.mockReset();
|
||||
guardedFetchMock.mockReset();
|
||||
buildGuardedModelFetchMock.mockReturnValue(guardedFetchMock);
|
||||
});
|
||||
|
||||
it("uses the guarded fetch transport and parses Gemini SSE output", async () => {
|
||||
guardedFetchMock.mockResolvedValueOnce(
|
||||
buildSseResponse([
|
||||
{
|
||||
responseId: "resp_1",
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{ thought: true, text: "draft", thoughtSignature: "sig_1" },
|
||||
{ text: "answer" },
|
||||
{
|
||||
thoughtSignature: "call_sig_1",
|
||||
functionCall: { name: "lookup", args: { q: "hello" } },
|
||||
},
|
||||
],
|
||||
},
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
usageMetadata: {
|
||||
promptTokenCount: 10,
|
||||
cachedContentTokenCount: 2,
|
||||
candidatesTokenCount: 5,
|
||||
thoughtsTokenCount: 3,
|
||||
totalTokenCount: 18,
|
||||
},
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
const model = attachModelProviderRequestTransport(
|
||||
{
|
||||
id: "gemini-3.1-pro-preview",
|
||||
name: "Gemini 3.1 Pro Preview",
|
||||
api: "google-generative-ai",
|
||||
provider: "google",
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
headers: { "X-Provider": "google" },
|
||||
} satisfies Model<"google-generative-ai">,
|
||||
{
|
||||
proxy: {
|
||||
mode: "explicit-proxy",
|
||||
url: "http://proxy.internal:8443",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const streamFn = createGoogleGenerativeAiTransportStreamFn();
|
||||
const stream = await Promise.resolve(
|
||||
streamFn(
|
||||
model,
|
||||
{
|
||||
systemPrompt: "Follow policy.",
|
||||
messages: [{ role: "user", content: "hello", timestamp: 0 }],
|
||||
tools: [
|
||||
{
|
||||
name: "lookup",
|
||||
description: "Look up a value",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { q: { type: "string" } },
|
||||
required: ["q"],
|
||||
},
|
||||
},
|
||||
],
|
||||
} as unknown as Parameters<typeof streamFn>[1],
|
||||
{
|
||||
apiKey: "gemini-api-key",
|
||||
cachedContent: "cachedContents/request-cache",
|
||||
reasoning: "medium",
|
||||
toolChoice: "auto",
|
||||
} as Parameters<typeof streamFn>[2],
|
||||
),
|
||||
);
|
||||
const result = await stream.result();
|
||||
|
||||
expect(buildGuardedModelFetchMock).toHaveBeenCalledWith(model);
|
||||
expect(guardedFetchMock).toHaveBeenCalledWith(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-pro-preview:streamGenerateContent?alt=sse",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: expect.objectContaining({
|
||||
accept: "text/event-stream",
|
||||
"Content-Type": "application/json",
|
||||
"x-goog-api-key": "gemini-api-key",
|
||||
"X-Provider": "google",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const init = guardedFetchMock.mock.calls[0]?.[1] as RequestInit;
|
||||
const requestBody = init.body;
|
||||
if (typeof requestBody !== "string") {
|
||||
throw new Error("Expected Google transport request body to be serialized JSON");
|
||||
}
|
||||
const payload = JSON.parse(requestBody) as Record<string, unknown>;
|
||||
expect(payload.systemInstruction).toEqual({
|
||||
parts: [{ text: "Follow policy." }],
|
||||
});
|
||||
expect(payload.cachedContent).toBe("cachedContents/request-cache");
|
||||
expect(payload.generationConfig).toMatchObject({
|
||||
thinkingConfig: { includeThoughts: true, thinkingLevel: "HIGH" },
|
||||
});
|
||||
expect(payload.toolConfig).toMatchObject({
|
||||
functionCallingConfig: { mode: "AUTO" },
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
api: "google-generative-ai",
|
||||
provider: "google",
|
||||
responseId: "resp_1",
|
||||
stopReason: "toolUse",
|
||||
usage: {
|
||||
input: 8,
|
||||
output: 8,
|
||||
cacheRead: 2,
|
||||
totalTokens: 18,
|
||||
},
|
||||
content: [
|
||||
{ type: "thinking", thinking: "draft", thinkingSignature: "sig_1" },
|
||||
{ type: "text", text: "answer" },
|
||||
{
|
||||
type: "toolCall",
|
||||
name: "lookup",
|
||||
arguments: { q: "hello" },
|
||||
thoughtSignature: "call_sig_1",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("uses bearer auth when the Google api key is an OAuth JSON payload", async () => {
|
||||
guardedFetchMock.mockResolvedValueOnce(buildSseResponse([]));
|
||||
|
||||
const model = attachModelProviderRequestTransport(
|
||||
{
|
||||
id: "gemini-3-flash-preview",
|
||||
name: "Gemini 3 Flash Preview",
|
||||
api: "google-generative-ai",
|
||||
provider: "custom-google",
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"google-generative-ai">,
|
||||
{
|
||||
tls: {
|
||||
ca: "ca-pem",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const streamFn = createGoogleGenerativeAiTransportStreamFn();
|
||||
const stream = await Promise.resolve(
|
||||
streamFn(
|
||||
model,
|
||||
{
|
||||
messages: [{ role: "user", content: "hello", timestamp: 0 }],
|
||||
} as Parameters<typeof streamFn>[1],
|
||||
{
|
||||
apiKey: JSON.stringify({ token: "oauth-token", projectId: "demo" }),
|
||||
} as Parameters<typeof streamFn>[2],
|
||||
),
|
||||
);
|
||||
await stream.result();
|
||||
|
||||
expect(guardedFetchMock).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
Authorization: "Bearer oauth-token",
|
||||
"Content-Type": "application/json",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("coerces replayed malformed tool-call args to an object for Google payloads", () => {
|
||||
const params = buildGoogleGenerativeAiParams(buildGeminiModel(), {
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
provider: "openai",
|
||||
api: "openai-responses",
|
||||
model: "gpt-5.4",
|
||||
stopReason: "toolUse",
|
||||
timestamp: 0,
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "lookup",
|
||||
arguments: "{not valid json",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
} as never);
|
||||
|
||||
expect(params.contents[0]).toMatchObject({
|
||||
role: "model",
|
||||
parts: [{ functionCall: { name: "lookup", args: {} } }],
|
||||
});
|
||||
});
|
||||
|
||||
it("replays Gemini tool call thought signatures for same-model history", () => {
|
||||
const model = buildGeminiModel({
|
||||
id: "gemini-3-flash-preview",
|
||||
name: "Gemini 3 Flash Preview",
|
||||
});
|
||||
|
||||
const params = buildGoogleGenerativeAiParams(model, {
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
provider: "google",
|
||||
api: "google-generative-ai",
|
||||
model: "gemini-3-flash-preview",
|
||||
stopReason: "toolUse",
|
||||
timestamp: 0,
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "lookup",
|
||||
arguments: { q: "hello" },
|
||||
thoughtSignature: "call_sig_1",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
} as never);
|
||||
|
||||
expect(params.contents[0]).toMatchObject({
|
||||
role: "model",
|
||||
parts: [
|
||||
{
|
||||
thoughtSignature: "call_sig_1",
|
||||
functionCall: { name: "lookup", args: { q: "hello" } },
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("builds direct Gemini payloads without negative fallback thinking budgets", () => {
|
||||
const model = {
|
||||
id: "custom-gemini-model",
|
||||
name: "Custom Gemini",
|
||||
api: "google-generative-ai",
|
||||
provider: "custom-google",
|
||||
baseUrl: "https://proxy.example.com/gemini/v1beta",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"google-generative-ai">;
|
||||
|
||||
const params = buildGoogleGenerativeAiParams(
|
||||
model,
|
||||
{
|
||||
messages: [{ role: "user", content: "hello", timestamp: 0 }],
|
||||
} as never,
|
||||
{
|
||||
reasoning: "medium",
|
||||
},
|
||||
);
|
||||
|
||||
expect(params.generationConfig).toMatchObject({
|
||||
thinkingConfig: { includeThoughts: true },
|
||||
});
|
||||
expect(params.generationConfig).not.toMatchObject({
|
||||
thinkingConfig: { thinkingBudget: -1 },
|
||||
});
|
||||
});
|
||||
|
||||
it("omits disabled thinkingBudget=0 for Gemini 2.5 Pro direct payloads", () => {
|
||||
const params = buildGoogleGenerativeAiParams(
|
||||
buildGeminiModel(),
|
||||
{
|
||||
messages: [{ role: "user", content: "hello", timestamp: 0 }],
|
||||
} as never,
|
||||
{
|
||||
maxTokens: 128,
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(params.generationConfig).toMatchObject({
|
||||
maxOutputTokens: 128,
|
||||
});
|
||||
expect(params.generationConfig).not.toHaveProperty("thinkingConfig");
|
||||
});
|
||||
|
||||
it("strips explicit thinkingBudget=0 but preserves includeThoughts for Gemini 2.5 Pro", () => {
|
||||
const params = buildGoogleGenerativeAiParams(
|
||||
buildGeminiModel(),
|
||||
{
|
||||
messages: [{ role: "user", content: "hello", timestamp: 0 }],
|
||||
} as never,
|
||||
{
|
||||
thinking: {
|
||||
enabled: true,
|
||||
budgetTokens: 0,
|
||||
},
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(params.generationConfig).toMatchObject({
|
||||
thinkingConfig: { includeThoughts: true },
|
||||
});
|
||||
expect(params.generationConfig).not.toMatchObject({
|
||||
thinkingConfig: { thinkingBudget: 0 },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["gemini-pro-latest", "LOW"],
|
||||
["gemini-flash-latest", "MINIMAL"],
|
||||
["gemini-flash-lite-latest", "MINIMAL"],
|
||||
] as const)(
|
||||
"uses thinkingLevel instead of disabled thinkingBudget for %s defaults",
|
||||
(id, level) => {
|
||||
const params = buildGoogleGenerativeAiParams(
|
||||
buildGeminiModel({ id }),
|
||||
{
|
||||
messages: [{ role: "user", content: "hello", timestamp: 0 }],
|
||||
} as never,
|
||||
{
|
||||
maxTokens: 128,
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(params.generationConfig).toMatchObject({
|
||||
maxOutputTokens: 128,
|
||||
thinkingConfig: { thinkingLevel: level },
|
||||
});
|
||||
expect(params.generationConfig).not.toMatchObject({
|
||||
thinkingConfig: { thinkingBudget: 0 },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("maps explicit Gemini 3 thinking budgets to thinkingLevel", () => {
|
||||
const params = buildGoogleGenerativeAiParams(
|
||||
buildGeminiModel({ id: "gemini-3-flash-preview" }),
|
||||
{
|
||||
messages: [{ role: "user", content: "hello", timestamp: 0 }],
|
||||
} as never,
|
||||
{
|
||||
thinking: {
|
||||
enabled: true,
|
||||
budgetTokens: 8192,
|
||||
},
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(params.generationConfig).toMatchObject({
|
||||
thinkingConfig: { includeThoughts: true, thinkingLevel: "MEDIUM" },
|
||||
});
|
||||
expect(params.generationConfig).not.toMatchObject({
|
||||
thinkingConfig: { thinkingBudget: 8192 },
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes explicit Gemini 3 Pro thinking levels", () => {
|
||||
const params = buildGoogleGenerativeAiParams(
|
||||
buildGeminiModel({ id: "gemini-3.1-pro-preview" }),
|
||||
{
|
||||
messages: [{ role: "user", content: "hello", timestamp: 0 }],
|
||||
} as never,
|
||||
{
|
||||
thinking: {
|
||||
enabled: true,
|
||||
level: "MINIMAL",
|
||||
},
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(params.generationConfig).toMatchObject({
|
||||
thinkingConfig: { includeThoughts: true, thinkingLevel: "LOW" },
|
||||
});
|
||||
});
|
||||
|
||||
it("includes cachedContent in direct Gemini payloads when requested", () => {
|
||||
const params = buildGoogleGenerativeAiParams(
|
||||
buildGeminiModel(),
|
||||
{
|
||||
messages: [{ role: "user", content: "hello", timestamp: 0 }],
|
||||
} as never,
|
||||
{
|
||||
cachedContent: "cachedContents/prebuilt-context",
|
||||
},
|
||||
);
|
||||
|
||||
expect(params.cachedContent).toBe("cachedContents/prebuilt-context");
|
||||
});
|
||||
});
|
||||
761
openclaw/extensions/google/transport-stream.ts
Normal file
761
openclaw/extensions/google/transport-stream.ts
Normal file
|
|
@ -0,0 +1,761 @@
|
|||
import type { StreamFn } from "@mariozechner/pi-agent-core";
|
||||
import {
|
||||
calculateCost,
|
||||
getEnvApiKey,
|
||||
type Context,
|
||||
type Model,
|
||||
type SimpleStreamOptions,
|
||||
type ThinkingLevel,
|
||||
} from "@mariozechner/pi-ai";
|
||||
import {
|
||||
buildGuardedModelFetch,
|
||||
coerceTransportToolCallArguments,
|
||||
createEmptyTransportUsage,
|
||||
createWritableTransportEventStream,
|
||||
failTransportStream,
|
||||
finalizeTransportStream,
|
||||
mergeTransportHeaders,
|
||||
sanitizeTransportPayloadText,
|
||||
stripSystemPromptCacheBoundary,
|
||||
transformTransportMessages,
|
||||
type WritableTransportStream,
|
||||
} from "openclaw/plugin-sdk/provider-transport-runtime";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { parseGeminiAuth } from "./gemini-auth.js";
|
||||
import { normalizeGoogleApiBaseUrl } from "./provider-policy.js";
|
||||
import {
|
||||
isGoogleGemini3FlashModel,
|
||||
isGoogleGemini3ProModel,
|
||||
resolveGoogleGemini3ThinkingLevel,
|
||||
stripInvalidGoogleThinkingBudget,
|
||||
type GoogleThinkingInputLevel,
|
||||
type GoogleThinkingLevel,
|
||||
} from "./thinking-api.js";
|
||||
|
||||
type GoogleTransportModel = Model<"google-generative-ai"> & {
|
||||
headers?: Record<string, string>;
|
||||
provider: string;
|
||||
};
|
||||
|
||||
type GoogleTransportOptions = SimpleStreamOptions & {
|
||||
cachedContent?: string;
|
||||
toolChoice?:
|
||||
| "auto"
|
||||
| "none"
|
||||
| "any"
|
||||
| "required"
|
||||
| {
|
||||
type: "function";
|
||||
function: {
|
||||
name: string;
|
||||
};
|
||||
};
|
||||
thinking?: {
|
||||
enabled: boolean;
|
||||
budgetTokens?: number;
|
||||
level?: GoogleThinkingLevel;
|
||||
};
|
||||
};
|
||||
|
||||
type GoogleGenerateContentRequest = {
|
||||
cachedContent?: string;
|
||||
contents: Array<Record<string, unknown>>;
|
||||
generationConfig?: Record<string, unknown>;
|
||||
systemInstruction?: Record<string, unknown>;
|
||||
tools?: Array<Record<string, unknown>>;
|
||||
toolConfig?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type GoogleTransportContentBlock =
|
||||
| { type: "text"; text: string; textSignature?: string }
|
||||
| { type: "thinking"; thinking: string; thinkingSignature?: string }
|
||||
| {
|
||||
type: "toolCall";
|
||||
id: string;
|
||||
name: string;
|
||||
arguments: Record<string, unknown>;
|
||||
thoughtSignature?: string;
|
||||
};
|
||||
|
||||
type MutableAssistantOutput = {
|
||||
role: "assistant";
|
||||
content: Array<GoogleTransportContentBlock>;
|
||||
api: "google-generative-ai";
|
||||
provider: string;
|
||||
model: string;
|
||||
usage: {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
totalTokens: number;
|
||||
cost: { input: number; output: number; cacheRead: number; cacheWrite: number; total: number };
|
||||
};
|
||||
stopReason: string;
|
||||
timestamp: number;
|
||||
responseId?: string;
|
||||
errorMessage?: string;
|
||||
};
|
||||
|
||||
type GoogleSseChunk = {
|
||||
responseId?: string;
|
||||
candidates?: Array<{
|
||||
content?: {
|
||||
parts?: Array<{
|
||||
text?: string;
|
||||
thought?: boolean;
|
||||
thoughtSignature?: string;
|
||||
functionCall?: {
|
||||
id?: string;
|
||||
name?: string;
|
||||
args?: Record<string, unknown>;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
finishReason?: string;
|
||||
}>;
|
||||
usageMetadata?: {
|
||||
promptTokenCount?: number;
|
||||
cachedContentTokenCount?: number;
|
||||
candidatesTokenCount?: number;
|
||||
thoughtsTokenCount?: number;
|
||||
totalTokenCount?: number;
|
||||
};
|
||||
};
|
||||
|
||||
let toolCallCounter = 0;
|
||||
|
||||
function requiresToolCallId(modelId: string): boolean {
|
||||
return modelId.startsWith("claude-") || modelId.startsWith("gpt-oss-");
|
||||
}
|
||||
|
||||
function supportsMultimodalFunctionResponse(modelId: string): boolean {
|
||||
const match = normalizeLowercaseStringOrEmpty(modelId).match(/^gemini(?:-live)?-(\d+)/);
|
||||
if (!match) {
|
||||
return true;
|
||||
}
|
||||
return Number.parseInt(match[1] ?? "", 10) >= 3;
|
||||
}
|
||||
|
||||
function retainThoughtSignature(existing: string | undefined, incoming: string | undefined) {
|
||||
if (typeof incoming === "string" && incoming.length > 0) {
|
||||
return incoming;
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
function mapToolChoice(
|
||||
choice: GoogleTransportOptions["toolChoice"],
|
||||
): { mode: "AUTO" | "NONE" | "ANY"; allowedFunctionNames?: string[] } | undefined {
|
||||
if (!choice) {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof choice === "object" && choice.type === "function") {
|
||||
return { mode: "ANY", allowedFunctionNames: [choice.function.name] };
|
||||
}
|
||||
switch (choice) {
|
||||
case "none":
|
||||
return { mode: "NONE" };
|
||||
case "any":
|
||||
case "required":
|
||||
return { mode: "ANY" };
|
||||
default:
|
||||
return { mode: "AUTO" };
|
||||
}
|
||||
}
|
||||
|
||||
function mapStopReasonString(reason: string): "stop" | "length" | "error" {
|
||||
switch (reason) {
|
||||
case "STOP":
|
||||
return "stop";
|
||||
case "MAX_TOKENS":
|
||||
return "length";
|
||||
default:
|
||||
return "error";
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeToolCallId(id: string): string {
|
||||
return id.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64);
|
||||
}
|
||||
|
||||
function resolveGoogleModelPath(modelId: string): string {
|
||||
if (modelId.startsWith("models/") || modelId.startsWith("tunedModels/")) {
|
||||
return modelId;
|
||||
}
|
||||
return `models/${modelId}`;
|
||||
}
|
||||
|
||||
function buildGoogleRequestUrl(model: GoogleTransportModel): string {
|
||||
const baseUrl = normalizeGoogleApiBaseUrl(model.baseUrl);
|
||||
return `${baseUrl}/${resolveGoogleModelPath(model.id)}:streamGenerateContent?alt=sse`;
|
||||
}
|
||||
|
||||
function resolveThinkingLevel(level: ThinkingLevel, modelId: string): GoogleThinkingLevel {
|
||||
const resolved = resolveGoogleGemini3ThinkingLevel({ modelId, thinkingLevel: level });
|
||||
if (resolved) {
|
||||
return resolved;
|
||||
}
|
||||
throw new Error("Unsupported thinking level");
|
||||
}
|
||||
|
||||
function resolveExplicitThinkingLevel(
|
||||
level: GoogleThinkingLevel,
|
||||
modelId: string,
|
||||
): GoogleThinkingLevel {
|
||||
return (
|
||||
resolveGoogleGemini3ThinkingLevel({
|
||||
modelId,
|
||||
thinkingLevel: level.toLowerCase() as GoogleThinkingInputLevel,
|
||||
}) ?? level
|
||||
);
|
||||
}
|
||||
|
||||
function getDisabledThinkingConfig(modelId: string): Record<string, unknown> | undefined {
|
||||
const thinkingLevel = resolveGoogleGemini3ThinkingLevel({ modelId, thinkingLevel: "off" });
|
||||
if (thinkingLevel) {
|
||||
return { thinkingLevel };
|
||||
}
|
||||
return normalizeGoogleThinkingConfig(modelId, { thinkingBudget: 0 });
|
||||
}
|
||||
|
||||
function getGoogleThinkingBudget(
|
||||
modelId: string,
|
||||
effort: ThinkingLevel,
|
||||
customBudgets?: GoogleTransportOptions["thinkingBudgets"],
|
||||
): number | undefined {
|
||||
const normalizedEffort = effort === "xhigh" ? "high" : effort;
|
||||
if (customBudgets?.[normalizedEffort] !== undefined) {
|
||||
return customBudgets[normalizedEffort];
|
||||
}
|
||||
if (modelId.includes("2.5-pro")) {
|
||||
return { minimal: 128, low: 2048, medium: 8192, high: 32768 }[normalizedEffort];
|
||||
}
|
||||
if (modelId.includes("2.5-flash")) {
|
||||
return { minimal: 128, low: 2048, medium: 8192, high: 24576 }[normalizedEffort];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveGoogleThinkingConfig(
|
||||
model: GoogleTransportModel,
|
||||
options: GoogleTransportOptions | undefined,
|
||||
): Record<string, unknown> | undefined {
|
||||
if (!model.reasoning) {
|
||||
return undefined;
|
||||
}
|
||||
if (options?.thinking) {
|
||||
if (!options.thinking.enabled) {
|
||||
return getDisabledThinkingConfig(model.id);
|
||||
}
|
||||
const config: Record<string, unknown> = { includeThoughts: true };
|
||||
if (options.thinking.level) {
|
||||
config.thinkingLevel = resolveExplicitThinkingLevel(options.thinking.level, model.id);
|
||||
} else if (typeof options.thinking.budgetTokens === "number") {
|
||||
const thinkingLevel = resolveGoogleGemini3ThinkingLevel({
|
||||
modelId: model.id,
|
||||
thinkingBudget: options.thinking.budgetTokens,
|
||||
});
|
||||
if (thinkingLevel) {
|
||||
config.thinkingLevel = thinkingLevel;
|
||||
} else {
|
||||
config.thinkingBudget = options.thinking.budgetTokens;
|
||||
}
|
||||
}
|
||||
return normalizeGoogleThinkingConfig(model.id, config);
|
||||
}
|
||||
if (!options?.reasoning) {
|
||||
return getDisabledThinkingConfig(model.id);
|
||||
}
|
||||
if (isGoogleGemini3ProModel(model.id) || isGoogleGemini3FlashModel(model.id)) {
|
||||
return {
|
||||
includeThoughts: true,
|
||||
thinkingLevel: resolveThinkingLevel(options.reasoning, model.id),
|
||||
};
|
||||
}
|
||||
const budget = getGoogleThinkingBudget(model.id, options.reasoning, options.thinkingBudgets);
|
||||
return normalizeGoogleThinkingConfig(model.id, {
|
||||
includeThoughts: true,
|
||||
...(typeof budget === "number" ? { thinkingBudget: budget } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeGoogleThinkingConfig(
|
||||
modelId: string,
|
||||
thinkingConfig: Record<string, unknown>,
|
||||
): Record<string, unknown> | undefined {
|
||||
stripInvalidGoogleThinkingBudget({ thinkingConfig, modelId });
|
||||
return Object.keys(thinkingConfig).length > 0 ? thinkingConfig : undefined;
|
||||
}
|
||||
|
||||
function convertGoogleMessages(model: GoogleTransportModel, context: Context) {
|
||||
const contents: Array<Record<string, unknown>> = [];
|
||||
const transformedMessages = transformTransportMessages(context.messages, model, (id) =>
|
||||
requiresToolCallId(model.id) ? normalizeToolCallId(id) : id,
|
||||
);
|
||||
for (const msg of transformedMessages) {
|
||||
if (msg.role === "user") {
|
||||
if (typeof msg.content === "string") {
|
||||
contents.push({
|
||||
role: "user",
|
||||
parts: [{ text: sanitizeTransportPayloadText(msg.content) }],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const parts = msg.content
|
||||
.map((item) =>
|
||||
item.type === "text"
|
||||
? { text: sanitizeTransportPayloadText(item.text) }
|
||||
: {
|
||||
inlineData: {
|
||||
mimeType: item.mimeType,
|
||||
data: item.data,
|
||||
},
|
||||
},
|
||||
)
|
||||
.filter((item) => model.input.includes("image") || !("inlineData" in item));
|
||||
if (parts.length > 0) {
|
||||
contents.push({ role: "user", parts });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (msg.role === "assistant") {
|
||||
const isSameProviderAndModel = msg.provider === model.provider && msg.model === model.id;
|
||||
const parts: Array<Record<string, unknown>> = [];
|
||||
for (const block of msg.content) {
|
||||
if (block.type === "text") {
|
||||
if (!block.text.trim()) {
|
||||
continue;
|
||||
}
|
||||
parts.push({
|
||||
text: sanitizeTransportPayloadText(block.text),
|
||||
...(isSameProviderAndModel && block.textSignature
|
||||
? { thoughtSignature: block.textSignature }
|
||||
: {}),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (block.type === "thinking") {
|
||||
if (!block.thinking.trim()) {
|
||||
continue;
|
||||
}
|
||||
if (isSameProviderAndModel) {
|
||||
parts.push({
|
||||
thought: true,
|
||||
text: sanitizeTransportPayloadText(block.thinking),
|
||||
...(block.thinkingSignature ? { thoughtSignature: block.thinkingSignature } : {}),
|
||||
});
|
||||
} else {
|
||||
parts.push({ text: sanitizeTransportPayloadText(block.thinking) });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (block.type === "toolCall") {
|
||||
parts.push({
|
||||
functionCall: {
|
||||
name: block.name,
|
||||
args: coerceTransportToolCallArguments(block.arguments),
|
||||
...(requiresToolCallId(model.id) ? { id: block.id } : {}),
|
||||
},
|
||||
...(isSameProviderAndModel && block.thoughtSignature
|
||||
? { thoughtSignature: block.thoughtSignature }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (parts.length > 0) {
|
||||
contents.push({ role: "model", parts });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (msg.role === "toolResult") {
|
||||
const textResult = msg.content
|
||||
.filter(
|
||||
(item): item is Extract<(typeof msg.content)[number], { type: "text" }> =>
|
||||
item.type === "text",
|
||||
)
|
||||
.map((item) => item.text)
|
||||
.join("\n");
|
||||
const imageContent = model.input.includes("image")
|
||||
? msg.content.filter(
|
||||
(item): item is Extract<(typeof msg.content)[number], { type: "image" }> =>
|
||||
item.type === "image",
|
||||
)
|
||||
: [];
|
||||
const responseValue = textResult
|
||||
? sanitizeTransportPayloadText(textResult)
|
||||
: imageContent.length > 0
|
||||
? "(see attached image)"
|
||||
: "";
|
||||
const imageParts = imageContent.map((imageBlock) => ({
|
||||
inlineData: {
|
||||
mimeType: imageBlock.mimeType,
|
||||
data: imageBlock.data,
|
||||
},
|
||||
}));
|
||||
const functionResponse = {
|
||||
functionResponse: {
|
||||
name: msg.toolName,
|
||||
response: msg.isError ? { error: responseValue } : { output: responseValue },
|
||||
...(supportsMultimodalFunctionResponse(model.id) && imageParts.length > 0
|
||||
? { parts: imageParts }
|
||||
: {}),
|
||||
...(requiresToolCallId(model.id) ? { id: msg.toolCallId } : {}),
|
||||
},
|
||||
};
|
||||
const last = contents[contents.length - 1];
|
||||
if (
|
||||
last?.role === "user" &&
|
||||
Array.isArray(last.parts) &&
|
||||
last.parts.some((part) => "functionResponse" in part)
|
||||
) {
|
||||
(last.parts as Array<Record<string, unknown>>).push(functionResponse);
|
||||
} else {
|
||||
contents.push({ role: "user", parts: [functionResponse] });
|
||||
}
|
||||
if (imageParts.length > 0 && !supportsMultimodalFunctionResponse(model.id)) {
|
||||
contents.push({ role: "user", parts: [{ text: "Tool result image:" }, ...imageParts] });
|
||||
}
|
||||
}
|
||||
}
|
||||
return contents;
|
||||
}
|
||||
|
||||
function convertGoogleTools(tools: NonNullable<Context["tools"]>) {
|
||||
if (tools.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return [
|
||||
{
|
||||
functionDeclarations: tools.map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parametersJsonSchema: tool.parameters,
|
||||
})),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
export function buildGoogleGenerativeAiParams(
|
||||
model: GoogleTransportModel,
|
||||
context: Context,
|
||||
options?: GoogleTransportOptions,
|
||||
): GoogleGenerateContentRequest {
|
||||
const generationConfig: Record<string, unknown> = {};
|
||||
if (typeof options?.temperature === "number") {
|
||||
generationConfig.temperature = options.temperature;
|
||||
}
|
||||
if (typeof options?.maxTokens === "number") {
|
||||
generationConfig.maxOutputTokens = options.maxTokens;
|
||||
}
|
||||
const thinkingConfig = resolveGoogleThinkingConfig(model, options);
|
||||
if (thinkingConfig) {
|
||||
generationConfig.thinkingConfig = thinkingConfig;
|
||||
}
|
||||
|
||||
const params: GoogleGenerateContentRequest = {
|
||||
contents: convertGoogleMessages(model, context),
|
||||
};
|
||||
if (typeof options?.cachedContent === "string" && options.cachedContent.trim()) {
|
||||
params.cachedContent = options.cachedContent.trim();
|
||||
}
|
||||
if (Object.keys(generationConfig).length > 0) {
|
||||
params.generationConfig = generationConfig;
|
||||
}
|
||||
if (context.systemPrompt) {
|
||||
params.systemInstruction = {
|
||||
parts: [
|
||||
{
|
||||
text: sanitizeTransportPayloadText(stripSystemPromptCacheBoundary(context.systemPrompt)),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (context.tools?.length) {
|
||||
params.tools = convertGoogleTools(context.tools);
|
||||
const toolChoice = mapToolChoice(options?.toolChoice);
|
||||
if (toolChoice) {
|
||||
params.toolConfig = {
|
||||
functionCallingConfig: toolChoice,
|
||||
};
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
function buildGoogleHeaders(
|
||||
model: GoogleTransportModel,
|
||||
apiKey: string | undefined,
|
||||
optionHeaders: Record<string, string> | undefined,
|
||||
): Record<string, string> {
|
||||
const authHeaders = apiKey ? parseGeminiAuth(apiKey).headers : undefined;
|
||||
return (
|
||||
mergeTransportHeaders(
|
||||
{
|
||||
accept: "text/event-stream",
|
||||
},
|
||||
authHeaders,
|
||||
model.headers,
|
||||
optionHeaders,
|
||||
) ?? {
|
||||
accept: "text/event-stream",
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async function* parseGoogleSseChunks(
|
||||
response: Response,
|
||||
signal?: AbortSignal,
|
||||
): AsyncGenerator<GoogleSseChunk> {
|
||||
if (!response.body) {
|
||||
throw new Error("No response body");
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
const abortHandler = () => {
|
||||
void reader.cancel().catch(() => undefined);
|
||||
};
|
||||
signal?.addEventListener("abort", abortHandler);
|
||||
try {
|
||||
while (true) {
|
||||
if (signal?.aborted) {
|
||||
throw new Error("Request was aborted");
|
||||
}
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
buffer += decoder.decode(value, { stream: true }).replace(/\r/g, "");
|
||||
let boundary = buffer.indexOf("\n\n");
|
||||
while (boundary >= 0) {
|
||||
const rawEvent = buffer.slice(0, boundary);
|
||||
buffer = buffer.slice(boundary + 2);
|
||||
boundary = buffer.indexOf("\n\n");
|
||||
const data = rawEvent
|
||||
.split("\n")
|
||||
.filter((line) => line.startsWith("data:"))
|
||||
.map((line) => line.slice(5).trim())
|
||||
.join("\n");
|
||||
if (!data || data === "[DONE]") {
|
||||
continue;
|
||||
}
|
||||
yield JSON.parse(data) as GoogleSseChunk;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
signal?.removeEventListener("abort", abortHandler);
|
||||
}
|
||||
}
|
||||
|
||||
function updateUsage(
|
||||
output: MutableAssistantOutput,
|
||||
model: GoogleTransportModel,
|
||||
chunk: GoogleSseChunk,
|
||||
) {
|
||||
const usage = chunk.usageMetadata;
|
||||
if (!usage) {
|
||||
return;
|
||||
}
|
||||
const promptTokens = usage.promptTokenCount || 0;
|
||||
const cacheRead = usage.cachedContentTokenCount || 0;
|
||||
output.usage = {
|
||||
input: Math.max(0, promptTokens - cacheRead),
|
||||
output: (usage.candidatesTokenCount || 0) + (usage.thoughtsTokenCount || 0),
|
||||
cacheRead,
|
||||
cacheWrite: 0,
|
||||
totalTokens: usage.totalTokenCount || 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
calculateCost(model, output.usage);
|
||||
}
|
||||
|
||||
function pushTextBlockEnd(
|
||||
stream: WritableTransportStream,
|
||||
output: MutableAssistantOutput,
|
||||
blockIndex: number,
|
||||
) {
|
||||
const block = output.content[blockIndex];
|
||||
if (!block) {
|
||||
return;
|
||||
}
|
||||
if (block.type === "thinking") {
|
||||
stream.push({
|
||||
type: "thinking_end",
|
||||
contentIndex: blockIndex,
|
||||
content: block.thinking,
|
||||
partial: output as never,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (block.type === "text") {
|
||||
stream.push({
|
||||
type: "text_end",
|
||||
contentIndex: blockIndex,
|
||||
content: block.text,
|
||||
partial: output as never,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function createGoogleGenerativeAiTransportStreamFn(): StreamFn {
|
||||
return (rawModel, context, rawOptions) => {
|
||||
const model = rawModel as GoogleTransportModel;
|
||||
const options = rawOptions as GoogleTransportOptions | undefined;
|
||||
const { eventStream, stream } = createWritableTransportEventStream();
|
||||
void (async () => {
|
||||
const output: MutableAssistantOutput = {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: "google-generative-ai",
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: createEmptyTransportUsage(),
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
try {
|
||||
const apiKey = options?.apiKey ?? getEnvApiKey(model.provider) ?? undefined;
|
||||
const guardedFetch = buildGuardedModelFetch(model);
|
||||
let params = buildGoogleGenerativeAiParams(model, context, options);
|
||||
const nextParams = await options?.onPayload?.(params, model);
|
||||
if (nextParams !== undefined) {
|
||||
params = nextParams as GoogleGenerateContentRequest;
|
||||
}
|
||||
const response = await guardedFetch(buildGoogleRequestUrl(model), {
|
||||
method: "POST",
|
||||
headers: buildGoogleHeaders(model, apiKey, options?.headers),
|
||||
body: JSON.stringify(params),
|
||||
signal: options?.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
const message = await response.text().catch(() => "");
|
||||
throw new Error(`Google Generative AI API error (${response.status}): ${message}`);
|
||||
}
|
||||
stream.push({ type: "start", partial: output as never });
|
||||
let currentBlockIndex = -1;
|
||||
for await (const chunk of parseGoogleSseChunks(response, options?.signal)) {
|
||||
output.responseId ||= chunk.responseId;
|
||||
updateUsage(output, model, chunk);
|
||||
const candidate = chunk.candidates?.[0];
|
||||
if (candidate?.content?.parts) {
|
||||
for (const part of candidate.content.parts) {
|
||||
if (typeof part.text === "string") {
|
||||
const isThinking = part.thought === true;
|
||||
const currentBlock = output.content[currentBlockIndex];
|
||||
if (
|
||||
currentBlockIndex < 0 ||
|
||||
!currentBlock ||
|
||||
(isThinking && currentBlock.type !== "thinking") ||
|
||||
(!isThinking && currentBlock.type !== "text")
|
||||
) {
|
||||
if (currentBlockIndex >= 0) {
|
||||
pushTextBlockEnd(stream, output, currentBlockIndex);
|
||||
}
|
||||
if (isThinking) {
|
||||
output.content.push({ type: "thinking", thinking: "" });
|
||||
currentBlockIndex = output.content.length - 1;
|
||||
stream.push({
|
||||
type: "thinking_start",
|
||||
contentIndex: currentBlockIndex,
|
||||
partial: output as never,
|
||||
});
|
||||
} else {
|
||||
output.content.push({ type: "text", text: "" });
|
||||
currentBlockIndex = output.content.length - 1;
|
||||
stream.push({
|
||||
type: "text_start",
|
||||
contentIndex: currentBlockIndex,
|
||||
partial: output as never,
|
||||
});
|
||||
}
|
||||
}
|
||||
const activeBlock = output.content[currentBlockIndex];
|
||||
if (activeBlock?.type === "thinking") {
|
||||
activeBlock.thinking += part.text;
|
||||
activeBlock.thinkingSignature = retainThoughtSignature(
|
||||
activeBlock.thinkingSignature,
|
||||
part.thoughtSignature,
|
||||
);
|
||||
stream.push({
|
||||
type: "thinking_delta",
|
||||
contentIndex: currentBlockIndex,
|
||||
delta: part.text,
|
||||
partial: output as never,
|
||||
});
|
||||
} else if (activeBlock?.type === "text") {
|
||||
activeBlock.text += part.text;
|
||||
activeBlock.textSignature = retainThoughtSignature(
|
||||
activeBlock.textSignature,
|
||||
part.thoughtSignature,
|
||||
);
|
||||
stream.push({
|
||||
type: "text_delta",
|
||||
contentIndex: currentBlockIndex,
|
||||
delta: part.text,
|
||||
partial: output as never,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (part.functionCall) {
|
||||
if (currentBlockIndex >= 0) {
|
||||
pushTextBlockEnd(stream, output, currentBlockIndex);
|
||||
currentBlockIndex = -1;
|
||||
}
|
||||
const providedId = part.functionCall.id;
|
||||
const isDuplicate = output.content.some(
|
||||
(block) => block.type === "toolCall" && block.id === providedId,
|
||||
);
|
||||
const toolCallId =
|
||||
providedId && !isDuplicate
|
||||
? providedId
|
||||
: `${part.functionCall.name || "tool"}_${Date.now()}_${++toolCallCounter}`;
|
||||
const toolCall: GoogleTransportContentBlock = {
|
||||
type: "toolCall",
|
||||
id: toolCallId,
|
||||
name: part.functionCall.name || "",
|
||||
arguments: part.functionCall.args ?? {},
|
||||
thoughtSignature: part.thoughtSignature,
|
||||
};
|
||||
output.content.push(toolCall);
|
||||
const blockIndex = output.content.length - 1;
|
||||
stream.push({
|
||||
type: "toolcall_start",
|
||||
contentIndex: blockIndex,
|
||||
partial: output as never,
|
||||
});
|
||||
stream.push({
|
||||
type: "toolcall_delta",
|
||||
contentIndex: blockIndex,
|
||||
delta: JSON.stringify(toolCall.arguments),
|
||||
partial: output as never,
|
||||
});
|
||||
stream.push({
|
||||
type: "toolcall_end",
|
||||
contentIndex: blockIndex,
|
||||
toolCall,
|
||||
partial: output as never,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (typeof candidate?.finishReason === "string") {
|
||||
output.stopReason = mapStopReasonString(candidate.finishReason);
|
||||
if (output.content.some((block) => block.type === "toolCall")) {
|
||||
output.stopReason = "toolUse";
|
||||
}
|
||||
}
|
||||
}
|
||||
if (currentBlockIndex >= 0) {
|
||||
pushTextBlockEnd(stream, output, currentBlockIndex);
|
||||
}
|
||||
finalizeTransportStream({ stream, output, signal: options?.signal });
|
||||
} catch (error) {
|
||||
failTransportStream({ stream, output, signal: options?.signal, error });
|
||||
}
|
||||
})();
|
||||
return eventStream as unknown as ReturnType<StreamFn>;
|
||||
};
|
||||
}
|
||||
16
openclaw/extensions/google/tsconfig.json
Normal file
16
openclaw/extensions/google/tsconfig.json
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"extends": "../tsconfig.package-boundary.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["./*.ts", "./src/**/*.ts"],
|
||||
"exclude": [
|
||||
"./**/*.test.ts",
|
||||
"./dist/**",
|
||||
"./node_modules/**",
|
||||
"./src/test-support/**",
|
||||
"./src/**/*test-helpers.ts",
|
||||
"./src/**/*test-harness.ts",
|
||||
"./src/**/*test-support.ts"
|
||||
]
|
||||
}
|
||||
160
openclaw/extensions/google/video-generation-provider.test.ts
Normal file
160
openclaw/extensions/google/video-generation-provider.test.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { GoogleGenAIMock, generateVideosMock, getVideosOperationMock } = vi.hoisted(() => {
|
||||
const generateVideosMock = vi.fn();
|
||||
const getVideosOperationMock = vi.fn();
|
||||
const GoogleGenAIMock = vi.fn(function GoogleGenAI() {
|
||||
return {
|
||||
models: {
|
||||
generateVideos: generateVideosMock,
|
||||
},
|
||||
operations: {
|
||||
getVideosOperation: getVideosOperationMock,
|
||||
},
|
||||
files: {
|
||||
download: vi.fn(),
|
||||
},
|
||||
};
|
||||
});
|
||||
return { GoogleGenAIMock, generateVideosMock, getVideosOperationMock };
|
||||
});
|
||||
|
||||
vi.mock("@google/genai", () => ({
|
||||
GoogleGenAI: GoogleGenAIMock,
|
||||
}));
|
||||
|
||||
import * as providerAuthRuntime from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import { expectExplicitVideoGenerationCapabilities } from "../../test/helpers/media-generation/provider-capability-assertions.js";
|
||||
import { buildGoogleVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
|
||||
describe("google video generation provider", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
generateVideosMock.mockReset();
|
||||
getVideosOperationMock.mockReset();
|
||||
GoogleGenAIMock.mockClear();
|
||||
});
|
||||
|
||||
it("declares explicit mode capabilities", () => {
|
||||
expectExplicitVideoGenerationCapabilities(buildGoogleVideoGenerationProvider());
|
||||
});
|
||||
|
||||
it("submits generation and returns inline video bytes", async () => {
|
||||
vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "google-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
generateVideosMock.mockResolvedValue({
|
||||
done: true,
|
||||
name: "operations/123",
|
||||
response: {
|
||||
generatedVideos: [
|
||||
{
|
||||
video: {
|
||||
videoBytes: Buffer.from("mp4-bytes").toString("base64"),
|
||||
mimeType: "video/mp4",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const provider = buildGoogleVideoGenerationProvider();
|
||||
const result = await provider.generateVideo({
|
||||
provider: "google",
|
||||
model: "veo-3.1-fast-generate-preview",
|
||||
prompt: "A tiny robot watering a windowsill garden",
|
||||
cfg: {},
|
||||
aspectRatio: "16:9",
|
||||
resolution: "720P",
|
||||
durationSeconds: 3,
|
||||
audio: true,
|
||||
});
|
||||
|
||||
expect(generateVideosMock).toHaveBeenCalledTimes(1);
|
||||
const [request] = generateVideosMock.mock.calls[0] ?? [];
|
||||
expect(request).toEqual(
|
||||
expect.objectContaining({
|
||||
model: "veo-3.1-fast-generate-preview",
|
||||
prompt: "A tiny robot watering a windowsill garden",
|
||||
config: expect.objectContaining({
|
||||
durationSeconds: 4,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "720p",
|
||||
generateAudio: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(request?.config).not.toHaveProperty("numberOfVideos");
|
||||
expect(result.videos).toHaveLength(1);
|
||||
expect(result.videos[0]?.mimeType).toBe("video/mp4");
|
||||
expect(GoogleGenAIMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
apiKey: "google-key",
|
||||
httpOptions: expect.not.objectContaining({
|
||||
baseUrl: expect.anything(),
|
||||
apiVersion: expect.anything(),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects mixed image and video inputs", async () => {
|
||||
vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "google-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
const provider = buildGoogleVideoGenerationProvider();
|
||||
|
||||
await expect(
|
||||
provider.generateVideo({
|
||||
provider: "google",
|
||||
model: "veo-3.1-fast-generate-preview",
|
||||
prompt: "Animate",
|
||||
cfg: {},
|
||||
inputImages: [{ buffer: Buffer.from("img"), mimeType: "image/png" }],
|
||||
inputVideos: [{ buffer: Buffer.from("vid"), mimeType: "video/mp4" }],
|
||||
}),
|
||||
).rejects.toThrow("Google video generation does not support image and video inputs together.");
|
||||
});
|
||||
|
||||
it("rounds unsupported durations to the nearest Veo value", async () => {
|
||||
vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "google-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
generateVideosMock.mockResolvedValue({
|
||||
done: true,
|
||||
response: {
|
||||
generatedVideos: [
|
||||
{
|
||||
video: {
|
||||
videoBytes: Buffer.from("mp4-bytes").toString("base64"),
|
||||
mimeType: "video/mp4",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const provider = buildGoogleVideoGenerationProvider();
|
||||
await provider.generateVideo({
|
||||
provider: "google",
|
||||
model: "veo-3.1-fast-generate-preview",
|
||||
prompt: "A tiny robot watering a windowsill garden",
|
||||
cfg: {},
|
||||
durationSeconds: 5,
|
||||
});
|
||||
|
||||
expect(generateVideosMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
config: expect.objectContaining({
|
||||
durationSeconds: 6,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
257
openclaw/extensions/google/video-generation-provider.ts
Normal file
257
openclaw/extensions/google/video-generation-provider.ts
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { GoogleGenAI } from "@google/genai";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
createProviderOperationDeadline,
|
||||
resolveProviderOperationTimeoutMs,
|
||||
waitProviderOperationPollInterval,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
|
||||
import type {
|
||||
GeneratedVideoAsset,
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationRequest,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
import { normalizeGoogleApiBaseUrl } from "./api.js";
|
||||
import {
|
||||
createGoogleVideoGenerationProviderMetadata,
|
||||
DEFAULT_GOOGLE_VIDEO_MODEL,
|
||||
GOOGLE_VIDEO_ALLOWED_DURATION_SECONDS,
|
||||
GOOGLE_VIDEO_MAX_DURATION_SECONDS,
|
||||
GOOGLE_VIDEO_MIN_DURATION_SECONDS,
|
||||
} from "./generation-provider-metadata.js";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 180_000;
|
||||
const POLL_INTERVAL_MS = 10_000;
|
||||
const MAX_POLL_ATTEMPTS = 90;
|
||||
|
||||
function resolveConfiguredGoogleVideoBaseUrl(req: VideoGenerationRequest): string | undefined {
|
||||
const configured = normalizeOptionalString(req.cfg?.models?.providers?.google?.baseUrl);
|
||||
return configured ? normalizeGoogleApiBaseUrl(configured) : undefined;
|
||||
}
|
||||
|
||||
function parseVideoSize(size: string | undefined): { width: number; height: number } | undefined {
|
||||
const trimmed = normalizeOptionalString(size);
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
const match = /^(\d+)x(\d+)$/u.exec(trimmed);
|
||||
if (!match) {
|
||||
return undefined;
|
||||
}
|
||||
const width = Number.parseInt(match[1] ?? "", 10);
|
||||
const height = Number.parseInt(match[2] ?? "", 10);
|
||||
if (!Number.isFinite(width) || !Number.isFinite(height)) {
|
||||
return undefined;
|
||||
}
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function resolveAspectRatio(params: {
|
||||
aspectRatio?: string;
|
||||
size?: string;
|
||||
}): "16:9" | "9:16" | undefined {
|
||||
const direct = normalizeOptionalString(params.aspectRatio);
|
||||
if (direct === "16:9" || direct === "9:16") {
|
||||
return direct;
|
||||
}
|
||||
const parsedSize = parseVideoSize(params.size);
|
||||
if (!parsedSize) {
|
||||
return undefined;
|
||||
}
|
||||
return parsedSize.width >= parsedSize.height ? "16:9" : "9:16";
|
||||
}
|
||||
|
||||
function resolveResolution(params: {
|
||||
resolution?: string;
|
||||
size?: string;
|
||||
}): "720p" | "1080p" | undefined {
|
||||
if (params.resolution === "720P") {
|
||||
return "720p";
|
||||
}
|
||||
if (params.resolution === "1080P") {
|
||||
return "1080p";
|
||||
}
|
||||
const parsedSize = parseVideoSize(params.size);
|
||||
if (!parsedSize) {
|
||||
return undefined;
|
||||
}
|
||||
const maxEdge = Math.max(parsedSize.width, parsedSize.height);
|
||||
return maxEdge >= 1920 ? "1080p" : maxEdge >= 1280 ? "720p" : undefined;
|
||||
}
|
||||
|
||||
function resolveDurationSeconds(durationSeconds: number | undefined): number | undefined {
|
||||
if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) {
|
||||
return undefined;
|
||||
}
|
||||
const rounded = Math.min(
|
||||
GOOGLE_VIDEO_MAX_DURATION_SECONDS,
|
||||
Math.max(GOOGLE_VIDEO_MIN_DURATION_SECONDS, Math.round(durationSeconds)),
|
||||
);
|
||||
return GOOGLE_VIDEO_ALLOWED_DURATION_SECONDS.reduce((best, current) => {
|
||||
const currentDistance = Math.abs(current - rounded);
|
||||
const bestDistance = Math.abs(best - rounded);
|
||||
if (currentDistance < bestDistance) {
|
||||
return current;
|
||||
}
|
||||
if (currentDistance === bestDistance && current > best) {
|
||||
return current;
|
||||
}
|
||||
return best;
|
||||
});
|
||||
}
|
||||
|
||||
function resolveInputImage(req: VideoGenerationRequest) {
|
||||
const input = req.inputImages?.[0];
|
||||
if (!input?.buffer) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
imageBytes: input.buffer.toString("base64"),
|
||||
mimeType: normalizeOptionalString(input.mimeType) || "image/png",
|
||||
};
|
||||
}
|
||||
|
||||
function resolveInputVideo(req: VideoGenerationRequest) {
|
||||
const input = req.inputVideos?.[0];
|
||||
if (!input?.buffer) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
videoBytes: input.buffer.toString("base64"),
|
||||
mimeType: normalizeOptionalString(input.mimeType) || "video/mp4",
|
||||
};
|
||||
}
|
||||
|
||||
async function downloadGeneratedVideo(params: {
|
||||
client: GoogleGenAI;
|
||||
file: unknown;
|
||||
index: number;
|
||||
}): Promise<GeneratedVideoAsset> {
|
||||
const tempDir = await mkdtemp(
|
||||
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-google-video-"),
|
||||
);
|
||||
const downloadPath = path.join(tempDir, `video-${params.index + 1}.mp4`);
|
||||
try {
|
||||
await params.client.files.download({
|
||||
file: params.file as never,
|
||||
downloadPath,
|
||||
});
|
||||
const buffer = await readFile(downloadPath);
|
||||
return {
|
||||
buffer,
|
||||
mimeType: "video/mp4",
|
||||
fileName: `video-${params.index + 1}.mp4`,
|
||||
};
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export function buildGoogleVideoGenerationProvider(): VideoGenerationProvider {
|
||||
return {
|
||||
...createGoogleVideoGenerationProviderMetadata(),
|
||||
async generateVideo(req) {
|
||||
if ((req.inputImages?.length ?? 0) > 1) {
|
||||
throw new Error("Google video generation supports at most one input image.");
|
||||
}
|
||||
if ((req.inputVideos?.length ?? 0) > 1) {
|
||||
throw new Error("Google video generation supports at most one input video.");
|
||||
}
|
||||
if ((req.inputImages?.length ?? 0) > 0 && (req.inputVideos?.length ?? 0) > 0) {
|
||||
throw new Error(
|
||||
"Google video generation does not support image and video inputs together.",
|
||||
);
|
||||
}
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "google",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
});
|
||||
if (!auth.apiKey) {
|
||||
throw new Error("Google API key missing");
|
||||
}
|
||||
|
||||
const configuredBaseUrl = resolveConfiguredGoogleVideoBaseUrl(req);
|
||||
const durationSeconds = resolveDurationSeconds(req.durationSeconds);
|
||||
const deadline = createProviderOperationDeadline({
|
||||
timeoutMs: req.timeoutMs,
|
||||
label: "Google video generation",
|
||||
});
|
||||
const client = new GoogleGenAI({
|
||||
apiKey: auth.apiKey,
|
||||
httpOptions: {
|
||||
...(configuredBaseUrl ? { baseUrl: configuredBaseUrl } : {}),
|
||||
timeout: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
},
|
||||
});
|
||||
let operation = await client.models.generateVideos({
|
||||
model: normalizeOptionalString(req.model) || DEFAULT_GOOGLE_VIDEO_MODEL,
|
||||
prompt: req.prompt,
|
||||
image: resolveInputImage(req),
|
||||
video: resolveInputVideo(req),
|
||||
config: {
|
||||
...(typeof durationSeconds === "number" ? { durationSeconds } : {}),
|
||||
...(resolveAspectRatio({ aspectRatio: req.aspectRatio, size: req.size })
|
||||
? { aspectRatio: resolveAspectRatio({ aspectRatio: req.aspectRatio, size: req.size }) }
|
||||
: {}),
|
||||
...(resolveResolution({ resolution: req.resolution, size: req.size })
|
||||
? { resolution: resolveResolution({ resolution: req.resolution, size: req.size }) }
|
||||
: {}),
|
||||
...(req.audio === true ? { generateAudio: true } : {}),
|
||||
},
|
||||
});
|
||||
|
||||
for (let attempt = 0; !(operation.done ?? false); attempt += 1) {
|
||||
if (attempt >= MAX_POLL_ATTEMPTS) {
|
||||
throw new Error("Google video generation did not finish in time");
|
||||
}
|
||||
await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS });
|
||||
resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS });
|
||||
operation = await client.operations.getVideosOperation({ operation });
|
||||
}
|
||||
if (operation.error) {
|
||||
throw new Error(JSON.stringify(operation.error));
|
||||
}
|
||||
const generatedVideos = operation.response?.generatedVideos ?? [];
|
||||
if (generatedVideos.length === 0) {
|
||||
throw new Error("Google video generation response missing generated videos");
|
||||
}
|
||||
const videos = await Promise.all(
|
||||
generatedVideos.map(async (entry, index) => {
|
||||
const inline = entry.video;
|
||||
if (inline?.videoBytes) {
|
||||
return {
|
||||
buffer: Buffer.from(inline.videoBytes, "base64"),
|
||||
mimeType: normalizeOptionalString(inline.mimeType) || "video/mp4",
|
||||
fileName: `video-${index + 1}.mp4`,
|
||||
};
|
||||
}
|
||||
if (!inline) {
|
||||
throw new Error("Google generated video missing file handle");
|
||||
}
|
||||
return await downloadGeneratedVideo({
|
||||
client,
|
||||
file: inline,
|
||||
index,
|
||||
});
|
||||
}),
|
||||
);
|
||||
return {
|
||||
videos,
|
||||
model: normalizeOptionalString(req.model) || DEFAULT_GOOGLE_VIDEO_MODEL,
|
||||
metadata: operation.name
|
||||
? {
|
||||
operationName: operation.name,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
1
openclaw/extensions/google/web-search-contract-api.ts
Normal file
1
openclaw/extensions/google/web-search-contract-api.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export { createGeminiWebSearchProvider } from "./src/gemini-web-search-provider.js";
|
||||
35
openclaw/extensions/google/web-search-provider.test.ts
Normal file
35
openclaw/extensions/google/web-search-provider.test.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { withEnv } from "openclaw/plugin-sdk/testing";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../../src/config/config.js";
|
||||
import { __testing, createGeminiWebSearchProvider } from "./src/gemini-web-search-provider.js";
|
||||
|
||||
describe("google web search provider", () => {
|
||||
it("falls back to GEMINI_API_KEY from the environment", () => {
|
||||
withEnv({ GEMINI_API_KEY: "AIza-env-test" }, () => {
|
||||
expect(__testing.resolveGeminiApiKey()).toBe("AIza-env-test");
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers configured api keys over env fallbacks", () => {
|
||||
withEnv({ GEMINI_API_KEY: "AIza-env-test" }, () => {
|
||||
expect(__testing.resolveGeminiApiKey({ apiKey: "AIza-configured-test" })).toBe(
|
||||
"AIza-configured-test",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("stores configured credentials at the canonical plugin config path", () => {
|
||||
const provider = createGeminiWebSearchProvider();
|
||||
const config = {} as OpenClawConfig;
|
||||
|
||||
provider.setConfiguredCredentialValue?.(config, "AIza-plugin-test");
|
||||
|
||||
expect(provider.credentialPath).toBe("plugins.entries.google.config.webSearch.apiKey");
|
||||
expect(provider.getConfiguredCredentialValue?.(config)).toBe("AIza-plugin-test");
|
||||
});
|
||||
|
||||
it("defaults the Gemini web search model and trims explicit overrides", () => {
|
||||
expect(__testing.resolveGeminiModel()).toBe("gemini-2.5-flash");
|
||||
expect(__testing.resolveGeminiModel({ model: " gemini-2.5-pro " })).toBe("gemini-2.5-pro");
|
||||
});
|
||||
});
|
||||
1
openclaw/extensions/google/web-search-provider.ts
Normal file
1
openclaw/extensions/google/web-search-provider.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export { createGeminiWebSearchProvider } from "./src/gemini-web-search-provider.js";
|
||||
Loading…
Add table
Add a link
Reference in a new issue