mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-10 05:10:45 +08:00
重构主控编排与运行时预热链路，统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为（含 wiki worker）、更新 Admin 可观测性与相关测试，降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
4
openclaw/extensions/lmstudio/src/api.ts
Normal file
4
openclaw/extensions/lmstudio/src/api.ts
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
export * from "./defaults.js";
|
||||
export * from "./models.js";
|
||||
export * from "./runtime.js";
|
||||
export * from "./setup.js";
|
||||
12
openclaw/extensions/lmstudio/src/defaults.ts
Normal file
12
openclaw/extensions/lmstudio/src/defaults.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
/** Shared LM Studio defaults used by setup, runtime discovery, and embeddings paths. */
|
||||
export const LMSTUDIO_DEFAULT_BASE_URL = "http://localhost:1234";
|
||||
export const LMSTUDIO_DEFAULT_INFERENCE_BASE_URL = `${LMSTUDIO_DEFAULT_BASE_URL}/v1`;
|
||||
export const LMSTUDIO_DEFAULT_EMBEDDING_MODEL = "text-embedding-nomic-embed-text-v1.5";
|
||||
export const LMSTUDIO_PROVIDER_LABEL = "LM Studio";
|
||||
export const LMSTUDIO_DEFAULT_API_KEY_ENV_VAR = "LM_API_TOKEN";
|
||||
export const LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER = "lmstudio-local";
|
||||
export const LMSTUDIO_MODEL_PLACEHOLDER = "model-key-from-api-v1-models";
|
||||
// Default context length sent when requesting LM Studio to load a model.
|
||||
export const LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH = 64000;
|
||||
export const LMSTUDIO_DEFAULT_MODEL_ID = "qwen/qwen3.5-9b";
|
||||
export const LMSTUDIO_PROVIDER_ID = "lmstudio";
|
||||
147
openclaw/extensions/lmstudio/src/embedding-provider.ts
Normal file
147
openclaw/extensions/lmstudio/src/embedding-provider.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
import { createSubsystemLogger } from "openclaw/plugin-sdk/logging-core";
|
||||
import {
|
||||
buildRemoteBaseUrlPolicy,
|
||||
createRemoteEmbeddingProvider,
|
||||
normalizeEmbeddingModelWithPrefixes,
|
||||
type MemoryEmbeddingProvider,
|
||||
type MemoryEmbeddingProviderCreateOptions,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { resolveMemorySecretInputString } from "openclaw/plugin-sdk/memory-core-host-secret";
|
||||
import { formatErrorMessage, type SsrFPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { LMSTUDIO_DEFAULT_EMBEDDING_MODEL, LMSTUDIO_PROVIDER_ID } from "./defaults.js";
|
||||
import { ensureLmstudioModelLoaded } from "./models.fetch.js";
|
||||
import { resolveLmstudioInferenceBase } from "./models.js";
|
||||
import {
|
||||
buildLmstudioAuthHeaders,
|
||||
resolveLmstudioProviderHeaders,
|
||||
resolveLmstudioRuntimeApiKey,
|
||||
} from "./runtime.js";
|
||||
|
||||
const log = createSubsystemLogger("memory/embeddings");
|
||||
|
||||
export type LmstudioEmbeddingClient = {
|
||||
baseUrl: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
model: string;
|
||||
};
|
||||
export const DEFAULT_LMSTUDIO_EMBEDDING_MODEL = LMSTUDIO_DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
/** Normalizes LM Studio embedding model refs and accepts `lmstudio/` prefix. */
|
||||
function normalizeLmstudioModel(model: string): string {
|
||||
return normalizeEmbeddingModelWithPrefixes({
|
||||
model,
|
||||
defaultModel: DEFAULT_LMSTUDIO_EMBEDDING_MODEL,
|
||||
prefixes: ["lmstudio/"],
|
||||
});
|
||||
}
|
||||
|
||||
function hasAuthorizationHeader(headers: Record<string, string> | undefined): boolean {
|
||||
if (!headers) {
|
||||
return false;
|
||||
}
|
||||
return Object.entries(headers).some(
|
||||
([headerName, value]) =>
|
||||
headerName.trim().toLowerCase() === "authorization" && value.trim().length > 0,
|
||||
);
|
||||
}
|
||||
|
||||
/** Resolves API key (real or synthetic placeholder) from runtime/provider auth config. */
|
||||
async function resolveLmstudioApiKey(
|
||||
options: MemoryEmbeddingProviderCreateOptions,
|
||||
): Promise<string | undefined> {
|
||||
try {
|
||||
return await resolveLmstudioRuntimeApiKey({
|
||||
config: options.config,
|
||||
agentDir: options.agentDir,
|
||||
});
|
||||
} catch (error) {
|
||||
// Embeddings can target local LM Studio instances that do not require auth.
|
||||
if (/LM Studio API key is required/i.test(formatErrorMessage(error))) {
|
||||
return undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates the LM Studio embedding provider client and preloads the target model before return. */
|
||||
export async function createLmstudioEmbeddingProvider(
|
||||
options: MemoryEmbeddingProviderCreateOptions,
|
||||
): Promise<{ provider: MemoryEmbeddingProvider; client: LmstudioEmbeddingClient }> {
|
||||
const providerConfig = options.config.models?.providers?.lmstudio;
|
||||
const providerBaseUrl = providerConfig?.baseUrl?.trim();
|
||||
const isFallbackActivation = options.fallback === "lmstudio" && options.provider !== "lmstudio";
|
||||
const remoteBaseUrl = options.remote?.baseUrl?.trim();
|
||||
const remoteApiKey = !isFallbackActivation
|
||||
? resolveMemorySecretInputString({
|
||||
value: options.remote?.apiKey,
|
||||
path: "agents.*.memorySearch.remote.apiKey",
|
||||
})
|
||||
: undefined;
|
||||
// memorySearch.remote is shared across primary + fallback providers.
|
||||
// Ignore it during fallback activation to avoid inheriting another provider's
|
||||
// endpoint/headers/credentials when LM Studio activates as a fallback.
|
||||
const baseUrlSource = !isFallbackActivation ? remoteBaseUrl : undefined;
|
||||
const configuredBaseUrl =
|
||||
baseUrlSource && baseUrlSource.length > 0
|
||||
? baseUrlSource
|
||||
: providerBaseUrl && providerBaseUrl.length > 0
|
||||
? providerBaseUrl
|
||||
: undefined;
|
||||
const baseUrl = resolveLmstudioInferenceBase(configuredBaseUrl);
|
||||
const model = normalizeLmstudioModel(options.model);
|
||||
const providerHeaders = await resolveLmstudioProviderHeaders({
|
||||
config: options.config,
|
||||
env: process.env,
|
||||
headers: Object.assign(
|
||||
{},
|
||||
providerConfig?.headers,
|
||||
!isFallbackActivation ? options.remote?.headers : {},
|
||||
),
|
||||
});
|
||||
const apiKey = hasAuthorizationHeader(providerHeaders)
|
||||
? undefined
|
||||
: !isFallbackActivation
|
||||
? remoteApiKey?.trim() || (await resolveLmstudioApiKey(options))
|
||||
: await resolveLmstudioApiKey(options);
|
||||
const headerOverrides = Object.assign({}, providerHeaders);
|
||||
const headers =
|
||||
buildLmstudioAuthHeaders({
|
||||
apiKey,
|
||||
json: true,
|
||||
headers: headerOverrides,
|
||||
}) ?? {};
|
||||
const ssrfPolicy = buildRemoteBaseUrlPolicy(baseUrl);
|
||||
const client: LmstudioEmbeddingClient = {
|
||||
baseUrl,
|
||||
model,
|
||||
headers,
|
||||
ssrfPolicy,
|
||||
};
|
||||
|
||||
try {
|
||||
await ensureLmstudioModelLoaded({
|
||||
baseUrl,
|
||||
apiKey,
|
||||
headers: headerOverrides,
|
||||
ssrfPolicy,
|
||||
modelKey: model,
|
||||
timeoutMs: 120_000,
|
||||
});
|
||||
} catch (error) {
|
||||
log.warn("lmstudio embeddings warmup failed; continuing without preload", {
|
||||
baseUrl,
|
||||
model,
|
||||
error: formatErrorMessage(error),
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
provider: createRemoteEmbeddingProvider({
|
||||
id: LMSTUDIO_PROVIDER_ID,
|
||||
client,
|
||||
errorPrefix: "lmstudio embeddings failed",
|
||||
}),
|
||||
client,
|
||||
};
|
||||
}
|
||||
265
openclaw/extensions/lmstudio/src/models.fetch.ts
Normal file
265
openclaw/extensions/lmstudio/src/models.fetch.ts
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
import { createSubsystemLogger } from "openclaw/plugin-sdk/logging-core";
|
||||
import type { ModelDefinitionConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { SELF_HOSTED_DEFAULT_COST } from "openclaw/plugin-sdk/provider-setup";
|
||||
import { fetchWithSsrFGuard, type SsrFPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH } from "./defaults.js";
|
||||
import {
|
||||
buildLmstudioModelName,
|
||||
mapLmstudioWireEntry,
|
||||
resolveLmstudioServerBase,
|
||||
resolveLoadedContextWindow,
|
||||
type LmstudioModelWire,
|
||||
} from "./models.js";
|
||||
import { buildLmstudioAuthHeaders } from "./runtime.js";
|
||||
|
||||
const log = createSubsystemLogger("extensions/lmstudio/models");
|
||||
|
||||
type LmstudioLoadResponse = {
|
||||
status?: string;
|
||||
};
|
||||
|
||||
export type FetchLmstudioModelsResult = {
|
||||
reachable: boolean;
|
||||
status?: number;
|
||||
models: LmstudioModelWire[];
|
||||
error?: unknown;
|
||||
};
|
||||
|
||||
type LmstudioModelsResponseWire = {
|
||||
models?: LmstudioModelWire[];
|
||||
};
|
||||
|
||||
type DiscoverLmstudioModelsParams = {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
headers?: Record<string, string>;
|
||||
quiet: boolean;
|
||||
/** Injectable fetch implementation; defaults to the global fetch. */
|
||||
fetchImpl?: typeof fetch;
|
||||
};
|
||||
|
||||
async function fetchLmstudioEndpoint(params: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
timeoutMs: number;
|
||||
fetchImpl?: typeof fetch;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
auditContext: string;
|
||||
}): Promise<{ response: Response; release: () => Promise<void> }> {
|
||||
if (params.ssrfPolicy) {
|
||||
return await fetchWithSsrFGuard({
|
||||
url: params.url,
|
||||
init: params.init,
|
||||
timeoutMs: params.timeoutMs,
|
||||
fetchImpl: params.fetchImpl,
|
||||
policy: params.ssrfPolicy,
|
||||
auditContext: params.auditContext,
|
||||
});
|
||||
}
|
||||
const fetchFn = params.fetchImpl ?? fetch;
|
||||
return {
|
||||
response: await fetchFn(params.url, {
|
||||
...params.init,
|
||||
signal: AbortSignal.timeout(params.timeoutMs),
|
||||
}),
|
||||
release: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
/** Fetches /api/v1/models and reports transport reachability separately from HTTP status. */
|
||||
export async function fetchLmstudioModels(params: {
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
headers?: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
timeoutMs?: number;
|
||||
/** Injectable fetch implementation; defaults to the global fetch. */
|
||||
fetchImpl?: typeof fetch;
|
||||
}): Promise<FetchLmstudioModelsResult> {
|
||||
const baseUrl = resolveLmstudioServerBase(params.baseUrl);
|
||||
const timeoutMs = params.timeoutMs ?? 5000;
|
||||
try {
|
||||
const { response, release } = await fetchLmstudioEndpoint({
|
||||
url: `${baseUrl}/api/v1/models`,
|
||||
init: {
|
||||
headers: buildLmstudioAuthHeaders({
|
||||
apiKey: params.apiKey,
|
||||
headers: params.headers,
|
||||
}),
|
||||
},
|
||||
timeoutMs,
|
||||
fetchImpl: params.fetchImpl,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
auditContext: "lmstudio-model-discovery",
|
||||
});
|
||||
try {
|
||||
if (!response.ok) {
|
||||
return {
|
||||
reachable: true,
|
||||
status: response.status,
|
||||
models: [],
|
||||
};
|
||||
}
|
||||
// External service payload is untrusted JSON; parse with a permissive wire type.
|
||||
const payload = (await response.json()) as LmstudioModelsResponseWire;
|
||||
return {
|
||||
reachable: true,
|
||||
status: response.status,
|
||||
models: Array.isArray(payload.models) ? payload.models : [],
|
||||
};
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
reachable: false,
|
||||
models: [],
|
||||
error,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Discovers LLM models from LM Studio and maps them to OpenClaw model definitions. */
|
||||
export async function discoverLmstudioModels(
|
||||
params: DiscoverLmstudioModelsParams,
|
||||
): Promise<ModelDefinitionConfig[]> {
|
||||
const fetched = await fetchLmstudioModels({
|
||||
baseUrl: params.baseUrl,
|
||||
apiKey: params.apiKey,
|
||||
headers: params.headers,
|
||||
fetchImpl: params.fetchImpl,
|
||||
});
|
||||
const quiet = params.quiet;
|
||||
if (!fetched.reachable) {
|
||||
if (!quiet) {
|
||||
log.debug(`Failed to discover LM Studio models: ${String(fetched.error)}`);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
if (fetched.status !== undefined && fetched.status >= 400) {
|
||||
if (!quiet) {
|
||||
log.debug(`Failed to discover LM Studio models: ${fetched.status}`);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
const models = fetched.models;
|
||||
if (models.length === 0) {
|
||||
if (!quiet) {
|
||||
log.debug("No LM Studio models found on local instance");
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
return models
|
||||
.map((entry): ModelDefinitionConfig | null => {
|
||||
const base = mapLmstudioWireEntry(entry);
|
||||
if (!base) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
id: base.id,
|
||||
// Runtime display: include format/vision/tool-use/loaded tags in the name.
|
||||
name: buildLmstudioModelName(base),
|
||||
reasoning: base.reasoning,
|
||||
input: base.input,
|
||||
cost: SELF_HOSTED_DEFAULT_COST,
|
||||
compat: { supportsUsageInStreaming: true },
|
||||
contextWindow: base.contextWindow,
|
||||
contextTokens: base.contextTokens,
|
||||
maxTokens: base.maxTokens,
|
||||
};
|
||||
})
|
||||
.filter((entry): entry is ModelDefinitionConfig => entry !== null);
|
||||
}
|
||||
|
||||
/** Ensures a model is loaded in LM Studio before first real inference/embedding call. */
|
||||
export async function ensureLmstudioModelLoaded(params: {
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
headers?: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
modelKey: string;
|
||||
requestedContextLength?: number;
|
||||
timeoutMs?: number;
|
||||
/** Injectable fetch implementation; defaults to the global fetch. */
|
||||
fetchImpl?: typeof fetch;
|
||||
}): Promise<void> {
|
||||
const modelKey = params.modelKey.trim();
|
||||
if (!modelKey) {
|
||||
throw new Error("LM Studio model key is required");
|
||||
}
|
||||
|
||||
const timeoutMs = params.timeoutMs ?? 30_000;
|
||||
const baseUrl = resolveLmstudioServerBase(params.baseUrl);
|
||||
const preflight = await fetchLmstudioModels({
|
||||
baseUrl,
|
||||
apiKey: params.apiKey,
|
||||
headers: params.headers,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
timeoutMs,
|
||||
fetchImpl: params.fetchImpl,
|
||||
});
|
||||
if (!preflight.reachable) {
|
||||
throw new Error(`LM Studio model discovery failed: ${String(preflight.error)}`);
|
||||
}
|
||||
if (preflight.status !== undefined && preflight.status >= 400) {
|
||||
throw new Error(`LM Studio model discovery failed (${preflight.status})`);
|
||||
}
|
||||
const matchingModel = preflight.models.find((entry) => entry.key?.trim() === modelKey);
|
||||
const loadedContextWindow = matchingModel ? resolveLoadedContextWindow(matchingModel) : null;
|
||||
const advertisedContextLimit =
|
||||
matchingModel?.max_context_length !== undefined &&
|
||||
Number.isFinite(matchingModel.max_context_length) &&
|
||||
matchingModel.max_context_length > 0
|
||||
? Math.floor(matchingModel.max_context_length)
|
||||
: null;
|
||||
const requestedContextLength =
|
||||
params.requestedContextLength !== undefined &&
|
||||
Number.isFinite(params.requestedContextLength) &&
|
||||
params.requestedContextLength > 0
|
||||
? Math.floor(params.requestedContextLength)
|
||||
: null;
|
||||
const contextLengthForLoad =
|
||||
advertisedContextLimit === null
|
||||
? (requestedContextLength ?? LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH)
|
||||
: Math.min(
|
||||
requestedContextLength ?? LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH,
|
||||
advertisedContextLimit,
|
||||
);
|
||||
if (loadedContextWindow !== null && loadedContextWindow >= contextLengthForLoad) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { response, release } = await fetchLmstudioEndpoint({
|
||||
url: `${baseUrl}/api/v1/models/load`,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: buildLmstudioAuthHeaders({
|
||||
apiKey: params.apiKey,
|
||||
headers: params.headers,
|
||||
json: true,
|
||||
}),
|
||||
body: JSON.stringify({
|
||||
model: modelKey,
|
||||
// Ask LM Studio to load with our default target, capped to the model's own limit.
|
||||
context_length: contextLengthForLoad,
|
||||
}),
|
||||
},
|
||||
timeoutMs,
|
||||
fetchImpl: params.fetchImpl,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
auditContext: "lmstudio-model-load",
|
||||
});
|
||||
try {
|
||||
if (!response.ok) {
|
||||
const body = await response.text();
|
||||
throw new Error(`LM Studio model load failed (${response.status})${body ? `: ${body}` : ""}`);
|
||||
}
|
||||
const payload = (await response.json()) as LmstudioLoadResponse;
|
||||
if (typeof payload.status === "string" && payload.status.toLowerCase() !== "loaded") {
|
||||
throw new Error(`LM Studio model load returned unexpected status: ${payload.status}`);
|
||||
}
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
383
openclaw/extensions/lmstudio/src/models.test.ts
Normal file
383
openclaw/extensions/lmstudio/src/models.test.ts
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
import {
|
||||
SELF_HOSTED_DEFAULT_CONTEXT_WINDOW,
|
||||
SELF_HOSTED_DEFAULT_MAX_TOKENS,
|
||||
} from "openclaw/plugin-sdk/provider-setup";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH } from "./defaults.js";
|
||||
import { discoverLmstudioModels, ensureLmstudioModelLoaded } from "./models.fetch.js";
|
||||
import {
|
||||
resolveLmstudioInferenceBase,
|
||||
resolveLmstudioReasoningCapability,
|
||||
resolveLmstudioServerBase,
|
||||
} from "./models.js";
|
||||
|
||||
const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/ssrf-runtime")>();
|
||||
return {
|
||||
...actual,
|
||||
fetchWithSsrFGuard: (...args: unknown[]) => fetchWithSsrFGuardMock(...args),
|
||||
};
|
||||
});
|
||||
|
||||
describe("lmstudio-models", () => {
|
||||
const asFetch = (mock: unknown) => mock as typeof fetch;
|
||||
const parseJsonRequestBody = (init: RequestInit | undefined): unknown => {
|
||||
if (typeof init?.body !== "string") {
|
||||
throw new Error("Expected request body to be a JSON string");
|
||||
}
|
||||
return JSON.parse(init.body) as unknown;
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
fetchWithSsrFGuardMock.mockReset();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("normalizes LM Studio base URLs", () => {
|
||||
expect(resolveLmstudioServerBase()).toBe("http://localhost:1234");
|
||||
expect(resolveLmstudioInferenceBase()).toBe("http://localhost:1234/v1");
|
||||
expect(resolveLmstudioServerBase("http://localhost:1234/api/v1")).toBe("http://localhost:1234");
|
||||
expect(resolveLmstudioInferenceBase("http://localhost:1234/api/v1")).toBe(
|
||||
"http://localhost:1234/v1",
|
||||
);
|
||||
expect(resolveLmstudioServerBase("localhost:1234/api/v1")).toBe("http://localhost:1234");
|
||||
expect(resolveLmstudioInferenceBase("localhost:1234/api/v1")).toBe("http://localhost:1234/v1");
|
||||
});
|
||||
|
||||
it("resolves reasoning capability for supported and unsupported options", () => {
|
||||
expect(resolveLmstudioReasoningCapability({ capabilities: undefined })).toBe(false);
|
||||
expect(
|
||||
resolveLmstudioReasoningCapability({
|
||||
capabilities: {
|
||||
reasoning: {
|
||||
allowed_options: ["low", "medium", "high"],
|
||||
default: "low",
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
resolveLmstudioReasoningCapability({
|
||||
capabilities: {
|
||||
reasoning: {
|
||||
allowed_options: ["off"],
|
||||
default: "off",
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("discovers llm models and maps metadata", async () => {
|
||||
const fetchMock = vi.fn(async (_url: string | URL) => ({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{
|
||||
type: "llm",
|
||||
key: "qwen3-8b-instruct",
|
||||
display_name: "Qwen3 8B",
|
||||
max_context_length: 262144,
|
||||
format: "mlx",
|
||||
capabilities: {
|
||||
vision: true,
|
||||
trained_for_tool_use: true,
|
||||
reasoning: {
|
||||
allowed_options: ["off", "on"],
|
||||
default: "on",
|
||||
},
|
||||
},
|
||||
loaded_instances: [{ id: "inst-1", config: { context_length: 64000 } }],
|
||||
},
|
||||
{
|
||||
type: "llm",
|
||||
key: "deepseek-r1",
|
||||
},
|
||||
{
|
||||
type: "embedding",
|
||||
key: "text-embedding-nomic-embed-text-v1.5",
|
||||
},
|
||||
{
|
||||
type: "llm",
|
||||
key: " ",
|
||||
},
|
||||
],
|
||||
}),
|
||||
}));
|
||||
|
||||
const models = await discoverLmstudioModels({
|
||||
baseUrl: "http://localhost:1234/v1",
|
||||
apiKey: "lm-token",
|
||||
quiet: false,
|
||||
fetchImpl: asFetch(fetchMock),
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"http://localhost:1234/api/v1/models",
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
Authorization: "Bearer lm-token",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(models).toHaveLength(2);
|
||||
expect(models[0]).toEqual({
|
||||
id: "qwen3-8b-instruct",
|
||||
name: "Qwen3 8B (MLX, vision, tool-use, loaded)",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
compat: { supportsUsageInStreaming: true },
|
||||
contextWindow: 262144,
|
||||
contextTokens: LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH,
|
||||
maxTokens: SELF_HOSTED_DEFAULT_MAX_TOKENS,
|
||||
});
|
||||
expect(models[1]).toEqual({
|
||||
id: "deepseek-r1",
|
||||
name: "deepseek-r1",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
compat: { supportsUsageInStreaming: true },
|
||||
contextWindow: SELF_HOSTED_DEFAULT_CONTEXT_WINDOW,
|
||||
contextTokens: LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH,
|
||||
maxTokens: SELF_HOSTED_DEFAULT_MAX_TOKENS,
|
||||
});
|
||||
});
|
||||
|
||||
it("skips model load when already loaded", async () => {
|
||||
const fetchMock = vi.fn(async (_url: string | URL) => ({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{
|
||||
type: "llm",
|
||||
key: "qwen3-8b-instruct",
|
||||
loaded_instances: [{ id: "inst-1", config: { context_length: 64000 } }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
}));
|
||||
vi.stubGlobal("fetch", asFetch(fetchMock));
|
||||
|
||||
await expect(
|
||||
ensureLmstudioModelLoaded({
|
||||
baseUrl: "http://localhost:1234/v1",
|
||||
modelKey: "qwen3-8b-instruct",
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
const calledUrls = fetchMock.mock.calls.map((call) => String(call[0]));
|
||||
expect(calledUrls).not.toContain("http://localhost:1234/api/v1/models/load");
|
||||
});
|
||||
|
||||
it("reloads model when requested context length exceeds the loaded window", async () => {
|
||||
const fetchMock = vi.fn(async (url: string | URL, init?: RequestInit) => {
|
||||
if (String(url).endsWith("/api/v1/models")) {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{
|
||||
type: "llm",
|
||||
key: "qwen3-8b-instruct",
|
||||
max_context_length: 32768,
|
||||
loaded_instances: [{ id: "inst-1", config: { context_length: 4096 } }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (String(url).endsWith("/api/v1/models/load")) {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({ status: "loaded" }),
|
||||
requestInit: init,
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected fetch URL: ${String(url)}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", asFetch(fetchMock));
|
||||
|
||||
await expect(
|
||||
ensureLmstudioModelLoaded({
|
||||
baseUrl: "http://localhost:1234/v1",
|
||||
modelKey: "qwen3-8b-instruct",
|
||||
requestedContextLength: 8192,
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
const loadCall = fetchMock.mock.calls.find((call) => String(call[0]).endsWith("/models/load"));
|
||||
expect(loadCall).toBeDefined();
|
||||
const loadInit = loadCall?.[1] as RequestInit;
|
||||
const loadBody = parseJsonRequestBody(loadInit) as { context_length: number };
|
||||
expect(loadBody.context_length).toBe(8192);
|
||||
});
|
||||
|
||||
it("reloads model to the clamped default target when already loaded below the default window", async () => {
|
||||
const fetchMock = vi.fn(async (url: string | URL, init?: RequestInit) => {
|
||||
if (String(url).endsWith("/api/v1/models")) {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{
|
||||
type: "llm",
|
||||
key: "qwen3-8b-instruct",
|
||||
max_context_length: 32768,
|
||||
loaded_instances: [{ id: "inst-1", config: { context_length: 4096 } }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (String(url).endsWith("/api/v1/models/load")) {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({ status: "loaded" }),
|
||||
requestInit: init,
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected fetch URL: ${String(url)}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", asFetch(fetchMock));
|
||||
|
||||
await expect(
|
||||
ensureLmstudioModelLoaded({
|
||||
baseUrl: "http://localhost:1234/v1",
|
||||
modelKey: "qwen3-8b-instruct",
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
const loadCall = fetchMock.mock.calls.find((call) => String(call[0]).endsWith("/models/load"));
|
||||
expect(loadCall).toBeDefined();
|
||||
const loadInit = loadCall?.[1] as RequestInit;
|
||||
const loadBody = parseJsonRequestBody(loadInit) as { context_length: number };
|
||||
expect(loadBody.context_length).toBe(32768);
|
||||
});
|
||||
|
||||
it("loads model with clamped context length and merged headers", async () => {
|
||||
const fetchMock = vi.fn(async (url: string | URL, init?: RequestInit) => {
|
||||
if (String(url).endsWith("/api/v1/models")) {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{
|
||||
type: "llm",
|
||||
key: "qwen3-8b-instruct",
|
||||
max_context_length: 32768,
|
||||
loaded_instances: [],
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (String(url).endsWith("/api/v1/models/load")) {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({ status: "loaded" }),
|
||||
requestInit: init,
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected fetch URL: ${String(url)}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", asFetch(fetchMock));
|
||||
|
||||
await expect(
|
||||
ensureLmstudioModelLoaded({
|
||||
baseUrl: "http://localhost:1234/v1",
|
||||
apiKey: "lm-token",
|
||||
headers: {
|
||||
"X-Proxy-Auth": "required",
|
||||
Authorization: "Bearer override",
|
||||
},
|
||||
modelKey: " qwen3-8b-instruct ",
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
const loadCall = fetchMock.mock.calls.find((call) => String(call[0]).endsWith("/models/load"));
|
||||
expect(loadCall).toBeDefined();
|
||||
expect(loadCall?.[1]).toMatchObject({
|
||||
method: "POST",
|
||||
headers: {
|
||||
"X-Proxy-Auth": "required",
|
||||
Authorization: "Bearer lm-token",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: "qwen3-8b-instruct",
|
||||
context_length: 32768,
|
||||
}),
|
||||
});
|
||||
const loadInit = loadCall![1] as RequestInit;
|
||||
const loadBody = parseJsonRequestBody(loadInit) as { context_length: number };
|
||||
expect(loadBody.context_length).not.toBe(LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH);
|
||||
});
|
||||
|
||||
it("uses requested context length when provided for model load", async () => {
|
||||
const fetchMock = vi.fn(async (url: string | URL, _init?: RequestInit) => {
|
||||
if (String(url).endsWith("/api/v1/models")) {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
models: [
|
||||
{
|
||||
type: "llm",
|
||||
key: "qwen3-8b-instruct",
|
||||
max_context_length: 32768,
|
||||
loaded_instances: [],
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (String(url).endsWith("/api/v1/models/load")) {
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({ status: "loaded" }),
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected fetch URL: ${String(url)}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", asFetch(fetchMock));
|
||||
|
||||
await expect(
|
||||
ensureLmstudioModelLoaded({
|
||||
baseUrl: "http://localhost:1234/v1",
|
||||
modelKey: "qwen3-8b-instruct",
|
||||
requestedContextLength: 8192,
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
const loadCall = fetchMock.mock.calls.find((call) => String(call[0]).endsWith("/models/load"));
|
||||
expect(loadCall).toBeDefined();
|
||||
const loadInit = loadCall?.[1] as unknown as RequestInit;
|
||||
const loadBody = parseJsonRequestBody(loadInit) as { context_length: number };
|
||||
expect(loadBody.context_length).toBe(8192);
|
||||
});
|
||||
|
||||
it("throws when model discovery fails", async () => {
|
||||
const fetchMock = vi.fn(async () => ({
|
||||
ok: false,
|
||||
status: 401,
|
||||
}));
|
||||
vi.stubGlobal("fetch", asFetch(fetchMock));
|
||||
|
||||
await expect(
|
||||
ensureLmstudioModelLoaded({
|
||||
baseUrl: "http://localhost:1234/v1",
|
||||
modelKey: "qwen3-8b-instruct",
|
||||
}),
|
||||
).rejects.toThrow("LM Studio model discovery failed (401)");
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
351
openclaw/extensions/lmstudio/src/models.ts
Normal file
351
openclaw/extensions/lmstudio/src/models.ts
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
import type {
|
||||
ModelDefinitionConfig,
|
||||
ModelProviderConfig,
|
||||
} from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import {
|
||||
SELF_HOSTED_DEFAULT_CONTEXT_WINDOW,
|
||||
SELF_HOSTED_DEFAULT_COST,
|
||||
SELF_HOSTED_DEFAULT_MAX_TOKENS,
|
||||
} from "openclaw/plugin-sdk/provider-setup";
|
||||
import { LMSTUDIO_DEFAULT_BASE_URL, LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH } from "./defaults.js";
|
||||
|
||||
export type LmstudioModelWire = {
|
||||
type?: "llm" | "embedding";
|
||||
key?: string;
|
||||
display_name?: string;
|
||||
max_context_length?: number;
|
||||
format?: "gguf" | "mlx" | null;
|
||||
capabilities?: {
|
||||
vision?: boolean;
|
||||
trained_for_tool_use?: boolean;
|
||||
reasoning?: LmstudioReasoningCapabilityWire;
|
||||
};
|
||||
loaded_instances?: Array<{
|
||||
id?: string;
|
||||
config?: {
|
||||
context_length?: number;
|
||||
} | null;
|
||||
} | null>;
|
||||
};
|
||||
|
||||
type LmstudioReasoningCapabilityWire = {
|
||||
allowed_options?: unknown;
|
||||
default?: unknown;
|
||||
};
|
||||
|
||||
type LmstudioConfiguredCatalogEntry = {
|
||||
id: string;
|
||||
name?: string;
|
||||
contextWindow?: number;
|
||||
contextTokens?: number;
|
||||
reasoning?: boolean;
|
||||
input?: ("text" | "image" | "document")[];
|
||||
};
|
||||
|
||||
function normalizeReasoningOption(value: unknown): string | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized.length > 0 ? normalized : null;
|
||||
}
|
||||
|
||||
function isReasoningEnabledOption(value: unknown): boolean {
|
||||
const normalized = normalizeReasoningOption(value);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
return normalized !== "off";
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves LM Studio reasoning support from capabilities payloads.
|
||||
* Defaults to false when the server omits reasoning metadata.
|
||||
*/
|
||||
export function resolveLmstudioReasoningCapability(
|
||||
entry: Pick<LmstudioModelWire, "capabilities">,
|
||||
): boolean {
|
||||
const reasoning = entry.capabilities?.reasoning;
|
||||
if (reasoning === undefined || reasoning === null) {
|
||||
return false;
|
||||
}
|
||||
const allowedOptionsRaw = reasoning.allowed_options;
|
||||
const allowedOptions = Array.isArray(allowedOptionsRaw)
|
||||
? allowedOptionsRaw
|
||||
.map((option) => normalizeReasoningOption(option))
|
||||
.filter((option): option is string => option !== null)
|
||||
: [];
|
||||
if (allowedOptions.length > 0) {
|
||||
return allowedOptions.some((option) => isReasoningEnabledOption(option));
|
||||
}
|
||||
return isReasoningEnabledOption(reasoning.default);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads loaded LM Studio instances and returns the largest valid context window.
|
||||
* Returns null when no usable loaded context is present.
|
||||
*/
|
||||
export function resolveLoadedContextWindow(
|
||||
entry: Pick<LmstudioModelWire, "loaded_instances">,
|
||||
): number | null {
|
||||
const loadedInstances = Array.isArray(entry.loaded_instances) ? entry.loaded_instances : [];
|
||||
let contextWindow: number | null = null;
|
||||
for (const instance of loadedInstances) {
|
||||
// Discovery payload is external JSON, so tolerate malformed entries.
|
||||
const length = instance?.config?.context_length;
|
||||
if (length === undefined || !Number.isFinite(length) || length <= 0) {
|
||||
continue;
|
||||
}
|
||||
const normalized = Math.floor(length);
|
||||
contextWindow = contextWindow === null ? normalized : Math.max(contextWindow, normalized);
|
||||
}
|
||||
return contextWindow;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a server path by stripping trailing slash and inference suffixes.
|
||||
*
|
||||
* LM Studio users often copy their inference URL (e.g. "http://localhost:1234/v1") instead
|
||||
* of the server root. This function strips a trailing "/v1" or "/api/v1" so the caller always
|
||||
* receives a clean root base URL. The expected input is the server root without any API version
|
||||
* path (e.g. "http://localhost:1234").
|
||||
*/
|
||||
function normalizeUrlPath(pathname: string): string {
|
||||
const trimmed = pathname.replace(/\/+$/, "");
|
||||
if (!trimmed) {
|
||||
return "";
|
||||
}
|
||||
return trimmed.replace(/\/api\/v1$/i, "").replace(/\/v1$/i, "");
|
||||
}
|
||||
|
||||
function hasExplicitHttpScheme(value: string): boolean {
|
||||
return /^https?:\/\//i.test(value);
|
||||
}
|
||||
|
||||
function isLikelyHostBaseUrl(value: string): boolean {
|
||||
return (
|
||||
/^(?:localhost|(?:\d{1,3}\.){3}\d{1,3}|[a-z0-9.-]+\.[a-z]{2,}|[^/\s?#]+:\d+)(?:[/?#].*)?$/i.test(
|
||||
value,
|
||||
) && !value.startsWith("/")
|
||||
);
|
||||
}
|
||||
|
||||
function toFetchableLmstudioBaseUrl(value: string): string {
|
||||
if (hasExplicitHttpScheme(value) || !isLikelyHostBaseUrl(value)) {
|
||||
return value;
|
||||
}
|
||||
return `http://${value}`;
|
||||
}
|
||||
|
||||
/** Resolves LM Studio server base URL (without /v1 or /api/v1). */
|
||||
export function resolveLmstudioServerBase(configuredBaseUrl?: string): string {
|
||||
// Use configured value when present; otherwise target local LM Studio default.
|
||||
const configured = configuredBaseUrl?.trim();
|
||||
const resolved = configured && configured.length > 0 ? configured : LMSTUDIO_DEFAULT_BASE_URL;
|
||||
const fetchableBaseUrl = toFetchableLmstudioBaseUrl(resolved);
|
||||
try {
|
||||
const parsed = new URL(fetchableBaseUrl);
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new TypeError(`Unsupported LM Studio protocol: ${parsed.protocol}`);
|
||||
}
|
||||
const pathname = normalizeUrlPath(parsed.pathname);
|
||||
parsed.pathname = pathname.length > 0 ? pathname : "/";
|
||||
parsed.search = "";
|
||||
parsed.hash = "";
|
||||
return parsed.toString().replace(/\/$/, "");
|
||||
} catch {
|
||||
const trimmed = resolved.replace(/\/+$/, "");
|
||||
const normalized = normalizeUrlPath(trimmed);
|
||||
return normalized.length > 0 ? normalized : LMSTUDIO_DEFAULT_BASE_URL;
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolves LM Studio inference base URL and always appends /v1. */
|
||||
export function resolveLmstudioInferenceBase(configuredBaseUrl?: string): string {
|
||||
const serverBase = resolveLmstudioServerBase(configuredBaseUrl);
|
||||
return `${serverBase}/v1`;
|
||||
}
|
||||
|
||||
/** Canonicalizes persisted LM Studio provider config to the inference base URL form. */
|
||||
export function normalizeLmstudioProviderConfig(
|
||||
provider: ModelProviderConfig,
|
||||
): ModelProviderConfig {
|
||||
const configuredBaseUrl = typeof provider.baseUrl === "string" ? provider.baseUrl.trim() : "";
|
||||
if (!configuredBaseUrl) {
|
||||
return provider;
|
||||
}
|
||||
const normalizedBaseUrl = resolveLmstudioInferenceBase(configuredBaseUrl);
|
||||
return normalizedBaseUrl === provider.baseUrl
|
||||
? provider
|
||||
: { ...provider, baseUrl: normalizedBaseUrl };
|
||||
}
|
||||
|
||||
export function normalizeLmstudioConfiguredCatalogEntry(
|
||||
entry: unknown,
|
||||
): LmstudioConfiguredCatalogEntry | null {
|
||||
if (!entry || typeof entry !== "object") {
|
||||
return null;
|
||||
}
|
||||
const record = entry as Record<string, unknown>;
|
||||
if (typeof record.id !== "string" || record.id.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
const id = record.id.trim();
|
||||
const name = typeof record.name === "string" && record.name.trim().length > 0 ? record.name : id;
|
||||
const contextWindow =
|
||||
typeof record.contextWindow === "number" && record.contextWindow > 0
|
||||
? record.contextWindow
|
||||
: undefined;
|
||||
const contextTokens =
|
||||
typeof record.contextTokens === "number" && record.contextTokens > 0
|
||||
? record.contextTokens
|
||||
: undefined;
|
||||
const reasoning = typeof record.reasoning === "boolean" ? record.reasoning : undefined;
|
||||
const input = Array.isArray(record.input)
|
||||
? record.input.filter(
|
||||
(item): item is "text" | "image" | "document" =>
|
||||
item === "text" || item === "image" || item === "document",
|
||||
)
|
||||
: undefined;
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
contextWindow,
|
||||
contextTokens,
|
||||
reasoning,
|
||||
input: input && input.length > 0 ? input : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeLmstudioConfiguredCatalogEntries(
|
||||
models: unknown,
|
||||
): LmstudioConfiguredCatalogEntry[] {
|
||||
if (!Array.isArray(models)) {
|
||||
return [];
|
||||
}
|
||||
return models
|
||||
.map((entry) => normalizeLmstudioConfiguredCatalogEntry(entry))
|
||||
.filter((entry): entry is LmstudioConfiguredCatalogEntry => entry !== null);
|
||||
}
|
||||
|
||||
export function buildLmstudioModelName(model: {
|
||||
displayName: string;
|
||||
format: "gguf" | "mlx" | null;
|
||||
vision: boolean;
|
||||
trainedForToolUse: boolean;
|
||||
loaded: boolean;
|
||||
}): string {
|
||||
const tags: string[] = [];
|
||||
if (model.format === "mlx") {
|
||||
tags.push("MLX");
|
||||
} else if (model.format === "gguf") {
|
||||
tags.push("GGUF");
|
||||
}
|
||||
if (model.vision) {
|
||||
tags.push("vision");
|
||||
}
|
||||
if (model.trainedForToolUse) {
|
||||
tags.push("tool-use");
|
||||
}
|
||||
if (model.loaded) {
|
||||
tags.push("loaded");
|
||||
}
|
||||
if (tags.length === 0) {
|
||||
return model.displayName;
|
||||
}
|
||||
return `${model.displayName} (${tags.join(", ")})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Base model fields extracted from a single LM Studio wire entry.
|
||||
* Shared by the setup layer (persists simple names to config) and the runtime
|
||||
* discovery path (which enriches the name with format/state tags).
|
||||
*/
|
||||
export type LmstudioModelBase = {
|
||||
id: string;
|
||||
displayName: string;
|
||||
format: "gguf" | "mlx" | null;
|
||||
vision: boolean;
|
||||
trainedForToolUse: boolean;
|
||||
loaded: boolean;
|
||||
reasoning: boolean;
|
||||
input: ModelDefinitionConfig["input"];
|
||||
cost: ModelDefinitionConfig["cost"];
|
||||
contextWindow: number;
|
||||
contextTokens: number;
|
||||
maxTokens: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Maps a single LM Studio wire entry to its base model fields.
|
||||
* Returns null for non-LLM entries or entries with no usable key.
|
||||
*
|
||||
* Shared by both the setup layer (persists simple names to config) and the
|
||||
* runtime discovery path (which enriches the name with format/state tags via
|
||||
* buildLmstudioModelName).
|
||||
*/
|
||||
export function mapLmstudioWireEntry(entry: LmstudioModelWire): LmstudioModelBase | null {
|
||||
if (entry.type !== "llm") {
|
||||
return null;
|
||||
}
|
||||
const id = entry.key?.trim() ?? "";
|
||||
if (!id) {
|
||||
return null;
|
||||
}
|
||||
const loadedContextWindow = resolveLoadedContextWindow(entry);
|
||||
const advertisedContextWindow =
|
||||
entry.max_context_length !== undefined &&
|
||||
Number.isFinite(entry.max_context_length) &&
|
||||
entry.max_context_length > 0
|
||||
? Math.floor(entry.max_context_length)
|
||||
: null;
|
||||
const contextWindow = advertisedContextWindow ?? SELF_HOSTED_DEFAULT_CONTEXT_WINDOW;
|
||||
// Keep native/advertised context window metadata in catalog, but use a practical
|
||||
// default target for model loading unless callers explicitly override it.
|
||||
const contextTokens = Math.min(contextWindow, LMSTUDIO_DEFAULT_LOAD_CONTEXT_LENGTH);
|
||||
const rawDisplayName = entry.display_name?.trim();
|
||||
return {
|
||||
id,
|
||||
displayName: rawDisplayName && rawDisplayName.length > 0 ? rawDisplayName : id,
|
||||
format: entry.format ?? null,
|
||||
vision: entry.capabilities?.vision === true,
|
||||
trainedForToolUse: entry.capabilities?.trained_for_tool_use === true,
|
||||
// Use the same validity check as resolveLoadedContextWindow so malformed entries
|
||||
// like [null, {}] don't produce a false positive "loaded" tag.
|
||||
loaded: loadedContextWindow !== null,
|
||||
reasoning: resolveLmstudioReasoningCapability(entry),
|
||||
input: entry.capabilities?.vision ? ["text", "image"] : ["text"],
|
||||
cost: SELF_HOSTED_DEFAULT_COST,
|
||||
contextWindow,
|
||||
contextTokens,
|
||||
maxTokens: Math.max(1, Math.min(contextWindow, SELF_HOSTED_DEFAULT_MAX_TOKENS)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps LM Studio wire models to config entries using plain display names.
|
||||
* Use this for config persistence where runtime format/state tags are not needed.
|
||||
* For runtime discovery with enriched names, use discoverLmstudioModels from models.fetch.ts.
|
||||
*/
|
||||
export function mapLmstudioWireModelsToConfig(
|
||||
models: LmstudioModelWire[],
|
||||
): ModelDefinitionConfig[] {
|
||||
return models
|
||||
.map((entry): ModelDefinitionConfig | null => {
|
||||
const base = mapLmstudioWireEntry(entry);
|
||||
if (!base) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
id: base.id,
|
||||
name: base.displayName,
|
||||
reasoning: base.reasoning,
|
||||
input: base.input,
|
||||
cost: base.cost,
|
||||
contextWindow: base.contextWindow,
|
||||
contextTokens: base.contextTokens,
|
||||
maxTokens: base.maxTokens,
|
||||
};
|
||||
})
|
||||
.filter((entry): entry is ModelDefinitionConfig => entry !== null);
|
||||
}
|
||||
59
openclaw/extensions/lmstudio/src/provider-auth.ts
Normal file
59
openclaw/extensions/lmstudio/src/provider-auth.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import {
|
||||
CUSTOM_LOCAL_AUTH_MARKER,
|
||||
hasConfiguredSecretInput,
|
||||
normalizeOptionalSecretInput,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER } from "./defaults.js";
|
||||
|
||||
export function hasLmstudioAuthorizationHeader(headers: unknown): boolean {
|
||||
if (!headers || typeof headers !== "object" || Array.isArray(headers)) {
|
||||
return false;
|
||||
}
|
||||
for (const [headerName, headerValue] of Object.entries(headers)) {
|
||||
if (headerName.trim().toLowerCase() !== "authorization") {
|
||||
continue;
|
||||
}
|
||||
if (hasConfiguredSecretInput(headerValue)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function resolveLmstudioProviderAuthMode(
|
||||
apiKey: ModelProviderConfig["apiKey"] | undefined,
|
||||
): ModelProviderConfig["auth"] | undefined {
|
||||
const normalized = normalizeOptionalSecretInput(apiKey);
|
||||
if (normalized !== undefined) {
|
||||
const trimmed = normalized.trim();
|
||||
if (
|
||||
!trimmed ||
|
||||
trimmed === LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER ||
|
||||
trimmed === CUSTOM_LOCAL_AUTH_MARKER
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return "api-key";
|
||||
}
|
||||
return hasConfiguredSecretInput(apiKey) ? "api-key" : undefined;
|
||||
}
|
||||
|
||||
export function shouldUseLmstudioApiKeyPlaceholder(params: {
|
||||
hasModels: boolean;
|
||||
resolvedApiKey: ModelProviderConfig["apiKey"] | undefined;
|
||||
hasAuthorizationHeader?: boolean;
|
||||
}): boolean {
|
||||
return params.hasModels && !params.resolvedApiKey && !params.hasAuthorizationHeader;
|
||||
}
|
||||
|
||||
export function shouldUseLmstudioSyntheticAuth(
|
||||
providerConfig: ModelProviderConfig | undefined,
|
||||
): boolean {
|
||||
const hasModels = Array.isArray(providerConfig?.models) && providerConfig.models.length > 0;
|
||||
return (
|
||||
hasModels &&
|
||||
!resolveLmstudioProviderAuthMode(providerConfig?.apiKey) &&
|
||||
!hasLmstudioAuthorizationHeader(providerConfig?.headers)
|
||||
);
|
||||
}
|
||||
310
openclaw/extensions/lmstudio/src/runtime.test.ts
Normal file
310
openclaw/extensions/lmstudio/src/runtime.test.ts
Normal file
|
|
@ -0,0 +1,310 @@
|
|||
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { CUSTOM_LOCAL_AUTH_MARKER } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER } from "./defaults.js";
|
||||
import {
|
||||
buildLmstudioAuthHeaders,
|
||||
resolveLmstudioConfiguredApiKey,
|
||||
resolveLmstudioProviderHeaders,
|
||||
resolveLmstudioRuntimeApiKey,
|
||||
} from "./runtime.js";
|
||||
|
||||
const resolveApiKeyForProviderMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/provider-auth-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/provider-auth-runtime")>();
|
||||
return {
|
||||
...actual,
|
||||
resolveApiKeyForProvider: (...args: unknown[]) => resolveApiKeyForProviderMock(...args),
|
||||
};
|
||||
});
|
||||
|
||||
function buildLmstudioConfig(overrides?: {
|
||||
apiKey?: unknown;
|
||||
headers?: unknown;
|
||||
auth?: "api-key";
|
||||
}): OpenClawConfig {
|
||||
return {
|
||||
models: {
|
||||
providers: {
|
||||
lmstudio: {
|
||||
baseUrl: "http://localhost:1234/v1",
|
||||
api: "openai-completions",
|
||||
...(overrides?.auth ? { auth: overrides.auth } : {}),
|
||||
...(overrides?.apiKey !== undefined ? { apiKey: overrides.apiKey } : {}),
|
||||
...(overrides?.headers !== undefined ? { headers: overrides.headers } : {}),
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
}
|
||||
|
||||
describe("lmstudio-runtime", () => {
|
||||
beforeEach(() => {
|
||||
resolveApiKeyForProviderMock.mockReset();
|
||||
});
|
||||
|
||||
it("throws when runtime auth resolves to blank and no configured key exists", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: " ",
|
||||
source: "profile:lmstudio:default",
|
||||
mode: "api-key",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig({ auth: "api-key" }),
|
||||
}),
|
||||
).rejects.toThrow(/LM Studio API key is required/i);
|
||||
});
|
||||
|
||||
it("falls back to configured env marker key when profile resolution fails", async () => {
|
||||
resolveApiKeyForProviderMock.mockRejectedValueOnce(
|
||||
new Error('No API key found for provider "lmstudio". Auth store: /tmp/auth-profiles.json.'),
|
||||
);
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig({
|
||||
auth: "api-key",
|
||||
apiKey: "${LM_API_TOKEN}",
|
||||
}),
|
||||
env: {
|
||||
LM_API_TOKEN: "template-lmstudio-key",
|
||||
},
|
||||
}),
|
||||
).resolves.toBe("template-lmstudio-key");
|
||||
});
|
||||
|
||||
it("accepts synthesized lmstudio-local for non-explicit auth mode", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER,
|
||||
source: "models.providers.lmstudio (synthetic local key)",
|
||||
mode: "api-key",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig(),
|
||||
}),
|
||||
).resolves.toBe(LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER);
|
||||
});
|
||||
|
||||
it("accepts synthesized lmstudio-local for explicit api-key mode", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER,
|
||||
source: "models.providers.lmstudio (synthetic local key)",
|
||||
mode: "api-key",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig({ auth: "api-key" }),
|
||||
}),
|
||||
).resolves.toBe(LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER);
|
||||
});
|
||||
|
||||
it("accepts shared synthetic local marker for keyless runtime auth", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: CUSTOM_LOCAL_AUTH_MARKER,
|
||||
source: "models.providers.lmstudio (synthetic local key)",
|
||||
mode: "api-key",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig(),
|
||||
}),
|
||||
).resolves.toBe(CUSTOM_LOCAL_AUTH_MARKER);
|
||||
});
|
||||
|
||||
it("allows header-only runtime auth when Authorization is configured", async () => {
|
||||
resolveApiKeyForProviderMock.mockRejectedValueOnce(
|
||||
new Error('No API key found for provider "lmstudio". Auth store: /tmp/auth-profiles.json.'),
|
||||
);
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig({
|
||||
headers: {
|
||||
Authorization: "Bearer proxy-token",
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("suppresses profile runtime auth when Authorization is configured", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: "stale-profile-key",
|
||||
source: "profile:lmstudio:default",
|
||||
mode: "api-key",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig({
|
||||
headers: {
|
||||
Authorization: "Bearer proxy-token",
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("suppresses env runtime auth when Authorization is configured", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: "stale-env-key",
|
||||
source: "env:LM_API_TOKEN",
|
||||
mode: "api-key",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig({
|
||||
headers: {
|
||||
Authorization: "Bearer proxy-token",
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("suppresses shell env runtime auth when Authorization is configured", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: "stale-shell-env-key",
|
||||
source: "shell env: LM_API_TOKEN",
|
||||
mode: "api-key",
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig({
|
||||
headers: {
|
||||
Authorization: "Bearer proxy-token",
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("throws when explicit api-key mode cannot resolve any key", async () => {
|
||||
resolveApiKeyForProviderMock.mockRejectedValue(
|
||||
new Error('No API key found for provider "lmstudio". Auth store: /tmp/auth-profiles.json.'),
|
||||
);
|
||||
|
||||
await expect(
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: buildLmstudioConfig({ auth: "api-key" }),
|
||||
}),
|
||||
).rejects.toThrow(/LM Studio API key is required/i);
|
||||
|
||||
await expect(
|
||||
resolveLmstudioConfiguredApiKey({
|
||||
config: buildLmstudioConfig({ auth: "api-key" }),
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("resolves SecretRef api key and headers", async () => {
|
||||
const headerRef = {
|
||||
"X-Proxy-Auth": {
|
||||
source: "env" as const,
|
||||
provider: "default" as const,
|
||||
id: "LMSTUDIO_PROXY_TOKEN",
|
||||
},
|
||||
};
|
||||
await expect(
|
||||
resolveLmstudioConfiguredApiKey({
|
||||
config: buildLmstudioConfig({
|
||||
apiKey: {
|
||||
source: "env",
|
||||
provider: "default",
|
||||
id: "LM_API_TOKEN",
|
||||
},
|
||||
}),
|
||||
env: {
|
||||
LM_API_TOKEN: "secretref-lmstudio-key",
|
||||
},
|
||||
}),
|
||||
).resolves.toBe("secretref-lmstudio-key");
|
||||
|
||||
await expect(
|
||||
resolveLmstudioProviderHeaders({
|
||||
config: buildLmstudioConfig({ headers: headerRef }),
|
||||
env: {
|
||||
LMSTUDIO_PROXY_TOKEN: "proxy-token",
|
||||
},
|
||||
headers: headerRef,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
"X-Proxy-Auth": "proxy-token",
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves env-template api keys from config", async () => {
|
||||
await expect(
|
||||
resolveLmstudioConfiguredApiKey({
|
||||
config: buildLmstudioConfig({
|
||||
apiKey: "${LM_API_TOKEN}",
|
||||
}),
|
||||
env: {
|
||||
LM_API_TOKEN: "template-lmstudio-key",
|
||||
},
|
||||
}),
|
||||
).resolves.toBe("template-lmstudio-key");
|
||||
});
|
||||
|
||||
it("throws a path-specific error when a SecretRef header cannot be resolved", async () => {
|
||||
const headerRef = {
|
||||
"X-Proxy-Auth": {
|
||||
source: "env" as const,
|
||||
provider: "default" as const,
|
||||
id: "LMSTUDIO_PROXY_TOKEN",
|
||||
},
|
||||
};
|
||||
await expect(
|
||||
resolveLmstudioProviderHeaders({
|
||||
config: buildLmstudioConfig({ headers: headerRef }),
|
||||
env: {},
|
||||
headers: headerRef,
|
||||
}),
|
||||
).rejects.toThrow(/models\.providers\.lmstudio\.headers\.X-Proxy-Auth/i);
|
||||
});
|
||||
|
||||
it("builds auth headers with key precedence and json support", () => {
|
||||
expect(buildLmstudioAuthHeaders({})).toBeUndefined();
|
||||
expect(buildLmstudioAuthHeaders({ apiKey: " sk-test " })).toEqual({
|
||||
Authorization: "Bearer sk-test",
|
||||
});
|
||||
expect(buildLmstudioAuthHeaders({ apiKey: " " })).toBeUndefined();
|
||||
expect(
|
||||
buildLmstudioAuthHeaders({ apiKey: LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER }),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
buildLmstudioAuthHeaders({
|
||||
apiKey: LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER,
|
||||
headers: {
|
||||
Authorization: "Bearer proxy-token",
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
Authorization: "Bearer proxy-token",
|
||||
});
|
||||
expect(
|
||||
buildLmstudioAuthHeaders({
|
||||
apiKey: "sk-new",
|
||||
json: true,
|
||||
headers: {
|
||||
authorization: "Bearer sk-old",
|
||||
"X-Proxy": "proxy-token",
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
"Content-Type": "application/json",
|
||||
"X-Proxy": "proxy-token",
|
||||
Authorization: "Bearer sk-new",
|
||||
});
|
||||
});
|
||||
});
|
||||
250
openclaw/extensions/lmstudio/src/runtime.ts
Normal file
250
openclaw/extensions/lmstudio/src/runtime.ts
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
import { resolveConfiguredSecretInputString } from "openclaw/plugin-sdk/config-runtime";
|
||||
import {
|
||||
CUSTOM_LOCAL_AUTH_MARKER,
|
||||
isKnownEnvApiKeyMarker,
|
||||
isNonSecretApiKeyMarker,
|
||||
normalizeApiKeyConfig,
|
||||
normalizeOptionalSecretInput,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
LMSTUDIO_DEFAULT_API_KEY_ENV_VAR,
|
||||
LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER,
|
||||
LMSTUDIO_PROVIDER_ID,
|
||||
} from "./defaults.js";
|
||||
import { hasLmstudioAuthorizationHeader } from "./provider-auth.js";
|
||||
|
||||
type LmstudioAuthHeadersParams = {
|
||||
apiKey?: string;
|
||||
json?: boolean;
|
||||
headers?: Record<string, string>;
|
||||
};
|
||||
|
||||
export function buildLmstudioAuthHeaders(
|
||||
params: LmstudioAuthHeadersParams,
|
||||
): Record<string, string> | undefined {
|
||||
const headers: Record<string, string> = { ...params.headers };
|
||||
// Runtime auth resolution is strict, but guard known non-secret markers here.
|
||||
const apiKey = params.apiKey?.trim();
|
||||
const isSyntheticLocalKey = apiKey === LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER;
|
||||
if (apiKey && !isSyntheticLocalKey && !isNonSecretApiKeyMarker(apiKey)) {
|
||||
for (const headerName of Object.keys(headers)) {
|
||||
if (headerName.toLowerCase() === "authorization") {
|
||||
delete headers[headerName];
|
||||
}
|
||||
}
|
||||
headers.Authorization = `Bearer ${apiKey}`;
|
||||
}
|
||||
if (params.json) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
}
|
||||
return Object.keys(headers).length > 0 ? headers : undefined;
|
||||
}
|
||||
|
||||
function sanitizeStringHeaders(headers: unknown): Record<string, string> | undefined {
|
||||
if (!headers || typeof headers !== "object" || Array.isArray(headers)) {
|
||||
return undefined;
|
||||
}
|
||||
const next: Record<string, string> = {};
|
||||
for (const [headerName, headerValue] of Object.entries(headers)) {
|
||||
if (typeof headerValue !== "string") {
|
||||
continue;
|
||||
}
|
||||
const normalized = headerValue.trim();
|
||||
if (!normalized) {
|
||||
continue;
|
||||
}
|
||||
next[headerName] = normalized;
|
||||
}
|
||||
return Object.keys(next).length > 0 ? next : undefined;
|
||||
}
|
||||
|
||||
function shouldSuppressResolvedRuntimeApiKeyForHeaderAuth(
|
||||
source: string | undefined,
|
||||
hasAuthorizationHeader: boolean,
|
||||
): boolean {
|
||||
if (!hasAuthorizationHeader || !source) {
|
||||
return false;
|
||||
}
|
||||
return /^profile:|^(?:shell )?env(?::|$)/.test(source);
|
||||
}
|
||||
|
||||
export async function resolveLmstudioConfiguredApiKey(params: {
|
||||
config?: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
path?: string;
|
||||
}): Promise<string | undefined> {
|
||||
const providerConfig = params.config?.models?.providers?.[LMSTUDIO_PROVIDER_ID];
|
||||
const apiKeyInput = providerConfig?.apiKey;
|
||||
if (apiKeyInput === undefined || apiKeyInput === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const directApiKey = normalizeOptionalSecretInput(apiKeyInput);
|
||||
if (directApiKey !== undefined) {
|
||||
const trimmed = normalizeApiKeyConfig(directApiKey).trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
if (isKnownEnvApiKeyMarker(trimmed)) {
|
||||
const envValue = normalizeOptionalSecretInput((params.env ?? process.env)[trimmed]);
|
||||
return envValue;
|
||||
}
|
||||
return isNonSecretApiKeyMarker(trimmed) ? undefined : trimmed;
|
||||
}
|
||||
|
||||
if (!params.config) {
|
||||
return undefined;
|
||||
}
|
||||
const path = params.path ?? "models.providers.lmstudio.apiKey";
|
||||
const resolved = await resolveConfiguredSecretInputString({
|
||||
config: params.config,
|
||||
env: params.env ?? process.env,
|
||||
value: apiKeyInput,
|
||||
path,
|
||||
unresolvedReasonStyle: "detailed",
|
||||
});
|
||||
if (resolved.unresolvedRefReason) {
|
||||
throw new Error(`${path}: ${resolved.unresolvedRefReason}`);
|
||||
}
|
||||
const resolvedValue = normalizeOptionalSecretInput(resolved.value);
|
||||
const trimmedResolvedValue = resolvedValue ? normalizeApiKeyConfig(resolvedValue).trim() : "";
|
||||
if (!trimmedResolvedValue) {
|
||||
return undefined;
|
||||
}
|
||||
if (isNonSecretApiKeyMarker(trimmedResolvedValue)) {
|
||||
return undefined;
|
||||
}
|
||||
return trimmedResolvedValue;
|
||||
}
|
||||
|
||||
export async function resolveLmstudioProviderHeaders(params: {
|
||||
config?: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
headers?: unknown;
|
||||
path?: string;
|
||||
}): Promise<Record<string, string> | undefined> {
|
||||
const headerInputs = params.headers;
|
||||
if (!headerInputs || typeof headerInputs !== "object" || Array.isArray(headerInputs)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (!params.config) {
|
||||
return sanitizeStringHeaders(headerInputs);
|
||||
}
|
||||
|
||||
const pathPrefix = params.path ?? "models.providers.lmstudio.headers";
|
||||
const resolved: Record<string, string> = {};
|
||||
for (const [headerName, headerValue] of Object.entries(headerInputs)) {
|
||||
const resolvedHeader = await resolveConfiguredSecretInputString({
|
||||
config: params.config,
|
||||
env: params.env ?? process.env,
|
||||
value: headerValue,
|
||||
path: `${pathPrefix}.${headerName}`,
|
||||
unresolvedReasonStyle: "detailed",
|
||||
});
|
||||
if (resolvedHeader.unresolvedRefReason) {
|
||||
throw new Error(`${pathPrefix}.${headerName}: ${resolvedHeader.unresolvedRefReason}`);
|
||||
}
|
||||
const resolvedValue = resolvedHeader.value;
|
||||
if (!resolvedValue) {
|
||||
continue;
|
||||
}
|
||||
resolved[headerName] = resolvedValue;
|
||||
}
|
||||
return Object.keys(resolved).length > 0 ? resolved : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves LM Studio API key and provider headers in parallel.
|
||||
* Use this as the standard auth setup step before discovery or model load calls.
|
||||
*/
|
||||
export async function resolveLmstudioRequestContext(params: {
|
||||
config?: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
providerHeaders?: unknown;
|
||||
}): Promise<{ apiKey: string | undefined; headers: Record<string, string> | undefined }> {
|
||||
const providerHeaders =
|
||||
params.providerHeaders ?? params.config?.models?.providers?.[LMSTUDIO_PROVIDER_ID]?.headers;
|
||||
const [apiKey, headers] = await Promise.all([
|
||||
resolveLmstudioRuntimeApiKey({
|
||||
config: params.config,
|
||||
agentDir: params.agentDir,
|
||||
env: params.env,
|
||||
headers: providerHeaders,
|
||||
}),
|
||||
resolveLmstudioProviderHeaders({
|
||||
config: params.config,
|
||||
env: params.env,
|
||||
headers: providerHeaders,
|
||||
}),
|
||||
]);
|
||||
return { apiKey, headers };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves LM Studio runtime API key from config.
|
||||
*/
|
||||
export async function resolveLmstudioRuntimeApiKey(params: {
|
||||
config?: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
headers?: unknown;
|
||||
}): Promise<string | undefined> {
|
||||
const config = params.config;
|
||||
if (!config) {
|
||||
return undefined;
|
||||
}
|
||||
const providerHeaders =
|
||||
params.headers ?? config.models?.providers?.[LMSTUDIO_PROVIDER_ID]?.headers;
|
||||
const hasAuthorizationHeader = hasLmstudioAuthorizationHeader(providerHeaders);
|
||||
let configuredApiKeyPromise: Promise<string | undefined> | undefined;
|
||||
const getConfiguredApiKey = async () => {
|
||||
configuredApiKeyPromise ??= resolveLmstudioConfiguredApiKey({
|
||||
config,
|
||||
env: params.env,
|
||||
});
|
||||
return await configuredApiKeyPromise;
|
||||
};
|
||||
const resolveConfiguredApiKeyOrThrow = async () => {
|
||||
const configuredApiKey = await getConfiguredApiKey();
|
||||
if (configuredApiKey) {
|
||||
return configuredApiKey;
|
||||
}
|
||||
if (hasAuthorizationHeader) {
|
||||
return undefined;
|
||||
}
|
||||
const envMarker = `\${${LMSTUDIO_DEFAULT_API_KEY_ENV_VAR}}`;
|
||||
throw new Error(
|
||||
[
|
||||
"LM Studio API key is required.",
|
||||
`Set models.providers.lmstudio.apiKey (for example "${envMarker}")`,
|
||||
'or run "openclaw models auth lmstudio".',
|
||||
].join(" "),
|
||||
);
|
||||
};
|
||||
let resolved: Awaited<ReturnType<typeof resolveApiKeyForProvider>>;
|
||||
try {
|
||||
resolved = await resolveApiKeyForProvider({
|
||||
provider: LMSTUDIO_PROVIDER_ID,
|
||||
cfg: config,
|
||||
agentDir: params.agentDir,
|
||||
});
|
||||
} catch {
|
||||
return await resolveConfiguredApiKeyOrThrow();
|
||||
}
|
||||
// Normalize empty/whitespace keys to undefined for callers.
|
||||
const resolvedApiKey = resolved.apiKey?.trim();
|
||||
if (!resolvedApiKey || resolvedApiKey.length === 0) {
|
||||
return await resolveConfiguredApiKeyOrThrow();
|
||||
}
|
||||
if (shouldSuppressResolvedRuntimeApiKeyForHeaderAuth(resolved.source, hasAuthorizationHeader)) {
|
||||
return await resolveConfiguredApiKeyOrThrow();
|
||||
}
|
||||
if (isNonSecretApiKeyMarker(resolvedApiKey) && resolvedApiKey !== CUSTOM_LOCAL_AUTH_MARKER) {
|
||||
return await resolveConfiguredApiKeyOrThrow();
|
||||
}
|
||||
return resolvedApiKey;
|
||||
}
|
||||
1232
openclaw/extensions/lmstudio/src/setup.test.ts
Normal file
1232
openclaw/extensions/lmstudio/src/setup.test.ts
Normal file
File diff suppressed because it is too large
Load diff
827
openclaw/extensions/lmstudio/src/setup.ts
Normal file
827
openclaw/extensions/lmstudio/src/setup.ts
Normal file
|
|
@ -0,0 +1,827 @@
|
|||
import {
|
||||
removeProviderAuthProfilesWithLock,
|
||||
buildApiKeyCredential,
|
||||
ensureApiKeyFromEnvOrPrompt,
|
||||
normalizeOptionalSecretInput,
|
||||
type OpenClawConfig,
|
||||
type SecretInput,
|
||||
type SecretInputMode,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import type {
|
||||
ModelDefinitionConfig,
|
||||
ModelProviderConfig,
|
||||
} from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { withAgentModelAliases } from "openclaw/plugin-sdk/provider-onboard";
|
||||
import {
|
||||
applyProviderDefaultModel,
|
||||
configureOpenAICompatibleSelfHostedProviderNonInteractive,
|
||||
type ProviderAuthMethodNonInteractiveContext,
|
||||
type ProviderAuthResult,
|
||||
type ProviderCatalogContext,
|
||||
type ProviderPrepareDynamicModelContext,
|
||||
type ProviderRuntimeModel,
|
||||
} from "openclaw/plugin-sdk/provider-setup";
|
||||
import { WizardCancelledError, type WizardPrompter } from "openclaw/plugin-sdk/setup";
|
||||
import {
|
||||
LMSTUDIO_DEFAULT_API_KEY_ENV_VAR,
|
||||
LMSTUDIO_DEFAULT_INFERENCE_BASE_URL,
|
||||
LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER,
|
||||
LMSTUDIO_MODEL_PLACEHOLDER,
|
||||
LMSTUDIO_DEFAULT_BASE_URL,
|
||||
LMSTUDIO_PROVIDER_LABEL,
|
||||
LMSTUDIO_DEFAULT_MODEL_ID,
|
||||
LMSTUDIO_PROVIDER_ID as PROVIDER_ID,
|
||||
} from "./defaults.js";
|
||||
import { discoverLmstudioModels, fetchLmstudioModels } from "./models.fetch.js";
|
||||
import {
|
||||
mapLmstudioWireModelsToConfig,
|
||||
type LmstudioModelWire,
|
||||
resolveLmstudioInferenceBase,
|
||||
} from "./models.js";
|
||||
import {
|
||||
hasLmstudioAuthorizationHeader,
|
||||
resolveLmstudioProviderAuthMode,
|
||||
shouldUseLmstudioApiKeyPlaceholder,
|
||||
} from "./provider-auth.js";
|
||||
import {
|
||||
resolveLmstudioConfiguredApiKey,
|
||||
resolveLmstudioProviderHeaders,
|
||||
resolveLmstudioRequestContext,
|
||||
} from "./runtime.js";
|
||||
|
||||
type ProviderPromptText = (params: {
|
||||
message: string;
|
||||
initialValue?: string;
|
||||
placeholder?: string;
|
||||
validate?: (value: string | undefined) => string | undefined;
|
||||
}) => Promise<string | undefined>;
|
||||
|
||||
type ProviderPromptNote = (message: string, title?: string) => Promise<void> | void;
|
||||
type LmstudioDiscoveryResult = Awaited<ReturnType<typeof fetchLmstudioModels>>;
|
||||
type LmstudioSetupDiscovery = {
|
||||
discovery: LmstudioDiscoveryResult;
|
||||
models: ModelDefinitionConfig[];
|
||||
defaultModel: string | undefined;
|
||||
defaultModelId: string | undefined;
|
||||
};
|
||||
|
||||
function stripLmstudioStoredAuthConfig(cfg: OpenClawConfig): OpenClawConfig {
|
||||
const { profiles: _profiles, order: _order, ...restAuth } = cfg.auth ?? {};
|
||||
const nextProfiles = Object.fromEntries(
|
||||
Object.entries(cfg.auth?.profiles ?? {}).filter(
|
||||
([, profile]) => profile.provider !== PROVIDER_ID,
|
||||
),
|
||||
);
|
||||
const nextOrder = Object.fromEntries(
|
||||
Object.entries(cfg.auth?.order ?? {}).filter(([providerId]) => providerId !== PROVIDER_ID),
|
||||
);
|
||||
return {
|
||||
...cfg,
|
||||
auth:
|
||||
Object.keys(restAuth).length > 0 ||
|
||||
Object.keys(nextProfiles).length > 0 ||
|
||||
Object.keys(nextOrder).length > 0
|
||||
? {
|
||||
...restAuth,
|
||||
...(Object.keys(nextProfiles).length > 0 ? { profiles: nextProfiles } : {}),
|
||||
...(Object.keys(nextOrder).length > 0 ? { order: nextOrder } : {}),
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function resolvePositiveInteger(value: unknown): number | undefined {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
const normalized = Math.floor(value);
|
||||
return normalized > 0 ? normalized : undefined;
|
||||
}
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed || !/^\d+$/.test(trimmed)) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = Number.parseInt(trimmed, 10);
|
||||
return Number.isFinite(normalized) && normalized > 0 ? normalized : undefined;
|
||||
}
|
||||
|
||||
function buildLmstudioSetupProviderConfig(params: {
|
||||
existingProvider: ModelProviderConfig | undefined;
|
||||
sharedProvider?: ModelProviderConfig;
|
||||
baseUrl: string;
|
||||
apiKey?: ModelProviderConfig["apiKey"];
|
||||
headers: ModelProviderConfig["headers"] | undefined;
|
||||
models: ModelDefinitionConfig[];
|
||||
}): ModelProviderConfig {
|
||||
const existingWithoutAuth = params.existingProvider
|
||||
? (({ auth: _auth, apiKey: _apiKey, ...rest }) => rest)(params.existingProvider)
|
||||
: undefined;
|
||||
const sharedWithoutAuth = params.sharedProvider
|
||||
? (({ auth: _auth, apiKey: _apiKey, ...rest }) => rest)(params.sharedProvider)
|
||||
: undefined;
|
||||
const resolvedAuth = resolveLmstudioProviderAuthMode(params.apiKey);
|
||||
return {
|
||||
...existingWithoutAuth,
|
||||
...sharedWithoutAuth,
|
||||
baseUrl: params.baseUrl,
|
||||
api: params.sharedProvider?.api ?? params.existingProvider?.api ?? "openai-completions",
|
||||
...(resolvedAuth ? { auth: resolvedAuth } : {}),
|
||||
...(params.apiKey !== undefined ? { apiKey: params.apiKey } : {}),
|
||||
headers: params.headers,
|
||||
models: params.models,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveLmstudioModelAdvertisedContextLimit(entry: LmstudioModelWire): number | undefined {
|
||||
const raw = entry.max_context_length;
|
||||
if (raw === undefined || !Number.isFinite(raw) || raw <= 0) {
|
||||
return undefined;
|
||||
}
|
||||
return Math.floor(raw);
|
||||
}
|
||||
|
||||
function applyModelContextTokensOverride(
|
||||
model: ModelDefinitionConfig,
|
||||
contextTokens: number,
|
||||
): ModelDefinitionConfig {
|
||||
return {
|
||||
...model,
|
||||
contextTokens,
|
||||
maxTokens: Math.min(model.maxTokens, contextTokens),
|
||||
};
|
||||
}
|
||||
|
||||
function applyRequestedContextWindowToAllModels(params: {
|
||||
models: ModelDefinitionConfig[];
|
||||
discoveryModels: LmstudioModelWire[];
|
||||
requestedContextWindow?: number;
|
||||
}): ModelDefinitionConfig[] {
|
||||
const requestedContextWindow = params.requestedContextWindow;
|
||||
if (!requestedContextWindow) {
|
||||
return params.models;
|
||||
}
|
||||
const contextLimitByModelId = new Map(
|
||||
params.discoveryModels
|
||||
.map((entry) => {
|
||||
const modelId = entry.key?.trim();
|
||||
if (!modelId) {
|
||||
return null;
|
||||
}
|
||||
return [modelId, resolveLmstudioModelAdvertisedContextLimit(entry)] as const;
|
||||
})
|
||||
.filter((entry): entry is readonly [string, number | undefined] => Boolean(entry)),
|
||||
);
|
||||
return params.models.map((model) =>
|
||||
applyModelContextTokensOverride(
|
||||
model,
|
||||
Math.min(
|
||||
requestedContextWindow,
|
||||
contextLimitByModelId.get(model.id) ?? requestedContextWindow,
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function resolveLmstudioDiscoveryFailure(params: {
|
||||
baseUrl: string;
|
||||
discovery: LmstudioDiscoveryResult;
|
||||
}): { noteLines: [string, string]; reason: string } | null {
|
||||
const { baseUrl, discovery } = params;
|
||||
if (!discovery.reachable) {
|
||||
return {
|
||||
noteLines: [
|
||||
`LM Studio could not be reached at ${baseUrl}.`,
|
||||
"Start LM Studio (or run lms server start) and re-run setup.",
|
||||
],
|
||||
reason: "LM Studio not reachable",
|
||||
};
|
||||
}
|
||||
if (discovery.status !== undefined && discovery.status >= 400) {
|
||||
return {
|
||||
noteLines: [
|
||||
`LM Studio returned HTTP ${discovery.status} while listing models at ${baseUrl}.`,
|
||||
"Check the base URL and API key, then re-run setup.",
|
||||
],
|
||||
reason: `LM Studio discovery failed (${discovery.status})`,
|
||||
};
|
||||
}
|
||||
const hasUsableModel = discovery.models.some(
|
||||
(model) => model.type === "llm" && Boolean(model.key?.trim()),
|
||||
);
|
||||
if (!hasUsableModel) {
|
||||
return {
|
||||
noteLines: [
|
||||
`No LM Studio LLM models were found at ${baseUrl}.`,
|
||||
"Load at least one model in LM Studio (or run lms load), then re-run setup.",
|
||||
],
|
||||
reason: "No LM Studio models found",
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolvePersistedLmstudioApiKey(params: {
|
||||
currentApiKey: ModelProviderConfig["apiKey"] | undefined;
|
||||
explicitAuth: ModelProviderConfig["auth"] | undefined;
|
||||
fallbackApiKey: ModelProviderConfig["apiKey"] | undefined;
|
||||
preferFallbackApiKey?: boolean;
|
||||
hasModels: boolean;
|
||||
hasAuthorizationHeader?: boolean;
|
||||
}): ModelProviderConfig["apiKey"] | undefined {
|
||||
if (params.explicitAuth === "api-key") {
|
||||
if (params.preferFallbackApiKey && params.fallbackApiKey !== undefined) {
|
||||
return params.fallbackApiKey;
|
||||
}
|
||||
if (resolveLmstudioProviderAuthMode(params.currentApiKey)) {
|
||||
return params.currentApiKey;
|
||||
}
|
||||
return params.fallbackApiKey;
|
||||
}
|
||||
return shouldUseLmstudioApiKeyPlaceholder({
|
||||
hasModels: params.hasModels,
|
||||
resolvedApiKey: params.currentApiKey,
|
||||
hasAuthorizationHeader: params.hasAuthorizationHeader,
|
||||
})
|
||||
? LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Keeps explicit model entries first and appends unique discovered entries. */
|
||||
function mergeDiscoveredModels(params: {
|
||||
explicitModels?: ModelDefinitionConfig[];
|
||||
discoveredModels?: ModelDefinitionConfig[];
|
||||
}): ModelDefinitionConfig[] {
|
||||
const explicitModels = Array.isArray(params.explicitModels) ? params.explicitModels : [];
|
||||
const discoveredModels = Array.isArray(params.discoveredModels) ? params.discoveredModels : [];
|
||||
if (explicitModels.length === 0) {
|
||||
return discoveredModels;
|
||||
}
|
||||
if (discoveredModels.length === 0) {
|
||||
return explicitModels;
|
||||
}
|
||||
|
||||
const merged = [...explicitModels];
|
||||
const seen = new Set(explicitModels.map((model) => model.id.trim()).filter(Boolean));
|
||||
for (const model of discoveredModels) {
|
||||
const id = model.id.trim();
|
||||
if (!id || seen.has(id)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(id);
|
||||
merged.push(model);
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
async function discoverLmstudioProviderCatalog(params: {
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
headers?: Record<string, string>;
|
||||
quiet: boolean;
|
||||
}): Promise<ModelProviderConfig> {
|
||||
const baseUrl = resolveLmstudioInferenceBase(params.baseUrl);
|
||||
const models = await discoverLmstudioModels({
|
||||
baseUrl,
|
||||
apiKey: params.apiKey ?? "",
|
||||
headers: params.headers,
|
||||
quiet: params.quiet,
|
||||
});
|
||||
return {
|
||||
baseUrl,
|
||||
api: "openai-completions",
|
||||
models,
|
||||
};
|
||||
}
|
||||
|
||||
function isLmstudioDiscoveryConfigResolutionError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return (
|
||||
message.includes("models.providers.lmstudio.apiKey") ||
|
||||
message.includes("models.providers.lmstudio.headers.")
|
||||
);
|
||||
}
|
||||
|
||||
/** Preserves existing allowlist metadata and appends discovered LM Studio model refs. */
|
||||
function mergeDiscoveredLmstudioAllowlistEntries(params: {
|
||||
existing?: NonNullable<NonNullable<OpenClawConfig["agents"]>["defaults"]>["models"];
|
||||
discoveredModels: ModelDefinitionConfig[];
|
||||
}) {
|
||||
return withAgentModelAliases(
|
||||
params.existing,
|
||||
params.discoveredModels
|
||||
.map((model) => model.id.trim())
|
||||
.filter(Boolean)
|
||||
.map((id) => `${PROVIDER_ID}/${id}`),
|
||||
);
|
||||
}
|
||||
|
||||
function selectDefaultLmstudioModelId(
|
||||
discoveredModels: ModelDefinitionConfig[],
|
||||
): string | undefined {
|
||||
const ids = discoveredModels.map((model) => model.id.trim()).filter(Boolean);
|
||||
if (ids.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return ids.includes(LMSTUDIO_DEFAULT_MODEL_ID) ? LMSTUDIO_DEFAULT_MODEL_ID : ids[0];
|
||||
}
|
||||
|
||||
async function discoverLmstudioSetupModels(params: {
|
||||
baseUrl: string;
|
||||
apiKey?: string;
|
||||
headers?: Record<string, string>;
|
||||
timeoutMs?: number;
|
||||
}): Promise<
|
||||
| { value: LmstudioSetupDiscovery }
|
||||
| { failure: NonNullable<ReturnType<typeof resolveLmstudioDiscoveryFailure>> }
|
||||
> {
|
||||
const discovery = await fetchLmstudioModels({
|
||||
baseUrl: params.baseUrl,
|
||||
apiKey: params.apiKey,
|
||||
...(params.headers ? { headers: params.headers } : {}),
|
||||
timeoutMs: params.timeoutMs ?? 5000,
|
||||
});
|
||||
const failure = resolveLmstudioDiscoveryFailure({
|
||||
baseUrl: params.baseUrl,
|
||||
discovery,
|
||||
});
|
||||
if (failure) {
|
||||
return { failure };
|
||||
}
|
||||
const models = mapLmstudioWireModelsToConfig(discovery.models);
|
||||
const defaultModelId = selectDefaultLmstudioModelId(models);
|
||||
return {
|
||||
value: {
|
||||
discovery,
|
||||
models,
|
||||
defaultModel: defaultModelId ? `${PROVIDER_ID}/${defaultModelId}` : undefined,
|
||||
defaultModelId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Interactive LM Studio setup with connectivity and model-availability checks. */
|
||||
export async function promptAndConfigureLmstudioInteractive(params: {
|
||||
config: OpenClawConfig;
|
||||
prompter?: WizardPrompter;
|
||||
secretInputMode?: SecretInputMode;
|
||||
allowSecretRefPrompt?: boolean;
|
||||
promptText?: ProviderPromptText;
|
||||
note?: ProviderPromptNote;
|
||||
}): Promise<ProviderAuthResult> {
|
||||
const promptText = params.prompter?.text ?? params.promptText;
|
||||
if (!promptText) {
|
||||
throw new Error("LM Studio interactive setup requires a text prompter.");
|
||||
}
|
||||
const note = params.prompter?.note ?? params.note;
|
||||
const baseUrlRaw = await promptText({
|
||||
message: `${LMSTUDIO_PROVIDER_LABEL} base URL`,
|
||||
initialValue: LMSTUDIO_DEFAULT_BASE_URL,
|
||||
placeholder: LMSTUDIO_DEFAULT_BASE_URL,
|
||||
validate: (value) => (value?.trim() ? undefined : "Required"),
|
||||
});
|
||||
const baseUrl = resolveLmstudioInferenceBase(baseUrlRaw ?? "");
|
||||
let credentialInput: SecretInput | undefined;
|
||||
let credentialMode: SecretInputMode | undefined;
|
||||
const implicitRefMode = params.allowSecretRefPrompt === false && !params.secretInputMode;
|
||||
const autoRefEnvKey = process.env[LMSTUDIO_DEFAULT_API_KEY_ENV_VAR]?.trim();
|
||||
const apiKey =
|
||||
params.prompter && implicitRefMode && autoRefEnvKey
|
||||
? autoRefEnvKey
|
||||
: params.prompter
|
||||
? await ensureApiKeyFromEnvOrPrompt({
|
||||
config: params.config,
|
||||
provider: PROVIDER_ID,
|
||||
envLabel: LMSTUDIO_DEFAULT_API_KEY_ENV_VAR,
|
||||
promptMessage: `${LMSTUDIO_PROVIDER_LABEL} API key`,
|
||||
normalize: (value) => value.trim(),
|
||||
validate: (value) => (value.trim() ? undefined : "Required"),
|
||||
prompter: params.prompter,
|
||||
secretInputMode:
|
||||
params.allowSecretRefPrompt === false
|
||||
? (params.secretInputMode ?? "plaintext")
|
||||
: params.secretInputMode,
|
||||
setCredential: async (apiKeyValue, mode) => {
|
||||
credentialInput = apiKeyValue;
|
||||
credentialMode = mode;
|
||||
},
|
||||
})
|
||||
: String(
|
||||
await promptText({
|
||||
message: `${LMSTUDIO_PROVIDER_LABEL} API key`,
|
||||
placeholder: "sk-...",
|
||||
validate: (value) => (value?.trim() ? undefined : "Required"),
|
||||
}),
|
||||
).trim();
|
||||
const credential = params.prompter
|
||||
? buildApiKeyCredential(
|
||||
PROVIDER_ID,
|
||||
credentialInput ??
|
||||
(implicitRefMode && autoRefEnvKey ? `\${${LMSTUDIO_DEFAULT_API_KEY_ENV_VAR}}` : apiKey),
|
||||
undefined,
|
||||
credentialMode
|
||||
? { secretInputMode: credentialMode }
|
||||
: implicitRefMode && autoRefEnvKey
|
||||
? { secretInputMode: "ref" }
|
||||
: undefined,
|
||||
)
|
||||
: {
|
||||
type: "api_key" as const,
|
||||
provider: PROVIDER_ID,
|
||||
key: apiKey,
|
||||
};
|
||||
const existingProvider = params.config.models?.providers?.[PROVIDER_ID];
|
||||
// Auth setup updates auth/profile/provider model fields but does not mutate
|
||||
// user-provided header overrides. Runtime request assembly is the source of truth for auth.
|
||||
const persistedHeaders = existingProvider?.headers;
|
||||
const resolvedHeaders = await resolveLmstudioProviderHeaders({
|
||||
config: params.config,
|
||||
env: process.env,
|
||||
headers: persistedHeaders,
|
||||
});
|
||||
const setupDiscovery = await discoverLmstudioSetupModels({
|
||||
baseUrl,
|
||||
apiKey,
|
||||
...(resolvedHeaders ? { headers: resolvedHeaders } : {}),
|
||||
timeoutMs: 5000,
|
||||
});
|
||||
if ("failure" in setupDiscovery) {
|
||||
await note?.(setupDiscovery.failure.noteLines.join("\n"), "LM Studio");
|
||||
throw new WizardCancelledError(setupDiscovery.failure.reason);
|
||||
}
|
||||
let discoveredModels = setupDiscovery.value.models;
|
||||
if (params.prompter) {
|
||||
const requestedRaw = await params.prompter.text({
|
||||
message: "Preferred context length to load LM Studio models with (optional)",
|
||||
placeholder: "e.g. 32768 (leave blank to skip)",
|
||||
validate: (value) =>
|
||||
value?.trim()
|
||||
? resolvePositiveInteger(value)
|
||||
? undefined
|
||||
: "Enter a positive integer token count"
|
||||
: undefined,
|
||||
});
|
||||
const requestedContextWindow = resolvePositiveInteger(requestedRaw);
|
||||
discoveredModels = applyRequestedContextWindowToAllModels({
|
||||
models: discoveredModels,
|
||||
discoveryModels: setupDiscovery.value.discovery.models,
|
||||
requestedContextWindow,
|
||||
});
|
||||
}
|
||||
const allowlistEntries = mergeDiscoveredLmstudioAllowlistEntries({
|
||||
existing: params.config.agents?.defaults?.models,
|
||||
discoveredModels,
|
||||
});
|
||||
const defaultModel = setupDiscovery.value.defaultModel;
|
||||
const persistedApiKey =
|
||||
resolvePersistedLmstudioApiKey({
|
||||
currentApiKey: existingProvider?.apiKey,
|
||||
explicitAuth: resolveLmstudioProviderAuthMode(apiKey),
|
||||
fallbackApiKey: LMSTUDIO_DEFAULT_API_KEY_ENV_VAR,
|
||||
preferFallbackApiKey: true,
|
||||
hasModels: discoveredModels.length > 0,
|
||||
hasAuthorizationHeader: hasLmstudioAuthorizationHeader(resolvedHeaders),
|
||||
}) ?? LMSTUDIO_DEFAULT_API_KEY_ENV_VAR;
|
||||
|
||||
return {
|
||||
profiles: [
|
||||
{
|
||||
profileId: `${PROVIDER_ID}:default`,
|
||||
credential,
|
||||
},
|
||||
],
|
||||
configPatch: {
|
||||
agents: {
|
||||
defaults: {
|
||||
models: allowlistEntries,
|
||||
},
|
||||
},
|
||||
models: {
|
||||
// Respect existing global mode; self-hosted provider setup should merge by default.
|
||||
mode: params.config.models?.mode ?? "merge",
|
||||
providers: {
|
||||
[PROVIDER_ID]: buildLmstudioSetupProviderConfig({
|
||||
existingProvider,
|
||||
baseUrl,
|
||||
apiKey: persistedApiKey,
|
||||
headers: persistedHeaders,
|
||||
models: discoveredModels,
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
/** Non-interactive setup path backed by the shared self-hosted helper. */
|
||||
export async function configureLmstudioNonInteractive(
|
||||
ctx: ProviderAuthMethodNonInteractiveContext,
|
||||
): Promise<OpenClawConfig | null> {
|
||||
const customBaseUrl = normalizeOptionalSecretInput(ctx.opts.customBaseUrl);
|
||||
const baseUrl = resolveLmstudioInferenceBase(
|
||||
customBaseUrl || LMSTUDIO_DEFAULT_INFERENCE_BASE_URL,
|
||||
);
|
||||
const normalizedCtx = customBaseUrl
|
||||
? {
|
||||
...ctx,
|
||||
opts: {
|
||||
...ctx.opts,
|
||||
customBaseUrl: baseUrl,
|
||||
},
|
||||
}
|
||||
: ctx;
|
||||
const configureShared = async (configureCtx: ProviderAuthMethodNonInteractiveContext) =>
|
||||
await configureOpenAICompatibleSelfHostedProviderNonInteractive({
|
||||
ctx: configureCtx,
|
||||
providerId: PROVIDER_ID,
|
||||
providerLabel: LMSTUDIO_PROVIDER_LABEL,
|
||||
defaultBaseUrl: LMSTUDIO_DEFAULT_INFERENCE_BASE_URL,
|
||||
defaultApiKeyEnvVar: LMSTUDIO_DEFAULT_API_KEY_ENV_VAR,
|
||||
modelPlaceholder: LMSTUDIO_MODEL_PLACEHOLDER,
|
||||
});
|
||||
const requestedModelId = normalizeOptionalSecretInput(normalizedCtx.opts.customModelId);
|
||||
const resolved = await normalizedCtx.resolveApiKey({
|
||||
provider: PROVIDER_ID,
|
||||
flagValue:
|
||||
normalizeOptionalSecretInput(normalizedCtx.opts.lmstudioApiKey) ??
|
||||
normalizeOptionalSecretInput(normalizedCtx.opts.customApiKey),
|
||||
flagName:
|
||||
normalizeOptionalSecretInput(normalizedCtx.opts.lmstudioApiKey) !== undefined
|
||||
? "--lmstudio-api-key"
|
||||
: "--custom-api-key",
|
||||
envVar: LMSTUDIO_DEFAULT_API_KEY_ENV_VAR,
|
||||
envVarName: LMSTUDIO_DEFAULT_API_KEY_ENV_VAR,
|
||||
required: false,
|
||||
});
|
||||
|
||||
const existingProvider = normalizedCtx.config.models?.providers?.[PROVIDER_ID];
|
||||
// Auth setup updates auth/profile/provider model fields but does not mutate
|
||||
// user-provided header overrides. Runtime request assembly is the source of truth for auth.
|
||||
const persistedHeaders = existingProvider?.headers;
|
||||
const resolvedHeaders = await resolveLmstudioProviderHeaders({
|
||||
config: normalizedCtx.config,
|
||||
env: process.env,
|
||||
headers: persistedHeaders,
|
||||
});
|
||||
const hasAuthorizationHeader = hasLmstudioAuthorizationHeader(resolvedHeaders);
|
||||
const useHeaderOnlyAuth = hasAuthorizationHeader && (!resolved || resolved.source !== "flag");
|
||||
const setupDiscoveryApiKey =
|
||||
(useHeaderOnlyAuth ? undefined : resolved?.key) ??
|
||||
(shouldUseLmstudioApiKeyPlaceholder({
|
||||
hasModels: true,
|
||||
resolvedApiKey: undefined,
|
||||
hasAuthorizationHeader,
|
||||
})
|
||||
? LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER
|
||||
: undefined);
|
||||
if (!setupDiscoveryApiKey && !hasAuthorizationHeader) {
|
||||
normalizedCtx.runtime.error(
|
||||
`LM Studio API key is required. Set ${LMSTUDIO_DEFAULT_API_KEY_ENV_VAR} or pass --lmstudio-api-key.`,
|
||||
);
|
||||
normalizedCtx.runtime.exit(1);
|
||||
return null;
|
||||
}
|
||||
const setupDiscovery = await discoverLmstudioSetupModels({
|
||||
baseUrl,
|
||||
apiKey: setupDiscoveryApiKey,
|
||||
...(resolvedHeaders ? { headers: resolvedHeaders } : {}),
|
||||
timeoutMs: 5000,
|
||||
});
|
||||
if ("failure" in setupDiscovery) {
|
||||
normalizedCtx.runtime.error(setupDiscovery.failure.noteLines.join("\n"));
|
||||
normalizedCtx.runtime.exit(1);
|
||||
return null;
|
||||
}
|
||||
const discoveredModels = setupDiscovery.value.models;
|
||||
const selectedModelId = requestedModelId ?? setupDiscovery.value.defaultModelId;
|
||||
const selectedModel = selectedModelId
|
||||
? discoveredModels.find((model) => model.id === selectedModelId)
|
||||
: undefined;
|
||||
if (!selectedModelId || !selectedModel) {
|
||||
const availableModels = discoveredModels.map((model) => model.id).join(", ");
|
||||
normalizedCtx.runtime.error(
|
||||
requestedModelId
|
||||
? [
|
||||
`LM Studio model ${requestedModelId} was not found at ${baseUrl}.`,
|
||||
`Available models: ${availableModels}`,
|
||||
].join("\n")
|
||||
: [
|
||||
`LM Studio did not expose a usable default model at ${baseUrl}.`,
|
||||
`Available models: ${availableModels || "(none)"}`,
|
||||
].join("\n"),
|
||||
);
|
||||
normalizedCtx.runtime.exit(1);
|
||||
return null;
|
||||
}
|
||||
if (useHeaderOnlyAuth) {
|
||||
await removeProviderAuthProfilesWithLock({
|
||||
provider: PROVIDER_ID,
|
||||
agentDir: normalizedCtx.agentDir,
|
||||
});
|
||||
const configWithoutStoredLmstudioAuth = stripLmstudioStoredAuthConfig(normalizedCtx.config);
|
||||
return applyProviderDefaultModel(
|
||||
{
|
||||
...configWithoutStoredLmstudioAuth,
|
||||
models: {
|
||||
...configWithoutStoredLmstudioAuth.models,
|
||||
mode: configWithoutStoredLmstudioAuth.models?.mode ?? "merge",
|
||||
providers: {
|
||||
...configWithoutStoredLmstudioAuth.models?.providers,
|
||||
[PROVIDER_ID]: buildLmstudioSetupProviderConfig({
|
||||
existingProvider,
|
||||
baseUrl,
|
||||
headers: persistedHeaders,
|
||||
models: discoveredModels,
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
`${PROVIDER_ID}/${selectedModelId}`,
|
||||
);
|
||||
}
|
||||
const resolvedOrSynthetic =
|
||||
resolved ??
|
||||
(setupDiscoveryApiKey
|
||||
? {
|
||||
key: setupDiscoveryApiKey,
|
||||
source: "flag" as const,
|
||||
}
|
||||
: null);
|
||||
if (!resolvedOrSynthetic) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Delegate to the shared helper even when modelId is set so that onboarding
|
||||
// state and credential storage are handled consistently. The pre-resolved key
|
||||
// is injected via resolveApiKey to skip a second prompt. The returned config
|
||||
// is then post-patched below to add the discovered model list and base URL.
|
||||
const configured = await configureShared({
|
||||
...normalizedCtx,
|
||||
opts: {
|
||||
...normalizedCtx.opts,
|
||||
customModelId: selectedModelId,
|
||||
},
|
||||
resolveApiKey: async () => resolvedOrSynthetic,
|
||||
});
|
||||
if (!configured) {
|
||||
return null;
|
||||
}
|
||||
const sharedProvider = configured.models?.providers?.[PROVIDER_ID];
|
||||
const resolvedSyntheticLocalKey = resolvedOrSynthetic.key === LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER;
|
||||
const persistedApiKey = resolvePersistedLmstudioApiKey({
|
||||
// If this run resolved to keyless local mode, avoid preserving stale env markers.
|
||||
currentApiKey: resolvedSyntheticLocalKey ? undefined : existingProvider?.apiKey,
|
||||
explicitAuth: resolveLmstudioProviderAuthMode(resolvedOrSynthetic.key),
|
||||
fallbackApiKey: resolvedSyntheticLocalKey
|
||||
? LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER
|
||||
: (configured.models?.providers?.[PROVIDER_ID]?.apiKey ?? LMSTUDIO_DEFAULT_API_KEY_ENV_VAR),
|
||||
preferFallbackApiKey: true,
|
||||
hasModels: discoveredModels.length > 0,
|
||||
hasAuthorizationHeader: hasLmstudioAuthorizationHeader(resolvedHeaders),
|
||||
});
|
||||
|
||||
return {
|
||||
...configured,
|
||||
models: {
|
||||
...configured.models,
|
||||
providers: {
|
||||
...configured.models?.providers,
|
||||
[PROVIDER_ID]: buildLmstudioSetupProviderConfig({
|
||||
existingProvider,
|
||||
sharedProvider,
|
||||
baseUrl,
|
||||
apiKey: persistedApiKey,
|
||||
headers: persistedHeaders,
|
||||
models: discoveredModels,
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Discovers provider settings, merging explicit config with live model discovery. */
|
||||
export async function discoverLmstudioProvider(ctx: ProviderCatalogContext): Promise<{
|
||||
provider: ModelProviderConfig;
|
||||
} | null> {
|
||||
const explicit = ctx.config.models?.providers?.[PROVIDER_ID];
|
||||
const explicitAuth = explicit?.auth;
|
||||
let explicitWithoutHeaders: Omit<ModelProviderConfig, "headers" | "auth" | "apiKey"> | undefined;
|
||||
if (explicit) {
|
||||
const { headers: _headers, auth: _auth, apiKey: _apiKey, ...rest } = explicit;
|
||||
explicitWithoutHeaders = rest;
|
||||
}
|
||||
const hasExplicitModels = Array.isArray(explicit?.models) && explicit.models.length > 0;
|
||||
const { apiKey, discoveryApiKey } = ctx.resolveProviderApiKey(PROVIDER_ID);
|
||||
let configuredDiscoveryApiKey: string | undefined;
|
||||
try {
|
||||
configuredDiscoveryApiKey = await resolveLmstudioConfiguredApiKey({
|
||||
config: ctx.config,
|
||||
env: ctx.env,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isLmstudioDiscoveryConfigResolutionError(error)) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
let resolvedHeaders: Record<string, string> | undefined;
|
||||
try {
|
||||
resolvedHeaders = await resolveLmstudioProviderHeaders({
|
||||
config: ctx.config,
|
||||
env: ctx.env,
|
||||
headers: explicit?.headers,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isLmstudioDiscoveryConfigResolutionError(error)) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const hasAuthorizationHeader = hasLmstudioAuthorizationHeader(resolvedHeaders);
|
||||
const resolvedDiscoveryApiKey = hasAuthorizationHeader
|
||||
? undefined
|
||||
: (discoveryApiKey ?? configuredDiscoveryApiKey);
|
||||
// CLI/runtime-resolved key takes precedence over static provider config key.
|
||||
const resolvedApiKey = apiKey ?? explicit?.apiKey;
|
||||
if (hasExplicitModels && explicitWithoutHeaders) {
|
||||
const persistedApiKey = resolvePersistedLmstudioApiKey({
|
||||
currentApiKey: resolvedApiKey,
|
||||
explicitAuth,
|
||||
fallbackApiKey: LMSTUDIO_DEFAULT_API_KEY_ENV_VAR,
|
||||
hasModels: hasExplicitModels,
|
||||
hasAuthorizationHeader,
|
||||
});
|
||||
const persistedAuth = resolveLmstudioProviderAuthMode(persistedApiKey);
|
||||
return {
|
||||
provider: {
|
||||
...explicitWithoutHeaders,
|
||||
...(resolvedHeaders ? { headers: resolvedHeaders } : {}),
|
||||
baseUrl: resolveLmstudioInferenceBase(explicitWithoutHeaders.baseUrl),
|
||||
// Keep explicit API unless absent, then fall back to provider default.
|
||||
api: explicitWithoutHeaders.api ?? "openai-completions",
|
||||
...(persistedApiKey ? { apiKey: persistedApiKey } : {}),
|
||||
...(persistedAuth ? { auth: persistedAuth } : {}),
|
||||
models: explicitWithoutHeaders.models,
|
||||
},
|
||||
};
|
||||
}
|
||||
const provider = await discoverLmstudioProviderCatalog({
|
||||
baseUrl: explicit?.baseUrl,
|
||||
// Prefer resolved discovery auth, then configured provider auth.
|
||||
apiKey: resolvedDiscoveryApiKey,
|
||||
headers: resolvedHeaders,
|
||||
quiet: !apiKey && !explicit && !resolvedDiscoveryApiKey,
|
||||
});
|
||||
const models = mergeDiscoveredModels({
|
||||
explicitModels: explicit?.models,
|
||||
discoveredModels: provider.models,
|
||||
});
|
||||
if (models.length === 0 && !apiKey && !explicit?.apiKey) {
|
||||
return null;
|
||||
}
|
||||
const persistedApiKey = resolvePersistedLmstudioApiKey({
|
||||
currentApiKey: resolvedApiKey,
|
||||
explicitAuth,
|
||||
fallbackApiKey: LMSTUDIO_DEFAULT_API_KEY_ENV_VAR,
|
||||
hasModels: models.length > 0,
|
||||
hasAuthorizationHeader,
|
||||
});
|
||||
const persistedAuth = resolveLmstudioProviderAuthMode(persistedApiKey);
|
||||
return {
|
||||
provider: {
|
||||
...provider,
|
||||
...explicitWithoutHeaders,
|
||||
...(resolvedHeaders ? { headers: resolvedHeaders } : {}),
|
||||
baseUrl: resolveLmstudioInferenceBase(explicit?.baseUrl ?? provider.baseUrl),
|
||||
...(persistedApiKey ? { apiKey: persistedApiKey } : {}),
|
||||
...(persistedAuth ? { auth: persistedAuth } : {}),
|
||||
models,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function prepareLmstudioDynamicModels(
|
||||
ctx: ProviderPrepareDynamicModelContext,
|
||||
): Promise<ProviderRuntimeModel[]> {
|
||||
const baseUrl = resolveLmstudioInferenceBase(ctx.providerConfig?.baseUrl);
|
||||
const { apiKey, headers } = await resolveLmstudioRequestContext({
|
||||
config: ctx.config,
|
||||
agentDir: ctx.agentDir,
|
||||
env: process.env,
|
||||
providerHeaders: ctx.providerConfig?.headers,
|
||||
});
|
||||
const discoveredModels = await discoverLmstudioModels({
|
||||
baseUrl,
|
||||
apiKey: apiKey ?? "",
|
||||
headers,
|
||||
quiet: true,
|
||||
});
|
||||
return discoveredModels.map((model) =>
|
||||
Object.assign({}, model, {
|
||||
provider: PROVIDER_ID,
|
||||
api: ctx.providerConfig?.api ?? `openai-completions`,
|
||||
baseUrl,
|
||||
}),
|
||||
);
|
||||
}
|
||||
402
openclaw/extensions/lmstudio/src/stream.test.ts
Normal file
402
openclaw/extensions/lmstudio/src/stream.test.ts
Normal file
|
|
@ -0,0 +1,402 @@
|
|||
import type { StreamFn } from "@mariozechner/pi-agent-core";
|
||||
import { createAssistantMessageEventStream } from "@mariozechner/pi-ai";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { __resetLmstudioPreloadCooldownForTest, wrapLmstudioInferencePreload } from "./stream.js";
|
||||
|
||||
const ensureLmstudioModelLoadedMock = vi.hoisted(() => vi.fn());
|
||||
const resolveLmstudioProviderHeadersMock = vi.hoisted(() =>
|
||||
vi.fn(async (_params?: unknown) => undefined),
|
||||
);
|
||||
const resolveLmstudioRuntimeApiKeyMock = vi.hoisted(() =>
|
||||
vi.fn(async (_params?: unknown) => undefined),
|
||||
);
|
||||
|
||||
vi.mock("./models.fetch.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("./models.fetch.js")>();
|
||||
return {
|
||||
...actual,
|
||||
ensureLmstudioModelLoaded: (params: unknown) => ensureLmstudioModelLoadedMock(params),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./runtime.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("./runtime.js")>();
|
||||
return {
|
||||
...actual,
|
||||
resolveLmstudioProviderHeaders: (params: unknown) => resolveLmstudioProviderHeadersMock(params),
|
||||
resolveLmstudioRuntimeApiKey: (params: unknown) => resolveLmstudioRuntimeApiKeyMock(params),
|
||||
};
|
||||
});
|
||||
|
||||
type StreamEvent = { type: string };
|
||||
|
||||
async function collectEvents(stream: ReturnType<StreamFn>): Promise<StreamEvent[]> {
|
||||
const resolved = stream instanceof Promise ? await stream : stream;
|
||||
const events: StreamEvent[] = [];
|
||||
for await (const event of resolved) {
|
||||
events.push(event as StreamEvent);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
function buildDoneStreamFn(): StreamFn {
|
||||
return vi.fn((_model, _context, _options) => {
|
||||
const stream = createAssistantMessageEventStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({ type: "done", reason: "stop", message: {} as never });
|
||||
stream.end();
|
||||
});
|
||||
return stream;
|
||||
});
|
||||
}
|
||||
|
||||
describe("lmstudio stream wrapper", () => {
|
||||
beforeEach(() => {
|
||||
__resetLmstudioPreloadCooldownForTest();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
ensureLmstudioModelLoadedMock.mockReset();
|
||||
resolveLmstudioProviderHeadersMock.mockReset();
|
||||
resolveLmstudioRuntimeApiKeyMock.mockReset();
|
||||
resolveLmstudioProviderHeadersMock.mockResolvedValue(undefined);
|
||||
resolveLmstudioRuntimeApiKeyMock.mockResolvedValue(undefined);
|
||||
__resetLmstudioPreloadCooldownForTest();
|
||||
});
|
||||
|
||||
it("preloads LM Studio model before inference using model context window", async () => {
|
||||
const baseStream = buildDoneStreamFn();
|
||||
const wrapped = wrapLmstudioInferencePreload({
|
||||
provider: "lmstudio",
|
||||
modelId: "qwen3-8b-instruct",
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
lmstudio: {
|
||||
baseUrl: "http://lmstudio.internal:1234/v1",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
streamFn: baseStream,
|
||||
} as never);
|
||||
|
||||
const stream = wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "lmstudio/qwen3-8b-instruct",
|
||||
contextWindow: 131072,
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
{ apiKey: "lmstudio-token" } as never,
|
||||
);
|
||||
const events = await collectEvents(stream);
|
||||
|
||||
expect(events).toEqual([expect.objectContaining({ type: "done" })]);
|
||||
expect(ensureLmstudioModelLoadedMock).toHaveBeenCalledTimes(1);
|
||||
expect(ensureLmstudioModelLoadedMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
baseUrl: "http://lmstudio.internal:1234/v1",
|
||||
modelKey: "qwen3-8b-instruct",
|
||||
requestedContextLength: 131072,
|
||||
apiKey: "lmstudio-token",
|
||||
ssrfPolicy: { allowedHostnames: ["lmstudio.internal"] },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("prefers model contextTokens over contextWindow for preload requests", async () => {
|
||||
const baseStream = buildDoneStreamFn();
|
||||
const wrapped = wrapLmstudioInferencePreload({
|
||||
provider: "lmstudio",
|
||||
modelId: "qwen3-8b-instruct",
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
lmstudio: {
|
||||
baseUrl: "http://lmstudio.internal:1234/v1",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
streamFn: baseStream,
|
||||
} as never);
|
||||
|
||||
const stream = wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "lmstudio/qwen3-8b-instruct",
|
||||
contextWindow: 131072,
|
||||
contextTokens: 64000,
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
{ apiKey: "lmstudio-token" } as never,
|
||||
);
|
||||
const events = await collectEvents(stream);
|
||||
|
||||
expect(events).toEqual([expect.objectContaining({ type: "done" })]);
|
||||
expect(ensureLmstudioModelLoadedMock).toHaveBeenCalledTimes(1);
|
||||
expect(ensureLmstudioModelLoadedMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
baseUrl: "http://lmstudio.internal:1234/v1",
|
||||
modelKey: "qwen3-8b-instruct",
|
||||
requestedContextLength: 64000,
|
||||
apiKey: "lmstudio-token",
|
||||
ssrfPolicy: { allowedHostnames: ["lmstudio.internal"] },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("continues inference when preload fails", async () => {
|
||||
ensureLmstudioModelLoadedMock.mockRejectedValueOnce(new Error("load failed"));
|
||||
const baseStream = buildDoneStreamFn();
|
||||
const wrapped = wrapLmstudioInferencePreload({
|
||||
provider: "lmstudio",
|
||||
modelId: "qwen3-8b-instruct",
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
lmstudio: {
|
||||
baseUrl: "http://localhost:1234",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
streamFn: baseStream,
|
||||
} as never);
|
||||
|
||||
const stream = wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "qwen3-8b-instruct",
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
undefined as never,
|
||||
);
|
||||
const events = await collectEvents(stream);
|
||||
expect(events).toEqual([expect.objectContaining({ type: "done" })]);
|
||||
expect(baseStream).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("dedupes concurrent preload requests for the same model and context", async () => {
|
||||
let resolvePreload: (() => void) | undefined;
|
||||
ensureLmstudioModelLoadedMock.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
resolvePreload = resolve;
|
||||
}),
|
||||
);
|
||||
const baseStream = buildDoneStreamFn();
|
||||
const wrapped = wrapLmstudioInferencePreload({
|
||||
provider: "lmstudio",
|
||||
modelId: "qwen3-8b-instruct",
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
lmstudio: {
|
||||
baseUrl: "http://localhost:1234",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
streamFn: baseStream,
|
||||
} as never);
|
||||
|
||||
const first = wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "qwen3-8b-instruct",
|
||||
contextWindow: 32768,
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
undefined as never,
|
||||
);
|
||||
const second = wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "qwen3-8b-instruct",
|
||||
contextWindow: 32768,
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
undefined as never,
|
||||
);
|
||||
|
||||
const firstPromise = collectEvents(first);
|
||||
const secondPromise = collectEvents(second);
|
||||
await vi.waitFor(() => {
|
||||
if (!resolvePreload) {
|
||||
throw new Error("LM Studio preload resolver not initialized");
|
||||
}
|
||||
});
|
||||
if (!resolvePreload) {
|
||||
throw new Error("LM Studio preload resolver not initialized");
|
||||
}
|
||||
resolvePreload();
|
||||
const [firstEvents, secondEvents] = await Promise.all([firstPromise, secondPromise]);
|
||||
|
||||
expect(firstEvents).toEqual([expect.objectContaining({ type: "done" })]);
|
||||
expect(secondEvents).toEqual([expect.objectContaining({ type: "done" })]);
|
||||
expect(ensureLmstudioModelLoadedMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("skips preload on the second attempt while the failure backoff is active", async () => {
|
||||
ensureLmstudioModelLoadedMock.mockRejectedValue(new Error("out of memory"));
|
||||
const baseStream = buildDoneStreamFn();
|
||||
const wrapped = wrapLmstudioInferencePreload({
|
||||
provider: "lmstudio",
|
||||
modelId: "qwen3-8b-instruct",
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
lmstudio: {
|
||||
baseUrl: "http://localhost:1234",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
streamFn: baseStream,
|
||||
} as never);
|
||||
|
||||
const firstEvents = await collectEvents(
|
||||
wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "qwen3-8b-instruct",
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
undefined as never,
|
||||
),
|
||||
);
|
||||
expect(firstEvents).toEqual([expect.objectContaining({ type: "done" })]);
|
||||
expect(ensureLmstudioModelLoadedMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
const secondEvents = await collectEvents(
|
||||
wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "qwen3-8b-instruct",
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
undefined as never,
|
||||
),
|
||||
);
|
||||
expect(secondEvents).toEqual([expect.objectContaining({ type: "done" })]);
|
||||
// The second call must NOT retry preload because cooldown is active, but
|
||||
// the underlying stream must still run so the user gets a response.
|
||||
expect(ensureLmstudioModelLoadedMock).toHaveBeenCalledTimes(1);
|
||||
expect(baseStream).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("retries preload once the cooldown expires", async () => {
|
||||
ensureLmstudioModelLoadedMock.mockRejectedValueOnce(new Error("out of memory"));
|
||||
ensureLmstudioModelLoadedMock.mockResolvedValueOnce(undefined);
|
||||
const baseStream = buildDoneStreamFn();
|
||||
const wrapped = wrapLmstudioInferencePreload({
|
||||
provider: "lmstudio",
|
||||
modelId: "qwen3-8b-instruct",
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
lmstudio: {
|
||||
baseUrl: "http://localhost:1234",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
streamFn: baseStream,
|
||||
} as never);
|
||||
|
||||
// Freeze Date.now at a known base so we can jump past the first backoff
|
||||
// window (5s by default) between the two preload attempts.
|
||||
const baseTime = 1_000_000;
|
||||
const nowSpy = vi.spyOn(Date, "now");
|
||||
nowSpy.mockReturnValue(baseTime);
|
||||
await collectEvents(
|
||||
wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "qwen3-8b-instruct",
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
undefined as never,
|
||||
),
|
||||
);
|
||||
expect(ensureLmstudioModelLoadedMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Move the clock past the initial 5s cooldown window so the next call is
|
||||
// allowed to retry preload.
|
||||
nowSpy.mockReturnValue(baseTime + 6_000);
|
||||
await collectEvents(
|
||||
wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "qwen3-8b-instruct",
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
undefined as never,
|
||||
),
|
||||
);
|
||||
expect(ensureLmstudioModelLoadedMock).toHaveBeenCalledTimes(2);
|
||||
nowSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("forces supportsUsageInStreaming compat before calling the underlying stream", async () => {
|
||||
const baseStream = buildDoneStreamFn();
|
||||
const wrapped = wrapLmstudioInferencePreload({
|
||||
provider: "lmstudio",
|
||||
modelId: "qwen3-8b-instruct",
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
lmstudio: {
|
||||
baseUrl: "http://localhost:1234",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
streamFn: baseStream,
|
||||
} as never);
|
||||
|
||||
const stream = wrapped(
|
||||
{
|
||||
provider: "lmstudio",
|
||||
api: "openai-completions",
|
||||
id: "qwen3-8b-instruct",
|
||||
compat: { supportsDeveloperRole: false },
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
undefined as never,
|
||||
);
|
||||
const events = await collectEvents(stream);
|
||||
|
||||
expect(events).toEqual([expect.objectContaining({ type: "done" })]);
|
||||
expect(baseStream).toHaveBeenCalledTimes(1);
|
||||
expect(baseStream).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "lmstudio",
|
||||
compat: expect.objectContaining({
|
||||
supportsDeveloperRole: false,
|
||||
supportsUsageInStreaming: true,
|
||||
}),
|
||||
}),
|
||||
expect.anything(),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
});
|
||||
270
openclaw/extensions/lmstudio/src/stream.ts
Normal file
270
openclaw/extensions/lmstudio/src/stream.ts
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
import type { StreamFn } from "@mariozechner/pi-agent-core";
|
||||
import { streamSimple } from "@mariozechner/pi-ai";
|
||||
import { createSubsystemLogger } from "openclaw/plugin-sdk/logging-core";
|
||||
import type { ProviderWrapStreamFnContext } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { SsrFPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { LMSTUDIO_PROVIDER_ID } from "./defaults.js";
|
||||
import { ensureLmstudioModelLoaded } from "./models.fetch.js";
|
||||
import { resolveLmstudioInferenceBase } from "./models.js";
|
||||
import { resolveLmstudioProviderHeaders, resolveLmstudioRuntimeApiKey } from "./runtime.js";
|
||||
|
||||
const log = createSubsystemLogger("extensions/lmstudio/stream");
|
||||
|
||||
type StreamOptions = Parameters<StreamFn>[2];
|
||||
type StreamModel = Parameters<StreamFn>[0];
|
||||
|
||||
const preloadInFlight = new Map<string, Promise<void>>();
|
||||
|
||||
/**
|
||||
* Cooldown state for the LM Studio preload endpoint.
|
||||
*
|
||||
* Without this, every chat request would retry preload ~every 2s even when
|
||||
* LM Studio has rejected the load (for example the memory guardrail will keep
|
||||
* rejecting until the user adjusts the setting or frees RAM). That produced
|
||||
* hundreds of `LM Studio inference preload failed` WARN lines per hour without
|
||||
* actually helping the user. The cooldown applies an exponential backoff per
|
||||
* preloadKey and, while the cooldown is active, the wrapper skips the preload
|
||||
* step entirely and proceeds directly to streaming — the model is often
|
||||
* already loaded from the user's LM Studio UI, so inference can succeed even
|
||||
* when preload keeps being rejected.
|
||||
*/
|
||||
type PreloadCooldownEntry = {
|
||||
untilMs: number;
|
||||
consecutiveFailures: number;
|
||||
};
|
||||
|
||||
const preloadCooldown = new Map<string, PreloadCooldownEntry>();
|
||||
|
||||
const PRELOAD_BACKOFF_BASE_MS = 5_000;
|
||||
const PRELOAD_BACKOFF_MAX_MS = 300_000;
|
||||
|
||||
function computePreloadBackoffMs(consecutiveFailures: number): number {
|
||||
const exponent = Math.max(0, consecutiveFailures - 1);
|
||||
const raw = PRELOAD_BACKOFF_BASE_MS * 2 ** exponent;
|
||||
return Math.min(PRELOAD_BACKOFF_MAX_MS, raw);
|
||||
}
|
||||
|
||||
function recordPreloadSuccess(preloadKey: string): void {
|
||||
preloadCooldown.delete(preloadKey);
|
||||
}
|
||||
|
||||
function recordPreloadFailure(preloadKey: string, now: number): PreloadCooldownEntry {
|
||||
const existing = preloadCooldown.get(preloadKey);
|
||||
const consecutiveFailures = (existing?.consecutiveFailures ?? 0) + 1;
|
||||
const entry: PreloadCooldownEntry = {
|
||||
consecutiveFailures,
|
||||
untilMs: now + computePreloadBackoffMs(consecutiveFailures),
|
||||
};
|
||||
preloadCooldown.set(preloadKey, entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
function isPreloadCoolingDown(preloadKey: string, now: number): PreloadCooldownEntry | undefined {
|
||||
const entry = preloadCooldown.get(preloadKey);
|
||||
if (!entry) {
|
||||
return undefined;
|
||||
}
|
||||
if (entry.untilMs <= now) {
|
||||
preloadCooldown.delete(preloadKey);
|
||||
return undefined;
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** Test-only hook for clearing preload cooldown state between cases. */
|
||||
export function __resetLmstudioPreloadCooldownForTest(): void {
|
||||
preloadCooldown.clear();
|
||||
preloadInFlight.clear();
|
||||
}
|
||||
|
||||
function normalizeLmstudioModelKey(modelId: string): string {
|
||||
const trimmed = modelId.trim();
|
||||
if (trimmed.toLowerCase().startsWith("lmstudio/")) {
|
||||
return trimmed.slice("lmstudio/".length).trim();
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function resolveRequestedContextLength(model: StreamModel): number | undefined {
|
||||
const withContextTokens = model as StreamModel & { contextTokens?: unknown };
|
||||
const contextTokens =
|
||||
typeof withContextTokens.contextTokens === "number" &&
|
||||
Number.isFinite(withContextTokens.contextTokens)
|
||||
? Math.floor(withContextTokens.contextTokens)
|
||||
: undefined;
|
||||
if (contextTokens && contextTokens > 0) {
|
||||
return contextTokens;
|
||||
}
|
||||
const contextWindow =
|
||||
typeof model.contextWindow === "number" && Number.isFinite(model.contextWindow)
|
||||
? Math.floor(model.contextWindow)
|
||||
: undefined;
|
||||
if (contextWindow && contextWindow > 0) {
|
||||
return contextWindow;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveModelHeaders(model: StreamModel): Record<string, string> | undefined {
|
||||
if (!model.headers || typeof model.headers !== "object" || Array.isArray(model.headers)) {
|
||||
return undefined;
|
||||
}
|
||||
return model.headers;
|
||||
}
|
||||
|
||||
function createPreloadKey(params: {
|
||||
baseUrl: string;
|
||||
modelKey: string;
|
||||
requestedContextLength?: number;
|
||||
}) {
|
||||
return `${params.baseUrl}::${params.modelKey}::${params.requestedContextLength ?? "default"}`;
|
||||
}
|
||||
|
||||
function buildLmstudioPreloadSsrFPolicy(baseUrl: string): SsrFPolicy | undefined {
|
||||
const trimmed = baseUrl.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const parsed = new URL(trimmed);
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
return undefined;
|
||||
}
|
||||
return { allowedHostnames: [parsed.hostname] };
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureLmstudioModelLoadedBestEffort(params: {
|
||||
baseUrl: string;
|
||||
modelKey: string;
|
||||
requestedContextLength?: number;
|
||||
options: StreamOptions;
|
||||
ctx: ProviderWrapStreamFnContext;
|
||||
modelHeaders?: Record<string, string>;
|
||||
}): Promise<void> {
|
||||
const providerConfig = params.ctx.config?.models?.providers?.[LMSTUDIO_PROVIDER_ID];
|
||||
const providerHeaders = { ...providerConfig?.headers, ...params.modelHeaders };
|
||||
const runtimeApiKey =
|
||||
typeof params.options?.apiKey === "string" && params.options.apiKey.trim().length > 0
|
||||
? params.options.apiKey.trim()
|
||||
: undefined;
|
||||
const headers = await resolveLmstudioProviderHeaders({
|
||||
config: params.ctx.config,
|
||||
headers: providerHeaders,
|
||||
});
|
||||
const configuredApiKey =
|
||||
runtimeApiKey !== undefined
|
||||
? undefined
|
||||
: await resolveLmstudioRuntimeApiKey({
|
||||
config: params.ctx.config,
|
||||
agentDir: params.ctx.agentDir,
|
||||
headers: providerHeaders,
|
||||
});
|
||||
|
||||
await ensureLmstudioModelLoaded({
|
||||
baseUrl: params.baseUrl,
|
||||
apiKey: runtimeApiKey ?? configuredApiKey,
|
||||
headers,
|
||||
ssrfPolicy: buildLmstudioPreloadSsrFPolicy(params.baseUrl),
|
||||
modelKey: params.modelKey,
|
||||
requestedContextLength: params.requestedContextLength,
|
||||
});
|
||||
}
|
||||
|
||||
export function wrapLmstudioInferencePreload(ctx: ProviderWrapStreamFnContext): StreamFn {
|
||||
const underlying = ctx.streamFn ?? streamSimple;
|
||||
return (model, context, options) => {
|
||||
if (model.provider !== LMSTUDIO_PROVIDER_ID) {
|
||||
return underlying(model, context, options);
|
||||
}
|
||||
const modelKey = normalizeLmstudioModelKey(model.id);
|
||||
if (!modelKey) {
|
||||
return underlying(model, context, options);
|
||||
}
|
||||
const providerBaseUrl = ctx.config?.models?.providers?.[LMSTUDIO_PROVIDER_ID]?.baseUrl;
|
||||
const resolvedBaseUrl = resolveLmstudioInferenceBase(
|
||||
typeof model.baseUrl === "string" ? model.baseUrl : providerBaseUrl,
|
||||
);
|
||||
const requestedContextLength = resolveRequestedContextLength(model);
|
||||
const preloadKey = createPreloadKey({
|
||||
baseUrl: resolvedBaseUrl,
|
||||
modelKey,
|
||||
requestedContextLength,
|
||||
});
|
||||
|
||||
const cooldownEntry = isPreloadCoolingDown(preloadKey, Date.now());
|
||||
const existing = preloadInFlight.get(preloadKey);
|
||||
const preloadPromise: Promise<void> | undefined =
|
||||
existing ??
|
||||
(cooldownEntry
|
||||
? undefined
|
||||
: (() => {
|
||||
const created = ensureLmstudioModelLoadedBestEffort({
|
||||
baseUrl: resolvedBaseUrl,
|
||||
modelKey,
|
||||
requestedContextLength,
|
||||
options,
|
||||
ctx,
|
||||
modelHeaders: resolveModelHeaders(model),
|
||||
})
|
||||
.then(
|
||||
() => {
|
||||
recordPreloadSuccess(preloadKey);
|
||||
},
|
||||
(error) => {
|
||||
const entry = recordPreloadFailure(preloadKey, Date.now());
|
||||
throw Object.assign(new Error("preload-failed"), {
|
||||
cause: error,
|
||||
consecutiveFailures: entry.consecutiveFailures,
|
||||
cooldownMs: entry.untilMs - Date.now(),
|
||||
});
|
||||
},
|
||||
)
|
||||
.finally(() => {
|
||||
preloadInFlight.delete(preloadKey);
|
||||
});
|
||||
preloadInFlight.set(preloadKey, created);
|
||||
return created;
|
||||
})());
|
||||
|
||||
return (async () => {
|
||||
if (preloadPromise) {
|
||||
try {
|
||||
await preloadPromise;
|
||||
} catch (error) {
|
||||
const annotated = error as {
|
||||
cause?: unknown;
|
||||
consecutiveFailures?: number;
|
||||
cooldownMs?: number;
|
||||
};
|
||||
const cause = annotated.cause ?? error;
|
||||
const failures = annotated.consecutiveFailures ?? 1;
|
||||
const cooldownSec = Math.max(0, Math.round((annotated.cooldownMs ?? 0) / 1000));
|
||||
log.warn(
|
||||
`LM Studio inference preload failed for "${modelKey}" (${failures} consecutive failure${
|
||||
failures === 1 ? "" : "s"
|
||||
}, next preload attempt skipped for ~${cooldownSec}s); continuing without preload: ${String(cause)}`,
|
||||
);
|
||||
}
|
||||
} else if (cooldownEntry) {
|
||||
log.debug(
|
||||
`LM Studio inference preload for "${modelKey}" skipped while backoff active (${cooldownEntry.consecutiveFailures} prior failures)`,
|
||||
);
|
||||
}
|
||||
// LM Studio uses OpenAI-compatible streaming usage payloads when requested via
|
||||
// `stream_options.include_usage`. Force this compat flag at call time so usage
|
||||
// reporting remains enabled even when catalog entries omitted compat metadata.
|
||||
const modelWithUsageCompat = {
|
||||
...model,
|
||||
compat: {
|
||||
...(model.compat && typeof model.compat === "object" ? model.compat : {}),
|
||||
supportsUsageInStreaming: true,
|
||||
},
|
||||
};
|
||||
const stream = underlying(modelWithUsageCompat, context, options);
|
||||
return stream instanceof Promise ? await stream : stream;
|
||||
})();
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue