mirror of
https://github.com/hansjone/dsh-im-ops.git
synced 2026-10-10 18:03:18 +08:00
feat: support image prompts across IM channels
This commit is contained in:
xmanrui
parent
99a877c640
commit
65c842c045
40 changed files with 3806 additions and 274 deletions
|
|
@ -623,7 +623,7 @@ export class HarnessClient {
|
|||
return ownership ? { ownership, recovered: true } : null;
|
||||
}
|
||||
|
||||
async ask(sessionId, text, options = {}) {
|
||||
async ask(sessionId, prompt, options = {}) {
|
||||
if (typeof options === 'number') options = { timeoutMs: options };
|
||||
const timeoutMs = options.timeoutMs ?? 600_000;
|
||||
const signal = options.signal;
|
||||
|
|
@ -693,10 +693,16 @@ export class HarnessClient {
|
|||
]);
|
||||
}
|
||||
|
||||
const content = typeof prompt === 'string'
|
||||
? [{ type: 'text', text: prompt }]
|
||||
: prompt;
|
||||
if (!Array.isArray(content) || content.length === 0) {
|
||||
throw new TypeError('Harness prompt content is required');
|
||||
}
|
||||
await this.rpc('session.prompt', {
|
||||
sessionId,
|
||||
mode: 'queue',
|
||||
content: [{ type: 'text', text }],
|
||||
content,
|
||||
clientTimeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
||||
}, 30_000, { rpcId: promptRpcId, signal });
|
||||
|
||||
|
|
|
|||
268
src/channels/shared/image-prompt.mjs
Normal file
268
src/channels/shared/image-prompt.mjs
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
const DEFAULT_MAX_IMAGE_BYTES = 5 * 1024 * 1024;
|
||||
const DEFAULT_MAX_IMAGES = 20;
|
||||
const DEFAULT_MAX_TOTAL_IMAGE_BYTES = 20 * 1024 * 1024;
|
||||
|
||||
export const DEFAULT_IMAGE_PROMPT = '请分析这张图片。';
|
||||
|
||||
export class ImagePromptError extends Error {
|
||||
constructor(code, message, userMessage, options = {}) {
|
||||
super(message, options);
|
||||
this.name = 'ImagePromptError';
|
||||
this.code = code;
|
||||
this.userMessage = userMessage;
|
||||
}
|
||||
}
|
||||
|
||||
function requestSignal(signal, timeoutMs) {
|
||||
const timeout = AbortSignal.timeout(timeoutMs);
|
||||
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
||||
}
|
||||
|
||||
async function cancelResponseBody(response) {
|
||||
try {
|
||||
await response?.body?.cancel?.();
|
||||
} catch {
|
||||
// The original download error is more useful than a best-effort cleanup failure.
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchImageBuffer(url, {
|
||||
fetchImpl = fetch,
|
||||
headers,
|
||||
signal,
|
||||
maxBytes = DEFAULT_MAX_IMAGE_BYTES,
|
||||
timeoutMs = 15_000,
|
||||
allowedHosts,
|
||||
} = {}) {
|
||||
const target = new URL(url);
|
||||
if (target.protocol !== 'https:') throw new Error('Image download URL must use HTTPS');
|
||||
if (Array.isArray(allowedHosts) && !allowedHosts.some((rule) => (
|
||||
typeof rule === 'string'
|
||||
&& (target.hostname === rule
|
||||
|| (rule.startsWith('.')
|
||||
&& (target.hostname === rule.slice(1) || target.hostname.endsWith(rule))))
|
||||
))) {
|
||||
throw new Error('Image download URL is not hosted by the messaging platform');
|
||||
}
|
||||
const response = await fetchImpl(target, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: requestSignal(signal, timeoutMs),
|
||||
redirect: 'manual',
|
||||
});
|
||||
if (Number.isInteger(response?.status) && response.status >= 300 && response.status < 400) {
|
||||
await cancelResponseBody(response);
|
||||
throw new ImagePromptError(
|
||||
'image-redirect-blocked',
|
||||
`Image download redirect was blocked (HTTP ${response.status})`,
|
||||
'图片下载地址发生了重定向，暂时无法读取。',
|
||||
);
|
||||
}
|
||||
if (!response?.ok) {
|
||||
await cancelResponseBody(response);
|
||||
throw new ImagePromptError(
|
||||
'image-http-error',
|
||||
`Image download failed with HTTP ${response?.status ?? 'unknown'}`,
|
||||
`图片下载失败（HTTP ${response?.status ?? 'unknown'}），请重新发送后再试。`,
|
||||
);
|
||||
}
|
||||
const declaredLength = Number(response.headers?.get?.('content-length'));
|
||||
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
||||
await cancelResponseBody(response);
|
||||
throw new ImagePromptError(
|
||||
'image-too-large',
|
||||
`Image response declares ${declaredLength} bytes; the limit is ${maxBytes}`,
|
||||
'图片超过 5 MB，请压缩后重试。',
|
||||
);
|
||||
}
|
||||
|
||||
if (response.body?.[Symbol.asyncIterator]) {
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
for await (const chunk of response.body) {
|
||||
const data = Buffer.from(chunk);
|
||||
size += data.length;
|
||||
if (size > maxBytes) {
|
||||
await response.body.cancel?.().catch?.(() => undefined);
|
||||
throw new ImagePromptError(
|
||||
'image-too-large',
|
||||
`Image response exceeded ${maxBytes} bytes`,
|
||||
'图片超过 5 MB，请压缩后重试。',
|
||||
);
|
||||
}
|
||||
chunks.push(data);
|
||||
}
|
||||
return Buffer.concat(chunks, size);
|
||||
}
|
||||
|
||||
const data = Buffer.from(await response.arrayBuffer());
|
||||
if (data.length > maxBytes) {
|
||||
throw new ImagePromptError(
|
||||
'image-too-large',
|
||||
`Image response contains ${data.length} bytes; the limit is ${maxBytes}`,
|
||||
'图片超过 5 MB，请压缩后重试。',
|
||||
);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
function cleanText(value) {
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function imageSources(message) {
|
||||
return Array.isArray(message?.images) ? message.images.filter(Boolean) : [];
|
||||
}
|
||||
|
||||
function safeName(value) {
|
||||
if (typeof value !== 'string') return undefined;
|
||||
const name = value
|
||||
.replaceAll('\\', '/')
|
||||
.split('/')
|
||||
.at(-1)
|
||||
?.replace(/[\u0000-\u001f\u007f]/g, '')
|
||||
.trim()
|
||||
.slice(0, 255);
|
||||
return name || undefined;
|
||||
}
|
||||
|
||||
function detectedImageMediaType(data) {
|
||||
if (data.length >= 8
|
||||
&& data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47
|
||||
&& data[4] === 0x0d && data[5] === 0x0a && data[6] === 0x1a && data[7] === 0x0a) {
|
||||
return 'image/png';
|
||||
}
|
||||
if (data.length >= 3 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
|
||||
return 'image/jpeg';
|
||||
}
|
||||
if (data.length >= 6) {
|
||||
const signature = data.subarray(0, 6).toString('ascii');
|
||||
if (signature === 'GIF87a' || signature === 'GIF89a') return 'image/gif';
|
||||
}
|
||||
if (data.length >= 12
|
||||
&& data.subarray(0, 4).toString('ascii') === 'RIFF'
|
||||
&& data.subarray(8, 12).toString('ascii') === 'WEBP') {
|
||||
return 'image/webp';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function loadedImage(value) {
|
||||
if (Buffer.isBuffer(value) || value instanceof Uint8Array) {
|
||||
return { data: Buffer.from(value) };
|
||||
}
|
||||
const raw = value?.data ?? value?.buffer;
|
||||
if (Buffer.isBuffer(raw) || raw instanceof Uint8Array) {
|
||||
return {
|
||||
data: Buffer.from(raw),
|
||||
name: value?.name ?? value?.filename,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function hasInboundImages(message) {
|
||||
return imageSources(message).length > 0;
|
||||
}
|
||||
|
||||
export function hasInboundPrompt(message) {
|
||||
return Boolean(cleanText(message?.content)) || hasInboundImages(message);
|
||||
}
|
||||
|
||||
export async function promptContentForMessage(message, {
|
||||
signal,
|
||||
maxImageBytes = DEFAULT_MAX_IMAGE_BYTES,
|
||||
maxImages = DEFAULT_MAX_IMAGES,
|
||||
maxTotalImageBytes = DEFAULT_MAX_TOTAL_IMAGE_BYTES,
|
||||
} = {}) {
|
||||
const sources = imageSources(message);
|
||||
if (sources.length > maxImages) {
|
||||
throw new ImagePromptError(
|
||||
'too-many-images',
|
||||
`Image message contains ${sources.length} images; the limit is ${maxImages}`,
|
||||
`一次最多只能处理 ${maxImages} 张图片。`,
|
||||
);
|
||||
}
|
||||
|
||||
const text = cleanText(message?.content);
|
||||
const content = [];
|
||||
let totalImageBytes = 0;
|
||||
if (text) content.push({ type: 'text', text });
|
||||
else if (sources.length > 0) content.push({ type: 'text', text: DEFAULT_IMAGE_PROMPT });
|
||||
|
||||
for (const [index, source] of sources.entries()) {
|
||||
signal?.throwIfAborted();
|
||||
if (Number.isFinite(source?.size) && source.size > maxImageBytes) {
|
||||
throw new ImagePromptError(
|
||||
'image-too-large',
|
||||
`Image ${index + 1} declares ${source.size} bytes; the limit is ${maxImageBytes}`,
|
||||
'图片超过 5 MB，请压缩后重试。',
|
||||
);
|
||||
}
|
||||
if (Number.isFinite(source?.size) && totalImageBytes + source.size > maxTotalImageBytes) {
|
||||
throw new ImagePromptError(
|
||||
'images-too-large',
|
||||
`Images declare more than ${maxTotalImageBytes} bytes in total`,
|
||||
'一次发送的图片总大小过大，请减少图片数量或压缩后重试。',
|
||||
);
|
||||
}
|
||||
|
||||
let result;
|
||||
try {
|
||||
result = source?.data === undefined
|
||||
? await source?.load?.({ signal, maxBytes: maxImageBytes })
|
||||
: source.data;
|
||||
} catch (error) {
|
||||
if (signal?.aborted || error?.name === 'AbortError' || error?.name === 'TimeoutError') throw error;
|
||||
if (error instanceof ImagePromptError) throw error;
|
||||
throw new ImagePromptError(
|
||||
'image-download-failed',
|
||||
`Unable to download image ${index + 1}: ${error?.message ?? String(error)}`,
|
||||
'图片下载失败，请重新发送后再试。',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
const loaded = loadedImage(result);
|
||||
if (!loaded?.data.length) {
|
||||
throw new ImagePromptError(
|
||||
'invalid-image-data',
|
||||
`Image ${index + 1} returned no data`,
|
||||
'未能读取图片内容，请重新发送。',
|
||||
);
|
||||
}
|
||||
if (loaded.data.length > maxImageBytes) {
|
||||
throw new ImagePromptError(
|
||||
'image-too-large',
|
||||
`Image ${index + 1} contains ${loaded.data.length} bytes; the limit is ${maxImageBytes}`,
|
||||
'图片超过 5 MB，请压缩后重试。',
|
||||
);
|
||||
}
|
||||
if (totalImageBytes + loaded.data.length > maxTotalImageBytes) {
|
||||
throw new ImagePromptError(
|
||||
'images-too-large',
|
||||
`Images contain more than ${maxTotalImageBytes} bytes in total`,
|
||||
'一次发送的图片总大小过大，请减少图片数量或压缩后重试。',
|
||||
);
|
||||
}
|
||||
totalImageBytes += loaded.data.length;
|
||||
const mediaType = detectedImageMediaType(loaded.data);
|
||||
if (!mediaType) {
|
||||
throw new ImagePromptError(
|
||||
'unsupported-image-type',
|
||||
`Image ${index + 1} is not JPEG, PNG, GIF, or WebP`,
|
||||
'暂不支持该图片格式，请发送 JPEG、PNG、WebP 或 GIF 图片。',
|
||||
);
|
||||
}
|
||||
content.push({
|
||||
type: 'image',
|
||||
mediaType,
|
||||
data: loaded.data.toString('base64'),
|
||||
...(safeName(loaded.name ?? source?.name) ? { name: safeName(loaded.name ?? source?.name) } : {}),
|
||||
});
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
export function imagePromptUserMessage(error) {
|
||||
return error instanceof ImagePromptError ? error.userMessage : null;
|
||||
}
|
||||
|
|
@ -2,6 +2,11 @@ import { runWorkspaceCommand } from './workspace-command.mjs';
|
|||
import { runCompactCommand } from './compact-command.mjs';
|
||||
import { askInWorkspaceSession } from './workspace-session.mjs';
|
||||
import { HarnessApprovalQueue } from './harness-approval.mjs';
|
||||
import {
|
||||
hasInboundImages,
|
||||
imagePromptUserMessage,
|
||||
promptContentForMessage,
|
||||
} from './image-prompt.mjs';
|
||||
import {
|
||||
harnessAnswerForQuestion,
|
||||
harnessQuestionText,
|
||||
|
|
@ -17,6 +22,7 @@ function cleanText(value) {
|
|||
function canClaimInteractionReply(message, pending, senderId) {
|
||||
return pending.actor === senderId
|
||||
&& (message.kind !== 'group' || message.addressed === true)
|
||||
&& !hasInboundImages(message)
|
||||
&& Boolean(cleanText(message.content));
|
||||
}
|
||||
|
||||
|
|
@ -98,7 +104,7 @@ export class TextHarnessBridge {
|
|||
key,
|
||||
actor: senderId,
|
||||
messageId,
|
||||
text: normalized.content,
|
||||
text: hasInboundImages(normalized) ? '' : normalized.content,
|
||||
addressed: normalized.kind !== 'group' || normalized.addressed === true,
|
||||
hasPendingQuestion: Boolean(pending),
|
||||
questionCompletion: pending?.submitting || pending?.claimedReplyMessageId
|
||||
|
|
@ -208,16 +214,17 @@ export class TextHarnessBridge {
|
|||
this.#status.lastRejectedAt = new Date().toISOString();
|
||||
return;
|
||||
}
|
||||
if (!text) {
|
||||
await this.#bot.sendText(target, '目前仅支持文字消息。');
|
||||
const hasImages = hasInboundImages(message);
|
||||
if (!text && !hasImages) {
|
||||
await this.#bot.sendText(target, '目前支持文字和图片消息。');
|
||||
return;
|
||||
}
|
||||
const command = text.toLowerCase();
|
||||
if (command === '/help') {
|
||||
if (!hasImages && command === '/help') {
|
||||
await this.#bot.sendText(target, [
|
||||
`${this.#descriptor.label}机器人已连接 DeepSeek Harness。`,
|
||||
'',
|
||||
'直接发送文字即可继续当前会话。',
|
||||
'直接发送文字或图片即可继续当前会话。',
|
||||
'/new 开启一个全新会话',
|
||||
'/compact 压缩当前会话的较早上下文',
|
||||
'/workspace 工作区绝对路径 切换工作区',
|
||||
|
|
@ -229,30 +236,34 @@ export class TextHarnessBridge {
|
|||
].join('\n'));
|
||||
return;
|
||||
}
|
||||
if (command === '/status') {
|
||||
if (!hasImages && command === '/status') {
|
||||
await this.#harness.ensureRunning({ signal: this.#signal });
|
||||
await this.#bot.sendText(target, `${this.#descriptor.label}机器人与 DeepSeek Harness 连接正常。`);
|
||||
return;
|
||||
}
|
||||
const workspaceCommand = await runWorkspaceCommand(text, this.#harness, conversationKey);
|
||||
const workspaceCommand = !hasImages
|
||||
? await runWorkspaceCommand(text, this.#harness, conversationKey)
|
||||
: null;
|
||||
if (workspaceCommand) {
|
||||
for (const reply of workspaceCommand.messages ?? [workspaceCommand.message]) {
|
||||
await this.#bot.sendText(target, reply);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (command === '/new') {
|
||||
if (!hasImages && command === '/new') {
|
||||
await this.#state.clearSession(conversationKey);
|
||||
await this.#bot.sendText(target, '已开启新会话。请发送你的问题。');
|
||||
return;
|
||||
}
|
||||
const compactCommand = await runCompactCommand(
|
||||
text,
|
||||
this.#harness,
|
||||
this.#state,
|
||||
conversationKey,
|
||||
{ signal: this.#signal },
|
||||
);
|
||||
const compactCommand = !hasImages
|
||||
? await runCompactCommand(
|
||||
text,
|
||||
this.#harness,
|
||||
this.#state,
|
||||
conversationKey,
|
||||
{ signal: this.#signal },
|
||||
)
|
||||
: null;
|
||||
if (compactCommand) {
|
||||
await this.#bot.sendText(target, compactCommand.message);
|
||||
return;
|
||||
|
|
@ -272,11 +283,15 @@ export class TextHarnessBridge {
|
|||
);
|
||||
}
|
||||
}
|
||||
const content = hasImages
|
||||
? await promptContentForMessage(message, { signal: this.#signal })
|
||||
: undefined;
|
||||
const { answer } = await askInWorkspaceSession({
|
||||
harness: this.#harness,
|
||||
state: this.#state,
|
||||
key: conversationKey,
|
||||
text,
|
||||
content,
|
||||
createOptions: this.#signal ? { signal: this.#signal } : undefined,
|
||||
existsOptions: this.#signal ? { signal: this.#signal } : undefined,
|
||||
askOptions: {
|
||||
|
|
@ -316,6 +331,18 @@ export class TextHarnessBridge {
|
|||
stream?.cancel?.();
|
||||
if (this.#signal?.aborted) return;
|
||||
this.#status.lastError = error?.message ?? String(error);
|
||||
const imageErrorMessage = imagePromptUserMessage(error);
|
||||
if (imageErrorMessage) {
|
||||
try {
|
||||
await this.#bot.sendText(target, imageErrorMessage);
|
||||
} catch (sendError) {
|
||||
this.#logger.error?.(
|
||||
`[dsh-im:${this.#descriptor.key}] failed to send the image error reply:`,
|
||||
sendError,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
this.#logger.error?.(`[dsh-im:${this.#descriptor.key}] failed to process a message:`, error);
|
||||
try {
|
||||
await this.#bot.sendText(target, '消息处理失败，请稍后重试。');
|
||||
|
|
@ -358,7 +385,7 @@ export class TextHarnessBridge {
|
|||
|
||||
const target = message.replyTarget;
|
||||
const text = cleanText(message.content);
|
||||
if (!text) {
|
||||
if (!text || hasInboundImages(message)) {
|
||||
try {
|
||||
await this.#bot.sendText(target, '请用文字回答当前问题。');
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ export async function askInWorkspaceSession({
|
|||
state,
|
||||
key,
|
||||
text,
|
||||
content,
|
||||
createOptions,
|
||||
existsOptions,
|
||||
askOptions,
|
||||
|
|
@ -48,7 +49,7 @@ export async function askInWorkspaceSession({
|
|||
}
|
||||
return {
|
||||
sessionId,
|
||||
answer: await session.ask(text, askOptions),
|
||||
answer: await session.ask(content ?? text, askOptions),
|
||||
};
|
||||
} catch (error) {
|
||||
if (error?.code !== WORKSPACE_SESSION_STALE) throw error;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue