mirror of
https://github.com/hansjone/dsh-im-ops.git
synced 2026-10-16 10:30:45 +08:00
feat: support image prompts across IM channels
This commit is contained in:
xmanrui
parent
99a877c640
commit
65c842c045
40 changed files with 3806 additions and 274 deletions
|
|
@ -8,11 +8,17 @@ import { HarnessApprovalQueue } from '../shared/harness-approval.mjs';
|
|||
import { runCompactCommand } from '../shared/compact-command.mjs';
|
||||
import { runWorkspaceCommand } from '../shared/workspace-command.mjs';
|
||||
import { askInWorkspaceSession } from '../shared/workspace-session.mjs';
|
||||
import {
|
||||
hasInboundImages,
|
||||
ImagePromptError,
|
||||
imagePromptUserMessage,
|
||||
promptContentForMessage,
|
||||
} from '../shared/image-prompt.mjs';
|
||||
|
||||
const HELP_TEXT = [
|
||||
'企业微信机器人已连接 DeepSeek Harness。',
|
||||
'',
|
||||
'直接发送文字即可继续当前会话。',
|
||||
'直接发送文字或图片即可继续当前会话。',
|
||||
'/new 开启一个全新会话',
|
||||
'/compact 压缩当前会话的较早上下文',
|
||||
'/workspace 工作区绝对路径 切换工作区',
|
||||
|
|
@ -23,6 +29,8 @@ const HELP_TEXT = [
|
|||
'/help 显示本帮助',
|
||||
].join('\n');
|
||||
const MAX_REPLY_BYTES = 18_000;
|
||||
const MAX_IMAGE_BYTES = 5 * 1024 * 1024;
|
||||
const MAX_PREFETCHED_IMAGES = 4;
|
||||
const INTERACTION_RESOLVED_TEXT = '这个问题已在其他客户端处理，无需再次回答。';
|
||||
|
||||
function nonEmptyString(value) {
|
||||
|
|
@ -59,6 +67,102 @@ function messageText(frame) {
|
|||
: text;
|
||||
}
|
||||
|
||||
function imageContents(frame) {
|
||||
const body = bodyOf(frame);
|
||||
if (body.msgtype === 'image') return [body.image];
|
||||
if (body.msgtype !== 'mixed' || !Array.isArray(body.mixed?.msg_item)) return [];
|
||||
return body.mixed.msg_item
|
||||
.filter((item) => item?.msgtype === 'image')
|
||||
.map((item) => item.image);
|
||||
}
|
||||
|
||||
function imageSource(client, image) {
|
||||
const url = nonEmptyString(image?.url);
|
||||
if (!url) return null;
|
||||
const aeskey = nonEmptyString(image?.aeskey) ?? undefined;
|
||||
return {
|
||||
async load({ signal, maxBytes }) {
|
||||
signal?.throwIfAborted();
|
||||
if (typeof client?.downloadFile !== 'function') {
|
||||
throw new Error('Enterprise WeChat image download is unavailable');
|
||||
}
|
||||
const result = await client.downloadFile(url, aeskey);
|
||||
signal?.throwIfAborted();
|
||||
const raw = result?.buffer;
|
||||
if (!Buffer.isBuffer(raw) && !(raw instanceof Uint8Array)) {
|
||||
throw new Error('Enterprise WeChat image download returned no data');
|
||||
}
|
||||
const data = Buffer.from(raw);
|
||||
if (Number.isFinite(maxBytes) && data.length > maxBytes) {
|
||||
throw new ImagePromptError(
|
||||
'image-too-large',
|
||||
`Enterprise WeChat image exceeds ${maxBytes} bytes`,
|
||||
'图片超过 5 MB，请压缩后重试。',
|
||||
);
|
||||
}
|
||||
return { data, name: result?.filename };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function wecomInboundMessage(frame, client) {
|
||||
return {
|
||||
content: messageText(frame),
|
||||
images: imageContents(frame).map((image) => imageSource(client, image)).filter(Boolean),
|
||||
};
|
||||
}
|
||||
|
||||
function prefetchInboundImages(message, signal) {
|
||||
if (!hasInboundImages(message)) return message;
|
||||
return {
|
||||
...message,
|
||||
images: message.images.map((source) => {
|
||||
const download = source.load({ signal, maxBytes: MAX_IMAGE_BYTES });
|
||||
// The conversation queue may not consume this promise immediately. Keep
|
||||
// an attached rejection handler while preserving the original outcome.
|
||||
download.catch(() => undefined);
|
||||
return {
|
||||
...source,
|
||||
async load({ signal: loadSignal, maxBytes = MAX_IMAGE_BYTES } = {}) {
|
||||
loadSignal?.throwIfAborted();
|
||||
const result = await download;
|
||||
loadSignal?.throwIfAborted();
|
||||
const raw = result?.data ?? result?.buffer ?? result;
|
||||
const size = Buffer.isBuffer(raw) || raw instanceof Uint8Array ? raw.length : 0;
|
||||
if (size > maxBytes) {
|
||||
throw new ImagePromptError(
|
||||
'image-too-large',
|
||||
`Enterprise WeChat image exceeds ${maxBytes} bytes`,
|
||||
'图片超过 5 MB，请压缩后重试。',
|
||||
);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function imageQueueFullMessage(message) {
|
||||
return {
|
||||
...message,
|
||||
images: message.images.map((source) => ({
|
||||
...source,
|
||||
async load() {
|
||||
throw new ImagePromptError(
|
||||
'image-queue-full',
|
||||
`Enterprise WeChat already has ${MAX_PREFETCHED_IMAGES} prefetched images`,
|
||||
'当前待处理图片较多，请稍后重新发送。',
|
||||
);
|
||||
},
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function interactionReplyText(frame) {
|
||||
return bodyOf(frame).msgtype === 'text' ? messageText(frame) : '';
|
||||
}
|
||||
|
||||
function splitUtf8(text, maxBytes = MAX_REPLY_BYTES) {
|
||||
const source = String(text ?? '').trim();
|
||||
if (!source) return [];
|
||||
|
|
@ -89,7 +193,7 @@ function progressText(update) {
|
|||
function canClaimInteractionReply(frame, pending) {
|
||||
return pending.questions[pending.index]
|
||||
&& nonEmptyString(bodyOf(frame).from?.userid) === pending.actor
|
||||
&& nonEmptyString(messageText(frame));
|
||||
&& nonEmptyString(interactionReplyText(frame));
|
||||
}
|
||||
|
||||
export function createWecomBridgeStatus() {
|
||||
|
|
@ -119,6 +223,7 @@ export class WecomHarnessBridge {
|
|||
#acceptedMessageIds = new Set();
|
||||
#approvalTasks = new Set();
|
||||
#approvals;
|
||||
#prefetchedImageCount = 0;
|
||||
|
||||
constructor({
|
||||
client,
|
||||
|
|
@ -169,7 +274,7 @@ export class WecomHarnessBridge {
|
|||
key,
|
||||
actor: senderId,
|
||||
messageId,
|
||||
text: messageText(frame),
|
||||
text: interactionReplyText(frame),
|
||||
addressed: true,
|
||||
hasPendingQuestion: Boolean(pending),
|
||||
questionCompletion: pending?.submitting || pending?.claimedReplyMessageId
|
||||
|
|
@ -232,11 +337,28 @@ export class WecomHarnessBridge {
|
|||
releaseMessageId = true,
|
||||
alreadyRecorded = false,
|
||||
} = {}) {
|
||||
// WeCom image URLs expire after five minutes, while a conversation turn
|
||||
// may legally stay queued longer. Start the authenticated SDK download as
|
||||
// soon as the validated callback is accepted, then consume it in order.
|
||||
const inboundMessage = wecomInboundMessage(frame, this.#client);
|
||||
const imageCount = inboundMessage.images.length;
|
||||
let reservedImages = 0;
|
||||
let preparedMessage = inboundMessage;
|
||||
if (imageCount > 0) {
|
||||
if (this.#prefetchedImageCount + imageCount <= MAX_PREFETCHED_IMAGES) {
|
||||
reservedImages = imageCount;
|
||||
this.#prefetchedImageCount += reservedImages;
|
||||
preparedMessage = prefetchInboundImages(inboundMessage, this.#signal);
|
||||
} else {
|
||||
preparedMessage = imageQueueFullMessage(inboundMessage);
|
||||
}
|
||||
}
|
||||
const previous = this.#queues.get(key) ?? Promise.resolve();
|
||||
const current = previous
|
||||
.catch(() => undefined)
|
||||
.then(() => this.#process(frame, { alreadyRecorded }))
|
||||
.then(() => this.#process(frame, { alreadyRecorded, preparedMessage }))
|
||||
.finally(() => {
|
||||
this.#prefetchedImageCount -= reservedImages;
|
||||
if (releaseMessageId) this.#acceptedMessageIds.delete(messageId);
|
||||
if (this.#queues.get(key) === current) this.#queues.delete(key);
|
||||
});
|
||||
|
|
@ -275,7 +397,7 @@ export class WecomHarnessBridge {
|
|||
}
|
||||
}
|
||||
|
||||
async #process(frame, { alreadyRecorded = false } = {}) {
|
||||
async #process(frame, { alreadyRecorded = false, preparedMessage } = {}) {
|
||||
if (this.#signal?.aborted) return;
|
||||
const body = bodyOf(frame);
|
||||
const messageId = typeof body.msgid === 'string' ? body.msgid : '';
|
||||
|
|
@ -287,35 +409,39 @@ export class WecomHarnessBridge {
|
|||
this.#status.messagesReceived += 1;
|
||||
this.#status.lastMessageAt = new Date().toISOString();
|
||||
}
|
||||
const text = messageText(frame);
|
||||
const message = preparedMessage ?? wecomInboundMessage(frame, this.#client);
|
||||
const text = message.content;
|
||||
const hasImages = hasInboundImages(message);
|
||||
const key = conversationKey(frame);
|
||||
let streamId = null;
|
||||
let streamStarted = false;
|
||||
try {
|
||||
if (!text) {
|
||||
await this.#sendImmediate(frame, chatId, '目前支持文字、语音转写和图文混排中的文字消息。');
|
||||
if (!text && !hasImages) {
|
||||
await this.#sendImmediate(frame, chatId, '目前支持文字、图片和语音转写消息。');
|
||||
await this.#state.markSeen(messageId);
|
||||
return;
|
||||
}
|
||||
const command = text.toLowerCase();
|
||||
if (command === '/help') {
|
||||
if (!hasImages && command === '/help') {
|
||||
await this.#sendImmediate(frame, chatId, HELP_TEXT);
|
||||
await this.#state.markSeen(messageId);
|
||||
return;
|
||||
}
|
||||
if (command === '/status') {
|
||||
if (!hasImages && command === '/status') {
|
||||
await this.#harness.ensureRunning({ signal: this.#signal });
|
||||
await this.#sendImmediate(frame, chatId, '企业微信机器人与 DeepSeek Harness 连接正常。');
|
||||
await this.#state.markSeen(messageId);
|
||||
return;
|
||||
}
|
||||
if (command === '/new') {
|
||||
if (!hasImages && command === '/new') {
|
||||
await this.#state.clearSession(key);
|
||||
await this.#sendImmediate(frame, chatId, '已开启新会话。请发送你的问题。');
|
||||
await this.#state.markSeen(messageId);
|
||||
return;
|
||||
}
|
||||
const workspaceCommand = await runWorkspaceCommand(text, this.#harness, key);
|
||||
const workspaceCommand = hasImages
|
||||
? null
|
||||
: await runWorkspaceCommand(text, this.#harness, key);
|
||||
if (workspaceCommand) {
|
||||
for (const reply of workspaceCommand.messages ?? [workspaceCommand.message]) {
|
||||
await this.#sendImmediate(frame, chatId, reply);
|
||||
|
|
@ -323,13 +449,15 @@ export class WecomHarnessBridge {
|
|||
await this.#state.markSeen(messageId);
|
||||
return;
|
||||
}
|
||||
const compactCommand = await runCompactCommand(
|
||||
text,
|
||||
this.#harness,
|
||||
this.#state,
|
||||
key,
|
||||
{ signal: this.#signal },
|
||||
);
|
||||
const compactCommand = hasImages
|
||||
? null
|
||||
: await runCompactCommand(
|
||||
text,
|
||||
this.#harness,
|
||||
this.#state,
|
||||
key,
|
||||
{ signal: this.#signal },
|
||||
);
|
||||
if (compactCommand) {
|
||||
await this.#sendImmediate(frame, chatId, compactCommand.message);
|
||||
await this.#state.markSeen(messageId);
|
||||
|
|
@ -344,11 +472,15 @@ export class WecomHarnessBridge {
|
|||
this.#logger.warn?.('[dsh-im:wecom] unable to start a stream; using an active reply:', error);
|
||||
}
|
||||
|
||||
const content = hasImages
|
||||
? await promptContentForMessage(message, { signal: this.#signal })
|
||||
: undefined;
|
||||
const { answer } = await askInWorkspaceSession({
|
||||
harness: this.#harness,
|
||||
state: this.#state,
|
||||
key,
|
||||
text,
|
||||
content,
|
||||
createOptions: { signal: this.#signal },
|
||||
existsOptions: { signal: this.#signal },
|
||||
askOptions: {
|
||||
|
|
@ -392,11 +524,12 @@ export class WecomHarnessBridge {
|
|||
if (this.#signal?.aborted) return;
|
||||
this.#status.lastError = error?.message ?? String(error);
|
||||
this.#logger.error?.('[dsh-im:wecom] failed to process an inbound message');
|
||||
const errorText = imagePromptUserMessage(error) ?? '消息处理失败，请稍后重试。';
|
||||
try {
|
||||
if (streamStarted && streamId) {
|
||||
await this.#client.replyStream(frame, streamId, '消息处理失败，请稍后重试。', true);
|
||||
await this.#client.replyStream(frame, streamId, errorText, true);
|
||||
} else {
|
||||
await this.#sendImmediate(frame, chatId, '消息处理失败，请稍后重试。');
|
||||
await this.#sendImmediate(frame, chatId, errorText);
|
||||
}
|
||||
await this.#state.markSeen(messageId);
|
||||
} catch {
|
||||
|
|
@ -425,9 +558,9 @@ export class WecomHarnessBridge {
|
|||
this.#status.messagesReceived += 1;
|
||||
this.#status.lastMessageAt = new Date().toISOString();
|
||||
|
||||
const text = nonEmptyString(messageText(frame));
|
||||
const text = nonEmptyString(interactionReplyText(frame));
|
||||
if (!text) {
|
||||
await this.#sendImmediate(frame, chatId, '请用文字或语音回答当前问题。')
|
||||
await this.#sendImmediate(frame, chatId, '请用文字回答当前问题。')
|
||||
.catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue