Merge pull request #118 from GoldJohnKing/fix/non-vision-model-image-file-fallback

src/channels/shared/harness-client.mjs

@@ -7,6 +7,12 @@ import {
   appendInboundFilesToPrompt,
   InboundFileError,
 } from './inbound-file.mjs';
+import {
+  IMAGE_FILE_FALLBACK_PROMPT,
+  contentWithoutImages,
+  imageFileSourcesFromContent,
+  isModelImageRejection,
+} from './image-prompt.mjs';
 import { outboundArtifactRegistry } from './semantic/artifact.mjs';
 import { t } from './i18n.mjs';
 import { watchHarnessMux } from './harness-mux.mjs';
@@ -1242,6 +1248,31 @@ export class HarnessClient {
     return ownership ? { ownership, recovered: true } : null;
   }
 
+  /** Stage inbound file sources into the Session workspace via the Host executor. */
+  async #stageWorkspaceFiles(sessionId, files, signal) {
+    if (!this.#fileIngressExecutor) {
+      throw new InboundFileError(
+        'inbound-file-ingress-unavailable',
+        'Harness file ingress is unavailable in this Host process.',
+      );
+    }
+    const sessionList = await this.rpc(
+      'session.list',
+      {},
+      30_000,
+      { signal },
+    );
+    const sessionWorkspace = sessionList?.items?.find(
+      (item) => item?.sessionId === sessionId,
+    )?.cwd;
+    return this.#fileIngressExecutor({
+      sessionId,
+      workspace: sessionWorkspace,
+      files,
+      signal,
+    });
+  }
+
   async ask(sessionId, prompt, options = {}) {
     if (typeof options === 'number') options = { timeoutMs: options };
     const timeoutMs = options.timeoutMs ?? 600_000;
@@ -1298,7 +1329,7 @@ export class HarnessClient {
     let interactionTask = null;
     let artifactsDelivered = false;
     let deliveredArtifactCount = 0;
-    let stagedInboundFiles = null;
+    const stagedBatches = [];
     let promptAccepted = false;
     let turnFinished = false;
 
@@ -1329,29 +1360,11 @@ export class HarnessClient {
     const closeArtifactConsumer = outboundArtifactRegistry.openConsumer(sessionId, promptRpcId);
 
     try {
+      const basePrompt = prompt;
       if (inboundFiles.length > 0) {
-        if (!this.#fileIngressExecutor) {
-          throw new InboundFileError(
-            'inbound-file-ingress-unavailable',
-            'Harness file ingress is unavailable in this Host process.',
-          );
-        }
-        const sessionList = await this.rpc(
-          'session.list',
-          {},
-          30_000,
-          { signal },
-        );
-        const sessionWorkspace = sessionList?.items?.find(
-          (item) => item?.sessionId === sessionId,
-        )?.cwd;
-        stagedInboundFiles = await this.#fileIngressExecutor({
-          sessionId,
-          workspace: sessionWorkspace,
-          files: inboundFiles,
-          signal,
-        });
-        prompt = appendInboundFilesToPrompt(prompt, stagedInboundFiles);
+        const staged = await this.#stageWorkspaceFiles(sessionId, inboundFiles, signal);
+        stagedBatches.push(staged);
+        prompt = appendInboundFilesToPrompt(prompt, staged);
       }
       if (interactionSignal) {
         let markOpen;
@@ -1378,12 +1391,47 @@ export class HarnessClient {
       if (!Array.isArray(content) || content.length === 0) {
         throw new TypeError('Harness prompt content is required');
       }
-      await this.rpc('session.prompt', {
+      const clientTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+      const sendPrompt = (promptContent) => this.rpc('session.prompt', {
         sessionId,
         mode: 'queue',
-        content,
-        clientTimeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+        content: promptContent,
+        clientTimeZone,
       }, 30_000, { rpcId: promptRpcId, signal });
+      try {
+        await sendPrompt(content);
+      } catch (error) {
+        // The Host refuses image blocks for a non-vision model before any
+        // durable user message exists. Re-deliver the same bytes the way
+        // ordinary uploads (zip, documents) already travel — staged into the
+        // Session workspace and named in a text manifest — then retry once
+        // with a text-only prompt. The retry reuses promptRpcId so reply
+        // tracking, control and interaction ownership stay bound to this ask.
+        const imageSources = isModelImageRejection(error)
+          ? imageFileSourcesFromContent(content)
+          : [];
+        if (imageSources.length === 0) throw error;
+        let stagedImages;
+        try {
+          stagedImages = await this.#stageWorkspaceFiles(sessionId, imageSources, signal);
+        } catch (stagingError) {
+          if (signal?.aborted) throw error;
+          console.warn(
+            `[${this.#logPrefix}] unable to restage rejected images as workspace files:`,
+            stagingError?.message ?? String(stagingError),
+          );
+          throw error;
+        }
+        stagedBatches.push(stagedImages);
+        const baseContent = typeof basePrompt === 'string'
+          ? [{ type: 'text', text: basePrompt }]
+          : basePrompt;
+        const fallbackPrompt = appendInboundFilesToPrompt([
+          ...contentWithoutImages(baseContent),
+          { type: 'text', text: t(IMAGE_FILE_FALLBACK_PROMPT) },
+        ], { files: stagedBatches.flatMap((batch) => batch?.files ?? []) });
+        await sendPrompt(fallbackPrompt);
+      }
       promptAccepted = true;
 
       try {
@@ -1441,10 +1489,14 @@ export class HarnessClient {
         throw turnStoppedError();
       }
     } finally {
-      if (stagedInboundFiles && (!promptAccepted || turnFinished)) {
-        await stagedInboundFiles.cleanup().catch((error) => {
-          console.warn(`[${this.#logPrefix}] unable to clean inbound files:`, error.message);
-        });
+      if (!promptAccepted || turnFinished) {
+        for (const staged of stagedBatches) {
+          try {
+            await staged?.cleanup?.();
+          } catch (error) {
+            console.warn(`[${this.#logPrefix}] unable to clean inbound files:`, error.message);
+          }
+        }
       }
       closeArtifactConsumer();
       if (ownership) {

src/channels/shared/i18n-en/shared-c.mjs

@@ -81,6 +81,8 @@ export default {
   // image-prompt.mjs
   '当前模型不支持图片，请用 /models 查看可用模型，再用 /model <序号> 切换后重发。':
     'The current model does not support images. Use /models to list available models, switch with /model <number>, then resend.',
+  '当前会话模型不支持直接接收图片输入。用户发送的图片已作为文件保存到工作区（见下方文件清单）。请使用可用工具分析这些图片文件后回答，例如 run_code 或 pwsh 读取字节、解析元数据、调用图像处理或 OCR 库；不要假设自己能直接看到图片内容。':
+    'The current session model does not accept direct image input. The images sent by the user were saved into the workspace as files (see the file manifest below). Answer by analyzing those image files with the available tools — for example run_code or pwsh to read bytes, parse metadata, or call image-processing or OCR libraries — and do not assume you can see the images directly.',
   '图片超过宿主允许的大小，请压缩后重试。':
     'The image exceeds the size allowed by the host; compress it and try again.',
   '图片分辨率过高，请压缩后重试。':

src/channels/shared/image-prompt.mjs

@@ -6,6 +6,12 @@ const DEFAULT_MAX_TOTAL_IMAGE_BYTES = 20 * 1024 * 1024;
 
 export const DEFAULT_IMAGE_PROMPT = '请分析这张图片。';
 
+/**
+ * Model-facing guidance appended when the Host refuses image input for the
+ * current model and the same images are re-delivered as workspace files.
+ */
+export const IMAGE_FILE_FALLBACK_PROMPT = '当前会话模型不支持直接接收图片输入。用户发送的图片已作为文件保存到工作区（见下方文件清单）。请使用可用工具分析这些图片文件后回答，例如 run_code 或 pwsh 读取字节、解析元数据、调用图像处理或 OCR 库；不要假设自己能直接看到图片内容。';
+
 export class ImagePromptError extends Error {
   constructor(code, message, userMessage, options = {}) {
     super(message, options);
@@ -299,3 +305,48 @@ export function imagePromptDiagnostic(error) {
 export function imagePromptUserMessage(error) {
   return imagePromptDiagnostic(error)?.userMessage ?? null;
 }
+
+const IMAGE_FILE_EXTENSIONS = new Map([
+  ['image/png', '.png'],
+  ['image/jpeg', '.jpg'],
+  ['image/gif', '.gif'],
+  ['image/webp', '.webp'],
+]);
+
+const IMAGE_EXTENSION_PATTERN = /\.(?:png|jpe?g|gif|webp)$/i;
+
+function imageStorageName(name, mediaType, index) {
+  const extension = IMAGE_FILE_EXTENSIONS.get(mediaType) ?? '.img';
+  const cleaned = safeName(name);
+  if (cleaned && IMAGE_EXTENSION_PATTERN.test(cleaned)) return cleaned;
+  return `${cleaned ?? `image-${index + 1}`}${extension}`;
+}
+
+/**
+ * Convert already-admitted image content blocks into inbound file sources so
+ * the same bytes can reach a non-vision model as workspace files — the path
+ * ordinary uploads such as zip archives already take.
+ */
+export function imageFileSourcesFromContent(content) {
+  if (!Array.isArray(content)) return [];
+  return content
+    .filter((part) => part?.type === 'image')
+    .map((part, index) => ({
+      name: imageStorageName(part.name, part.mediaType, index),
+      ...(typeof part.mediaType === 'string' && part.mediaType.trim()
+        ? { mediaType: part.mediaType.trim() }
+        : {}),
+      data: Buffer.from(typeof part.data === 'string' ? part.data : '', 'base64'),
+    }));
+}
+
+/** Return the same content with every image block removed. */
+export function contentWithoutImages(content) {
+  return Array.isArray(content) ? content.filter((part) => part?.type !== 'image') : content;
+}
+
+/** Whether an error is the Host rejecting image input for a non-vision model. */
+export function isModelImageRejection(error) {
+  return error?.code === 'attachment-error'
+    && error?.details?.reason === 'MODEL_DOES_NOT_SUPPORT_IMAGES';
+}