feat: admit photos and image documents into the bound session

Telegram 图片消息此前一律回复「暂只支持文本消息」。本改动:

- photo 取最大尺寸(TG 压成 jpeg),document 仅接受 image/png|jpeg|webp|gif
- caption 作为伴随文本块进入同一条消息(不解析为命令)
- 相册(同一 media_group_id)去抖 1.2s 合并为一条多图消息,避免 N 连回合
- 派发走宿主 sessions.prompt RPC(Web 端同通道):模型图像能力校验 +
  附件服务持久化 + mode queue 排队,与纯文本路径的 agent.followup 语义一致
- 错误映射:MODEL_DOES_NOT_SUPPORT_IMAGES → 提示换多模态模型;其余回退
  图片发送失败 + [code] 详情;下载失败即时提示
- 客户端新增 getFile/downloadFile(GET file 端点,复用代理感知 fetch)

tests +7(58/58):photo+caption、png document、pdf 拒绝、相册合并、
模型不支持、下载失败、未绑定。无 emoji 文案。
This commit is contained in:
RRRosmontis
2026-09-05 17:51:21 +08:00
parent 8600835436
commit f3f6a3ebaf
13 changed files with 723 additions and 4 deletions
+8
View File
@@ -22,6 +22,14 @@ export type ApiProxySessions = {
rpcId: string;
payload: unknown;
}) => Promise<unknown>;
/**
* Prompt dispatch (Web 端同通道):文本 + 图片字节(base64)经宿主附件服务持久化后
* 进入会话;含图片时会校验当前模型是否支持图像输入。
*/
prompt?: (req: {
rpcId: string;
payload: unknown;
}) => Promise<unknown>;
};
export type ApiProxyLike = {
workspace?: {
+24
View File
@@ -53,6 +53,8 @@ export declare class TelegramBridge {
private readonly compacting;
/** sessionId → compaction abort controller (cancelled on bridge stop) */
private readonly compactAborts;
/** Album accumulation: chatId:media_group_id → parts buffered into one prompt. */
private readonly mediaGroups;
constructor(ctx: Context, options: TelegramBridgeOptions);
/** Effective rendering mode for a chat: per-chat override, else the global mode. */
private modeFor;
@@ -83,6 +85,28 @@ export declare class TelegramBridge {
/** 等待压缩落定并汇报结果(不阻塞轮询循环)。 */
private runCompaction;
private followupBound;
/**
* Extract wire prompt parts from a media message.
* Returns undefined when the message carries no media at all; returns [] for
* an unsupported media type (the user was already notified); otherwise the
* caption text part (when present) followed by the image part.
*/
private collectMediaParts;
/** Download one TG file and encode it as a canonical-base64 wire image part. */
private downloadImagePart;
/**
* Album photos arrive as separate updates sharing a media_group_id — debounce
* them into ONE prompt so an album does not spawn N sequential turns.
*/
private accumulateMediaGroup;
/**
* Dispatch an image-bearing message to the bound session through the host
* prompt RPC — the exact channel the Web composer uses. The host checks
* whether the current model accepts image input, admits the bytes into the
* durable attachment store, then queues the message on the agent (mode
* 'queue' — same semantics as the text path's agent.followup).
*/
private sendImageFollowup;
private pollLoop;
private interruptibleDelay;
private interruptibleSleep;
+146 -2
View File
@@ -1,3 +1,5 @@
import { Buffer } from 'node:buffer';
import { randomUUID } from 'node:crypto';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { dirname, join } from 'node:path';
@@ -14,6 +16,10 @@ import { formatLastTurn, loadLastTurn } from './history.js';
import { describeAgent, displayLabel, workspaceName } from './label.js';
import { formatModel, loadSessionModels, selectSessionModel, } from './models.js';
import { formatStatusText, } from './status.js';
/** Host-admitted raster media types (@deepseek-ai/dsh-attachment). */
const IMAGE_MEDIA_TYPES = ['image/png', 'image/jpeg', 'image/webp', 'image/gif'];
/** Album debounce window: TG emits each photo of an album as its own update. */
const MEDIA_GROUP_WINDOW_MS = 1200;
/**
* 把 command-compact 返回的英文结果翻译成 TG 中文回复。
* 文案以宿主 @deepseek-ai/dsh-command-compact 的固定输出为准;无法识别时返回
@@ -115,6 +121,8 @@ export class TelegramBridge {
compacting = new Set();
/** sessionId → compaction abort controller (cancelled on bridge stop) */
compactAborts = new Map();
/** Album accumulation: chatId:media_group_id → parts buffered into one prompt. */
mediaGroups = new Map();
constructor(ctx, options) {
this.ctx = ctx;
this.token = options.token;
@@ -195,6 +203,9 @@ export class TelegramBridge {
controller.abort();
this.compactAborts.clear();
this.compacting.clear();
for (const group of this.mediaGroups.values())
clearTimeout(group.timer);
this.mediaGroups.clear();
if (this.hookTimer) {
clearTimeout(this.hookTimer);
this.hookTimer = undefined;
@@ -220,9 +231,23 @@ export class TelegramBridge {
await this.client.sendMessage(chatId, MSG.DENIED);
return;
}
// Non-text messages: tell the (authorized) user instead of dropping silently.
// Media messages: photos / image documents (optionally with a caption) are
// admitted into the bound session; other media get an explicit notice.
// A caption is treated as accompanying text — never parsed as a command.
const mediaParts = await this.collectMediaParts(chatId, message);
if (mediaParts !== undefined) {
if (mediaParts.length > 0) {
if (message.media_group_id)
this.accumulateMediaGroup(chatId, message.media_group_id, mediaParts);
else
await this.sendImageFollowup(chatId, mediaParts);
}
return;
}
// Non-text messages without recognized media: tell the (authorized) user
// instead of dropping silently.
if (!message.text) {
await this.enqueueNotice(chatId, '暂只支持文本消息(图片/语音/文件等媒体暂不处理)');
await this.enqueueNotice(chatId, MSG.MEDIA_UNSUPPORTED);
return;
}
const parsed = parseCommand(message.text);
@@ -883,6 +908,125 @@ export class TelegramBridge {
await this.enqueueNotice(chatId, '已加入队列:当前任务完成后将处理这条消息');
}
}
/**
* Extract wire prompt parts from a media message.
* Returns undefined when the message carries no media at all; returns [] for
* an unsupported media type (the user was already notified); otherwise the
* caption text part (when present) followed by the image part.
*/
async collectMediaParts(chatId, message) {
const caption = message.caption?.trim();
const textPart = caption ? { type: 'text', text: caption } : undefined;
const withText = (part) => textPart ? [textPart, part] : [part];
if (message.photo && message.photo.length > 0) {
// TG photo sizes are ordered smallest → largest; the last is the best quality.
const best = message.photo[message.photo.length - 1];
const part = await this.downloadImagePart(chatId, best.file_id, 'image/jpeg');
return part ? withText(part) : [];
}
const doc = message.document;
if (doc) {
if (!doc.mime_type || !IMAGE_MEDIA_TYPES.includes(doc.mime_type)) {
await this.enqueueNotice(chatId, MSG.MEDIA_UNSUPPORTED);
return [];
}
const part = await this.downloadImagePart(chatId, doc.file_id, doc.mime_type, doc.file_name);
return part ? withText(part) : [];
}
return undefined;
}
/** Download one TG file and encode it as a canonical-base64 wire image part. */
async downloadImagePart(chatId, fileId, mediaType, name) {
try {
const file = await this.client.getFile(fileId);
if (!file.file_path)
throw new Error('TG 未返回文件路径');
const bytes = await this.client.downloadFile(file.file_path);
// Buffer base64 is canonical (standard alphabet, no whitespace) — the host
// attachment admission rejects non-canonical forms outright.
const data = Buffer.from(bytes).toString('base64');
return { type: 'image', mediaType, data, ...(name === undefined ? {} : { name }) };
}
catch (err) {
await this.enqueueNotice(chatId, MSG.IMAGE_FAILED(this.redact(err)));
return undefined;
}
}
/**
* Album photos arrive as separate updates sharing a media_group_id — debounce
* them into ONE prompt so an album does not spawn N sequential turns.
*/
accumulateMediaGroup(chatId, groupId, parts) {
const key = `${String(chatId)}:${groupId}`;
const existing = this.mediaGroups.get(key);
if (existing) {
clearTimeout(existing.timer);
existing.parts.push(...parts);
}
const entry = existing ?? { chatId, parts: [...parts] };
entry.timer = setTimeout(() => {
this.mediaGroups.delete(key);
this.sendImageFollowup(entry.chatId, entry.parts).catch((err) => {
this.ctx.logger.error(this.redact(err));
});
}, MEDIA_GROUP_WINDOW_MS);
this.mediaGroups.set(key, entry);
}
/**
* Dispatch an image-bearing message to the bound session through the host
* prompt RPC — the exact channel the Web composer uses. The host checks
* whether the current model accepts image input, admits the bytes into the
* durable attachment store, then queues the message on the agent (mode
* 'queue' — same semantics as the text path's agent.followup).
*/
async sendImageFollowup(chatId, parts) {
const binding = this.bindings.get(String(chatId));
if (!binding) {
await this.client.sendMessage(chatId, MSG.NEED_BIND);
return;
}
const agent = await this.ensureLiveAgent(binding.sessionId);
if (!agent) {
this.bindings.delete(String(chatId));
await this.client.sendMessage(chatId, MSG.GONE);
return;
}
const proxy = resolveApiProxy(this.ctx);
if (!proxy?.sessions?.prompt) {
await this.enqueueNotice(chatId, '当前宿主不支持会话消息接口(sessions.prompt),无法发送图片。');
return;
}
const busy = this.busySessions.has(binding.sessionId)
|| Boolean(agent.phase?.kind
&& agent.phase?.kind !== 'idle');
let res;
try {
res = await proxy.sessions.prompt({
rpcId: randomUUID(),
payload: { sessionId: binding.sessionId, mode: 'queue', content: parts },
});
}
catch (err) {
await this.enqueueNotice(chatId, MSG.IMAGE_FAILED(this.redact(err)));
return;
}
const result = res?.result;
if (!result?.ok) {
const code = result?.error?.code;
const message = String(result?.error?.message ?? '宿主未确认接收');
if (code === 'attachment-error' && result?.error?.details?.reason === 'MODEL_DOES_NOT_SUPPORT_IMAGES') {
await this.enqueueNotice(chatId, MSG.IMAGE_MODEL_UNSUPPORTED);
}
else {
const detail = `${code ? `[${code}] ` : ''}${message}`.trim();
await this.enqueueNotice(chatId, MSG.IMAGE_FAILED(detail));
}
return;
}
if (busy) {
await this.enqueueNotice(chatId, '已加入队列:当前任务完成后将处理这条消息');
}
}
async pollLoop() {
const signal = this.pollAbort?.signal;
let errorCount = 0;
+32
View File
@@ -10,12 +10,38 @@ export interface TelegramChat {
title?: string;
username?: string;
}
export interface TelegramPhotoSize {
file_id: string;
file_unique_id: string;
width: number;
height: number;
file_size?: number;
}
export interface TelegramDocument {
file_id: string;
file_unique_id: string;
file_name?: string;
mime_type?: string;
file_size?: number;
}
export interface TelegramMessage {
message_id: number;
date: number;
chat: TelegramChat;
from?: TelegramUser;
text?: string;
/** Media caption (photos/documents); TG puts it here, not in `text`. */
caption?: string;
photo?: TelegramPhotoSize[];
document?: TelegramDocument;
/** Album grouping id: several photo updates share one media_group_id. */
media_group_id?: string;
}
export interface TelegramFile {
file_id: string;
file_unique_id: string;
file_size?: number;
file_path?: string;
}
export interface TelegramCallbackQuery {
id: string;
@@ -56,6 +82,10 @@ export interface TelegramClientLike {
sendChatAction(chatId: number, action: string): Promise<boolean>;
answerCallbackQuery(callbackQueryId: string, text?: string): Promise<boolean>;
setMyCommands(commands: TelegramBotCommand[]): Promise<boolean>;
/** Resolve a file_id to a file_path (downloadable URL suffix). */
getFile(fileId: string): Promise<TelegramFile>;
/** Download file bytes from the Bot API file endpoint (GET, proxy-aware). */
downloadFile(filePath: string): Promise<Uint8Array>;
}
export declare class TelegramClient implements TelegramClientLike {
private readonly token;
@@ -72,4 +102,6 @@ export declare class TelegramClient implements TelegramClientLike {
sendChatAction(chatId: number, action: string): Promise<boolean>;
answerCallbackQuery(callbackQueryId: string, text?: string): Promise<boolean>;
setMyCommands(commands: TelegramBotCommand[]): Promise<boolean>;
getFile(fileId: string): Promise<TelegramFile>;
downloadFile(filePath: string): Promise<Uint8Array>;
}
+17
View File
@@ -104,4 +104,21 @@ export class TelegramClient {
async setMyCommands(commands) {
return this.call('setMyCommands', { commands });
}
async getFile(fileId) {
return this.call('getFile', { file_id: fileId });
}
async downloadFile(filePath) {
const url = `${this.baseUrl}/file/bot${this.token}/${filePath}`;
try {
const response = await this.fetchImpl(url);
if (!response.ok) {
throw new Error(`Telegram file download failed: HTTP ${String(response.status)}`);
}
return new Uint8Array(await response.arrayBuffer());
}
catch (err) {
const message = err instanceof Error ? err.message : String(err);
throw new Error(this.redact(message));
}
}
}
+3
View File
@@ -28,6 +28,9 @@ export declare const MSG: {
readonly COMPACT_PERSIST_FAILED: "压缩完成但保存失败。请检查本机存储与权限后重试。";
readonly COMPACT_FAILED: (detail?: string) => string;
readonly GONE: "绑定的会话已不可用。请重新 /sessions。";
readonly MEDIA_UNSUPPORTED: "暂不支持该消息类型(支持文本与 png/jpeg/webp/gif 图片;语音/视频/其他文件请在 Web 端处理)。";
readonly IMAGE_MODEL_UNSUPPORTED: "当前模型不支持图片输入,请在 Web 端或 /model 切换到多模态模型后重试。";
readonly IMAGE_FAILED: (detail?: string) => string;
readonly LAST_FAILED: "无法读取上次对话。请确认已绑定,且本机 dsh web / apiProxy 可用。";
readonly MODEL_UNAVAILABLE: (detail?: string) => string;
readonly MODEL_UNROUTABLE: (current: string) => string;
+9
View File
@@ -16,6 +16,7 @@ export const MSG = {
'/help — 显示帮助',
'',
'绑定后直接发文字,消息会进入该本机会话;Web 与手机看到同一条轨迹。',
'直接发送图片(或带文字说明的图片)也会进入该会话,说明文字会一并作为消息文本。',
'仅白名单用户可用。无会话时请先在 dsh web 开对话或保留历史会话。',
].join('\n'),
NEED_BIND: '尚未绑定本机会话。请先发送 /sessions 选择一个。',
@@ -59,6 +60,14 @@ export const MSG = {
return `${tip}\n详情:${detail}`;
},
GONE: '绑定的会话已不可用。请重新 /sessions。',
MEDIA_UNSUPPORTED: '暂不支持该消息类型(支持文本与 png/jpeg/webp/gif 图片;语音/视频/其他文件请在 Web 端处理)。',
IMAGE_MODEL_UNSUPPORTED: '当前模型不支持图片输入,请在 Web 端或 /model 切换到多模态模型后重试。',
IMAGE_FAILED(detail) {
const tip = '图片发送失败。';
if (!detail)
return tip;
return `${tip}\n详情:${detail}`;
},
LAST_FAILED: '无法读取上次对话。请确认已绑定,且本机 dsh web / apiProxy 可用。',
MODEL_UNAVAILABLE(detail) {
const tip = '无法读取模型列表。请确认已绑定会话,且本机 dsh web 已加载 host-apiproxy。';