feat: admit photos and image documents into the bound session
Telegram 图片消息此前一律回复「暂只支持文本消息」。本改动: - photo 取最大尺寸(TG 压成 jpeg),document 仅接受 image/png|jpeg|webp|gif - caption 作为伴随文本块进入同一条消息(不解析为命令) - 相册(同一 media_group_id)去抖 1.2s 合并为一条多图消息,避免 N 连回合 - 派发走宿主 sessions.prompt RPC(Web 端同通道):模型图像能力校验 + 附件服务持久化 + mode queue 排队,与纯文本路径的 agent.followup 语义一致 - 错误映射:MODEL_DOES_NOT_SUPPORT_IMAGES → 提示换多模态模型;其余回退 图片发送失败 + [code] 详情;下载失败即时提示 - 客户端新增 getFile/downloadFile(GET file 端点,复用代理感知 fetch) tests +7(58/58):photo+caption、png document、pdf 拒绝、相册合并、 模型不支持、下载失败、未绑定。无 emoji 文案。
This commit is contained in:
Vendored
+8
@@ -22,6 +22,14 @@ export type ApiProxySessions = {
|
||||
rpcId: string;
|
||||
payload: unknown;
|
||||
}) => Promise<unknown>;
|
||||
/**
|
||||
* Prompt dispatch (Web 端同通道):文本 + 图片字节(base64)经宿主附件服务持久化后
|
||||
* 进入会话;含图片时会校验当前模型是否支持图像输入。
|
||||
*/
|
||||
prompt?: (req: {
|
||||
rpcId: string;
|
||||
payload: unknown;
|
||||
}) => Promise<unknown>;
|
||||
};
|
||||
export type ApiProxyLike = {
|
||||
workspace?: {
|
||||
|
||||
Vendored
+24
@@ -53,6 +53,8 @@ export declare class TelegramBridge {
|
||||
private readonly compacting;
|
||||
/** sessionId → compaction abort controller (cancelled on bridge stop) */
|
||||
private readonly compactAborts;
|
||||
/** Album accumulation: chatId:media_group_id → parts buffered into one prompt. */
|
||||
private readonly mediaGroups;
|
||||
constructor(ctx: Context, options: TelegramBridgeOptions);
|
||||
/** Effective rendering mode for a chat: per-chat override, else the global mode. */
|
||||
private modeFor;
|
||||
@@ -83,6 +85,28 @@ export declare class TelegramBridge {
|
||||
/** 等待压缩落定并汇报结果(不阻塞轮询循环)。 */
|
||||
private runCompaction;
|
||||
private followupBound;
|
||||
/**
|
||||
* Extract wire prompt parts from a media message.
|
||||
* Returns undefined when the message carries no media at all; returns [] for
|
||||
* an unsupported media type (the user was already notified); otherwise the
|
||||
* caption text part (when present) followed by the image part.
|
||||
*/
|
||||
private collectMediaParts;
|
||||
/** Download one TG file and encode it as a canonical-base64 wire image part. */
|
||||
private downloadImagePart;
|
||||
/**
|
||||
* Album photos arrive as separate updates sharing a media_group_id — debounce
|
||||
* them into ONE prompt so an album does not spawn N sequential turns.
|
||||
*/
|
||||
private accumulateMediaGroup;
|
||||
/**
|
||||
* Dispatch an image-bearing message to the bound session through the host
|
||||
* prompt RPC — the exact channel the Web composer uses. The host checks
|
||||
* whether the current model accepts image input, admits the bytes into the
|
||||
* durable attachment store, then queues the message on the agent (mode
|
||||
* 'queue' — same semantics as the text path's agent.followup).
|
||||
*/
|
||||
private sendImageFollowup;
|
||||
private pollLoop;
|
||||
private interruptibleDelay;
|
||||
private interruptibleSleep;
|
||||
|
||||
+146
-2
@@ -1,3 +1,5 @@
|
||||
import { Buffer } from 'node:buffer';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
@@ -14,6 +16,10 @@ import { formatLastTurn, loadLastTurn } from './history.js';
|
||||
import { describeAgent, displayLabel, workspaceName } from './label.js';
|
||||
import { formatModel, loadSessionModels, selectSessionModel, } from './models.js';
|
||||
import { formatStatusText, } from './status.js';
|
||||
/** Host-admitted raster media types (@deepseek-ai/dsh-attachment). */
|
||||
const IMAGE_MEDIA_TYPES = ['image/png', 'image/jpeg', 'image/webp', 'image/gif'];
|
||||
/** Album debounce window: TG emits each photo of an album as its own update. */
|
||||
const MEDIA_GROUP_WINDOW_MS = 1200;
|
||||
/**
|
||||
* 把 command-compact 返回的英文结果翻译成 TG 中文回复。
|
||||
* 文案以宿主 @deepseek-ai/dsh-command-compact 的固定输出为准;无法识别时返回
|
||||
@@ -115,6 +121,8 @@ export class TelegramBridge {
|
||||
compacting = new Set();
|
||||
/** sessionId → compaction abort controller (cancelled on bridge stop) */
|
||||
compactAborts = new Map();
|
||||
/** Album accumulation: chatId:media_group_id → parts buffered into one prompt. */
|
||||
mediaGroups = new Map();
|
||||
constructor(ctx, options) {
|
||||
this.ctx = ctx;
|
||||
this.token = options.token;
|
||||
@@ -195,6 +203,9 @@ export class TelegramBridge {
|
||||
controller.abort();
|
||||
this.compactAborts.clear();
|
||||
this.compacting.clear();
|
||||
for (const group of this.mediaGroups.values())
|
||||
clearTimeout(group.timer);
|
||||
this.mediaGroups.clear();
|
||||
if (this.hookTimer) {
|
||||
clearTimeout(this.hookTimer);
|
||||
this.hookTimer = undefined;
|
||||
@@ -220,9 +231,23 @@ export class TelegramBridge {
|
||||
await this.client.sendMessage(chatId, MSG.DENIED);
|
||||
return;
|
||||
}
|
||||
// Non-text messages: tell the (authorized) user instead of dropping silently.
|
||||
// Media messages: photos / image documents (optionally with a caption) are
|
||||
// admitted into the bound session; other media get an explicit notice.
|
||||
// A caption is treated as accompanying text — never parsed as a command.
|
||||
const mediaParts = await this.collectMediaParts(chatId, message);
|
||||
if (mediaParts !== undefined) {
|
||||
if (mediaParts.length > 0) {
|
||||
if (message.media_group_id)
|
||||
this.accumulateMediaGroup(chatId, message.media_group_id, mediaParts);
|
||||
else
|
||||
await this.sendImageFollowup(chatId, mediaParts);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Non-text messages without recognized media: tell the (authorized) user
|
||||
// instead of dropping silently.
|
||||
if (!message.text) {
|
||||
await this.enqueueNotice(chatId, '暂只支持文本消息(图片/语音/文件等媒体暂不处理)');
|
||||
await this.enqueueNotice(chatId, MSG.MEDIA_UNSUPPORTED);
|
||||
return;
|
||||
}
|
||||
const parsed = parseCommand(message.text);
|
||||
@@ -883,6 +908,125 @@ export class TelegramBridge {
|
||||
await this.enqueueNotice(chatId, '已加入队列:当前任务完成后将处理这条消息');
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Extract wire prompt parts from a media message.
|
||||
* Returns undefined when the message carries no media at all; returns [] for
|
||||
* an unsupported media type (the user was already notified); otherwise the
|
||||
* caption text part (when present) followed by the image part.
|
||||
*/
|
||||
async collectMediaParts(chatId, message) {
|
||||
const caption = message.caption?.trim();
|
||||
const textPart = caption ? { type: 'text', text: caption } : undefined;
|
||||
const withText = (part) => textPart ? [textPart, part] : [part];
|
||||
if (message.photo && message.photo.length > 0) {
|
||||
// TG photo sizes are ordered smallest → largest; the last is the best quality.
|
||||
const best = message.photo[message.photo.length - 1];
|
||||
const part = await this.downloadImagePart(chatId, best.file_id, 'image/jpeg');
|
||||
return part ? withText(part) : [];
|
||||
}
|
||||
const doc = message.document;
|
||||
if (doc) {
|
||||
if (!doc.mime_type || !IMAGE_MEDIA_TYPES.includes(doc.mime_type)) {
|
||||
await this.enqueueNotice(chatId, MSG.MEDIA_UNSUPPORTED);
|
||||
return [];
|
||||
}
|
||||
const part = await this.downloadImagePart(chatId, doc.file_id, doc.mime_type, doc.file_name);
|
||||
return part ? withText(part) : [];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
/** Download one TG file and encode it as a canonical-base64 wire image part. */
|
||||
async downloadImagePart(chatId, fileId, mediaType, name) {
|
||||
try {
|
||||
const file = await this.client.getFile(fileId);
|
||||
if (!file.file_path)
|
||||
throw new Error('TG 未返回文件路径');
|
||||
const bytes = await this.client.downloadFile(file.file_path);
|
||||
// Buffer base64 is canonical (standard alphabet, no whitespace) — the host
|
||||
// attachment admission rejects non-canonical forms outright.
|
||||
const data = Buffer.from(bytes).toString('base64');
|
||||
return { type: 'image', mediaType, data, ...(name === undefined ? {} : { name }) };
|
||||
}
|
||||
catch (err) {
|
||||
await this.enqueueNotice(chatId, MSG.IMAGE_FAILED(this.redact(err)));
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Album photos arrive as separate updates sharing a media_group_id — debounce
|
||||
* them into ONE prompt so an album does not spawn N sequential turns.
|
||||
*/
|
||||
accumulateMediaGroup(chatId, groupId, parts) {
|
||||
const key = `${String(chatId)}:${groupId}`;
|
||||
const existing = this.mediaGroups.get(key);
|
||||
if (existing) {
|
||||
clearTimeout(existing.timer);
|
||||
existing.parts.push(...parts);
|
||||
}
|
||||
const entry = existing ?? { chatId, parts: [...parts] };
|
||||
entry.timer = setTimeout(() => {
|
||||
this.mediaGroups.delete(key);
|
||||
this.sendImageFollowup(entry.chatId, entry.parts).catch((err) => {
|
||||
this.ctx.logger.error(this.redact(err));
|
||||
});
|
||||
}, MEDIA_GROUP_WINDOW_MS);
|
||||
this.mediaGroups.set(key, entry);
|
||||
}
|
||||
/**
|
||||
* Dispatch an image-bearing message to the bound session through the host
|
||||
* prompt RPC — the exact channel the Web composer uses. The host checks
|
||||
* whether the current model accepts image input, admits the bytes into the
|
||||
* durable attachment store, then queues the message on the agent (mode
|
||||
* 'queue' — same semantics as the text path's agent.followup).
|
||||
*/
|
||||
async sendImageFollowup(chatId, parts) {
|
||||
const binding = this.bindings.get(String(chatId));
|
||||
if (!binding) {
|
||||
await this.client.sendMessage(chatId, MSG.NEED_BIND);
|
||||
return;
|
||||
}
|
||||
const agent = await this.ensureLiveAgent(binding.sessionId);
|
||||
if (!agent) {
|
||||
this.bindings.delete(String(chatId));
|
||||
await this.client.sendMessage(chatId, MSG.GONE);
|
||||
return;
|
||||
}
|
||||
const proxy = resolveApiProxy(this.ctx);
|
||||
if (!proxy?.sessions?.prompt) {
|
||||
await this.enqueueNotice(chatId, '当前宿主不支持会话消息接口(sessions.prompt),无法发送图片。');
|
||||
return;
|
||||
}
|
||||
const busy = this.busySessions.has(binding.sessionId)
|
||||
|| Boolean(agent.phase?.kind
|
||||
&& agent.phase?.kind !== 'idle');
|
||||
let res;
|
||||
try {
|
||||
res = await proxy.sessions.prompt({
|
||||
rpcId: randomUUID(),
|
||||
payload: { sessionId: binding.sessionId, mode: 'queue', content: parts },
|
||||
});
|
||||
}
|
||||
catch (err) {
|
||||
await this.enqueueNotice(chatId, MSG.IMAGE_FAILED(this.redact(err)));
|
||||
return;
|
||||
}
|
||||
const result = res?.result;
|
||||
if (!result?.ok) {
|
||||
const code = result?.error?.code;
|
||||
const message = String(result?.error?.message ?? '宿主未确认接收');
|
||||
if (code === 'attachment-error' && result?.error?.details?.reason === 'MODEL_DOES_NOT_SUPPORT_IMAGES') {
|
||||
await this.enqueueNotice(chatId, MSG.IMAGE_MODEL_UNSUPPORTED);
|
||||
}
|
||||
else {
|
||||
const detail = `${code ? `[${code}] ` : ''}${message}`.trim();
|
||||
await this.enqueueNotice(chatId, MSG.IMAGE_FAILED(detail));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (busy) {
|
||||
await this.enqueueNotice(chatId, '已加入队列:当前任务完成后将处理这条消息');
|
||||
}
|
||||
}
|
||||
async pollLoop() {
|
||||
const signal = this.pollAbort?.signal;
|
||||
let errorCount = 0;
|
||||
|
||||
Vendored
+32
@@ -10,12 +10,38 @@ export interface TelegramChat {
|
||||
title?: string;
|
||||
username?: string;
|
||||
}
|
||||
export interface TelegramPhotoSize {
|
||||
file_id: string;
|
||||
file_unique_id: string;
|
||||
width: number;
|
||||
height: number;
|
||||
file_size?: number;
|
||||
}
|
||||
export interface TelegramDocument {
|
||||
file_id: string;
|
||||
file_unique_id: string;
|
||||
file_name?: string;
|
||||
mime_type?: string;
|
||||
file_size?: number;
|
||||
}
|
||||
export interface TelegramMessage {
|
||||
message_id: number;
|
||||
date: number;
|
||||
chat: TelegramChat;
|
||||
from?: TelegramUser;
|
||||
text?: string;
|
||||
/** Media caption (photos/documents); TG puts it here, not in `text`. */
|
||||
caption?: string;
|
||||
photo?: TelegramPhotoSize[];
|
||||
document?: TelegramDocument;
|
||||
/** Album grouping id: several photo updates share one media_group_id. */
|
||||
media_group_id?: string;
|
||||
}
|
||||
export interface TelegramFile {
|
||||
file_id: string;
|
||||
file_unique_id: string;
|
||||
file_size?: number;
|
||||
file_path?: string;
|
||||
}
|
||||
export interface TelegramCallbackQuery {
|
||||
id: string;
|
||||
@@ -56,6 +82,10 @@ export interface TelegramClientLike {
|
||||
sendChatAction(chatId: number, action: string): Promise<boolean>;
|
||||
answerCallbackQuery(callbackQueryId: string, text?: string): Promise<boolean>;
|
||||
setMyCommands(commands: TelegramBotCommand[]): Promise<boolean>;
|
||||
/** Resolve a file_id to a file_path (downloadable URL suffix). */
|
||||
getFile(fileId: string): Promise<TelegramFile>;
|
||||
/** Download file bytes from the Bot API file endpoint (GET, proxy-aware). */
|
||||
downloadFile(filePath: string): Promise<Uint8Array>;
|
||||
}
|
||||
export declare class TelegramClient implements TelegramClientLike {
|
||||
private readonly token;
|
||||
@@ -72,4 +102,6 @@ export declare class TelegramClient implements TelegramClientLike {
|
||||
sendChatAction(chatId: number, action: string): Promise<boolean>;
|
||||
answerCallbackQuery(callbackQueryId: string, text?: string): Promise<boolean>;
|
||||
setMyCommands(commands: TelegramBotCommand[]): Promise<boolean>;
|
||||
getFile(fileId: string): Promise<TelegramFile>;
|
||||
downloadFile(filePath: string): Promise<Uint8Array>;
|
||||
}
|
||||
|
||||
@@ -104,4 +104,21 @@ export class TelegramClient {
|
||||
async setMyCommands(commands) {
|
||||
return this.call('setMyCommands', { commands });
|
||||
}
|
||||
async getFile(fileId) {
|
||||
return this.call('getFile', { file_id: fileId });
|
||||
}
|
||||
async downloadFile(filePath) {
|
||||
const url = `${this.baseUrl}/file/bot${this.token}/${filePath}`;
|
||||
try {
|
||||
const response = await this.fetchImpl(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Telegram file download failed: HTTP ${String(response.status)}`);
|
||||
}
|
||||
return new Uint8Array(await response.arrayBuffer());
|
||||
}
|
||||
catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
throw new Error(this.redact(message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+3
@@ -28,6 +28,9 @@ export declare const MSG: {
|
||||
readonly COMPACT_PERSIST_FAILED: "压缩完成但保存失败。请检查本机存储与权限后重试。";
|
||||
readonly COMPACT_FAILED: (detail?: string) => string;
|
||||
readonly GONE: "绑定的会话已不可用。请重新 /sessions。";
|
||||
readonly MEDIA_UNSUPPORTED: "暂不支持该消息类型(支持文本与 png/jpeg/webp/gif 图片;语音/视频/其他文件请在 Web 端处理)。";
|
||||
readonly IMAGE_MODEL_UNSUPPORTED: "当前模型不支持图片输入,请在 Web 端或 /model 切换到多模态模型后重试。";
|
||||
readonly IMAGE_FAILED: (detail?: string) => string;
|
||||
readonly LAST_FAILED: "无法读取上次对话。请确认已绑定,且本机 dsh web / apiProxy 可用。";
|
||||
readonly MODEL_UNAVAILABLE: (detail?: string) => string;
|
||||
readonly MODEL_UNROUTABLE: (current: string) => string;
|
||||
|
||||
@@ -16,6 +16,7 @@ export const MSG = {
|
||||
'/help — 显示帮助',
|
||||
'',
|
||||
'绑定后直接发文字,消息会进入该本机会话;Web 与手机看到同一条轨迹。',
|
||||
'直接发送图片(或带文字说明的图片)也会进入该会话,说明文字会一并作为消息文本。',
|
||||
'仅白名单用户可用。无会话时请先在 dsh web 开对话或保留历史会话。',
|
||||
].join('\n'),
|
||||
NEED_BIND: '尚未绑定本机会话。请先发送 /sessions 选择一个。',
|
||||
@@ -59,6 +60,14 @@ export const MSG = {
|
||||
return `${tip}\n详情:${detail}`;
|
||||
},
|
||||
GONE: '绑定的会话已不可用。请重新 /sessions。',
|
||||
MEDIA_UNSUPPORTED: '暂不支持该消息类型(支持文本与 png/jpeg/webp/gif 图片;语音/视频/其他文件请在 Web 端处理)。',
|
||||
IMAGE_MODEL_UNSUPPORTED: '当前模型不支持图片输入,请在 Web 端或 /model 切换到多模态模型后重试。',
|
||||
IMAGE_FAILED(detail) {
|
||||
const tip = '图片发送失败。';
|
||||
if (!detail)
|
||||
return tip;
|
||||
return `${tip}\n详情:${detail}`;
|
||||
},
|
||||
LAST_FAILED: '无法读取上次对话。请确认已绑定,且本机 dsh web / apiProxy 可用。',
|
||||
MODEL_UNAVAILABLE(detail) {
|
||||
const tip = '无法读取模型列表。请确认已绑定会话,且本机 dsh web 已加载 host-apiproxy。';
|
||||
|
||||
Reference in New Issue
Block a user