支持上传文档或图像并解读

2026-01-13 07:00:39 +08:00 · 2024-04-01 00:48:16 +08:00
parent dc1481f069
commit 19ecdcf7db
4 changed files with 276 additions and 15 deletions
--- a/README.md
+++ b/README.md
@@ -239,11 +239,129 @@ Authorization: Bearer [refresh_token]

 ### 文档解读

-接口开发中...
+提供一个可访问的文件URL或者BASE64_URL进行解析。
+
+**POST /v1/chat/completions**
+
+header 需要设置 Authorization 头部：
+
+```
+Authorization: Bearer [refresh_token]
+```
+
+请求数据：
+```json
+{
+    // 模型名称随意填写，如果不希望输出检索过程模型名称请包含silent_search
+    "model": "kimi",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "file",
+                    "file_url": {
+                        "url": "https://mj101-1317487292.cos.ap-shanghai.myqcloud.com/ai/test.pdf"
+                    }
+                },
+                {
+                    "type": "text",
+                    "text": "文档里说了什么？"
+                }
+            ]
+        }
+    ]
+}
+```
+
+响应数据：
+```json
+{
+    "id": "85774360661086208",
+    "model": "step",
+    "object": "chat.completion",
+    "choices": [
+        {
+            "index": 0,
+            "message": {
+                "role": "assistant",
+                "content": "这是一个关于爱情魔法的文档。它包含了四个部分：\n\n1. **PMG 4.1390 – 1495**：这是一个使用面包和咒语来吸引心仪女性的仪式。仪式中需要将面包分成七个小块，并在特定地点进行咒语的念诵和投掷。\n2. **PMG 4.1342 – 57**：这是一个召唤恶魔来使一个名叫Tereous的女性受到折磨，直到她与一个名叫Didymos的人相爱并结合的咒语。\n3. **PGM 4.1265 – 74**：这是关于如何赢得一个美丽的女人的咒语。它涉及到连续三天保持纯洁，向女神阿佛洛狄特（Aphrodite）供奉乳香，并在心中默念她的神秘名字。\n4. **PGM 4.1496 – 1**：这是一个使用没药来吸引一个特定女性的咒语。这个咒语需要在煤上焚烧没药的同时念诵，目的是让这个女性心中只想着施咒者，并最终与施咒者相爱。"
+            },
+            "finish_reason": "stop"
+        }
+    ],
+    "usage": {
+        "prompt_tokens": 1,
+        "completion_tokens": 1,
+        "total_tokens": 2
+    },
+    "created": 1711903489
+}
+```

 ### 图像解析

-接口开发中...
+提供一个可访问的图像URL或者BASE64_URL进行解析。
+
+此格式兼容 [gpt-4-vision-preview](https://platform.openai.com/docs/guides/vision) API格式，您也可以用这个格式传送文档进行解析。
+
+**POST /v1/chat/completions**
+
+header 需要设置 Authorization 头部：
+
+```
+Authorization: Bearer [refresh_token]
+```
+
+请求数据：
+```json
+{
+    // 模型名称随意填写
+    "model": "step",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "https://k.sinaimg.cn/n/sinakd20111/106/w1024h682/20240327/babd-2ce15fdcfbd6ddbdc5ab588c29b3d3d9.jpg/w700d1q75cms.jpg"
+                    }
+                },
+                {
+                    "type": "text",
+                    "text": "图像描述了什么？"
+                }
+            ]
+        }
+    ]
+}
+```
+
+响应数据：
+```json
+{
+    "id": "85773574417829888",
+    "model": "step",
+    "object": "chat.completion",
+    "choices": [
+        {
+        "index": 0,
+        "message": {
+            "role": "assistant",
+            "content": "这张图片展示了一个活动现场，似乎是某种新产品或技术的发布会。图片中央有一个大屏幕，上面写着“创新技术及产品首发”，屏幕上还展示了一些公司的标志或名称，如“RWKV”、“财跃星辰”、“阶跃星辰”、“商汤”和“零方科技”。在屏幕下方的舞台上，有几位穿着正装的人士正在进行互动，可能是在进行产品发布或演示。整个场景给人一种正式且科技感十足的印象。"
+        },
+        "finish_reason": "stop"
+        }
+    ],
+    "usage": {
+        "prompt_tokens": 1,
+        "completion_tokens": 1,
+        "total_tokens": 2
+    },
+    "created": 1711903302
+}
+```

 ## 注意事项

--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "step-free-api",
-  "version": "0.0.4",
+  "version": "0.0.5",
  "description": "Stepchat Free API Server",
  "type": "module",
  "main": "dist/index.js",
--- a/src/api/consts/exceptions.ts
+++ b/src/api/consts/exceptions.ts
@@ -5,5 +5,7 @@ export default {
    API_TOKEN_EXPIRES: [-2002, 'Token已失效'],
    API_FILE_URL_INVALID: [-2003, '远程文件URL非法'],
    API_FILE_EXECEEDS_SIZE: [-2004, '远程文件超出大小'],
-    API_CHAT_STREAM_PUSHING: [-2005, '已有对话流正在输出']
+    API_CHAT_STREAM_PUSHING: [-2005, '已有对话流正在输出'],
+    API_FILE_UPLOAD_FAILED: [-2006, '文件上传失败'],
+    API_FILE_UPLOAD_TIMEOUT: [-2007, '文件上传超时']
 }
--- a/src/api/controllers/chat.ts
+++ b/src/api/controllers/chat.ts
@@ -207,8 +207,12 @@ async function createCompletion(
    logger.info(messages);

    // 提取引用文件URL并上传step获得引用的文件ID列表
-    // const refFileUrls = extractRefFileUrls(messages);
-    // const refs = refFileUrls.length ? await Promise.all(refFileUrls.map(fileUrl => uploadFile(fileUrl, refreshToken))) : [];
+    const refFileUrls = extractRefFileUrls(messages);
+    const refs = refFileUrls.length
+      ? await Promise.all(
+          refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
+        )
+      : [];

    // 创建会话
    const convId = await createConversation("新会话", refreshToken);
@@ -217,7 +221,7 @@ async function createCompletion(
    const { deviceId, token } = await acquireToken(refreshToken);
    const result = await axios.post(
      `https://stepchat.cn/api/proto.chat.v1.ChatMessageService/SendMessageStream`,
-      messagesPrepare(convId, messages),
+      messagesPrepare(convId, messages, refs),
      {
        headers: {
          "Content-Type": "application/connect+json",
@@ -283,12 +287,12 @@ async function createCompletionStream(
    logger.info(messages);

    // 提取引用文件URL并上传step获得引用的文件ID列表
-    // const refFileUrls = extractRefFileUrls(messages);
-    // const refs = refFileUrls.length
-    //   ? await Promise.all(
-    //       refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
-    //     )
-    //   : [];
+    const refFileUrls = extractRefFileUrls(messages);
+    const refs = refFileUrls.length
+      ? await Promise.all(
+          refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
+        )
+      : [];

    // 创建会话
    const convId = await createConversation("新会话", refreshToken);
@@ -297,7 +301,7 @@ async function createCompletionStream(
    const { deviceId, token } = await acquireToken(refreshToken);
    const result = await axios.post(
      `https://stepchat.cn/api/proto.chat.v1.ChatMessageService/SendMessageStream`,
-      messagesPrepare(convId, messages),
+      messagesPrepare(convId, messages, refs),
      {
        headers: {
          "Content-Type": "application/connect+json",
@@ -384,7 +388,7 @@ function extractRefFileUrls(messages: any[]) {
 *
 * @param messages 参考gpt系列消息格式，多轮对话请完整提供上下文
 */
-function messagesPrepare(convId: string, messages: any[]) {
+function messagesPrepare(convId: string, messages: any[], refs: any[]) {
  const content = messages.reduce((content, message) => {
    if (_.isArray(message.content)) {
      return message.content.reduce((_content, v) => {
@@ -398,6 +402,7 @@ function messagesPrepare(convId: string, messages: any[]) {
    chatId: convId,
    messageInfo: {
      text: content,
+      attachments: refs.length > 0 ? refs : undefined,
    },
  });
  const data = wrapData(json);
@@ -698,6 +703,142 @@ function generateCookie(deviceId: string, accessToken: string) {
  return [`Oasis-Token=${accessToken}`, `Oasis-Webid=${deviceId}`].join("; ");
 }

+/**
+ * 预检查文件URL有效性
+ *
+ * @param fileUrl 文件URL
+ */
+async function checkFileUrl(fileUrl: string) {
+  if (util.isBASE64Data(fileUrl)) return;
+  const result = await axios.head(fileUrl, {
+    timeout: 15000,
+    headers: {
+      Cookie:
+        "INGRESSCOOKIE=1711735363.098.26095.391795|cdfd1cd25bff0dd747986e2907e40a4e; Oasis-Webid=9b4315f03ff3e2abdf7a0b6eb1d41b293ac6fa12; Oasis-Token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY3RpdmF0ZWQiOmZhbHNlLCJhZ2UiOjgsImJhbmVkIjpmYWxzZSwiZXhwIjoxNzExOTAzODkxLCJtb2RlIjoyLCJvYXNpc19pZCI6ODM1NDA2NzE4ODA5NTM4NTYsInZlcnNpb24iOjF9.KrkngKk8drUVfgRBnEE3A07JKjmqgHL3c7J5PlMxKWw...eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhcHBfaWQiOjEwMzAwLCJkZXZpY2VfaWQiOiI5YjQzMTVmMDNmZjNlMmFiZGY3YTBiNmViMWQ0MWIyOTNhYzZmYTEyIiwiZXhwIjoxNzEzMDMxMzkxLCJvYXNpc19pZCI6ODM1NDA2NzE4ODA5NTM4NTYsInZlcnNpb24iOjF9.HRfkpUOFNGO0Jm6wvijGg9PzxD9d9-j4gXh4eqOkAKk",
+      Referer: "https://platform.stepfun.com/",
+      UserAgent:
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+    },
+    validateStatus: () => true,
+  });
+  if (result.status >= 400)
+    throw new APIException(
+      EX.API_FILE_URL_INVALID,
+      `File ${fileUrl} is not valid: [${result.status}] ${result.statusText}`
+    );
+  // 检查文件大小
+  if (result.headers && result.headers["content-length"]) {
+    const fileSize = parseInt(result.headers["content-length"], 10);
+    if (fileSize > FILE_MAX_SIZE)
+      throw new APIException(
+        EX.API_FILE_EXECEEDS_SIZE,
+        `File ${fileUrl} is not valid`
+      );
+  }
+}
+
+/**
+ * 上传文件
+ *
+ * @param fileUrl 文件URL
+ * @param refreshToken 用于刷新access_token的refresh_token
+ */
+async function uploadFile(fileUrl: string, refreshToken: string) {
+  // 预检查远程文件URL可用性
+  await checkFileUrl(fileUrl);
+
+  let filename, fileData: Buffer, mimeType;
+  // 如果是BASE64数据则直接转换为Buffer
+  if (util.isBASE64Data(fileUrl)) {
+    mimeType = util.extractBASE64DataFormat(fileUrl);
+    const ext = mime.getExtension(mimeType);
+    filename = `${util.uuid()}.${ext}`;
+    fileData = Buffer.from(util.removeBASE64DataHeader(fileUrl), "base64");
+  }
+  // 下载文件到内存，如果您的服务器内存很小，建议考虑改造为流直传到下一个接口上，避免停留占用内存
+  else {
+    filename = path.basename(fileUrl);
+    const queryIndex = filename.indexOf("?");
+    if (queryIndex != -1) filename = filename.substring(0, queryIndex);
+    ({ data: fileData } = await axios.get(fileUrl, {
+      responseType: "arraybuffer",
+      // 100M限制
+      maxContentLength: FILE_MAX_SIZE,
+      headers: {
+        Cookie:
+          "INGRESSCOOKIE=1711735363.098.26095.391795|cdfd1cd25bff0dd747986e2907e40a4e; Oasis-Webid=9b4315f03ff3e2abdf7a0b6eb1d41b293ac6fa12; Oasis-Token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY3RpdmF0ZWQiOmZhbHNlLCJhZ2UiOjgsImJhbmVkIjpmYWxzZSwiZXhwIjoxNzExOTAzODkxLCJtb2RlIjoyLCJvYXNpc19pZCI6ODM1NDA2NzE4ODA5NTM4NTYsInZlcnNpb24iOjF9.KrkngKk8drUVfgRBnEE3A07JKjmqgHL3c7J5PlMxKWw...eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhcHBfaWQiOjEwMzAwLCJkZXZpY2VfaWQiOiI5YjQzMTVmMDNmZjNlMmFiZGY3YTBiNmViMWQ0MWIyOTNhYzZmYTEyIiwiZXhwIjoxNzEzMDMxMzkxLCJvYXNpc19pZCI6ODM1NDA2NzE4ODA5NTM4NTYsInZlcnNpb24iOjF9.HRfkpUOFNGO0Jm6wvijGg9PzxD9d9-j4gXh4eqOkAKk",
+        Referer: "https://platform.stepfun.com/",
+        UserAgent:
+          "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+      },
+      // 60秒超时
+      timeout: 60000,
+    }));
+  }
+
+  // 获取文件的MIME类型
+  mimeType = mimeType || mime.getType(filename);
+  // 上传文件到目标OSS
+  const { deviceId, token } = await acquireToken(refreshToken);
+  let result = await axios.request({
+    method: "PUT",
+    url: `https://stepchat.cn/api/storage?file_name=${filename}`,
+    data: fileData,
+    // 100M限制
+    maxBodyLength: FILE_MAX_SIZE,
+    // 60秒超时
+    timeout: 60000,
+    headers: {
+      Cookie: generateCookie(deviceId, token),
+      "Oasis-Webid": deviceId,
+      Referer: "https://stepchat.cn/chats/new",
+      "Stepchat-Meta-Width": "undefined",
+      "Stepchat-Meta-Height": "undefined",
+      "Stepchat-Meta-Size": fileData.byteLength,
+      ...FAKE_HEADERS,
+    },
+    validateStatus: () => true,
+  });
+  const { id: fileId } = checkResult(result, refreshToken);
+
+  let fileStatus;
+  const startTime = util.unixTimestamp();
+  while (fileStatus != 1) {
+    // 获取文件上传结果
+    result = await axios.post(
+      "https://stepchat.cn/api/proto.file.v1.FileService/GetFileStatus",
+      {
+        id: fileId,
+      },
+      {
+        headers: {
+          Cookie: generateCookie(deviceId, token),
+          "Oasis-Webid": deviceId,
+          Referer: "https://stepchat.cn/chats/new",
+          ...FAKE_HEADERS,
+        },
+        timeout: 15000,
+      }
+    );
+    ({ fileStatus } = checkResult(result, refreshToken));
+    // 上传失败处理
+    if ([12, 22, 59, 404].includes(fileStatus))
+      throw new APIException(EX.API_FILE_UPLOAD_FAILED);
+    // 上传超时处理
+    if (util.unixTimestamp() - startTime > 60)
+      throw new APIException(EX.API_FILE_UPLOAD_TIMEOUT);
+  }
+
+  return {
+    attachmentType: mimeType,
+    attachmentId: fileId,
+    name: filename,
+    width: "undefined",
+    height: "undefined",
+    size: `${fileData.byteLength}`,
+  };
+}
+
 /**
 * Token切分
 *