v4.1 (#183)

* chat item table * perf: chat item save * docs * limit * docs * docs * perf: node card * docs * docs
2025-07-30 18:48:55 +00:00 · 2023-08-17 16:57:22 +08:00
parent ce61ac3fac
commit 324e4a0e75
49 changed files with 617 additions and 359 deletions
--- a/client/src/pages/api/admin/initChat.ts
+++ b/client/src/pages/api/admin/initChat.ts
@@ -45,9 +45,7 @@ async function init(limit: number, skip: number) {
      chatId: { $exists: false }
    },
    '_id'
-  )
-    .limit(limit)
-    .skip(skip);
+  ).limit(limit);

  await Promise.all(
    chats.map((chat) =>
--- a/client/src/pages/api/admin/initChatItem.ts
+++ b/client/src/pages/api/admin/initChatItem.ts
@@ -0,0 +1,98 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { authUser } from '@/service/utils/auth';
+import { connectToDatabase, Chat, ChatItem } from '@/service/mongo';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    await authUser({ req, authRoot: true });
+    await connectToDatabase();
+
+    const { limit = 100 } = req.body as { limit: number };
+    let skip = 0;
+
+    const total = await Chat.countDocuments({
+      content: { $exists: true, $not: { $size: 0 } },
+      isInit: { $ne: true }
+    });
+    const totalChat = await Chat.aggregate([
+      {
+        $project: {
+          contentLength: { $size: '$content' }
+        }
+      },
+      {
+        $group: {
+          _id: null,
+          totalLength: { $sum: '$contentLength' }
+        }
+      }
+    ]);
+
+    console.log('chatLen:', total, totalChat);
+
+    let promise = Promise.resolve();
+
+    for (let i = 0; i < total; i += limit) {
+      const skipVal = skip;
+      skip += limit;
+      promise = promise
+        .then(() => init(limit))
+        .then(() => {
+          console.log(skipVal);
+        });
+    }
+
+    await promise;
+
+    jsonRes(res, {});
+  } catch (error) {
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+async function init(limit: number) {
+  // 遍历 app
+  const chats = await Chat.find(
+    {
+      content: { $exists: true, $not: { $size: 0 } },
+      isInit: { $ne: true }
+    },
+    '_id userId appId chatId content'
+  )
+    .sort({ updateTime: -1 })
+    .limit(limit);
+
+  await Promise.all(
+    chats.map(async (chat) => {
+      const inserts = chat.content
+        .map((item) => ({
+          dataId: nanoid(),
+          chatId: chat.chatId,
+          userId: chat.userId,
+          appId: chat.appId,
+          obj: item.obj,
+          value: item.value,
+          responseData: item.responseData
+        }))
+        .filter((item) => item.chatId && item.userId && item.appId && item.obj && item.value);
+
+      try {
+        await Promise.all(inserts.map((item) => ChatItem.create(item)));
+        await Chat.findByIdAndUpdate(chat._id, {
+          isInit: true
+        });
+      } catch (error) {
+        console.log(error);
+
+        await ChatItem.deleteMany({ chatId: chat.chatId });
+      }
+    })
+  );
+}
--- a/client/src/pages/api/admin/initv4.ts
+++ b/client/src/pages/api/admin/initv4.ts
@@ -408,9 +408,7 @@ async function init(limit: number, skip: number) {
      // userId: '63f9a14228d2a688d8dc9e1b'
    },
    '_id chat'
-  )
-    .limit(limit)
-    .skip(skip);
+  ).limit(limit);

  return Promise.all(
    apps.map(async (app) => {
--- a/client/src/pages/api/chat/delChatRecordByContentId.ts
+++ b/client/src/pages/api/chat/delChatRecordByContentId.ts
@@ -1,35 +1,23 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, Chat } from '@/service/mongo';
+import { connectToDatabase, ChatItem } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';

 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
  try {
    const { chatId, contentId } = req.query as { chatId: string; contentId: string };

-    if (!chatId || !contentId) {
-      throw new Error('缺少参数');
-    }
-
    await connectToDatabase();

    // 凭证校验
    const { userId } = await authUser({ req, authToken: true });

-    const chatRecord = await Chat.findOne({ chatId });
-
-    if (!chatRecord) {
-      throw new Error('找不到对话');
-    }
-
    // 删除一条数据库记录
-    await Chat.updateOne(
-      {
-        chatId,
-        userId
-      },
-      { $pull: { content: { _id: contentId } } }
-    );
+    await ChatItem.deleteOne({
+      dataId: contentId,
+      chatId,
+      userId
+    });

    jsonRes(res);
  } catch (err) {
--- a/client/src/pages/api/chat/history/updateHistoryQuote.ts
+++ b/client/src/pages/api/chat/history/updateHistoryQuote.ts
@@ -1,57 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, Chat } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
-import { Types } from 'mongoose';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    let {
-      chatId,
-      contentId,
-      quoteId,
-      sourceText = ''
-    } = req.body as {
-      chatId: string;
-      contentId: string;
-      quoteId: string;
-      sourceText: string;
-    };
-    await connectToDatabase();
-
-    const { userId } = await authUser({ req, authToken: true });
-
-    if (!contentId || !chatId || !quoteId) {
-      throw new Error('params is error');
-    }
-
-    await Chat.updateOne(
-      {
-        chatId,
-        userId: new Types.ObjectId(userId),
-        'content._id': new Types.ObjectId(contentId)
-      },
-      {
-        $set: {
-          'content.$.rawSearch.$[quoteElem].source': sourceText
-        }
-      },
-      {
-        arrayFilters: [
-          {
-            'quoteElem.id': quoteId
-          }
-        ]
-      }
-    );
-
-    jsonRes(res, {
-      data: ''
-    });
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}
--- a/client/src/pages/api/chat/init.ts
+++ b/client/src/pages/api/chat/init.ts
@@ -1,11 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, Chat } from '@/service/mongo';
+import { Chat, ChatItem } from '@/service/mongo';
 import type { InitChatResponse } from '@/api/response/chat';
 import { authUser } from '@/service/utils/auth';
 import { ChatItemType } from '@/types/chat';
 import { authApp } from '@/service/utils/auth';
-import mongoose from 'mongoose';
 import type { ChatSchema } from '@/types/mongoSchema';
 import { getSpecialModule, getChatModelNameList } from '@/components/ChatBox/utils';
 import { TaskResponseKeyEnum } from '@/constants/chat';
@@ -27,8 +26,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
      });
    }

-    await connectToDatabase();
-
    // 校验使用权限
    const app = (
      await authApp({
@@ -39,49 +36,42 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
      })
    ).app;

-    // 历史记录
+    // get app and history
    const { chat, history = [] }: { chat?: ChatSchema; history?: ChatItemType[] } =
      await (async () => {
        if (chatId) {
          // auth chatId
          const [chat, history] = await Promise.all([
-            Chat.findOne({
-              chatId,
-              userId
-            }),
-            Chat.aggregate([
+            Chat.findOne(
              {
-                $match: {
-                  chatId,
-                  userId: new mongoose.Types.ObjectId(userId)
-                }
+                chatId,
+                userId
              },
+              'title variables'
+            ),
+            ChatItem.find(
              {
-                $project: {
-                  content: {
-                    $slice: ['$content', -30] // 返回 content 数组的最后 30 个元素
-                  }
-                }
+                chatId,
+                userId
              },
-              { $unwind: '$content' },
-              {
-                $project: {
-                  _id: '$content._id',
-                  obj: '$content.obj',
-                  value: '$content.value',
-                  [TaskResponseKeyEnum.responseData]: `$content.${TaskResponseKeyEnum.responseData}`
-                }
-              }
-            ])
+              `dataId obj value ${TaskResponseKeyEnum.responseData}`
+            )
+              .sort({ _id: -1 })
+              .limit(30)
          ]);
          if (!chat) {
            throw new Error('聊天框不存在');
          }
-          return { history, chat };
+          history.reverse();
+          return { app, history, chat };
        }
        return {};
      })();

+    if (!app) {
+      throw new Error('Auth App Error');
+    }
+
    const isOwner = String(app.userId) === userId;

    jsonRes<InitChatResponse>(res, {
@@ -108,3 +98,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    });
  }
 }
+
+export const config = {
+  api: {
+    bodyParser: {
+      sizeLimit: '10mb'
+    }
+  }
+};
--- a/client/src/pages/api/chat/removeHistory.ts
+++ b/client/src/pages/api/chat/removeHistory.ts
@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, Chat } from '@/service/mongo';
+import { connectToDatabase, Chat, ChatItem } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';

 type Props = {
@@ -17,16 +17,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    await connectToDatabase();

    if (chatId) {
-      await Chat.findOneAndRemove({
-        chatId,
-        userId
-      });
+      await Promise.all([
+        Chat.findOneAndRemove({
+          chatId,
+          userId
+        }),
+        ChatItem.deleteMany({
+          userId,
+          chatId
+        })
+      ]);
    }
    if (appId) {
-      await Chat.deleteMany({
-        appId,
-        userId
-      });
+      await Promise.all([
+        Chat.deleteMany({
+          appId,
+          userId
+        }),
+        ChatItem.deleteMany({
+          userId,
+          appId
+        })
+      ]);
    }

    jsonRes(res);
--- a/client/src/pages/api/openapi/v1/chat/completions.ts
+++ b/client/src/pages/api/openapi/v1/chat/completions.ts
@@ -28,7 +28,7 @@ import { BillSourceEnum } from '@/constants/user';
 import { ChatHistoryItemResType } from '@/types/chat';
 import { UserModelSchema } from '@/types/mongoSchema';

-export type MessageItemType = ChatCompletionRequestMessage & { _id?: string };
+export type MessageItemType = ChatCompletionRequestMessage & { dataId?: string };
 type FastGptWebChatProps = {
  chatId?: string; // undefined: nonuse history, '': new chat, 'xxxxx': use history
  appId?: string;
@@ -172,7 +172,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
        content: [
          prompt,
          {
-            _id: messages[messages.length - 1]._id,
+            dataId: messages[messages.length - 1].dataId,
            obj: ChatRoleEnum.AI,
            value: answerText,
            responseData
--- a/client/src/pages/api/openapi/v1/chat/getHistory.ts
+++ b/client/src/pages/api/openapi/v1/chat/getHistory.ts
@@ -2,10 +2,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
-import { connectToDatabase, Chat } from '@/service/mongo';
+import { connectToDatabase, ChatItem } from '@/service/mongo';
 import { Types } from 'mongoose';
 import type { ChatItemType } from '@/types/chat';
-import { TaskResponseKeyEnum } from '@/constants/chat';

 export type Props = {
  chatId?: string;
@@ -37,30 +36,37 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 export async function getChatHistory({
  chatId,
  userId,
-  limit = 20
+  limit = 30
 }: Props & { userId: string }): Promise<Response> {
  if (!chatId) {
    return { history: [] };
  }

-  const history = await Chat.aggregate([
-    { $match: { chatId, userId: new Types.ObjectId(userId) } },
+  const history = await ChatItem.aggregate([
    {
-      $project: {
-        content: {
-          $slice: ['$content', -limit] // 返回 content 数组的最后20个元素
-        }
+      $match: {
+        chatId,
+        userId: new Types.ObjectId(userId)
      }
    },
-    { $unwind: '$content' },
+    {
+      $sort: {
+        _id: -1
+      }
+    },
+    {
+      $limit: limit
+    },
    {
      $project: {
-        obj: '$content.obj',
-        value: '$content.value',
-        [TaskResponseKeyEnum.responseData]: `$content.responseData`
+        dataId: 1,
+        obj: 1,
+        value: 1
      }
    }
  ]);

+  history.reverse();
+
  return { history };
 }