new framwork

client/src/pages/api/plugins/kb/create.ts

@@ -0,0 +1,35 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, KB } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    const { name, tags } = req.body as {
+      name: string;
+      tags: string[];
+    };
+
+    if (!name) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    await connectToDatabase();
+
+    const { _id } = await KB.create({
+      name,
+      userId,
+      tags
+    });
+
+    jsonRes(res, { data: _id });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

client/src/pages/api/plugins/kb/data/exportModelData.ts

@@ -0,0 +1,79 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, User } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+import { PgClient } from '@/service/pg';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    let { kbId } = req.query as {
+      kbId: string;
+    };
+
+    if (!kbId) {
+      throw new Error('缺少参数');
+    }
+
+    await connectToDatabase();
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000);
+
+    // auth export times
+    const authTimes = await User.findOne(
+      {
+        _id: userId,
+        $or: [
+          { 'limit.exportKbTime': { $exists: false } },
+          { 'limit.exportKbTime': { $lte: thirtyMinutesAgo } }
+        ]
+      },
+      '_id limit'
+    );
+
+    if (!authTimes) {
+      throw new Error('上次导出未到半小时，每半小时仅可导出一次。');
+    }
+
+    // 统计数据
+    const count = await PgClient.count('modelData', {
+      where: [['kb_id', kbId], 'AND', ['user_id', userId]]
+    });
+    // 从 pg 中获取所有数据
+    const pgData = await PgClient.select<{ q: string; a: string }>('modelData', {
+      where: [['kb_id', kbId], 'AND', ['user_id', userId]],
+      fields: ['q', 'a'],
+      order: [{ field: 'id', mode: 'DESC' }],
+      limit: count
+    });
+
+    const data: [string, string][] = pgData.rows.map((item) => [
+      item.q.replace(/\n/g, '\\n'),
+      item.a.replace(/\n/g, '\\n')
+    ]);
+
+    // update export time
+    await User.findByIdAndUpdate(userId, {
+      'limit.exportKbTime': new Date()
+    });
+
+    jsonRes(res, {
+      data
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}
+
+export const config = {
+  api: {
+    bodyParser: {
+      sizeLimit: '100mb'
+    }
+  }
+};

client/src/pages/api/plugins/kb/data/getDataById.ts

@@ -0,0 +1,39 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+import { PgClient } from '@/service/pg';
+import type { KbDataItemType } from '@/types/plugin';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    let { dataId } = req.query as {
+      dataId: string;
+    };
+    if (!dataId) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    await connectToDatabase();
+
+    const where: any = [['user_id', userId], 'AND', ['id', dataId]];
+
+    const searchRes = await PgClient.select<KbDataItemType>('modelData', {
+      fields: ['id', 'q', 'a', 'source'],
+      where,
+      limit: 1
+    });
+
+    jsonRes(res, {
+      data: searchRes.rows[0]
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

client/src/pages/api/plugins/kb/data/getDataList.ts

@@ -0,0 +1,70 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+import { PgClient } from '@/service/pg';
+import type { KbDataItemType } from '@/types/plugin';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    let {
+      kbId,
+      pageNum = 1,
+      pageSize = 10,
+      searchText = ''
+    } = req.body as {
+      kbId: string;
+      pageNum: number;
+      pageSize: number;
+      searchText: string;
+    };
+    if (!kbId) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    await connectToDatabase();
+
+    const where: any = [
+      ['user_id', userId],
+      'AND',
+      ['kb_id', kbId],
+      ...(searchText
+        ? [
+            'AND',
+            `(q LIKE '%${searchText}%' OR a LIKE '%${searchText}%' OR source LIKE '%${searchText}%')`
+          ]
+        : [])
+    ];
+
+    const [searchRes, total] = await Promise.all([
+      PgClient.select<KbDataItemType>('modelData', {
+        fields: ['id', 'q', 'a', 'source'],
+        where,
+        order: [{ field: 'id', mode: 'DESC' }],
+        limit: pageSize,
+        offset: pageSize * (pageNum - 1)
+      }),
+      PgClient.count('modelData', {
+        fields: ['id'],
+        where
+      })
+    ]);
+
+    jsonRes(res, {
+      data: {
+        pageNum,
+        pageSize,
+        data: searchRes.rows,
+        total
+      }
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

client/src/pages/api/plugins/kb/data/getTrainingData.ts

@@ -0,0 +1,52 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, TrainingData } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { Types } from 'mongoose';
+import { startQueue } from '@/service/utils/tools';
+
+/* 拆分数据成QA */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { kbId, init = false } = req.body as { kbId: string; init: boolean };
+    if (!kbId) {
+      throw new Error('参数错误');
+    }
+    await connectToDatabase();
+
+    const { userId } = await authUser({ req, authToken: true });
+
+    // split queue data
+    const result = await TrainingData.aggregate([
+      {
+        $match: {
+          userId: new Types.ObjectId(userId),
+          kbId: new Types.ObjectId(kbId)
+        }
+      },
+      {
+        $group: {
+          _id: '$mode',
+          count: { $sum: 1 }
+        }
+      }
+    ]);
+
+    jsonRes(res, {
+      data: {
+        qaListLen: result.find((item) => item._id === TrainingModeEnum.qa)?.count || 0,
+        vectorListLen: result.find((item) => item._id === TrainingModeEnum.index)?.count || 0
+      }
+    });
+
+    if (init) {
+      startQueue();
+    }
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

client/src/pages/api/plugins/kb/delete.ts

@@ -0,0 +1,55 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, KB, Model, TrainingData } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+import { PgClient } from '@/service/pg';
+import { Types } from 'mongoose';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    const { id } = req.query as {
+      id: string;
+    };
+
+    if (!id) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    await connectToDatabase();
+
+    // delete all pg data
+    await PgClient.delete('modelData', {
+      where: [['user_id', userId], 'AND', ['kb_id', id]]
+    });
+
+    // delete training data
+    await TrainingData.deleteMany({
+      userId,
+      kbId: id
+    });
+
+    // delete related model
+    await Model.updateMany(
+      {
+        userId
+      },
+      { $pull: { 'chat.relatedKbs': new Types.ObjectId(id) } }
+    );
+
+    // delete kb data
+    await KB.findOneAndDelete({
+      _id: id,
+      userId
+    });
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

client/src/pages/api/plugins/kb/detail.ts

@@ -0,0 +1,46 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, KB } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    const { id } = req.query as {
+      id: string;
+    };
+
+    if (!id) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    await connectToDatabase();
+
+    const data = await KB.findOne({
+      _id: id,
+      userId
+    });
+
+    if (!data) {
+      throw new Error('kb is not exist');
+    }
+
+    jsonRes(res, {
+      data: {
+        _id: data._id,
+        avatar: data.avatar,
+        name: data.name,
+        userId: data.userId,
+        updateTime: data.updateTime,
+        tags: data.tags.join(' ')
+      }
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

client/src/pages/api/plugins/kb/list.ts

@@ -0,0 +1,42 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, KB } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+import { PgClient } from '@/service/pg';
+import { KbItemType } from '@/types/plugin';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    await connectToDatabase();
+
+    const kbList = await KB.find({
+      userId
+    }).sort({ updateTime: -1 });
+
+    const data = await Promise.all(
+      kbList.map(async (item) => ({
+        _id: item._id,
+        avatar: item.avatar,
+        name: item.name,
+        userId: item.userId,
+        updateTime: item.updateTime,
+        tags: item.tags.join(' '),
+        totalData: await PgClient.count('modelData', {
+          where: [['user_id', userId], 'AND', ['kb_id', item._id]]
+        })
+      }))
+    );
+
+    jsonRes<KbItemType[]>(res, {
+      data
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

client/src/pages/api/plugins/kb/update.ts

@@ -0,0 +1,39 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, KB } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+import type { KbUpdateParams } from '@/api/plugins/kb';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    const { id, name, tags, avatar } = req.body as KbUpdateParams;
+
+    if (!id || !name) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    await connectToDatabase();
+
+    await KB.findOneAndUpdate(
+      {
+        _id: id,
+        userId
+      },
+      {
+        avatar,
+        name,
+        tags: tags.split(' ').filter((item) => item)
+      }
+    );
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}