v4.6-3 (#471)

packages/service/common/api/plusRequest.ts

@@ -15,9 +15,6 @@ interface ResponseDataType {
  * 请求开始
  */
 function requestStart(config: InternalAxiosRequestConfig): InternalAxiosRequestConfig {
-  if (config.headers) {
-    config.headers.rootkey = process.env.ROOT_KEY;
-  }
   return config;
 }
 
@@ -62,7 +59,8 @@ const instance = axios.create({
   timeout: 60000, // 超时时间
   headers: {
     'content-type': 'application/json',
-    'Cache-Control': 'no-cache'
+    'Cache-Control': 'no-cache',
+    rootkey: process.env.ROOT_KEY
   }
 });
 

packages/service/common/pg/index.ts

@@ -171,8 +171,7 @@ export async function initPg() {
           tmb_id VARCHAR(50) NOT NULL,
           dataset_id VARCHAR(50) NOT NULL,
           collection_id VARCHAR(50) NOT NULL,
-          q TEXT NOT NULL,
-          a TEXT
+          data_id VARCHAR(50) NOT NULL
       );
       CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 24, ef_construction = 64);
     `);

packages/service/core/chat/utils.ts

@@ -0,0 +1,53 @@
+import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
+import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
+import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
+import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
+
+/* slice chat context by tokens */
+export function ChatContextFilter({
+  messages = [],
+  maxTokens
+}: {
+  messages: ChatItemType[];
+  maxTokens: number;
+}) {
+  if (!Array.isArray(messages)) {
+    return [];
+  }
+  const rawTextLen = messages.reduce((sum, item) => sum + item.value.length, 0);
+
+  // If the text length is less than half of the maximum token, no calculation is required
+  if (rawTextLen < maxTokens * 0.5) {
+    return messages;
+  }
+
+  // filter startWith system prompt
+  const chatStartIndex = messages.findIndex((item) => item.obj !== ChatRoleEnum.System);
+  const systemPrompts: ChatItemType[] = messages.slice(0, chatStartIndex);
+  const chatPrompts: ChatItemType[] = messages.slice(chatStartIndex);
+
+  // reduce token of systemPrompt
+  maxTokens -= countMessagesTokens({
+    messages: systemPrompts
+  });
+
+  // 根据 tokens 截断内容
+  const chats: ChatItemType[] = [];
+
+  // 从后往前截取对话内容
+  for (let i = chatPrompts.length - 1; i >= 0; i--) {
+    const item = chatPrompts[i];
+    chats.unshift(item);
+
+    const tokens = countPromptTokens(item.value, adaptRole_Chat2Message(item.obj));
+    maxTokens -= tokens;
+
+    /* 整体 tokens 超出范围, system必须保留 */
+    if (maxTokens <= 0) {
+      chats.shift();
+      break;
+    }
+  }
+
+  return [...systemPrompts, ...chats];
+}

packages/service/core/dataset/collection/schema.ts

@@ -56,8 +56,7 @@ const DatasetCollectionSchema = new Schema({
         ref: 'dataset.files'
       },
       rawLink: {
-        type: String,
-        default: ''
+        type: String
       },
       // 451 初始化
       pgCollectionId: {

packages/service/core/dataset/controller.ts

@@ -1,5 +1,25 @@
 import { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type';
 import { MongoDatasetCollection } from './collection/schema';
+import { MongoDataset } from './schema';
+
+/* ============= dataset ========== */
+/* find all datasetId by top datasetId */
+export async function findDatasetIdTreeByTopDatasetId(
+  id: string,
+  result: string[] = []
+): Promise<string[]> {
+  let allChildrenIds = [...result];
+
+  // find children
+  const children = await MongoDataset.find({ parentId: id });
+
+  for (const child of children) {
+    const grandChildrenIds = await findDatasetIdTreeByTopDatasetId(child._id, result);
+    allChildrenIds = allChildrenIds.concat(grandChildrenIds);
+  }
+
+  return [String(id), ...allChildrenIds];
+}
 
 export async function getCollectionWithDataset(collectionId: string) {
   const data = (

packages/service/core/dataset/data/schema.ts

@@ -0,0 +1,78 @@
+import { connectionMongo, type Model } from '../../../common/mongo';
+const { Schema, model, models } = connectionMongo;
+import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
+import {
+  TeamCollectionName,
+  TeamMemberCollectionName
+} from '@fastgpt/global/support/user/team/constant';
+import { DatasetCollectionName } from '../schema';
+import { DatasetColCollectionName } from '../collection/schema';
+import { DatasetDataIndexTypeMap } from '@fastgpt/global/core/dataset/constant';
+
+export const DatasetDataCollectionName = 'dataset.datas';
+
+const DatasetDataSchema = new Schema({
+  teamId: {
+    type: Schema.Types.ObjectId,
+    ref: TeamCollectionName,
+    required: true
+  },
+  tmbId: {
+    type: Schema.Types.ObjectId,
+    ref: TeamMemberCollectionName,
+    required: true
+  },
+  datasetId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetCollectionName,
+    required: true
+  },
+  collectionId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetColCollectionName,
+    required: true
+  },
+  q: {
+    type: String,
+    required: true
+  },
+  a: {
+    type: String,
+    default: ''
+  },
+  indexes: {
+    type: [
+      {
+        defaultIndex: {
+          type: Boolean,
+          default: false
+        },
+        type: {
+          type: String,
+          enum: Object.keys(DatasetDataIndexTypeMap),
+          required: true
+        },
+        dataId: {
+          type: String,
+          required: true
+        },
+        text: {
+          type: String,
+          required: true
+        }
+      }
+    ],
+    default: []
+  }
+});
+
+try {
+  DatasetDataSchema.index({ userId: 1 });
+  DatasetDataSchema.index({ datasetId: 1 });
+  DatasetDataSchema.index({ collectionId: 1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoDatasetData: Model<DatasetDataSchemaType> =
+  models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);

packages/service/core/dataset/training/schema.ts

@@ -2,7 +2,7 @@
 import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
-import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
+import { DatasetDataIndexTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
 import { DatasetColCollectionName } from '../collection/schema';
 import { DatasetCollectionName } from '../schema';
 import {
@@ -33,12 +33,13 @@ const TrainingDataSchema = new Schema({
     ref: DatasetCollectionName,
     required: true
   },
-  datasetCollectionId: {
+  collectionId: {
     type: Schema.Types.ObjectId,
     ref: DatasetColCollectionName,
     required: true
   },
   billId: {
+    // concat bill
     type: String,
     default: ''
   },
@@ -48,6 +49,7 @@ const TrainingDataSchema = new Schema({
     required: true
   },
   expireAt: {
+    // It will be deleted after 7 days
     type: Date,
     default: () => new Date()
   },
@@ -56,6 +58,7 @@ const TrainingDataSchema = new Schema({
     default: () => new Date('2000/1/1')
   },
   model: {
+    // ai model
     type: String,
     required: true
   },
@@ -71,13 +74,29 @@ const TrainingDataSchema = new Schema({
   a: {
     type: String,
     default: ''
+  },
+  indexes: {
+    type: [
+      {
+        type: {
+          type: String,
+          enum: Object.keys(DatasetDataIndexTypeMap),
+          required: true
+        },
+        text: {
+          type: String,
+          required: true
+        }
+      }
+    ],
+    default: []
   }
 });
 
 try {
   TrainingDataSchema.index({ lockTime: 1 });
   TrainingDataSchema.index({ userId: 1 });
-  TrainingDataSchema.index({ datasetCollectionId: 1 });
+  TrainingDataSchema.index({ collectionId: 1 });
   TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 });
 } catch (error) {
   console.log(error);

packages/service/support/activity/promotion/schema.ts

@@ -23,6 +23,7 @@ const PromotionRecordSchema = new Schema({
     enum: ['pay', 'register']
   },
   amount: {
+    // 1 * PRICE_SCALE
     type: Number,
     required: true
   }

packages/service/support/outLink/tools.ts

@@ -30,7 +30,7 @@ export const pushResult2Remote = async ({
   shareId?: string;
   responseData?: any[];
 }) => {
-  if (!shareId || !authToken) return;
+  if (!shareId || !authToken || !global.systemEnv.pluginBaseUrl) return;
   try {
     const outLink = await MongoOutLink.findOne({
       shareId

packages/service/support/permission/auth/common.ts

@@ -1,5 +1,7 @@
+import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
 import { parseHeaderCert } from '../controller';
 import { AuthModeType } from '../type';
+import { authOutLinkValid } from './outLink';
 
 export const authCert = async (props: AuthModeType) => {
   const result = await parseHeaderCert(props);
@@ -10,3 +12,22 @@ export const authCert = async (props: AuthModeType) => {
     canWrite: true
   };
 };
+export async function authCertAndShareId({
+  shareId,
+  ...props
+}: AuthModeType & { shareId?: string }) {
+  if (!shareId) {
+    return authCert(props);
+  }
+
+  const { app } = await authOutLinkValid({ shareId });
+
+  return {
+    teamId: String(app.teamId),
+    tmbId: String(app.tmbId),
+    authType: AuthUserTypeEnum.outLink,
+    apikey: '',
+    isOwner: false,
+    canWrite: false
+  };
+}

packages/service/support/permission/auth/dataset.ts

@@ -27,11 +27,11 @@ export async function authDataset({
   }
 > {
   const result = await parseHeaderCert(props);
-  const { userId, teamId, tmbId } = result;
+  const { teamId, tmbId } = result;
   const { role } = await getTeamInfoByTmbId({ tmbId });
 
   const { dataset, isOwner, canWrite } = await (async () => {
-    const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toJSON();
+    const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toObject();
 
     if (!dataset) {
       return Promise.reject(DatasetErrEnum.unAuthDataset);

packages/service/support/permission/auth/openapi.ts

@@ -6,7 +6,6 @@ import { getTeamInfoByTmbId } from '../../user/team/controller';
 import { MongoOpenApi } from '../../openapi/schema';
 import { OpenApiErrEnum } from '@fastgpt/global/common/error/code/openapi';
 import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
-import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
 
 export async function authOpenApiKeyCrud({
   id,

packages/service/support/permission/auth/user.ts

@@ -37,13 +37,11 @@ export async function authUserRole(props: AuthModeType): Promise<
     teamOwner: boolean;
   }
 > {
-  const { userId, teamId, tmbId } = await parseHeaderCert(props);
-  const { role: userRole, canWrite } = await getTeamInfoByTmbId({ tmbId });
+  const result = await parseHeaderCert(props);
+  const { role: userRole, canWrite } = await getTeamInfoByTmbId({ tmbId: result.tmbId });
 
   return {
-    userId,
-    teamId,
-    tmbId,
+    ...result,
     isOwner: true,
     role: userRole,
     teamOwner: userRole === TeamMemberRoleEnum.owner,

packages/service/support/user/team/controller.ts

@@ -4,57 +4,42 @@ import {
   TeamMemberRoleEnum,
   TeamMemberStatusEnum,
   TeamCollectionName,
-  TeamMemberCollectionName
+  TeamMemberCollectionName,
+  leaveStatus
 } from '@fastgpt/global/support/user/team/constant';
 
-export async function getTeamInfoByTmbId({
-  tmbId,
-  userId
-}: {
-  tmbId?: string;
-  userId?: string;
-}): Promise<TeamItemType> {
-  if (!tmbId && !userId) {
-    return Promise.reject('tmbId or userId is required');
-  }
-
+async function getTeam(match: Record<string, any>): Promise<TeamItemType> {
   const db = connectionMongo?.connection?.db;
 
   const TeamMember = db.collection(TeamMemberCollectionName);
 
   const results = await TeamMember.aggregate([
     {
-      $match: tmbId
-        ? {
-            _id: new Types.ObjectId(tmbId)
-          }
-        : {
-            userId: new Types.ObjectId(userId),
-            defaultTeam: true
-          }
+      $match: match
     },
     {
       $lookup: {
-        from: TeamCollectionName, // 关联的集合名
-        localField: 'teamId', // TeamMember 集合中用于关联的字段
-        foreignField: '_id', // Team 集合中用于关联的字段
-        as: 'team' // 查询结果中的字段名，存放关联查询的结果
+        from: TeamCollectionName,
+        localField: 'teamId',
+        foreignField: '_id',
+        as: 'team'
       }
     },
     {
-      $unwind: '$team' // 将查询结果中的 team 字段展开，变成一个对象
+      $unwind: '$team'
     }
   ]).toArray();
   const tmb = results[0];
 
   if (!tmb) {
-    return Promise.reject('team not exist');
+    return Promise.reject('member not exist');
   }
 
   return {
     userId: String(tmb.userId),
     teamId: String(tmb.teamId),
     teamName: tmb.team.name,
+    memberName: tmb.name,
     avatar: tmb.team.avatar,
     balance: tmb.team.balance,
     tmbId: String(tmb._id),
@@ -65,11 +50,31 @@ export async function getTeamInfoByTmbId({
     maxSize: tmb.team.maxSize
   };
 }
+
+export async function getTeamInfoByTmbId({ tmbId }: { tmbId: string }) {
+  if (!tmbId) {
+    return Promise.reject('tmbId or userId is required');
+  }
+  return getTeam({
+    _id: new Types.ObjectId(tmbId),
+    status: leaveStatus
+  });
+}
+
+export async function getUserDefaultTeam({ userId }: { userId: string }) {
+  if (!userId) {
+    return Promise.reject('tmbId or userId is required');
+  }
+  return getTeam({
+    userId: new Types.ObjectId(userId),
+    defaultTeam: true
+  });
+}
 export async function createDefaultTeam({
   userId,
   teamName = 'My Team',
   avatar = '/icon/logo.svg',
-  balance = 0,
+  balance,
   maxSize = 5
 }: {
   userId: string;
@@ -103,6 +108,7 @@ export async function createDefaultTeam({
     await TeamMember.insertOne({
       teamId: insertedId,
       userId,
+      name: 'Owner',
       role: TeamMemberRoleEnum.owner,
       status: TeamMemberStatusEnum.active,
       createTime: new Date(),
@@ -116,7 +122,7 @@ export async function createDefaultTeam({
       },
       {
         $set: {
-          balance,
+          ...(balance !== undefined && { balance }),
           maxSize
         }
       }

packages/service/support/wallet/bill/schema.ts

@@ -36,6 +36,7 @@ const BillSchema = new Schema({
     default: () => new Date()
   },
   total: {
+    // 1 * PRICE_SCALE
     type: Number,
     required: true
   },