4.6.4 (#588)

packages/global/common/system/constants.ts

@@ -0,0 +1,2 @@
+export const HUMAN_ICON = `/icon/human.svg`;
+export const LOGO_ICON = `/icon/logo.svg`;

packages/global/common/system/types/index.d.ts

@@ -9,6 +9,7 @@ export type FeConfigsType = {
   hide_app_flow?: boolean;
   concatMd?: string;
   docUrl?: string;
+  chatbotUrl?: string;
   openAPIDocUrl?: string;
   systemTitle?: string;
   googleClientVerKey?: string;

packages/global/core/chat/api.d.ts

@@ -0,0 +1,8 @@
+export type UpdateChatFeedbackProps = {
+  chatId: string;
+  chatItemId: string;
+  shareId?: string;
+  outLinkUid?: string;
+  userBadFeedback?: string;
+  userGoodFeedback?: string;
+};

packages/global/core/chat/constants.ts

@@ -50,9 +50,6 @@ export enum ChatStatusEnum {
   finish = 'finish'
 }
 
-export const HUMAN_ICON = `/icon/human.svg`;
-export const LOGO_ICON = `/icon/logo.svg`;
-
 export const IMG_BLOCK_KEY = 'img-block';
 export const FILE_BLOCK_KEY = 'file-block';
 

packages/global/core/chat/type.d.ts

@@ -38,7 +38,9 @@ export type ChatItemSchema = {
   time: Date;
   obj: `${ChatRoleEnum}`;
   value: string;
-  userFeedback?: string;
+  userGoodFeedback?: string;
+  userBadFeedback?: string;
+  robotBadFeedback?: string;
   adminFeedback?: AdminFbkType;
   [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };
@@ -56,7 +58,8 @@ export type ChatItemType = {
   dataId?: string;
   obj: ChatItemSchema['obj'];
   value: any;
-  userFeedback?: string;
+  userGoodFeedback?: string;
+  userBadFeedback?: string;
   adminFeedback?: ChatItemSchema['feedback'];
   [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };

packages/global/core/dataset/api.d.ts

@@ -25,6 +25,8 @@ export type CreateDatasetCollectionParams = {
   chunkSize?: number;
   fileId?: string;
   rawLink?: string;
+  qaPrompt?: string;
+  hashRawText?: string;
   metadata?: Record<string, any>;
 };
 
@@ -40,6 +42,7 @@ export type PgSearchRawType = {
 export type PushDatasetDataChunkProps = {
   q: string; // embedding content
   a?: string; // bonus content
+  chunkIndex?: number;
   indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
 };
 

packages/global/core/dataset/type.d.ts

@@ -45,6 +45,8 @@ export type DatasetCollectionSchemaType = {
   chunkSize: number;
   fileId?: string;
   rawLink?: string;
+  qaPrompt?: string;
+  hashRawText?: string;
   metadata?: Record<string, any>;
 };
 
@@ -134,6 +136,7 @@ export type DatasetDataItemType = {
   sourceId?: string;
   q: string;
   a: string;
+  chunkIndex: number;
   indexes: DatasetDataIndexItemType[];
   isOwner: boolean;
   canWrite: boolean;

packages/global/support/user/constant.ts

@@ -1,4 +1,22 @@
+export enum UserStatusEnum {
+  active = 'active',
+  forbidden = 'forbidden'
+}
+export const userStatusMap = {
+  [UserStatusEnum.active]: {
+    label: 'support.user.status.active'
+  },
+  [UserStatusEnum.forbidden]: {
+    label: 'support.user.status.forbidden'
+  }
+};
+
 export enum OAuthEnum {
   github = 'github',
   google = 'google'
 }
+
+export enum UserAuthTypeEnum {
+  register = 'register',
+  findPassword = 'findPassword'
+}

packages/global/support/user/type.d.ts

@@ -1,4 +1,4 @@
-import { InformTypeEnum } from './constant';
+import { InformTypeEnum, UserStatusEnum } from './constant';
 import { TeamItemType } from './team/type';
 
 export type UserModelSchema = {
@@ -12,6 +12,7 @@ export type UserModelSchema = {
   openaiKey: string;
   createTime: number;
   timezone: string;
+  status: `${UserStatusEnum}`;
   openaiAccount?: {
     key: string;
     baseUrl: string;

packages/service/common/api/plusRequest.ts

@@ -70,7 +70,7 @@ instance.interceptors.request.use(requestStart, (err) => Promise.reject(err));
 instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err));
 
 export function request(url: string, data: any, config: ConfigType, method: Method): any {
-  if (!global.systemEnv?.pluginBaseUrl) {
+  if (global.systemEnv && !global.systemEnv?.pluginBaseUrl) {
     return Promise.reject('该功能为商业版特有...');
   }
 

packages/service/common/file/image/schema.ts

@@ -8,6 +8,10 @@ const ImageSchema = new Schema({
     ref: TeamCollectionName,
     required: true
   },
+  createTime: {
+    type: Date,
+    default: () => new Date()
+  },
   binary: {
     type: Buffer
   },
@@ -25,7 +29,10 @@ try {
   console.log(error);
 }
 
-export const MongoImage: Model<{ teamId: string; binary: Buffer; metadata?: Record<string, any> }> =
-  models['image'] || model('image', ImageSchema);
+export const MongoImage: Model<{
+  teamId: string;
+  binary: Buffer;
+  metadata?: { fileId?: string };
+}> = models['image'] || model('image', ImageSchema);
 
 MongoImage.syncIndexes();

packages/service/common/mongo/controller.ts

@@ -1,26 +0,0 @@
-/* add logger */
-export const addLog = {
-  info: (msg: string, obj?: Record<string, any>) => {
-    global.logger?.info(msg, { meta: obj });
-  },
-  error: (msg: string, error?: any) => {
-    global.logger?.error(msg, {
-      meta: {
-        stack: error?.stack,
-        ...(error?.config && {
-          config: {
-            headers: error.config.headers,
-            url: error.config.url,
-            data: error.config.data
-          }
-        }),
-        ...(error?.response && {
-          response: {
-            status: error.response.status,
-            statusText: error.response.statusText
-          }
-        })
-      }
-    });
-  }
-};

packages/service/common/mongo/init.ts

@@ -1,6 +1,4 @@
 import mongoose from './index';
-import { createLogger, format, transports } from 'winston';
-import 'winston-mongodb';
 
 /**
  * connect MongoDB and init data
@@ -36,7 +34,6 @@ export async function connectMongo({
     });
 
     console.log('mongo connected');
-    initLogger();
 
     afterHook && (await afterHook());
   } catch (error) {
@@ -45,35 +42,3 @@ export async function connectMongo({
     global.mongodb = undefined;
   }
 }
-
-function initLogger() {
-  global.logger = createLogger({
-    transports: [
-      new transports.MongoDB({
-        db: process.env.MONGODB_URI as string,
-        collection: 'server_logs',
-        options: {
-          useUnifiedTopology: true
-        },
-        cappedSize: 500000000,
-        tryReconnect: true,
-        metaKey: 'meta',
-        format: format.combine(format.timestamp(), format.json())
-      }),
-      new transports.Console({
-        format: format.combine(
-          format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
-          format.printf((info) => {
-            if (info.level === 'error') {
-              console.log(info.meta);
-              return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`;
-            }
-            return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${
-              info.meta ? `: ${JSON.stringify(info.meta)}` : ''
-            }`;
-          })
-        )
-      })
-    ]
-  });
-}

packages/service/common/mongo/type.d.ts

@@ -3,5 +3,4 @@ import type { Logger } from 'winston';
 
 declare global {
   var mongodb: Mongoose | undefined;
-  var logger: Logger;
 }

packages/service/common/pg/index.ts

@@ -171,9 +171,10 @@ export async function initPg() {
           tmb_id VARCHAR(50) NOT NULL,
           dataset_id VARCHAR(50) NOT NULL,
           collection_id VARCHAR(50) NOT NULL,
-          data_id VARCHAR(50) NOT NULL
+          data_id VARCHAR(50) NOT NULL,
+          createTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP
       );
-      CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 24, ef_construction = 64);
+      CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);
     `);
 
     console.log('init pg successful');

packages/service/common/response/index.ts

@@ -1,7 +1,7 @@
 import type { NextApiResponse } from 'next';
 import { sseResponseEventEnum } from './constant';
 import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
-import { addLog } from '../mongo/controller';
+import { addLog } from '../system/log';
 import { clearCookie } from '../../support/permission/controller';
 
 export interface ResponseType<T = any> {

packages/service/common/system/config/controller.ts

@@ -0,0 +1,15 @@
+import { SystemConfigsTypeEnum } from '@fastgpt/global/common/system/config/constants';
+import { MongoSystemConfigs } from './schema';
+import { FeConfigsType } from '@fastgpt/global/common/system/types';
+
+export const getFastGPTFeConfig = async () => {
+  const res = await MongoSystemConfigs.findOne({
+    type: SystemConfigsTypeEnum.fastgpt
+  }).sort({
+    createTime: -1
+  });
+
+  const config: FeConfigsType = res?.value?.FeConfig || {};
+
+  return config;
+};

packages/service/common/system/config/schema.ts

@@ -19,14 +19,15 @@ const systemConfigSchema = new Schema({
     type: Date,
     default: () => new Date()
   }
-})
+});
 
 try {
   systemConfigSchema.index({ createTime: -1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 });
+  systemConfigSchema.index({ type: 1 });
 } catch (error) {
   console.log(error);
 }
 
-export const MongoSystemConfigs: Model<SystemConfigsType>=
+export const MongoSystemConfigs: Model<SystemConfigsType> =
   models[collectionName] || model(collectionName, systemConfigSchema);
-  MongoSystemConfigs.syncIndexes();
+MongoSystemConfigs.syncIndexes();

packages/service/common/system/log.ts

@@ -0,0 +1,66 @@
+import dayjs from 'dayjs';
+
+/* add logger */
+export const addLog = {
+  log(level: 'info' | 'warn' | 'error', msg: string, obj: Record<string, any> = {}) {
+    console.log(
+      `[${level.toLocaleUpperCase()}] ${dayjs().format(
+        'YYYY-MM-DD HH:mm:ss'
+      )} ${msg}: ${JSON.stringify(obj)}`
+    );
+
+    const lokiUrl = process.env.LOKI_LOG_URL as string;
+    if (!lokiUrl) return;
+
+    try {
+      fetch(lokiUrl, {
+        method: 'POST',
+        headers: {
+          'Content-type': 'application/json'
+        },
+        body: JSON.stringify({
+          streams: [
+            {
+              stream: {
+                level
+              },
+              values: [
+                [
+                  `${Date.now() * 1000000}`,
+                  JSON.stringify({
+                    message: msg,
+                    ...obj
+                  })
+                ]
+              ]
+            }
+          ]
+        })
+      });
+    } catch (error) {}
+  },
+  info(msg: string, obj?: Record<string, any>) {
+    this.log('info', msg, obj);
+  },
+  warn(msg: string, obj?: Record<string, any>) {
+    this.log('warn', msg, obj);
+  },
+  error(msg: string, error?: any) {
+    this.log('error', msg, {
+      stack: error?.stack,
+      ...(error?.config && {
+        config: {
+          headers: error.config.headers,
+          url: error.config.url,
+          data: error.config.data
+        }
+      }),
+      ...(error?.response && {
+        response: {
+          status: error.response.status,
+          statusText: error.response.statusText
+        }
+      })
+    });
+  }
+};

packages/service/core/chat/chatItemSchema.ts

@@ -54,7 +54,14 @@ const ChatItemSchema = new Schema({
     type: String,
     default: ''
   },
-  userFeedback: {
+  userGoodFeedback: {
+    type: String
+  },
+  userFeedback: String,
+  userBadFeedback: {
+    type: String
+  },
+  robotBadFeedback: {
     type: String
   },
   adminFeedback: {
@@ -77,7 +84,10 @@ try {
   ChatItemSchema.index({ userId: 1 });
   ChatItemSchema.index({ appId: 1 });
   ChatItemSchema.index({ chatId: 1 });
-  ChatItemSchema.index({ userFeedback: 1 });
+  ChatItemSchema.index({ userGoodFeedback: 1 });
+  ChatItemSchema.index({ userBadFeedback: 1 });
+  ChatItemSchema.index({ robotBadFeedback: 1 });
+  ChatItemSchema.index({ adminFeedback: 1 });
 } catch (error) {
   console.log(error);
 }

packages/service/core/dataset/collection/controller.ts

@@ -6,6 +6,8 @@ import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset
 import { MongoDatasetCollection } from './schema';
 
 export async function createOneCollection({
+  teamId,
+  tmbId,
   name,
   parentId,
   datasetId,
@@ -14,8 +16,8 @@ export async function createOneCollection({
   chunkSize = 0,
   fileId,
   rawLink,
-  teamId,
-  tmbId,
+  qaPrompt,
+  hashRawText,
   metadata = {}
 }: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
   const { _id } = await MongoDatasetCollection.create({
@@ -29,6 +31,8 @@ export async function createOneCollection({
     chunkSize,
     fileId,
     rawLink,
+    qaPrompt,
+    hashRawText,
     metadata
   });
 
@@ -71,3 +75,19 @@ export function createDefaultCollection({
     updateTime: new Date('2099')
   });
 }
+
+// check same collection
+export const getSameRawTextCollection = async ({
+  datasetId,
+  hashRawText
+}: {
+  datasetId: string;
+  hashRawText?: string;
+}) => {
+  const collection = await MongoDatasetCollection.findOne({
+    datasetId,
+    hashRawText
+  });
+
+  return collection;
+};

packages/service/core/dataset/collection/schema.ts

@@ -72,6 +72,12 @@ const DatasetCollectionSchema = new Schema({
   rawLink: {
     type: String
   },
+  qaPrompt: {
+    type: String
+  },
+  hashRawText: {
+    type: String
+  },
   metadata: {
     type: Object,
     default: {}
@@ -82,6 +88,7 @@ try {
   DatasetCollectionSchema.index({ datasetId: 1 });
   DatasetCollectionSchema.index({ datasetId: 1, parentId: 1 });
   DatasetCollectionSchema.index({ updateTime: -1 });
+  DatasetCollectionSchema.index({ hashRawText: -1 });
 } catch (error) {
   console.log(error);
 }

packages/service/core/dataset/data/controller.ts

@@ -16,22 +16,22 @@ export async function delDatasetRelevantData({ datasetIds }: { datasetIds: strin
     datasetId: { $in: datasetIds }
   });
 
-  // delete related files
-  await Promise.all(
-    datasetIds.map((id) => delFileByMetadata({ bucketName: BucketNameEnum.dataset, datasetId: id }))
-  );
+  await delay(2000);
 
-  await delay(500);
-
-  // delete pg data
-  await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`);
   // delete dataset.datas
   await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } });
+  // delete pg data
+  await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`);
 
   // delete collections
   await MongoDatasetCollection.deleteMany({
     datasetId: { $in: datasetIds }
   });
+
+  // delete related files
+  await Promise.all(
+    datasetIds.map((id) => delFileByMetadata({ bucketName: BucketNameEnum.dataset, datasetId: id }))
+  );
 }
 /**
  * delete all data by collectionIds
@@ -51,6 +51,18 @@ export async function delCollectionRelevantData({
     collectionId: { $in: collectionIds }
   });
 
+  await delay(2000);
+
+  // delete dataset.datas
+  await MongoDatasetData.deleteMany({ collectionId: { $in: collectionIds } });
+  // delete pg data
+  await deletePgDataById(`collection_id IN ('${collectionIds.join("','")}')`);
+
+  // delete collections
+  await MongoDatasetCollection.deleteMany({
+    _id: { $in: collectionIds }
+  });
+
   // delete file and imgs
   await Promise.all([
     delImgByFileIdList(filterFileIds),
@@ -59,13 +71,6 @@ export async function delCollectionRelevantData({
       fileIdList: filterFileIds
     })
   ]);
-
-  await delay(500);
-
-  // delete pg data
-  await deletePgDataById(`collection_id IN ('${collectionIds.join("','")}')`);
-  // delete dataset.datas
-  await MongoDatasetData.deleteMany({ collectionId: { $in: collectionIds } });
 }
 /**
  * delete one data by mongoDataId

packages/service/core/dataset/data/schema.ts

@@ -71,7 +71,6 @@ const DatasetDataSchema = new Schema({
     ],
     default: []
   },
-  // metadata
   updateTime: {
     type: Date,
     default: () => new Date()
@@ -89,6 +88,7 @@ try {
   DatasetDataSchema.index({ teamId: 1 });
   DatasetDataSchema.index({ datasetId: 1 });
   DatasetDataSchema.index({ collectionId: 1 });
+  DatasetDataSchema.index({ updateTime: -1 });
   // full text index
   DatasetDataSchema.index({ datasetId: 1, fullTextToken: 'text' });
   DatasetDataSchema.index({ inited: 1 });

packages/service/package.json

@@ -13,8 +13,7 @@
     "nextjs-cors": "^2.1.2",
     "pg": "^8.10.0",
     "tunnel": "^0.0.6",
-    "winston": "^3.10.0",
-    "winston-mongodb": "^5.1.1"
+    "dayjs": "^1.11.7"
   },
   "devDependencies": {
     "@types/cookie": "^0.5.2",

packages/service/support/user/schema.ts

@@ -3,10 +3,16 @@ const { Schema, model, models } = connectionMongo;
 import { hashStr } from '@fastgpt/global/common/string/tools';
 import { PRICE_SCALE } from '@fastgpt/global/support/wallet/bill/constants';
 import type { UserModelSchema } from '@fastgpt/global/support/user/type';
+import { UserStatusEnum, userStatusMap } from '@fastgpt/global/support/user/constant';
 
 export const userCollectionName = 'users';
 
 const UserSchema = new Schema({
+  status: {
+    type: String,
+    enum: Object.keys(userStatusMap),
+    default: UserStatusEnum.active
+  },
   username: {
     // 可以是手机/邮箱，新的验证都只用手机
     type: String,