File input (#2270)

* doc

* feat: file upload  config

* perf: chat box file params

* feat: markdown show file

* feat: chat file store and clear

* perf: read file contentType

* feat: llm vision config

* feat: file url output

* perf: plugin error text

* perf: image load

* feat: ai chat document

* perf: file block ui

* feat: read file node

* feat: file read response field

* feat: simple mode support read files

* feat: tool call

* feat: read file histories

* perf: select file

* perf: select file config

* i18n

* i18n

* fix: ts; feat: tool response preview result

projects/app/src/service/common/system/cron.ts

@@ -1,7 +1,12 @@
 import { setCron } from '@fastgpt/service/common/system/cron';
 import { startTrainingQueue } from '@/service/core/dataset/training/utils';
 import { clearTmpUploadFiles } from '@fastgpt/service/common/file/utils';
-import { checkInvalidDatasetFiles, checkInvalidDatasetData, checkInvalidVector } from './cronTask';
+import {
+  checkInvalidDatasetFiles,
+  checkInvalidDatasetData,
+  checkInvalidVector,
+  removeExpiredChatFiles
+} from './cronTask';
 import { checkTimerLock } from '@fastgpt/service/common/system/timerLock/utils';
 import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants';
 import { addHours } from 'date-fns';
@@ -28,7 +33,8 @@ const clearInvalidDataCron = () => {
         lockMinuted: 59
       })
     ) {
-      checkInvalidDatasetFiles(addHours(new Date(), -6), addHours(new Date(), -2));
+      await checkInvalidDatasetFiles(addHours(new Date(), -6), addHours(new Date(), -2));
+      removeExpiredChatFiles();
     }
   });
 

projects/app/src/service/common/system/cronTask.ts

@@ -1,3 +1,4 @@
+import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import {
   delFileByFileIdList,
   getGFSCollection
@@ -11,15 +12,16 @@ import {
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { addDays } from 'date-fns';
 
 /* 
   check dataset.files data. If there is no match in dataset.collections, delete it
-  可能异常情况
+  可能异常情况:
   1. 上传了文件，未成功创建集合
 */
 export async function checkInvalidDatasetFiles(start: Date, end: Date) {
   let deleteFileAmount = 0;
-  const collection = getGFSCollection('dataset');
+  const collection = getGFSCollection(BucketNameEnum.dataset);
   const where = {
     uploadDate: { $gte: start, $lte: end }
   };
@@ -46,7 +48,10 @@ export async function checkInvalidDatasetFiles(start: Date, end: Date) {
 
       // 3. if not found, delete file
       if (hasCollection === 0) {
-        await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(file._id)] });
+        await delFileByFileIdList({
+          bucketName: BucketNameEnum.dataset,
+          fileIdList: [String(file._id)]
+        });
         console.log('delete file', file._id);
         deleteFileAmount++;
       }
@@ -59,6 +64,35 @@ export async function checkInvalidDatasetFiles(start: Date, end: Date) {
   addLog.info(`Clear invalid dataset files finish, remove ${deleteFileAmount} files`);
 }
 
+/* 
+  Remove 7 days ago chat files
+*/
+export const removeExpiredChatFiles = async () => {
+  let deleteFileAmount = 0;
+  const collection = getGFSCollection(BucketNameEnum.chat);
+  const where = {
+    uploadDate: { $lte: addDays(new Date(), -7) }
+  };
+
+  // get all file _id
+  const files = await collection.find(where, { projection: { _id: 1 } }).toArray();
+
+  // Delete file one by one
+  for await (const file of files) {
+    try {
+      await delFileByFileIdList({
+        bucketName: BucketNameEnum.chat,
+        fileIdList: [String(file._id)]
+      });
+      deleteFileAmount++;
+    } catch (error) {
+      console.log(error);
+    }
+  }
+
+  addLog.info(`Remove expired chat files finish, remove ${deleteFileAmount} files`);
+};
+
 /* 
   检测无效的 Mongo 数据
   异常情况：

projects/app/src/service/core/app/utils.ts

@@ -1,4 +1,5 @@
 import { getUserChatInfoAndAuthTeamPoints } from '@/service/support/permission/auth/team';
+import { defaultApp } from '@/web/core/app/constants';
 import { getNextTimeByCronStringAndTimezone } from '@fastgpt/global/common/string/time';
 import { getNanoid } from '@fastgpt/global/common/string/tools';
 import { delay } from '@fastgpt/global/common/system/utils';
@@ -46,6 +47,7 @@ export const getScheduleTriggerApp = async () => {
               }
             }
           ],
+          chatConfig: defaultApp.chatConfig,
           histories: [],
           stream: false,
           detail: false,

projects/app/src/service/events/generateQA.ts

@@ -14,6 +14,7 @@ import { checkInvalidChunkAndLock } from '@fastgpt/service/core/dataset/training
 import { addMinutes } from 'date-fns';
 import { countGptMessagesTokens } from '@fastgpt/service/common/string/tiktoken/index';
 import { pushDataListToTrainingQueueByCollectionId } from '@fastgpt/service/core/dataset/training/controller';
+import { loadRequestMessages } from '@fastgpt/service/core/chat/utils';
 
 const reduceQueue = () => {
   global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -113,7 +114,7 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
     const chatResponse = await ai.chat.completions.create({
       model,
       temperature: 0.3,
-      messages,
+      messages: await loadRequestMessages({ messages, useVision: false }),
       stream: false
     });
     const answer = chatResponse.choices?.[0].message?.content || '';