Add image index and pdf parse (#3956)

* feat: think tag parse

* feat: parse think tag test

* feat: pdf parse ux

* feat: doc2x parse

* perf: rewrite training mode setting

* feat: image parse queue

* perf: image index

* feat: image parse process

* feat: add init sh

* fix: ts

packages/service/core/dataset/training/schema.ts

@@ -1,14 +1,15 @@
 /* 模型的知识库 */
-import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
-const { Schema, model, models } = connectionMongo;
+import { connectionMongo, getMongoModel } from '../../../common/mongo';
+const { Schema } = connectionMongo;
 import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
-import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { DatasetColCollectionName } from '../collection/schema';
 import { DatasetCollectionName } from '../schema';
 import {
   TeamCollectionName,
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
+import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
 
 export const DatasetTrainingCollectionName = 'dataset_trainings';
 
@@ -25,7 +26,6 @@ const TrainingDataSchema = new Schema({
   },
   datasetId: {
     type: Schema.Types.ObjectId,
-    ref: DatasetCollectionName,
     required: true
   },
   collectionId: {
@@ -33,15 +33,13 @@ const TrainingDataSchema = new Schema({
     ref: DatasetColCollectionName,
     required: true
   },
-  billId: {
-    // concat bill
-    type: String
-  },
+  billId: String,
   mode: {
     type: String,
-    enum: Object.keys(TrainingTypeMap),
+    enum: Object.values(TrainingModeEnum),
     required: true
   },
+
   expireAt: {
     // It will be deleted after 7 days
     type: Date,
@@ -88,6 +86,10 @@ const TrainingDataSchema = new Schema({
   indexes: {
     type: [
       {
+        type: {
+          type: String,
+          enum: Object.values(DatasetDataIndexTypeEnum)
+        },
         text: {
           type: String,
           required: true
@@ -98,6 +100,19 @@ const TrainingDataSchema = new Schema({
   }
 });
 
+TrainingDataSchema.virtual('dataset', {
+  ref: DatasetCollectionName,
+  localField: 'datasetId',
+  foreignField: '_id',
+  justOne: true
+});
+TrainingDataSchema.virtual('collection', {
+  ref: DatasetColCollectionName,
+  localField: 'collectionId',
+  foreignField: '_id',
+  justOne: true
+});
+
 try {
   // lock training data(teamId); delete training data
   TrainingDataSchema.index({ teamId: 1, datasetId: 1 });