4.6.2-alpha (#517)

packages/global/common/string/tools.ts

@@ -13,7 +13,8 @@ export const hashStr = (str: string) => {
 };
 
 /* simple text, remove chinese space and extra \n */
-export const simpleText = (text: string) => {
+export const simpleText = (text = '') => {
+  text = text.trim();
   text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
   text = text.replace(/\r\n|\r/g, '\n');
   text = text.replace(/\n{3,}/g, '\n\n');

packages/global/core/dataset/type.d.ts

@@ -59,6 +59,7 @@ export type DatasetDataSchemaType = {
   collectionId: string;
   q: string; // large chunks or question
   a: string; // answer or custom content
+  fullTextToken: string;
   indexes: DatasetDataIndexItemType[];
 };
 
@@ -83,6 +84,9 @@ export type DatasetTrainingSchemaType = {
 export type CollectionWithDatasetType = Omit<DatasetCollectionSchemaType, 'datasetId'> & {
   datasetId: DatasetSchemaType;
 };
+export type DatasetDataWithCollectionType = Omit<DatasetDataSchemaType, 'collectionId'> & {
+  collectionId: DatasetCollectionSchemaType;
+};
 
 /* ================= dataset ===================== */
 export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel'> & {
@@ -130,6 +134,6 @@ export type DatasetFileSchema = {
 };
 
 /* ============= search =============== */
-export type SearchDataResponseItemType = DatasetDataItemType & {
+export type SearchDataResponseItemType = Omit<DatasetDataItemType, 'isOwner' | 'canWrite'> & {
   score: number;
 };

packages/global/core/module/node/type.d.ts

@@ -20,7 +20,7 @@ export type FlowNodeChangeProps = {
 };
 
 export type FlowNodeInputItemType = {
-  key: `${ModuleInputKeyEnum}`;
+  key: `${ModuleInputKeyEnum}` | string;
   type: `${FlowNodeInputTypeEnum}`; // Decide on a render style
   value?: any;
   valueType?: `${ModuleDataTypeEnum}`; // data type

packages/global/core/module/type.d.ts

@@ -25,7 +25,7 @@ export type moduleTemplateListType = {
 // store module type
 export type ModuleItemType = {
   name: string;
-  logo?: string;
+  avatar?: string;
   intro?: string;
   moduleId: string;
   position?: {

packages/service/core/dataset/data/schema.ts

@@ -43,13 +43,9 @@ const DatasetDataSchema = new Schema({
     type: String,
     default: ''
   },
-  qToken: {
+  fullTextToken: {
     type: String,
-    default: ''
-  },
-  aToken: {
-    type: String,
-    default: ''
+    required: true
   },
   indexes: {
     type: [
@@ -82,7 +78,7 @@ try {
   DatasetDataSchema.index({ datasetId: 1 });
   DatasetDataSchema.index({ collectionId: 1 });
   // full text index
-  DatasetDataSchema.index({ qToken: 'text', aToken: 'text' });
+  DatasetDataSchema.index({ fullTextToken: 'text' });
 } catch (error) {
   console.log(error);
 }

packages/service/package.json

@@ -4,23 +4,23 @@
   "dependencies": {
     "@fastgpt/global": "workspace:*",
     "axios": "^1.5.1",
-    "nextjs-cors": "^2.1.2",
-    "next": "13.5.2",
     "cookie": "^0.5.0",
+    "encoding": "^0.1.13",
     "jsonwebtoken": "^9.0.2",
     "mongoose": "^7.0.2",
-    "winston": "^3.10.0",
-    "winston-mongodb": "^5.1.1",
-    "tunnel": "^0.0.6",
-    "encoding": "^0.1.13",
+    "nanoid": "^4.0.1",
+    "next": "13.5.2",
+    "nextjs-cors": "^2.1.2",
     "pg": "^8.10.0",
-    "nanoid": "^4.0.1"
+    "tunnel": "^0.0.6",
+    "winston": "^3.10.0",
+    "winston-mongodb": "^5.1.1"
   },
   "devDependencies": {
-    "@types/tunnel": "^0.0.4",
-    "@types/pg": "^8.6.6",
-    "@types/node": "^20.8.5",
     "@types/cookie": "^0.5.2",
-    "@types/jsonwebtoken": "^9.0.3"
+    "@types/jsonwebtoken": "^9.0.3",
+    "@types/node": "^20.8.5",
+    "@types/pg": "^8.6.6",
+    "@types/tunnel": "^0.0.4"
   }
 }