Change embedding (#1463)

* rebuild embedding queue

* dataset menu

* feat: rebuild data api

* feat: ui change embedding model

* dataset ui

* feat: rebuild index ui

* rename collection
This commit is contained in:
Archer
2024-05-13 14:51:42 +08:00
committed by GitHub
parent 59fd94384d
commit 80a84a5733
37 changed files with 1260 additions and 419 deletions

View File

@@ -11,6 +11,7 @@ import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/con
import { DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { ClientSession } from '@fastgpt/service/common/mongo';
/* insert data.
* 1. create data id
@@ -26,9 +27,11 @@ export async function insertData2Dataset({
a = '',
chunkIndex = 0,
indexes,
model
model,
session
}: CreateDatasetDataProps & {
model: string;
session?: ClientSession;
}) {
if (!q || !datasetId || !collectionId || !model) {
console.log(q, a, datasetId, collectionId, model);
@@ -70,20 +73,25 @@ export async function insertData2Dataset({
);
// create mongo data
const { _id } = await MongoDatasetData.create({
teamId,
tmbId,
datasetId,
collectionId,
q,
a,
fullTextToken: jiebaSplit({ text: qaStr }),
chunkIndex,
indexes: indexes?.map((item, i) => ({
...item,
dataId: result[i].insertId
}))
});
const [{ _id }] = await MongoDatasetData.create(
[
{
teamId,
tmbId,
datasetId,
collectionId,
q,
a,
fullTextToken: jiebaSplit({ text: qaStr }),
chunkIndex,
indexes: indexes?.map((item, i) => ({
...item,
dataId: result[i].insertId
}))
}
],
{ session }
);
return {
insertId: _id,