ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-29 09:44:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

4.6.3-website dataset (#532)

Browse Source
This commit is contained in:
Archer
2023-12-03 20:45:57 +08:00
committed by GitHub
parent b916183848
commit a9ae270335
122 changed files with 3793 additions and 1360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
projects/app/src/pages/account/components/BillDetail.tsx
View File

@@ -35,7 +35,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
<ModalBody>
<Flex alignItems={'center'} pb={4}>
<Box flex={'0 0 80px'}>用户:</Box>
<Box>{bill.memberName}</Box>
<Box>{t(bill.memberName)}</Box>
</Flex>
<Flex alignItems={'center'} pb={4}>
<Box flex={'0 0 80px'}>订单号:</Box>

4
projects/app/src/pages/account/components/Info.tsx
View File

@@ -96,8 +96,8 @@ const UserInfo = () => {
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
onclickSave({

2
projects/app/src/pages/api/admin/initv46-2.ts
View File

@@ -1,7 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/pg';
import {
DatasetDataIndexTypeEnum,

7
projects/app/src/pages/api/admin/initv46-fix.ts
View File

@@ -1,12 +1,9 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/pg';
import {
DatasetDataIndexTypeEnum,
PgDatasetTableName
} from '@fastgpt/global/core/dataset/constant';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';

2
projects/app/src/pages/api/admin/initv46.ts
View File

@@ -8,7 +8,7 @@ import {
} from '@fastgpt/service/support/user/team/controller';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { UserModelSchema } from '@fastgpt/global/support/user/type';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';

2
projects/app/src/pages/api/admin/initv462-2.ts
View File

@@ -1,7 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';

20
projects/app/src/pages/api/admin/initv462.ts
View File

@@ -1,7 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { jiebaSplit } from '@/service/core/dataset/utils';
@@ -17,10 +17,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
console.log(
'total',
await MongoDatasetData.countDocuments({ fullTextToken: { $exists: false } })
await MongoDatasetData.countDocuments({
fullTextToken: { $exists: false },
updateTime: { $lt: new Date() }
})
);
await initFullTextToken(limit);
await initFullTextToken(limit, new Date());
jsonRes(res, {
message: 'success'
@@ -34,9 +37,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
}
}
export async function initFullTextToken(limit = 50): Promise<any> {
export async function initFullTextToken(limit = 50, endDate: Date): Promise<any> {
try {
const dataList = await MongoDatasetData.find({ fullTextToken: { $exists: false } }, '_id q a')
const dataList = await MongoDatasetData.find(
{ fullTextToken: { $exists: false }, updateTime: { $lt: endDate } },
'_id q a'
)
.limit(limit)
.lean();
if (dataList.length === 0) return;
@@ -56,9 +62,9 @@ export async function initFullTextToken(limit = 50): Promise<any> {
success += result.filter((item) => item.status === 'fulfilled').length;
console.log(`success: ${success}`);
return initFullTextToken(limit);
return initFullTextToken(limit, endDate);
} catch (error) {
await delay(1000);
return initFullTextToken(limit);
return initFullTextToken(limit, endDate);
}
}

62
projects/app/src/pages/api/admin/initv463-2.ts Normal file
View File

@@ -0,0 +1,62 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { jiebaSplit } from '@/service/core/dataset/utils';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
console.log('total', await MongoDatasetData.countDocuments({ inited: { $exists: false } }));
await initFullTextToken(limit);
jsonRes(res, {
message: 'success'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function initFullTextToken(limit = 50): Promise<any> {
try {
const dataList = await MongoDatasetData.find({ inited: { $exists: false } }, '_id q a')
.limit(limit)
.lean();
if (dataList.length === 0) return;
const result = await Promise.allSettled(
dataList.map((item) => {
const text = item.q + (item.a || '');
const tokens = jiebaSplit({ text });
return MongoDatasetData.findByIdAndUpdate(item._id, {
$set: {
inited: true,
fullTextToken: tokens
}
});
})
);
success += result.filter((item) => item.status === 'fulfilled').length;
console.log(`success: ${success}`);
return initFullTextToken(limit);
} catch (error) {
await delay(1000);
return initFullTextToken(limit);
}
}

70
projects/app/src/pages/api/admin/initv463.ts
View File

@@ -4,7 +4,8 @@ import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
@@ -15,32 +16,85 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await connectToDatabase();
success = 0;
await MongoDatasetCollection.updateMany({}, [
await MongoDatasetCollection.updateMany({ createTime: { $exists: false } }, [
{
$set: {
createTime: '$updateTime'
}
}
]);
await MongoDatasetCollection.updateMany({ trainingType: { $exists: false } }, [
{
$set: {
createTime: '$updateTime',
trainingType: {
$cond: {
if: { $ifNull: ['$a', false] },
then: TrainingModeEnum.qa,
else: TrainingModeEnum.chunk
}
},
chunkSize: 0,
fileId: '$metadata.fileId',
}
}
}
]);
await MongoDatasetCollection.updateMany({ chunkSize: { $exists: false } }, [
{
$set: {
chunkSize: 0
}
}
]);
await MongoDatasetCollection.updateMany({ fileId: { $exists: false } }, [
{
$set: {
fileId: '$metadata.fileId'
}
}
]);
await MongoDatasetCollection.updateMany({ rawLink: { $exists: false } }, [
{
$set: {
rawLink: '$metadata.rawLink'
}
}
]);
await MongoDatasetData.updateMany(
{},
{ chunkIndex: { $exists: false } },
{
chunkIndex: 0
}
);
await MongoDatasetData.updateMany(
{ updateTime: { $exists: false } },
{
chunkIndex: 0,
updateTime: new Date()
}
);
await MongoDataset.updateMany(
{ status: { $exists: false } },
{
$set: {
status: DatasetStatusEnum.active
}
}
);
// dataset tags to intro
await MongoDataset.updateMany({ tags: { $exists: true } }, [
{
$set: {
intro: {
$reduce: {
input: '$tags',
initialValue: '',
in: { $concat: ['$$value', ' ', '$$this'] }
}
}
}
}
]);
jsonRes(res, {
message: 'success'
});

92
projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts Normal file
View File

@@ -0,0 +1,92 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { delFileById, getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
import { addLog } from '@fastgpt/service/common/mongo/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { delay } from '@fastgpt/global/common/system/utils';
/*
check dataset.files data. If there is no match in dataset.collections, delete it
*/
let deleteFileAmount = 0;
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const {
startDay = 10,
endDay = 3,
limit = 30
} = req.body as { startDay?: number; endDay?: number; limit?: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
// start: now - maxDay, end: now - 3 day
const start = new Date(Date.now() - startDay * 24 * 60 * 60 * 1000);
const end = new Date(Date.now() - endDay * 24 * 60 * 60 * 1000);
deleteFileAmount = 0;
checkFiles(start, end, limit);
jsonRes(res, {
message: 'success'
});
} catch (error) {
addLog.error(`check valid dataset files error`, error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function checkFiles(start: Date, end: Date, limit: number) {
const collection = getGFSCollection('dataset');
const where = {
uploadDate: { $gte: start, $lte: end }
};
// 1. get all _id
const ids = await collection
.find(where, {
projection: {
_id: 1
}
})
.toArray();
console.log('total files', ids.length);
for (let i = 0; i < limit; i++) {
check(i);
}
async function check(index: number): Promise<any> {
const id = ids[index];
if (!id) {
console.log(`检测完成，共删除 ${deleteFileAmount} 个无效文件`);
return;
}
try {
const { _id } = id;
// 2. find fileId in dataset.collections
const hasCollection = await MongoDatasetCollection.countDocuments({ fileId: _id });
// 3. if not found, delete file
if (hasCollection === 0) {
await delFileById({ bucketName: 'dataset', fileId: String(_id) });
console.log('delete file', _id);
deleteFileAmount++;
}
index % 100 === 0 && console.log(index);
return check(index + limit);
} catch (error) {
console.log(error);
await delay(2000);
return check(index);
}
}
}

33
projects/app/src/pages/api/common/file/read.ts
View File

@@ -2,8 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authFileToken } from '@fastgpt/service/support/permission/controller';
import jschardet from 'jschardet';
import { getDownloadBuf, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
import { detect } from 'jschardet';
import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -11,24 +11,43 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { token } = req.query as { token: string };
const { fileId, teamId, tmbId, bucketName } = await authFileToken(token);
const { fileId, bucketName } = await authFileToken(token);
if (!fileId) {
throw new Error('fileId is empty');
}
const [file, buffer] = await Promise.all([
const [file, encodeStream] = await Promise.all([
getFileById({ bucketName, fileId }),
getDownloadBuf({ bucketName, fileId })
getDownloadStream({ bucketName, fileId })
]);
const encoding = jschardet.detect(buffer)?.encoding;
// get encoding
let buffers: Buffer = Buffer.from([]);
for await (const chunk of encodeStream) {
buffers = Buffer.concat([buffers, chunk]);
if (buffers.length > 10) {
encodeStream.abort();
break;
}
}
const encoding = detect(buffers)?.encoding || 'utf-8';
res.setHeader('Content-Type', `${file.contentType}; charset=${encoding}`);
res.setHeader('Cache-Control', 'public, max-age=3600');
res.setHeader('Content-Disposition', `inline; filename="${encodeURIComponent(file.filename)}"`);
res.end(buffer);
const fileStream = await getDownloadStream({ bucketName, fileId });
fileStream.pipe(res);
fileStream.on('error', () => {
res.status(500).end();
});
fileStream.on('end', () => {
res.end();
});
} catch (error) {
jsonRes(res, {
code: 500,

6
projects/app/src/pages/api/core/chat/chatTest.ts
View File

@@ -52,12 +52,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
/* start process */
const { responseData } = await dispatchModules({
res,
appId,
modules,
variables,
teamId,
tmbId,
user,
appId,
modules,
variables,
params: {
history,
userChatInput: prompt

18
projects/app/src/pages/api/core/dataset/allDataset.ts
View File

@@ -2,10 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import { getVectorModel } from '@/service/core/ai/model';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
/* get all dataset by teamId or tmbId */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -16,18 +17,23 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const datasets = await MongoDataset.find({
...mongoRPermission({ teamId, tmbId, role }),
type: 'dataset'
type: { $ne: DatasetTypeEnum.folder }
}).lean();
const data = datasets.map((item) => ({
...item,
_id: item._id,
parentId: item.parentId,
avatar: item.avatar,
name: item.name,
intro: item.intro,
type: item.type,
permission: item.permission,
vectorModel: getVectorModel(item.vectorModel),
agentModel: getQAModel(item.agentModel),
canWrite: String(item.tmbId) === tmbId,
isOwner: teamOwner || String(item.tmbId) === tmbId
}));
jsonRes<DatasetItemType[]>(res, {
jsonRes<DatasetListItemType[]>(res, {
data
});
} catch (err) {

74
projects/app/src/pages/api/core/dataset/collection/create.ts
View File

@@ -4,15 +4,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { CreateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum,
DatasetCollectionTrainingModeEnum
} from '@fastgpt/global/core/dataset/constant';
import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -42,68 +37,3 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
}
}
export async function createOneCollection({
name,
parentId,
datasetId,
type,
trainingType = DatasetCollectionTrainingModeEnum.manual,
chunkSize = 0,
fileId,
rawLink,
teamId,
tmbId
}: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
const { _id } = await MongoDatasetCollection.create({
name,
teamId,
tmbId,
datasetId,
parentId: parentId || null,
type,
trainingType,
chunkSize,
fileId,
rawLink
});
// create default collection
if (type === DatasetCollectionTypeEnum.folder) {
await createDefaultCollection({
datasetId,
parentId: _id,
teamId,
tmbId
});
}
return _id;
}
// create default collection
export function createDefaultCollection({
name = '手动录入',
datasetId,
parentId,
teamId,
tmbId
}: {
name?: '手动录入' | '手动标注';
datasetId: string;
parentId?: string;
teamId: string;
tmbId: string;
}) {
return MongoDatasetCollection.create({
name,
teamId,
tmbId,
datasetId,
parentId,
type: DatasetCollectionTypeEnum.virtual,
trainingType: DatasetCollectionTrainingModeEnum.manual,
chunkSize: 0,
updateTime: new Date('2099')
});
}

31
projects/app/src/pages/api/core/dataset/collection/delById.ts → projects/app/src/pages/api/core/dataset/collection/delete.ts
View File

@@ -1,13 +1,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils';
import { delDataByCollectionId } from '@/service/core/dataset/data/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { delFileById } from '@fastgpt/service/common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -19,7 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('CollectionIdId is required');
}
const { teamId } = await authDatasetCollection({
await authDatasetCollection({
req,
authToken: true,
collectionId,
@@ -30,26 +27,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const collections = await findCollectionAndChild(collectionId, '_id metadata');
const delIdList = collections.map((item) => item._id);
// delete training data
await MongoDatasetTraining.deleteMany({
collectionId: { $in: delIdList },
teamId
// delete
await delCollectionRelevantData({
collectionIds: delIdList,
fileIds: collections.map((item) => String(item.metadata?.fileId)).filter(Boolean)
});
// delete pg data
await delDataByCollectionId({ collectionIds: delIdList });
// delete file
await Promise.all(
collections.map((collection) => {
if (!collection?.fileId) return;
return delFileById({
bucketName: BucketNameEnum.dataset,
fileId: collection.fileId
});
})
);
// delete collection
await MongoDatasetCollection.deleteMany({
_id: { $in: delIdList }

11
projects/app/src/pages/api/core/dataset/collection/list.ts
View File

@@ -6,7 +6,10 @@ import { Types } from '@fastgpt/service/common/mongo';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
import { PagingData } from '@/types';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import {
DatasetColCollectionName,
MongoDatasetCollection
} from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { startQueue } from '@/service/utils/tools';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
@@ -45,7 +48,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// not count data amount
if (simple) {
const collections = await MongoDatasetCollection.find(match, '_id name type parentId')
const collections = await MongoDatasetCollection.find(match, '_id parentId type name')
.sort({
updateTime: -1
})
@@ -72,6 +75,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{
$match: match
},
// count training data
{
$lookup: {
from: DatasetTrainingCollectionName,
@@ -89,6 +93,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
as: 'trainings'
}
},
// count collection total data
{
$lookup: {
from: DatasetDataCollectionName,
@@ -106,7 +111,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
as: 'datas'
}
},
// 统计子集合的数量和子训练的数量
{
$project: {
_id: 1,
@@ -114,6 +118,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
tmbId: 1,
name: 1,
type: 1,
status: 1,
updateTime: 1,
dataAmount: { $size: '$datas' },
trainingAmount: { $size: '$trainings' },

23
projects/app/src/pages/api/core/dataset/create.ts
View File

@@ -3,20 +3,20 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import { createDefaultCollection } from './collection/create';
import { createDefaultCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
parentId,
name,
tags,
type,
avatar,
vectorModel = global.vectorModels[0].model,
agentModel,
parentId,
type
agentModel
} = req.body as CreateDatasetParams;
// 凭证校验
@@ -26,7 +26,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
name,
teamId,
tmbId,
tags,
vectorModel,
agentModel,
avatar,
@@ -34,11 +33,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
type
});
await createDefaultCollection({
datasetId: _id,
teamId,
tmbId
});
if (type === DatasetTypeEnum.dataset) {
await createDefaultCollection({
datasetId: _id,
teamId,
tmbId
});
}
jsonRes(res, { data: _id });
} catch (err) {

4
projects/app/src/pages/api/core/dataset/data/delete.ts
View File

@@ -3,7 +3,7 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { deleteDataByDataId } from '@/service/core/dataset/data/controller';
import { delDatasetDataByDataId } from '@fastgpt/service/core/dataset/data/controller';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -19,7 +19,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// 凭证校验
await authDatasetData({ req, authToken: true, dataId, per: 'w' });
await deleteDataByDataId(dataId);
await delDatasetDataByDataId(dataId);
jsonRes(res, {
data: 'success'

22
projects/app/src/pages/api/core/dataset/delete.ts
View File

@@ -1,13 +1,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { delDatasetFiles } from '@fastgpt/service/core/dataset/file/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { delDataByDatasetId } from '@/service/core/dataset/data/controller';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -25,21 +22,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const deletedIds = await findDatasetIdTreeByTopDatasetId(id);
// delete training data(There could be a training mission)
await MongoDatasetTraining.deleteMany({
datasetId: { $in: deletedIds }
});
// delete all dataset.data and pg data
await delDataByDatasetId({ datasetIds: deletedIds });
// delete related files
await delDatasetFiles({ datasetId: id });
// delete collections
await MongoDatasetCollection.deleteMany({
datasetId: { $in: deletedIds }
});
await delDatasetRelevantData({ datasetIds: deletedIds });
// delete dataset data
await MongoDataset.deleteMany({

21
projects/app/src/pages/api/core/dataset/list.ts
View File

@@ -1,12 +1,12 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { getVectorModel } from '@/service/core/ai/model';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -27,16 +27,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
.lean();
const data = await Promise.all(
datasets.map(async (item) => ({
...item,
vectorModel: getVectorModel(item.vectorModel),
agentModel: getQAModel(item.agentModel),
datasets.map<DatasetListItemType>((item) => ({
_id: item._id,
parentId: item.parentId,
avatar: item.avatar,
name: item.name,
intro: item.intro,
type: item.type,
permission: item.permission,
canWrite,
isOwner: teamOwner || String(item.tmbId) === tmbId
isOwner: teamOwner || String(item.tmbId) === tmbId,
vectorModel: getVectorModel(item.vectorModel)
}))
);
jsonRes<DatasetItemType[]>(res, {
jsonRes<DatasetListItemType[]>(res, {
data
});
} catch (err) {

8
projects/app/src/pages/api/core/dataset/searchTest.ts
View File

@@ -44,7 +44,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
// push bill
pushGenerateVectorBill({
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
@@ -54,11 +54,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
if (apikey) {
updateApiKeyUsage({
apikey,
usage: countModelPrice({
model: dataset.vectorModel,
tokens: tokenLen,
type: ModelTypeEnum.vector
})
usage: total
});
}

10
projects/app/src/pages/api/core/dataset/update.ts
View File

@@ -2,14 +2,14 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d';
import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, parentId, name, avatar, tags, permission, agentModel } =
req.body as DatasetUpdateParams;
const { id, parentId, name, avatar, tags, permission, agentModel, websiteConfig, status } =
req.body as DatasetUpdateBody;
if (!id) {
throw new Error('缺少参数');
@@ -28,7 +28,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
...(avatar && { avatar }),
...(tags && { tags }),
...(permission && { permission }),
...(agentModel && { agentModel: agentModel.model })
...(agentModel && { agentModel: agentModel.model }),
...(websiteConfig && { websiteConfig }),
...(status && { status })
}
);

73
projects/app/src/pages/api/plugins/urlFetch.ts
View File

@@ -1,73 +0,0 @@
// pages/api/fetchContent.ts
import { NextApiRequest, NextApiResponse } from 'next';
import axios from 'axios';
import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import { simpleText } from '@fastgpt/global/common/string/tools';
import { connectToDatabase } from '@/service/mongo';
export type UrlFetchResponse = FetchResultItem[];
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
try {
await connectToDatabase();
let { urlList = [] } = req.body as { urlList: string[] };
if (!urlList || urlList.length === 0) {
throw new Error('urlList is empty');
}
await authCert({ req, authToken: true });
urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
const response = (
await Promise.allSettled(
urlList.map(async (url) => {
try {
const fetchRes = await axios.get(url, {
timeout: 30000
});
const dom = new JSDOM(fetchRes.data, {
url,
contentType: 'text/html'
});
const reader = new Readability(dom.window.document);
const article = reader.parse();
const content = article?.textContent || '';
return {
url,
content: simpleText(`${article?.title}\n${content}`)
};
} catch (error) {
return {
url,
content: ''
};
}
})
)
)
.filter((item) => item.status === 'fulfilled')
.map((item: any) => item.value)
.filter((item) => item.content);
jsonRes<UrlFetchResponse>(res, {
data: response
});
} catch (error: any) {
jsonRes(res, {
code: 500,
error: error
});
}
};
export default fetchContent;

4
projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts
View File

@@ -24,13 +24,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
source: BillSourceEnum.training,
list: [
{
moduleName: '索引生成',
moduleName: 'wallet.moduleName.index',
model: vectorModelData.name,
amount: 0,
tokenLen: 0
},
{
moduleName: 'QA 拆分',
moduleName: 'wallet.moduleName.qa',
model: agentModelData.name,
amount: 0,
tokenLen: 0

34
projects/app/src/pages/api/tools/urlFetch.ts Normal file
View File

@@ -0,0 +1,34 @@
// pages/api/fetchContent.ts
import { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { connectToDatabase } from '@/service/mongo';
import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
import { urlsFetch } from '@fastgpt/global/common/file/tools';
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
try {
await connectToDatabase();
let { urlList = [], selector } = req.body as UrlFetchParams;
if (!urlList || urlList.length === 0) {
throw new Error('urlList is empty');
}
await authCert({ req, authToken: true });
jsonRes<UrlFetchResponse>(res, {
data: await urlsFetch({
urlList,
selector
})
});
} catch (error: any) {
jsonRes(res, {
code: 500,
error: error
});
}
};
export default fetchContent;

8
projects/app/src/pages/api/v1/chat/completions.ts
View File

@@ -14,10 +14,10 @@ import { getChatHistory } from './getHistory';
import { saveChat } from '@/service/utils/chat/saveChat';
import { responseWrite } from '@fastgpt/service/common/response';
import { pushChatBill } from '@/service/support/wallet/bill/push';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { authOutLinkChat } from '@/service/support/permission/auth/outLink';
import { pushResult2Remote, updateOutLinkUsage } from '@fastgpt/service/support/outLink/tools';
import requestIp from 'request-ip';
import { getBillSourceByAuthType } from '@fastgpt/global/support/wallet/bill/tools';
import { selectShareResponse } from '@/utils/service/core/chat';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
@@ -276,11 +276,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
appId: app._id,
teamId: user.team.teamId,
tmbId: user.team.tmbId,
source: (() => {
if (authType === 'apikey') return BillSourceEnum.api;
if (shareId) return BillSourceEnum.shareLink;
return BillSourceEnum.fastgpt;
})(),
source: getBillSourceByAuthType({ shareId, authType }),
response: responseData
});

35
projects/app/src/pages/api/v1/embeddings.ts
View File

@@ -6,6 +6,8 @@ import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
import { connectToDatabase } from '@/service/mongo';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { getVectorsByText, GetVectorProps } from '@/service/core/ai/vector';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
import { getBillSourceByAuthType } from '@fastgpt/global/support/wallet/bill/tools';
type Props = GetVectorProps & {
billId?: string;
@@ -15,24 +17,21 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
try {
let { input, model, billId } = req.body as Props;
await connectToDatabase();
const { teamId, tmbId } = await authCert({ req, authToken: true, authApiKey: true });
if (!Array.isArray(input) || typeof input !== 'string') {
if (!Array.isArray(input) && typeof input !== 'string') {
throw new Error('input is nor array or string');
}
const { teamId, tmbId, apikey, authType } = await authCert({
req,
authToken: true,
authApiKey: true
});
await authTeamBalance(teamId);
const { tokenLen, vectors } = await getVectorsByText({ input, model });
pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
model,
billId
});
jsonRes(res, {
data: {
object: 'list',
@@ -48,6 +47,22 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
}
}
});
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokenLen,
model,
billId,
source: getBillSourceByAuthType({ authType })
});
if (apikey) {
updateApiKeyUsage({
apikey,
usage: total
});
}
} catch (err) {
console.log(err);
jsonRes(res, {

12
projects/app/src/pages/api/v1/rerank.ts
View File

@@ -7,12 +7,13 @@ import { connectToDatabase } from '@/service/mongo';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { PostReRankProps, PostReRankResponse } from '@fastgpt/global/core/ai/api';
import { reRankRecall } from '@/service/core/ai/rerank';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let { query, inputs } = req.body as PostReRankProps;
try {
await connectToDatabase();
const { teamId, tmbId } = await authCert({
const { teamId, tmbId, apikey } = await authCert({
req,
authApiKey: true
});
@@ -23,12 +24,19 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
const result = await reRankRecall({ query, inputs });
pushReRankBill({
const { total } = pushReRankBill({
teamId,
tmbId,
source: 'api'
});
if (apikey) {
updateApiKeyUsage({
apikey,
usage: total
});
}
jsonRes<PostReRankResponse>(res, {
data: result
});

4
projects/app/src/pages/app/detail/components/InfoModal.tsx
View File

@@ -103,8 +103,8 @@ const InfoModal = ({
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
setRefresh((state) => !state);

4
projects/app/src/pages/app/list/component/CreateModal.tsx
View File

@@ -60,8 +60,8 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
setRefresh((state) => !state);

366
projects/app/src/pages/dataset/detail/components/CollectionCard.tsx
View File

@@ -11,14 +11,18 @@ import {
Tbody,
Image,
MenuButton,
useDisclosure
useDisclosure,
Button,
Link,
useTheme
} from '@chakra-ui/react';
import {
getDatasetCollections,
delDatasetCollectionById,
putDatasetCollectionById,
postDatasetCollection,
getDatasetCollectionPathById
getDatasetCollectionPathById,
postWebsiteSync
} from '@/web/core/dataset/api';
import { useQuery } from '@tanstack/react-query';
import { debounce } from 'lodash';
@@ -39,7 +43,10 @@ import EmptyTip from '@/components/EmptyTip';
import {
FolderAvatarSrc,
DatasetCollectionTypeEnum,
TrainingModeEnum
DatasetCollectionTrainingModeEnum,
DatasetTypeEnum,
DatasetTypeMap,
DatasetStatusEnum
} from '@fastgpt/global/core/dataset/constant';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
@@ -52,13 +59,18 @@ import { useToast } from '@/web/common/hooks/useToast';
import MyTooltip from '@/components/MyTooltip';
import { useUserStore } from '@/web/support/user/useUserStore';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
const FileImportModal = dynamic(() => import('./Import/ImportModal'), {});
const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {});
const CollectionCard = () => {
const BoxRef = useRef<HTMLDivElement>(null);
const lastSearch = useRef('');
const router = useRouter();
const theme = useTheme();
const { toast } = useToast();
const { parentId = '', datasetId } = router.query as { parentId: string; datasetId: string };
const { t } = useTranslation();
@@ -66,7 +78,7 @@ const CollectionCard = () => {
const { isPc } = useSystemStore();
const { userInfo } = useUserStore();
const [searchText, setSearchText] = useState('');
const { setLoading } = useSystemStore();
const { datasetDetail, updateDataset, loadDatasetDetail } = useDatasetStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: t('dataset.Confirm to delete the file')
@@ -76,11 +88,18 @@ const CollectionCard = () => {
onOpen: onOpenFileImportModal,
onClose: onCloseFileImportModal
} = useDisclosure();
const {
isOpen: isOpenWebsiteModal,
onOpen: onOpenWebsiteModal,
onClose: onCloseWebsiteModal
} = useDisclosure();
const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } =
useEditTitle({
title: t('dataset.Create Virtual File'),
tip: t('dataset.Virtual File Tip')
tip: t('dataset.Virtual File Tip'),
canEmpty: false
});
const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('Rename')
});
@@ -128,48 +147,60 @@ const CollectionCard = () => {
() =>
collections.map((collection) => {
const icon = getCollectionIcon(collection.type, collection.name);
const status = (() => {
if (collection.trainingAmount > 0) {
return {
statusText: t('dataset.collections.Collection Embedding', {
total: collection.trainingAmount
}),
color: 'myGray.500'
};
}
return {
statusText: t('core.dataset.collection.status.active'),
color: 'green.500'
};
})();
return {
...collection,
icon,
...(collection.trainingAmount > 0
? {
statusText: t('dataset.collections.Collection Embedding', {
total: collection.trainingAmount
}),
color: 'myGray.500'
}
: {
statusText: t('dataset.collections.Ready'),
color: 'green.500'
})
...status
};
}),
[collections, t]
);
const hasTrainingData = useMemo(
() => !!formatCollections.find((item) => item.trainingAmount > 0),
[formatCollections]
);
const { mutate: onCreateVirtualFile } = useRequest({
mutationFn: ({ name }: { name: string }) => {
setLoading(true);
return postDatasetCollection({
const { mutate: onCreateCollection, isLoading: isCreating } = useRequest({
mutationFn: async ({
name,
type,
callback,
...props
}: {
name: string;
type: `${DatasetCollectionTypeEnum}`;
callback?: (id: string) => void;
trainingType?: `${DatasetCollectionTrainingModeEnum}`;
rawLink?: string;
chunkSize?: number;
}) => {
const id = await postDatasetCollection({
parentId,
datasetId,
name,
type: DatasetCollectionTypeEnum.virtual
type,
...props
});
callback?.(id);
return id;
},
onSuccess() {
getData(pageNum);
},
onSettled() {
setLoading(false);
},
successToast: t('dataset.collections.Create Virtual File Success'),
errorToast: t('common.Create Virtual File Failed')
successToast: t('common.Create Success'),
errorToast: t('common.Create Failed')
});
const { mutate: onUpdateCollectionName } = useRequest({
mutationFn: ({ collectionId, name }: { collectionId: string; name: string }) => {
@@ -185,9 +216,8 @@ const CollectionCard = () => {
successToast: t('common.Rename Success'),
errorToast: t('common.Rename Failed')
});
const { mutate: onDelCollection } = useRequest({
const { mutate: onDelCollection, isLoading: isDeleting } = useRequest({
mutationFn: (collectionId: string) => {
setLoading(true);
return delDatasetCollectionById({
collectionId
});
@@ -195,26 +225,54 @@ const CollectionCard = () => {
onSuccess() {
getData(pageNum);
},
onSettled() {
setLoading(false);
},
successToast: t('common.Delete Success'),
errorToast: t('common.Delete Failed')
});
const { mutate: onUpdateDatasetWebsiteConfig, isLoading: isUpdating } = useRequest({
mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => {
onCloseWebsiteModal();
const [_, billId] = await Promise.all([
updateDataset({
id: datasetDetail._id,
websiteConfig,
status: DatasetStatusEnum.syncing
}),
postCreateTrainingBill({
name: 'core.dataset.training.Website Sync',
vectorModel: datasetDetail.vectorModel.model,
agentModel: datasetDetail.agentModel.model
})
]);
return billId;
},
onSuccess(billId: string) {
try {
postWebsiteSync({ datasetId: datasetDetail._id, billId });
} catch (error) {}
},
errorToast: t('common.Update Failed')
});
const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
getDatasetCollectionPathById(parentId)
);
const hasTrainingData = useMemo(
() => !!formatCollections.find((item) => item.trainingAmount > 0),
[formatCollections]
);
useQuery(
['refreshCollection'],
() => {
getData(1);
if (datasetDetail.status === DatasetStatusEnum.syncing) {
loadDatasetDetail(datasetId, true);
}
return null;
},
{
refetchInterval: 6000,
enabled: hasTrainingData
enabled: hasTrainingData || datasetDetail.status === DatasetStatusEnum.syncing
}
);
@@ -224,17 +282,33 @@ const CollectionCard = () => {
return (
<Flex flexDirection={'column'} ref={BoxRef} py={[1, 3]} h={'100%'}>
<Flex px={[2, 5]} alignItems={['flex-start', 'center']}>
<Flex px={[2, 5]} alignItems={['flex-start', 'center']} h={'35px'}>
<Box flex={1}>
<ParentPath
paths={paths.map((path, i) => ({
parentId: path.parentId,
parentName: i === paths.length - 1 ? `${path.parentName}(${total})` : path.parentName
parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName
}))}
FirstPathDom={
<Box fontWeight={'bold'} fontSize={['sm', 'lg']}>
{t('common.File')}({total})
</Box>
<>
<Box fontWeight={'bold'} fontSize={['sm', 'lg']}>
{t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total})
</Box>
{datasetDetail?.websiteConfig?.url && (
<Flex fontSize={'sm'}>
{t('core.dataset.website.Base Url')}:
<Link
href={datasetDetail.websiteConfig.url}
target="_blank"
mr={2}
textDecoration={'underline'}
color={'myBlue.700'}
>
{datasetDetail.websiteConfig.url}
</Link>
</Flex>
)}
</>
}
onClick={(e) => {
router.replace({
@@ -279,68 +353,109 @@ const CollectionCard = () => {
/>
</Flex>
)}
{userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
<MyMenu
offset={[-40, 10]}
width={120}
Button={
<MenuButton
_hover={{
color: 'myBlue.600'
}}
fontSize={['sm', 'md']}
>
<Flex
alignItems={'center'}
px={5}
py={2}
borderRadius={'md'}
cursor={'pointer'}
bg={'myBlue.600'}
overflow={'hidden'}
color={'white'}
h={['28px', '35px']}
>
<MyIcon name={'importLight'} mr={2} w={'14px'} />
<Box>{t('dataset.collections.Create And Import')}</Box>
</Flex>
</MenuButton>
}
menuList={[
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
{t('Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
child: (
<Flex>
<Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.Create Virtual File')}
</Flex>
),
onClick: () => {
onOpenCreateVirtualFileModal({
defaultVal: '',
onSuccess: (name) => onCreateVirtualFile({ name })
});
{datasetDetail?.type === DatasetTypeEnum.dataset && (
<>
{userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
<MyMenu
offset={[-40, 10]}
width={120}
Button={
<MenuButton
_hover={{
color: 'myBlue.600'
}}
fontSize={['sm', 'md']}
>
<Flex
alignItems={'center'}
px={5}
py={2}
borderRadius={'md'}
cursor={'pointer'}
bg={'myBlue.600'}
overflow={'hidden'}
color={'white'}
h={['28px', '35px']}
>
<MyIcon name={'importLight'} mr={2} w={'14px'} />
<Box>{t('dataset.collections.Create And Import')}</Box>
</Flex>
</MenuButton>
}
},
{
child: (
<Flex>
<Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.File Input')}
menuList={[
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
{t('Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
child: (
<Flex>
<Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.Create Virtual File')}
</Flex>
),
onClick: () => {
onOpenCreateVirtualFileModal({
defaultVal: '',
onSuccess: (name) => {
onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual });
}
});
}
},
{
child: (
<Flex>
<Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.File Input')}
</Flex>
),
onClick: onOpenFileImportModal
}
]}
/>
)}
</>
)}
{datasetDetail?.type === DatasetTypeEnum.websiteDataset && (
<>
{datasetDetail?.websiteConfig?.url ? (
<Flex alignItems={'center'}>
{datasetDetail.status === DatasetStatusEnum.active && (
<Button onClick={onOpenWebsiteModal}>{t('common.Config')}</Button>
)}
{datasetDetail.status === DatasetStatusEnum.syncing && (
<Flex
ml={3}
alignItems={'center'}
px={3}
py={1}
borderRadius="md"
border={theme.borders.base}
>
<Box
animation={'zoomStopIcon 0.5s infinite alternate'}
bg={'myGray.700'}
w="8px"
h="8px"
borderRadius={'50%'}
mt={'1px'}
></Box>
<Box ml={2} color={'myGray.600'}>
{t('core.dataset.status.syncing')}
</Box>
</Flex>
),
onClick: onOpenFileImportModal
}
]}
/>
)}
</Flex>
) : (
<Button onClick={onOpenWebsiteModal}>{t('core.dataset.Set Website Config')}</Button>
)}
</>
)}
</Flex>
@@ -428,11 +543,7 @@ const CollectionCard = () => {
</MyTooltip>
</Flex>
</Td>
<Td fontSize={'md'}>
{collection.type === DatasetCollectionTypeEnum.folder
? '-'
: collection.dataAmount}
</Td>
<Td fontSize={'md'}>{collection.dataAmount || '-'}</Td>
<Td>{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>
<Flex
@@ -443,10 +554,10 @@ const CollectionCard = () => {
h: '10px',
mr: 2,
borderRadius: 'lg',
bg: collection?.color
bg: collection.color
}}
>
{collection?.statusText}
{t(collection.statusText)}
</Flex>
</Td>
<Td onClick={(e) => e.stopPropagation()}>
@@ -536,14 +647,31 @@ const CollectionCard = () => {
))}
</Tbody>
</Table>
<Loading loading={isLoading && collections.length === 0} fixed={false} />
{total > pageSize && (
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
)}
{total === 0 && <EmptyTip text="数据集空空如也" />}
{total === 0 && (
<EmptyTip
text={
datasetDetail.type === DatasetTypeEnum.dataset ? (
t('core.dataset.collection.Empty Tip')
) : (
<Flex>
{t('core.dataset.collection.Website Empty Tip')}
<Box textDecoration={'underline'} cursor={'pointer'} onClick={onOpenWebsiteModal}>
{t('core.dataset.collection.Click top config website')}
</Box>
</Flex>
)
}
/>
)}
</TableContainer>
<Loading
loading={isCreating || isDeleting || isUpdating || (isLoading && collections.length === 0)}
/>
<ConfirmModal />
<EditTitleModal />
@@ -559,7 +687,6 @@ const CollectionCard = () => {
onClose={onCloseFileImportModal}
/>
)}
{!!editFolderData && (
<EditFolderModal
onClose={() => setEditFolderData(undefined)}
@@ -570,15 +697,13 @@ const CollectionCard = () => {
id: editFolderData.id,
name
});
getData(pageNum);
} else {
await postDatasetCollection({
parentId,
datasetId,
onCreateCollection({
name,
type: DatasetCollectionTypeEnum.folder
});
}
getData(pageNum);
} catch (error) {
return Promise.reject(error);
}
@@ -587,7 +712,6 @@ const CollectionCard = () => {
name={editFolderData.name}
/>
)}
{!!moveCollectionData && (
<SelectCollections
datasetId={datasetId}
@@ -608,6 +732,16 @@ const CollectionCard = () => {
}}
/>
)}
{isOpenWebsiteModal && (
<WebSiteConfigModal
onClose={onCloseWebsiteModal}
onSuccess={onUpdateDatasetWebsiteConfig}
defaultValue={{
url: datasetDetail?.websiteConfig?.url,
selector: datasetDetail?.websiteConfig?.selector
}}
/>
)}
</Flex>
);
};

20
projects/app/src/pages/dataset/detail/components/DataCard.tsx
View File

@@ -55,7 +55,10 @@ const DataCard = () => {
const router = useRouter();
const { userInfo } = useUserStore();
const { isPc } = useSystemStore();
const { collectionId = '' } = router.query as { collectionId: string };
const { collectionId = '', datasetId } = router.query as {
collectionId: string;
datasetId: string;
};
const { Loading, setIsLoading } = useLoading({ defaultLoading: true });
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
@@ -99,8 +102,18 @@ const DataCard = () => {
);
// get file info
const { data: collection } = useQuery(['getDatasetCollectionById', collectionId], () =>
getDatasetCollectionById(collectionId)
const { data: collection } = useQuery(
['getDatasetCollectionById', collectionId],
() => getDatasetCollectionById(collectionId),
{
onError: () => {
router.replace({
query: {
datasetId
}
});
}
}
);
const canWrite = useMemo(
@@ -290,6 +303,7 @@ const DataCard = () => {
</Flex>
<Box
maxH={'135px'}
minH={'90px'}
overflow={'hidden'}
wordBreak={'break-all'}
pt={1}

4
projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
View File

@@ -18,13 +18,13 @@ import { useTranslation } from 'next-i18next';
import { customAlphabet } from 'nanoid';
import dynamic from 'next/dynamic';
import MyTooltip from '@/components/MyTooltip';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
const CreateFileModal = dynamic(() => import('./CreateFileModal'));
@@ -215,7 +215,7 @@ const FileSelect = ({
);
// link fetch
const onUrlFetch = useCallback(
(e: FetchResultItem[]) => {
(e: UrlFetchResponse) => {
const result: FileItemType[] = e.map<FileItemType>(({ url, content }) => {
const splitRes = splitText2Chunks({
text: content,

3
projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
View File

@@ -188,6 +188,8 @@ const Provider = ({
const onReSplitChunks = useCallback(async () => {
try {
setPreviewFile(undefined);
setFiles((state) =>
state.map((file) => {
const splitRes = splitText2Chunks({
@@ -490,6 +492,7 @@ export const SelectorContainer = ({
display={['block', 'none']}
onClick={(e) => {
e.stopPropagation();
setPreviewFile(undefined);
setFiles((state) => state.filter((file) => file.id !== item.id));
}}
/>

65
projects/app/src/pages/dataset/detail/components/Import/UrlFetchModal.tsx
View File

@@ -1,31 +1,39 @@
import React, { useRef } from 'react';
import React from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import { Box, Button, Input, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
import { useRequest } from '@/web/common/hooks/useRequest';
import { postFetchUrls } from '@/web/common/plugin/api';
import { postFetchUrls } from '@/web/common/tools/api';
import { useForm } from 'react-hook-form';
import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
const UrlFetchModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: FetchResultItem[]) => void;
onSuccess: (e: UrlFetchResponse) => void;
}) => {
const { t } = useTranslation();
const Dom = useRef<HTMLTextAreaElement>(null);
const { register, handleSubmit } = useForm({
defaultValues: {
urls: '',
selector: ''
}
});
const { mutate, isLoading } = useRequest({
mutationFn: async () => {
const val = Dom.current?.value || '';
const urls = val.split('\n').filter((e) => e);
const res = await postFetchUrls(urls);
mutationFn: async ({ urls, selector }: { urls: string; selector: string }) => {
const urlList = urls.split('\n').filter((e) => e);
const res = await postFetchUrls({
urlList,
selector
});
onSuccess(res);
onClose();
},
errorToast: '获取链接失败'
errorToast: t('core.dataset.import.Fetch Error')
});
return (
@@ -34,8 +42,8 @@ const UrlFetchModal = ({
title={
<Box>
<Box>{t('file.Fetch Url')}</Box>
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'} mt={1}>
目前仅支持读取静态链接，请注意检查结果
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'}>
{t('core.dataset.import.Fetch url tip')}
</Box>
</Box>
}
@@ -45,20 +53,31 @@ const UrlFetchModal = ({
w={'600px'}
>
<ModalBody>
<Textarea
ref={Dom}
rows={12}
whiteSpace={'nowrap'}
resize={'both'}
placeholder={'最多10个链接，每行一个。'}
/>
<Box>
<Box fontWeight={'bold'}>{t('core.dataset.import.Fetch Url')}</Box>
<Textarea
{...register('urls', {
required: true
})}
rows={11}
whiteSpace={'nowrap'}
resize={'both'}
placeholder={t('core.dataset.import.Fetch url placeholder')}
/>
</Box>
<Box mt={4}>
<Box fontWeight={'bold'}>
{t('core.dataset.website.Selector')}({t('common.choosable')})
</Box>{' '}
<Input {...register('selector')} placeholder="body .content #document" />
</Box>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
取消
{t('common.Close')}
</Button>
<Button isLoading={isLoading} onClick={mutate}>
确认
<Button isLoading={isLoading} onClick={handleSubmit((data) => mutate(data))}>
{t('common.Confirm')}
</Button>
</ModalFooter>
</MyModal>

101
projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx Normal file
View File

@@ -0,0 +1,101 @@
import React from 'react';
import MyModal from '@/components/MyModal';
import { useTranslation } from 'next-i18next';
import { Box, Button, Input, ModalBody, ModalFooter } from '@chakra-ui/react';
import { strIsLink } from '@fastgpt/global/common/string/tools';
import { useToast } from '@/web/common/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useConfirm } from '@/web/common/hooks/useConfirm';
type FormType = {
url?: string | undefined;
selector?: string | undefined;
};
const WebsiteConfigModal = ({
onClose,
onSuccess,
defaultValue = {
url: '',
selector: ''
}
}: {
onClose: () => void;
onSuccess: (data: FormType) => void;
defaultValue?: FormType;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { register, handleSubmit } = useForm({
defaultValues: defaultValue
});
const isEdit = !!defaultValue.url;
const confirmTip = isEdit
? t('core.dataset.website.Confirm Update Tips')
: t('core.dataset.website.Confirm Create Tips');
const { ConfirmModal, openConfirm } = useConfirm({
type: 'common'
});
return (
<MyModal
isOpen
iconSrc="core/dataset/websiteDataset"
title={t('core.dataset.website.Config')}
onClose={onClose}
maxW={'500px'}
>
<ModalBody>
<Box fontSize={'sm'} color={'myGray.600'}>
{t('core.dataset.website.Config Description')}
</Box>
<Box mt={2}>
<Box>{t('core.dataset.website.Base Url')}</Box>
<Input
placeholder={t('core.dataset.collection.Website Link')}
{...register('url', {
required: true
})}
/>
</Box>
<Box mt={3}>
<Box>
{t('core.dataset.website.Selector')}({t('common.choosable')})
</Box>
<Input {...register('selector')} placeholder="body .content #document" />
</Box>
</ModalBody>
<ModalFooter>
<Button variant={'base'} onClick={onClose}>
{t('common.Close')}
</Button>
<Button
ml={2}
onClick={handleSubmit((data) => {
if (!data.url) return;
// check is link
if (!strIsLink(data.url)) {
return toast({
status: 'warning',
title: t('common.link.UnValid')
});
}
openConfirm(
() => {
onSuccess(data);
},
undefined,
confirmTip
)();
})}
>
{t('core.dataset.website.Start Sync')}
</Button>
</ModalFooter>
<ConfirmModal />
</MyModal>
);
};
export default WebsiteConfigModal;

93
projects/app/src/pages/dataset/detail/components/Info.tsx
View File

@@ -7,7 +7,7 @@ import React, {
ForwardedRef
} from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, Button, FormControl, IconButton, Input } from '@chakra-ui/react';
import { Box, Flex, Button, FormControl, IconButton, Input, Textarea } from '@chakra-ui/react';
import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
import { delDatasetById } from '@/web/core/dataset/api';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
@@ -18,21 +18,19 @@ import { UseFormReturn } from 'react-hook-form';
import { compressImgFileAndUpload } from '@/web/common/file/controller';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import Avatar from '@/components/Avatar';
import Tag from '@/components/Tag';
import MyTooltip from '@/components/MyTooltip';
import { useTranslation } from 'next-i18next';
import PermissionRadio from '@/components/support/permission/Radio';
import MySelect from '@/components/Select';
import { qaModelList } from '@/web/common/system/staticData';
export interface ComponentRef {
initInput: (tags: string) => void;
}
const Info = (
{ datasetId, form }: { datasetId: string; form: UseFormReturn<DatasetItemType, any> },
ref: ForwardedRef<ComponentRef>
) => {
const Info = ({
datasetId,
form
}: {
datasetId: string;
form: UseFormReturn<DatasetItemType, any>;
}) => {
const { t } = useTranslation();
const { getValues, formState, setValue, register, handleSubmit } = form;
const InputRef = useRef<HTMLInputElement>(null);
@@ -52,7 +50,7 @@ const Info = (
multiple: false
});
const { datasetDetail, loadDatasetDetail, loadDatasets, updateDataset } = useDatasetStore();
const { datasetDetail, loadDatasets, updateDataset } = useDatasetStore();
/* 点击删除 */
const onclickDelKb = useCallback(async () => {
@@ -121,8 +119,8 @@ const Info = (
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
@@ -138,14 +136,6 @@ const Info = (
[setRefresh, setValue, toast]
);
useImperativeHandle(ref, () => ({
initInput: (tags: string) => {
if (InputRef.current) {
InputRef.current.value = tags;
}
}
}));
return (
<Box py={5} px={[5, 10]}>
<Flex mt={5} w={'100%'} alignItems={'center'}>
@@ -154,18 +144,7 @@ const Info = (
</Box>
<Box flex={1}>{datasetDetail._id}</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
索引模型
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
MaxTokens
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
</Flex>
<Flex mt={5} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
知识库头像
@@ -183,7 +162,7 @@ const Info = (
</MyTooltip>
</Box>
</Flex>
<FormControl mt={8} w={'100%'} display={'flex'} alignItems={'center'}>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
知识库名称
</Box>
@@ -194,7 +173,19 @@ const Info = (
required: '知识库名称不能为空'
})}
/>
</FormControl>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
索引模型
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.Max Token')}
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
</Flex>
<Flex mt={6} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('dataset.Agent Model')}
@@ -216,33 +207,9 @@ const Info = (
/>
</Box>
</Flex>
<Flex mt={8} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
标签
<MyTooltip label={'用空格隔开多个标签，便于搜索'} forceShow>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Input
flex={[1, '0 0 300px']}
ref={InputRef}
defaultValue={getValues('tags')}
placeholder={'标签,使用空格分割。'}
maxLength={30}
onChange={(e) => {
setValue('tags', e.target.value.split(' ').filter(Boolean));
setRefresh(!refresh);
}}
/>
<Flex w={'100%'} pl={['90px', '160px']} mt={2}>
{getValues('tags')
.filter(Boolean)
.map((item, i) => (
<Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
{item}
</Tag>
))}
</Flex>
<Flex mt={8} alignItems={'center'} w={'100%'}>
<Box flex={['0 0 90px', '0 0 160px']}>{t('common.Intro')}</Box>
<Textarea flex={[1, '0 0 300px']} {...register('intro')} placeholder={t('common.Intro')} />
</Flex>
{datasetDetail.isOwner && (
<Flex mt={5} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
@@ -292,4 +259,4 @@ const Info = (
);
};
export default forwardRef(Info);
export default React.memo(Info);

12
projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
View File

@@ -199,12 +199,12 @@ const InputDataModal = ({
<Box p={5} borderRight={theme.borders.base}>
<RawSourceText
w={'200px'}
className=""
className="textEllipsis3"
whiteSpace={'pre-wrap'}
sourceName={collection.sourceName}
sourceId={collection.sourceId}
mb={6}
fontSize={['14px', '16px']}
fontSize={'sm'}
/>
<SideTabs
list={tabList}
@@ -220,14 +220,14 @@ const InputDataModal = ({
}}
/>
</Box>
<Flex flexDirection={'column'} px={5} py={3} flex={1} h={'100%'}>
<Box fontSize={'lg'} fontWeight={'bold'} mb={4}>
<Flex flexDirection={'column'} py={3} flex={1} h={'100%'}>
<Box fontSize={'lg'} px={5} fontWeight={'bold'} mb={4}>
{currentTab === TabEnum.content && (
<>{defaultValue.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')}</>
)}
{currentTab === TabEnum.index && <> {t('dataset.data.Index Edit')}</>}
</Box>
<Box flex={1} overflow={'auto'}>
<Box flex={1} px={5} overflow={'auto'}>
{currentTab === TabEnum.content && (
<>
<Box>
@@ -358,7 +358,7 @@ const InputDataModal = ({
</Grid>
)}
</Box>
<Flex justifyContent={'flex-end'} mt={4}>
<Flex justifyContent={'flex-end'} px={5} mt={4}>
<Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
{t('common.Close')}
</Button>

6
projects/app/src/pages/dataset/detail/components/Test.tsx
View File

@@ -187,7 +187,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
</Box>
</Box>
{/* result show */}
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={1}>
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={'1 0 0'}>
{!datasetTestItem?.results || datasetTestItem.results.length === 0 ? (
<Flex
mt={[10, 0]}
@@ -275,7 +275,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
<MyIcon name={'kbTest'} w={'14px'} />
<Progress
mx={2}
flex={1}
flex={'1 0 0'}
value={item.score * 100}
size="sm"
borderRadius={'20px'}
@@ -283,7 +283,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
/>
<Box>{item.score.toFixed(4)}</Box>
</Flex>
<Box px={2} fontSize={'xs'} color={'myGray.600'}>
<Box px={2} fontSize={'xs'} color={'myGray.600'} wordBreak={'break-word'}>
<Box>{item.q}</Box>
<Box>{item.a}</Box>
</Box>

26
projects/app/src/pages/dataset/detail/index.tsx
View File

@@ -7,7 +7,6 @@ import { useQuery } from '@tanstack/react-query';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { type ComponentRef } from './components/Info';
import Tabs from '@/components/Tabs';
import dynamic from 'next/dynamic';
import MyIcon from '@/components/Icon';
@@ -25,6 +24,7 @@ import Script from 'next/script';
import CollectionCard from './components/CollectionCard';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useUserStore } from '@/web/support/user/useUserStore';
import { DatasetTypeMap } from '../../../../../../packages/global/core/dataset/constant';
const DataCard = dynamic(() => import('./components/DataCard'), {
ssr: false
@@ -41,7 +41,6 @@ export enum TabEnum {
}
const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${TabEnum}` }) => {
const InfoRef = useRef<ComponentRef>(null);
const theme = useTheme();
const { t } = useTranslation();
const { toast } = useToast();
@@ -77,7 +76,6 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
useQuery([datasetId], () => loadDatasetDetail(datasetId), {
onSuccess(res) {
form.reset(res);
InfoRef.current?.initInput(res.tags?.join(' '));
},
onError(err: any) {
router.replace(`/dataset/list`);
@@ -107,14 +105,24 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
>
<Flex mb={4} alignItems={'center'}>
<Avatar src={datasetDetail.avatar} w={'34px'} borderRadius={'lg'} />
<Box ml={2} fontWeight={'bold'}>
{datasetDetail.name}
<Box ml={2}>
<Box fontWeight={'bold'}>{datasetDetail.name}</Box>
</Box>
</Flex>
{DatasetTypeMap[datasetDetail.type] && (
<Flex alignItems={'center'} pl={2}>
<MyIcon
name={DatasetTypeMap[datasetDetail.type]?.icon as any}
mr={1}
w={'16px'}
/>
<Box>{t(DatasetTypeMap[datasetDetail.type]?.label)}</Box>
</Flex>
)}
<SideTabs
flex={1}
mx={'auto'}
mt={2}
mt={3}
w={'100%'}
list={tabList}
activeId={currentTab}
@@ -156,7 +164,7 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
borderRadius={'50%'}
aria-label={''}
/>
全部知识库
{t('core.dataset.All Dataset')}
</Flex>
</Flex>
) : (
@@ -180,9 +188,7 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
{currentTab === TabEnum.collectionCard && <CollectionCard />}
{currentTab === TabEnum.dataCard && <DataCard />}
{currentTab === TabEnum.test && <Test datasetId={datasetId} />}
{currentTab === TabEnum.info && (
<Info ref={InfoRef} datasetId={datasetId} form={form} />
)}
{currentTab === TabEnum.info && <Info datasetId={datasetId} form={form} />}
</Box>
)}
</Flex>

148
projects/app/src/pages/dataset/list/component/CreateModal.tsx
View File

@@ -1,14 +1,5 @@
import React, { useCallback, useState, useRef } from 'react';
import {
Box,
Flex,
Button,
ModalHeader,
ModalFooter,
ModalBody,
Input,
Image
} from '@chakra-ui/react';
import React, { useCallback, useState } from 'react';
import { Box, Flex, Button, ModalFooter, ModalBody, Input } from '@chakra-ui/react';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useForm } from 'react-hook-form';
import { compressImgFileAndUpload } from '@/web/common/file/controller';
@@ -23,10 +14,11 @@ import MyModal from '@/components/MyModal';
import { postCreateDataset } from '@/web/core/dataset/api';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import MySelect from '@/components/Select';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { vectorModelList, qaModelList } from '@/web/common/system/staticData';
import Tag from '@/components/Tag';
import { useTranslation } from 'next-i18next';
import MyRadio from '@/components/common/MyRadio';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { feConfigs } from '@/web/common/system/staticData';
const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: string }) => {
const { t } = useTranslation();
@@ -36,16 +28,15 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
const { isPc } = useSystemStore();
const { register, setValue, getValues, handleSubmit } = useForm<CreateDatasetParams>({
defaultValues: {
parentId,
type: DatasetTypeEnum.dataset,
avatar: '/icon/logo.svg',
name: '',
tags: '',
intro: '',
vectorModel: vectorModelList[0].model,
agentModel: qaModelList[0].model,
type: 'dataset',
parentId
agentModel: qaModelList[0].model
}
});
const InputRef = useRef<HTMLInputElement>(null);
const { File, onOpen: onOpenSelectFile } = useSelectFile({
fileType: '.jpg,.png',
@@ -59,8 +50,8 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
setRefresh((state) => !state);
@@ -97,32 +88,67 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
w={'450px'}
>
<ModalBody>
<Box color={'myGray.800'} fontWeight={'bold'}>
取个名字
</Box>
<Flex mt={3} alignItems={'center'}>
<MyTooltip label={'点击设置头像'}>
<Avatar
flexShrink={0}
src={getValues('avatar')}
w={['28px', '32px']}
h={['28px', '32px']}
cursor={'pointer'}
borderRadius={'md'}
onClick={onOpenSelectFile}
/>
</MyTooltip>
<Input
ml={3}
flex={1}
autoFocus
bg={'myWhite.600'}
maxLength={30}
{...register('name', {
required: '知识库名称不能为空~'
})}
<>
<Box mb={1} color={'myGray.800'} fontWeight={'bold'}>
{t('core.dataset.Dataset Type')}
</Box>
<MyRadio
gridGap={2}
gridTemplateColumns={'repeat(1,1fr)'}
list={[
{
title: t('core.dataset.Common Dataset'),
value: DatasetTypeEnum.dataset,
icon: 'core/dataset/commonDataset',
desc: t('core.dataset.Common Dataset Desc')
},
...(feConfigs.isPlus
? [
{
title: t('core.dataset.Website Dataset'),
value: DatasetTypeEnum.websiteDataset,
icon: 'core/dataset/websiteDataset',
desc: t('core.dataset.Website Dataset Desc')
}
]
: [])
]}
value={getValues('type')}
onChange={(e) => {
setValue('type', e as `${DatasetTypeEnum}`);
setRefresh(!refresh);
}}
/>
</Flex>
</>
<Box mt={5}>
<Box color={'myGray.800'} fontWeight={'bold'}>
取个名字
</Box>
<Flex mt={1} alignItems={'center'}>
<MyTooltip label={'点击设置头像'}>
<Avatar
flexShrink={0}
src={getValues('avatar')}
w={['28px', '32px']}
h={['28px', '32px']}
cursor={'pointer'}
borderRadius={'md'}
onClick={onOpenSelectFile}
/>
</MyTooltip>
<Input
ml={3}
flex={1}
autoFocus
bg={'myWhite.600'}
placeholder={t('common.Name')}
maxLength={30}
{...register('name', {
required: '知识库名称不能为空~'
})}
/>
</Flex>
</Box>
<Flex mt={6} alignItems={'center'}>
<Box flex={'0 0 100px'}>索引模型</Box>
<Box flex={1}>
@@ -157,42 +183,14 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
/>
</Box>
</Flex>
<Flex mt={6} alignItems={'center'} w={'100%'}>
<Box flex={'0 0 100px'}>
标签
<MyTooltip label={'用空格隔开多个标签，便于搜索'} forceShow>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Input
flex={1}
ref={InputRef}
placeholder={'标签,使用空格分割。'}
maxLength={30}
onChange={(e) => {
setValue('tags', e.target.value);
setRefresh(!refresh);
}}
/>
</Flex>
<Flex mt={2} flexWrap={'wrap'}>
{getValues('tags')
.split(' ')
.filter(Boolean)
.map((item, i) => (
<Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
{item}
</Tag>
))}
</Flex>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={3} onClick={onClose}>
取消
{t('common.Close')}
</Button>
<Button isLoading={creating} onClick={handleSubmit((data) => onclickCreate(data))}>
确认创建
{t('common.Confirm Create')}
</Button>
</ModalFooter>

95
projects/app/src/pages/dataset/list/index.tsx
View File

@@ -7,8 +7,7 @@ import {
useDisclosure,
Card,
MenuButton,
Image,
Link
Image
} from '@chakra-ui/react';
import { useRouter } from 'next/router';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
@@ -28,8 +27,11 @@ import Avatar from '@/components/Avatar';
import MyIcon from '@/components/Icon';
import { serviceSideProps } from '@/web/common/utils/i18n';
import dynamic from 'next/dynamic';
import { FolderAvatarSrc, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import Tag from '@/components/Tag';
import {
FolderAvatarSrc,
DatasetTypeEnum,
DatasetTypeMap
} from '@fastgpt/global/core/dataset/constant';
import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/web/common/hooks/useRequest';
import { useSystemStore } from '@/web/common/system/useSystemStore';
@@ -55,12 +57,12 @@ const Kb = () => {
const DeleteTipsMap = useRef({
[DatasetTypeEnum.folder]: t('dataset.deleteFolderTips'),
[DatasetTypeEnum.dataset]: t('dataset.deleteDatasetTips')
[DatasetTypeEnum.dataset]: t('dataset.deleteDatasetTips'),
[DatasetTypeEnum.websiteDataset]: t('core.dataset.Delete Website Tips')
});
const { openConfirm, ConfirmModal } = useConfirm({
title: t('common.Delete Warning'),
content: ''
type: 'delete'
});
const { myDatasets, loadDatasets, setDatasets, updateDataset } = useDatasetStore();
const { onOpenModal: onOpenTitleModal, EditModal: EditTitleModal } = useEditTitle({
@@ -110,14 +112,26 @@ const Kb = () => {
errorToast: t('dataset.Export Dataset Limit Error')
});
const { data, refetch } = useQuery(['loadDataset', parentId], () => {
const { data, refetch, isFetching } = useQuery(['loadDataset', parentId], () => {
return Promise.all([loadDatasets(parentId), getDatasetPaths(parentId)]);
});
const paths = data?.[1] || [];
const formatDatasets = useMemo(
() =>
myDatasets.map((item) => {
return {
...item,
label: DatasetTypeMap[item.type]?.label,
icon: DatasetTypeMap[item.type]?.icon
};
}),
[myDatasets]
);
return (
<PageContainer>
<PageContainer isLoading={isFetching}>
<Flex pt={3} px={5} alignItems={'center'}>
{/* url path */}
<ParentPaths
@@ -179,7 +193,7 @@ const Kb = () => {
child: (
<Flex>
<Image src={'/imgs/module/db.png'} alt={''} w={'20px'} mr={1} />
{t('Dataset')}
{t('core.dataset.Dataset')}
</Flex>
),
onClick: onOpenCreateModal
@@ -194,15 +208,15 @@ const Kb = () => {
gridGap={5}
userSelect={'none'}
>
{myDatasets.map((dataset) => (
{formatDatasets.map((dataset) => (
<Card
display={'flex'}
flexDirection={'column'}
key={dataset._id}
py={4}
py={3}
px={5}
cursor={'pointer'}
h={'130px'}
minH={'130px'}
border={theme.borders.md}
boxShadow={'none'}
position={'relative'}
@@ -250,7 +264,7 @@ const Kb = () => {
parentId: dataset._id
}
});
} else if (dataset.type === DatasetTypeEnum.dataset) {
} else {
router.push({
pathname: '/dataset/detail',
query: {
@@ -388,27 +402,22 @@ const Kb = () => {
{dataset.name}
</Box>
</Flex>
<Box flex={'1 0 0'} overflow={'hidden'} pt={2}>
<Flex>
{dataset.tags.filter(Boolean).map((tag, i) => (
<Tag key={i} mr={2} mb={2}>
{tag}
</Tag>
))}
</Flex>
<Box
flex={1}
className={'textEllipsis3'}
py={1}
wordBreak={'break-all'}
fontSize={'sm'}
color={'myGray.500'}
>
{dataset.intro || t('core.dataset.Intro Placeholder')}
</Box>
<Flex alignItems={'center'} fontSize={'sm'}>
<Box flex={1}>
<PermissionIconText permission={dataset.permission} color={'myGray.600'} />
</Box>
{dataset.type === DatasetTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box>
) : (
<>
<MyIcon mr={1} name="kbTest" w={'12px'} />
<Box color={'myGray.500'}>{dataset.vectorModel.name}</Box>
</>
)}
<MyIcon mr={1} name={dataset.icon as any} w={'12px'} />
<Box color={'myGray.500'}>{t(dataset.label)}</Box>
</Flex>
</Card>
))}
@@ -417,7 +426,7 @@ const Kb = () => {
<Flex mt={'35vh'} flexDirection={'column'} alignItems={'center'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
还没有知识库，快去创建一个吧！
{t('core.dataset.Empty Dataset Tips')}
</Box>
</Flex>
)}
@@ -429,27 +438,19 @@ const Kb = () => {
onClose={() => setEditFolderData(undefined)}
editCallback={async (name) => {
try {
if (editFolderData.id) {
await putDatasetById({
id: editFolderData.id,
name
});
} else {
await postCreateDataset({
parentId,
name,
type: DatasetTypeEnum.folder,
avatar: FolderAvatarSrc,
tags: ''
});
}
await postCreateDataset({
parentId,
name,
type: DatasetTypeEnum.folder,
avatar: FolderAvatarSrc,
intro: ''
});
refetch();
} catch (error) {
return Promise.reject(error);
}
}}
isEdit={!!editFolderData.id}
name={editFolderData.name}
isEdit={false}
/>
)}
{!!moveDataId && (

4
projects/app/src/pages/plugin/list/component/EditModal.tsx
View File

@@ -138,8 +138,8 @@ const CreateModal = ({
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
setRefresh((state) => !state);