mirror of
https://github.com/labring/FastGPT.git
synced 2025-08-05 22:55:27 +00:00
4.7-alpha2 (#1027)
* feat: stop toolCall and rename some field. (#46) * perf: node delete tip;pay tip * fix: toolCall cannot save child answer * feat: stop tool * fix: team modal * fix feckbackMoal auth bug (#47) * 简单的支持提示词运行tool。优化workflow模板 (#49) * remove templates * fix: request body undefined * feat: prompt tool run * feat: workflow tamplates modal * perf: plugin start * 4.7 (#50) * fix docker-compose download url (#994) original code is a bad url with '404 NOT FOUND' return. fix docker-compose download url, add 'v' before docker-compose version * Update ai_settings.md (#1000) * Update configuration.md * Update configuration.md * Fix history in classifyQuestion and extract modules (#1012) * Fix history in classifyQuestion and extract modules * Add chatValue2RuntimePrompt import and update text formatting * flow controller to packages * fix: rerank select * modal ui * perf: modal code path * point not sufficient * feat: http url support variable * fix http key * perf: prompt * perf: ai setting modal * simple edit ui --------- Co-authored-by: entorick <entorick11@qq.com> Co-authored-by: liujianglc <liujianglc@163.com> Co-authored-by: Fengrui Liu <liufengrui.work@bytedance.com> * fix team share redirect to login (#51) * feat: support openapi import plugins (#48) * feat: support openapi import plugins * feat: import from url * fix: add body params parse * fix build * fix * fix * fix * tool box ui (#52) * fix: training queue * feat: simple edit tool select * perf: simple edit dataset prompt * fix: chatbox tool ux * feat: quote prompt module * perf: plugin tools sign * perf: model avatar * tool selector ui * feat: max histories * perf: http plugin import (#53) * perf: plugin http import * chatBox ui * perf: name * fix: Node template card (#54) * fix: ts * setting modal * package * package * feat: add plugins search (#57) * feat: add plugins search * perf: change http plugin header input * Yjl (#56) * perf: prompt tool call * perf: chat box ux * doc * doc * price tip * perf: tool selector * ui' * fix: vector queue * fix: empty tool and empty response * fix: empty msg * perf: pg index * perf: ui tip * doc * tool tip --------- Co-authored-by: yst <77910600+yu-and-liu@users.noreply.github.com> Co-authored-by: entorick <entorick11@qq.com> Co-authored-by: liujianglc <liujianglc@163.com> Co-authored-by: Fengrui Liu <liufengrui.work@bytedance.com> Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
@@ -1,70 +0,0 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { delay } from '@fastgpt/global/common/system/utils';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { jiebaSplit } from '@/service/common/string/jieba';
|
||||
|
||||
let success = 0;
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { limit = 50 } = req.body as { limit: number };
|
||||
await authCert({ req, authRoot: true });
|
||||
await connectToDatabase();
|
||||
success = 0;
|
||||
|
||||
console.log(
|
||||
'total',
|
||||
await MongoDatasetData.countDocuments({
|
||||
fullTextToken: { $exists: false },
|
||||
updateTime: { $lt: new Date() }
|
||||
})
|
||||
);
|
||||
|
||||
await initFullTextToken(limit, new Date());
|
||||
|
||||
jsonRes(res, {
|
||||
message: 'success'
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error
|
||||
});
|
||||
}
|
||||
}
|
||||
export async function initFullTextToken(limit = 50, endDate: Date): Promise<any> {
|
||||
try {
|
||||
const dataList = await MongoDatasetData.find(
|
||||
{ fullTextToken: { $exists: false }, updateTime: { $lt: endDate } },
|
||||
'_id q a'
|
||||
)
|
||||
.limit(limit)
|
||||
.lean();
|
||||
if (dataList.length === 0) return;
|
||||
|
||||
const result = await Promise.allSettled(
|
||||
dataList.map((item) => {
|
||||
const text = item.q + (item.a || '');
|
||||
const tokens = jiebaSplit({ text });
|
||||
|
||||
return MongoDatasetData.findByIdAndUpdate(item._id, {
|
||||
$set: {
|
||||
fullTextToken: tokens
|
||||
}
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
success += result.filter((item) => item.status === 'fulfilled').length;
|
||||
console.log(`success: ${success}`);
|
||||
return initFullTextToken(limit, endDate);
|
||||
} catch (error) {
|
||||
await delay(1000);
|
||||
return initFullTextToken(limit, endDate);
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { delay } from '@fastgpt/global/common/system/utils';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { jiebaSplit } from '@/service/common/string/jieba';
|
||||
|
||||
let success = 0;
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { limit = 50 } = req.body as { limit: number };
|
||||
await authCert({ req, authRoot: true });
|
||||
await connectToDatabase();
|
||||
success = 0;
|
||||
|
||||
console.log('total', await MongoDatasetData.countDocuments({ inited: { $exists: false } }));
|
||||
|
||||
await initFullTextToken(limit);
|
||||
|
||||
jsonRes(res, {
|
||||
message: 'success'
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error
|
||||
});
|
||||
}
|
||||
}
|
||||
export async function initFullTextToken(limit = 50): Promise<any> {
|
||||
try {
|
||||
const dataList = await MongoDatasetData.find({ inited: { $exists: false } }, '_id q a')
|
||||
.limit(limit)
|
||||
.lean();
|
||||
if (dataList.length === 0) return;
|
||||
|
||||
const result = await Promise.allSettled(
|
||||
dataList.map((item) => {
|
||||
const text = item.q + (item.a || '');
|
||||
const tokens = jiebaSplit({ text });
|
||||
|
||||
return MongoDatasetData.findByIdAndUpdate(item._id, {
|
||||
$set: {
|
||||
inited: true,
|
||||
fullTextToken: tokens
|
||||
}
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
success += result.filter((item) => item.status === 'fulfilled').length;
|
||||
console.log(`success: ${success}`);
|
||||
return initFullTextToken(limit);
|
||||
} catch (error) {
|
||||
await delay(1000);
|
||||
return initFullTextToken(limit);
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
|
||||
import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
|
||||
|
||||
let success = 0;
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { limit = 50 } = req.body as { limit: number };
|
||||
await authCert({ req, authRoot: true });
|
||||
await connectToDatabase();
|
||||
success = 0;
|
||||
|
||||
await MongoDatasetCollection.updateMany({ createTime: { $exists: false } }, [
|
||||
{
|
||||
$set: {
|
||||
createTime: '$updateTime'
|
||||
}
|
||||
}
|
||||
]);
|
||||
await MongoDatasetCollection.updateMany({ trainingType: { $exists: false } }, [
|
||||
{
|
||||
$set: {
|
||||
trainingType: {
|
||||
$cond: {
|
||||
if: { $ifNull: ['$a', false] },
|
||||
then: TrainingModeEnum.qa,
|
||||
else: TrainingModeEnum.chunk
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]);
|
||||
await MongoDatasetCollection.updateMany({ chunkSize: { $exists: false } }, [
|
||||
{
|
||||
$set: {
|
||||
chunkSize: 0
|
||||
}
|
||||
}
|
||||
]);
|
||||
await MongoDatasetCollection.updateMany({ fileId: { $exists: false } }, [
|
||||
{
|
||||
$set: {
|
||||
fileId: '$metadata.fileId'
|
||||
}
|
||||
}
|
||||
]);
|
||||
await MongoDatasetCollection.updateMany({ rawLink: { $exists: false } }, [
|
||||
{
|
||||
$set: {
|
||||
rawLink: '$metadata.rawLink'
|
||||
}
|
||||
}
|
||||
]);
|
||||
|
||||
await MongoDatasetData.updateMany(
|
||||
{ chunkIndex: { $exists: false } },
|
||||
{
|
||||
chunkIndex: 0
|
||||
}
|
||||
);
|
||||
await MongoDatasetData.updateMany(
|
||||
{ updateTime: { $exists: false } },
|
||||
{
|
||||
updateTime: new Date()
|
||||
}
|
||||
);
|
||||
|
||||
await MongoDataset.updateMany(
|
||||
{ status: { $exists: false } },
|
||||
{
|
||||
$set: {
|
||||
status: DatasetStatusEnum.active
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// dataset tags to intro
|
||||
await MongoDataset.updateMany({ tags: { $exists: true } }, [
|
||||
{
|
||||
$set: {
|
||||
intro: {
|
||||
$reduce: {
|
||||
input: '$tags',
|
||||
initialValue: '',
|
||||
in: { $concat: ['$$value', ' ', '$$this'] }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]);
|
||||
|
||||
jsonRes(res, {
|
||||
message: 'success'
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error
|
||||
});
|
||||
}
|
||||
}
|
||||
49
projects/app/src/pages/api/admin/initv47.ts
Normal file
49
projects/app/src/pages/api/admin/initv47.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoUsage } from '@fastgpt/service/support/wallet/usage/schema';
|
||||
import { connectionMongo } from '@fastgpt/service/common/mongo';
|
||||
import { checkFiles } from '../timerTask/dataset/checkInValidDatasetFiles';
|
||||
import { addHours } from 'date-fns';
|
||||
import { checkInvalid as checkInvalidImg } from '../timerTask/dataset/checkInvalidDatasetImage';
|
||||
import { checkInvalidCollection } from '../timerTask/dataset/checkInvalidMongoCollection';
|
||||
import { checkInvalidVector } from '../timerTask/dataset/checkInvalidVector';
|
||||
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
|
||||
import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
|
||||
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
await authCert({ req, authRoot: true });
|
||||
|
||||
await MongoPlugin.updateMany(
|
||||
{ type: { $exists: false } },
|
||||
{
|
||||
$set: {
|
||||
type: PluginTypeEnum.custom
|
||||
}
|
||||
}
|
||||
);
|
||||
await MongoPlugin.updateMany(
|
||||
{ parentId: { $exists: false } },
|
||||
{
|
||||
$set: {
|
||||
parentId: null
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
jsonRes(res, {
|
||||
message: 'success'
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user