* fix if-else find variables (#92)

* fix if-else find variables

* change workflow output type

* fix tooltip style

* fix

* 4.8 (#93)

* api middleware

* perf: app version histories

* faq

* perf: value type show

* fix: ts

* fix: Run the same node multiple times

* feat: auto save workflow

* perf: auto save workflow

---------

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-04-27 12:21:01 +08:00
committed by GitHub
parent c8412e7dc9
commit d407e87dd9
87 changed files with 1607 additions and 1779 deletions

View File

@@ -1,52 +1,68 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { CreateAppParams } from '@fastgpt/global/core/app/api.d';
import type { CreateAppParams } from '@/global/core/app/api.d';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { checkTeamAppLimit } from '@fastgpt/service/support/permission/teamLimit';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { MongoAppVersion } from '@fastgpt/service/core/app/versionSchema';
import { NextAPI } from '@/service/middle/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
name = 'APP',
avatar,
type = AppTypeEnum.advanced,
modules,
edges
} = req.body as CreateAppParams;
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const {
name = 'APP',
avatar,
type = AppTypeEnum.advanced,
modules,
edges
} = req.body as CreateAppParams;
if (!name || !Array.isArray(modules)) {
throw new Error('缺少参数');
}
// 凭证校验
const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true });
// 上限校验
await checkTeamAppLimit(teamId);
// 创建模型
const response = await MongoApp.create({
avatar,
name,
teamId,
tmbId,
modules,
edges,
type,
version: 'v2'
});
jsonRes(res, {
data: response._id
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
if (!name || !Array.isArray(modules)) {
throw new Error('缺少参数');
}
// 凭证校验
const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true });
// 上限校验
await checkTeamAppLimit(teamId);
// 创建模型
const appId = await mongoSessionRun(async (session) => {
const [{ _id: appId }] = await MongoApp.create(
[
{
avatar,
name,
teamId,
tmbId,
modules,
edges,
type,
version: 'v2'
}
],
{ session }
);
await MongoAppVersion.create(
[
{
appId,
nodes: modules,
edges
}
],
{ session }
);
return appId;
});
jsonRes(res, {
data: appId
});
}
export default NextAPI(handler);

View File

@@ -1,60 +1,59 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { MongoAppVersion } from '@fastgpt/service/core/app/versionSchema';
import { NextAPI } from '@/service/middle/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { appId } = req.query as { appId: string };
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { appId } = req.query as { appId: string };
if (!appId) {
throw new Error('参数错误');
}
// 凭证校验
await authApp({ req, authToken: true, appId, per: 'owner' });
// 删除对应的聊天
await mongoSessionRun(async (session) => {
await MongoChatItem.deleteMany(
{
appId
},
{ session }
);
await MongoChat.deleteMany(
{
appId
},
{ session }
);
// 删除分享链接
await MongoOutLink.deleteMany(
{
appId
},
{ session }
);
// delete app
await MongoApp.deleteOne(
{
_id: appId
},
{ session }
);
});
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
if (!appId) {
throw new Error('参数错误');
}
// 凭证校验
await authApp({ req, authToken: true, appId, per: 'owner' });
// 删除对应的聊天
await mongoSessionRun(async (session) => {
await MongoChatItem.deleteMany(
{
appId
},
{ session }
);
await MongoChat.deleteMany(
{
appId
},
{ session }
);
// 删除分享链接
await MongoOutLink.deleteMany(
{
appId
},
{ session }
);
// delete version
await MongoAppVersion.deleteMany(
{
appId
},
{ session }
);
// delete app
await MongoApp.deleteOne(
{
_id: appId
},
{ session }
);
});
}
export default NextAPI(handler);

View File

@@ -2,27 +2,20 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { NextAPI } from '@/service/middle/entry';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { appId } = req.query as { appId: string };
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { appId } = req.query as { appId: string };
if (!appId) {
throw new Error('参数错误');
}
// 凭证校验
const { app } = await authApp({ req, authToken: true, appId, per: 'w' });
jsonRes(res, {
data: app
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
if (!appId) {
throw new Error('参数错误');
}
// 凭证校验
const { app } = await authApp({ req, authToken: true, appId, per: 'w' });
return app;
}
export default NextAPI(handler);

View File

@@ -1,6 +1,4 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import type { PagingData } from '@/types';
import { AppLogsListItemType } from '@/types/app';
@@ -9,144 +7,140 @@ import { addDays } from 'date-fns';
import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema';
import { NextAPI } from '@/service/middle/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const {
pageNum = 1,
pageSize = 20,
appId,
dateStart = addDays(new Date(), -7),
dateEnd = new Date()
} = req.body as GetAppChatLogsParams;
async function handler(
req: NextApiRequest,
res: NextApiResponse
): Promise<PagingData<AppLogsListItemType>> {
const {
pageNum = 1,
pageSize = 20,
appId,
dateStart = addDays(new Date(), -7),
dateEnd = new Date()
} = req.body as GetAppChatLogsParams;
if (!appId) {
throw new Error('缺少参数');
if (!appId) {
throw new Error('缺少参数');
}
// 凭证校验
const { teamId } = await authApp({ req, authToken: true, appId, per: 'w' });
const where = {
teamId: new Types.ObjectId(teamId),
appId: new Types.ObjectId(appId),
updateTime: {
$gte: new Date(dateStart),
$lte: new Date(dateEnd)
}
};
// 凭证校验
const { teamId } = await authApp({ req, authToken: true, appId, per: 'w' });
const where = {
teamId: new Types.ObjectId(teamId),
appId: new Types.ObjectId(appId),
updateTime: {
$gte: new Date(dateStart),
$lte: new Date(dateEnd)
}
};
const [data, total] = await Promise.all([
MongoChat.aggregate([
{ $match: where },
{
$sort: {
userBadFeedbackCount: -1,
userGoodFeedbackCount: -1,
customFeedbacksCount: -1,
updateTime: -1
}
},
{ $skip: (pageNum - 1) * pageSize },
{ $limit: pageSize },
{
$lookup: {
from: ChatItemCollectionName,
let: { chatId: '$chatId' },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$appId', new Types.ObjectId(appId)] },
{ $eq: ['$chatId', '$$chatId'] }
]
}
}
},
{
$project: {
userGoodFeedback: 1,
userBadFeedback: 1,
customFeedbacks: 1,
adminFeedback: 1
}
}
],
as: 'chatitems'
}
},
{
$addFields: {
userGoodFeedbackCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.userGoodFeedback', false] }
const [data, total] = await Promise.all([
MongoChat.aggregate([
{ $match: where },
{
$sort: {
userBadFeedbackCount: -1,
userGoodFeedbackCount: -1,
customFeedbacksCount: -1,
updateTime: -1
}
},
{ $skip: (pageNum - 1) * pageSize },
{ $limit: pageSize },
{
$lookup: {
from: ChatItemCollectionName,
let: { chatId: '$chatId' },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$appId', new Types.ObjectId(appId)] },
{ $eq: ['$chatId', '$$chatId'] }
]
}
}
},
userBadFeedbackCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.userBadFeedback', false] }
}
{
$project: {
userGoodFeedback: 1,
userBadFeedback: 1,
customFeedbacks: 1,
adminFeedback: 1
}
},
customFeedbacksCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $gt: [{ $size: { $ifNull: ['$$item.customFeedbacks', []] } }, 0] }
}
}
],
as: 'chatitems'
}
},
{
$addFields: {
userGoodFeedbackCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.userGoodFeedback', false] }
}
},
markCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.adminFeedback', false] }
}
}
},
userBadFeedbackCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.userBadFeedback', false] }
}
}
},
customFeedbacksCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $gt: [{ $size: { $ifNull: ['$$item.customFeedbacks', []] } }, 0] }
}
}
},
markCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.adminFeedback', false] }
}
}
}
},
{
$project: {
_id: 1,
id: '$chatId',
title: 1,
source: 1,
time: '$updateTime',
messageCount: { $size: '$chatitems' },
userGoodFeedbackCount: 1,
userBadFeedbackCount: 1,
customFeedbacksCount: 1,
markCount: 1
}
}
]),
MongoChat.countDocuments(where)
]);
jsonRes<PagingData<AppLogsListItemType>>(res, {
data: {
pageNum,
pageSize,
data,
total
},
{
$project: {
_id: 1,
id: '$chatId',
title: 1,
source: 1,
time: '$updateTime',
messageCount: { $size: '$chatitems' },
userGoodFeedbackCount: 1,
userBadFeedbackCount: 1,
customFeedbacksCount: 1,
markCount: 1
}
}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
]),
MongoChat.countDocuments(where)
]);
return {
pageNum,
pageSize,
data,
total
};
}
export default NextAPI(handler);

View File

@@ -1,38 +1,30 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { AppListItemType } from '@fastgpt/global/core/app/type';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { NextAPI } from '@/service/middle/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
// 凭证校验
const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true });
async function handler(req: NextApiRequest, res: NextApiResponse<any>): Promise<AppListItemType[]> {
// 凭证校验
const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true });
// 根据 userId 获取模型信息
const myApps = await MongoApp.find(
{ ...mongoRPermission({ teamId, tmbId, role }) },
'_id avatar name intro tmbId permission'
).sort({
updateTime: -1
});
jsonRes<AppListItemType[]>(res, {
data: myApps.map((app) => ({
_id: app._id,
avatar: app.avatar,
name: app.name,
intro: app.intro,
isOwner: teamOwner || String(app.tmbId) === tmbId,
permission: app.permission
}))
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
// 根据 userId 获取模型信息
const myApps = await MongoApp.find(
{ ...mongoRPermission({ teamId, tmbId, role }) },
'_id avatar name intro tmbId permission'
).sort({
updateTime: -1
});
return myApps.map((app) => ({
_id: app._id,
avatar: app.avatar,
name: app.name,
intro: app.intro,
isOwner: teamOwner || String(app.tmbId) === tmbId,
permission: app.permission
}));
}
export default NextAPI(handler);

View File

@@ -2,106 +2,50 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import type { AppUpdateParams } from '@fastgpt/global/core/app/api';
import type { AppUpdateParams } from '@/global/core/app/api';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { getLLMModel } from '@fastgpt/service/core/ai/model';
import { getGuideModule, splitGuideModule } from '@fastgpt/global/core/workflow/utils';
import { getNextTimeByCronStringAndTimezone } from '@fastgpt/global/common/string/time';
import { getScheduleTriggerApp } from '@/service/core/app/utils';
import { beforeUpdateAppFormat } from '@fastgpt/service/core/app/controller';
import { NextAPI } from '@/service/middle/entry';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
name,
avatar,
type,
intro,
modules: nodes,
edges,
permission,
teamTags
} = req.body as AppUpdateParams;
const { appId } = req.query as { appId: string };
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { name, avatar, type, intro, nodes, edges, permission, teamTags } =
req.body as AppUpdateParams;
const { appId } = req.query as { appId: string };
if (!appId) {
throw new Error('appId is empty');
}
// 凭证校验
await authApp({ req, authToken: true, appId, per: permission ? 'owner' : 'w' });
// format nodes data
// 1. dataset search limit, less than model quoteMaxToken
if (nodes) {
let maxTokens = 3000;
nodes.forEach((item) => {
if (
item.flowNodeType === FlowNodeTypeEnum.chatNode ||
item.flowNodeType === FlowNodeTypeEnum.tools
) {
const model =
item.inputs.find((item) => item.key === NodeInputKeyEnum.aiModel)?.value || '';
const chatModel = getLLMModel(model);
const quoteMaxToken = chatModel.quoteMaxToken || 3000;
maxTokens = Math.max(maxTokens, quoteMaxToken);
}
});
nodes.forEach((item) => {
if (item.flowNodeType === FlowNodeTypeEnum.datasetSearchNode) {
item.inputs.forEach((input) => {
if (input.key === NodeInputKeyEnum.datasetMaxTokens) {
const val = input.value as number;
if (val > maxTokens) {
input.value = maxTokens;
}
}
});
}
});
}
// 2. get schedule plan
const { scheduledTriggerConfig } = splitGuideModule(getGuideModule(nodes || []));
// 更新模型
await MongoApp.updateOne(
{
_id: appId
},
{
name,
type,
avatar,
intro,
permission,
version: 'v2',
teamTags: teamTags,
...(nodes && {
modules: nodes
}),
...(edges && {
edges
}),
scheduledTriggerConfig,
scheduledTriggerNextTime: scheduledTriggerConfig
? getNextTimeByCronStringAndTimezone(scheduledTriggerConfig)
: null
}
);
getScheduleTriggerApp();
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
if (!appId) {
throw new Error('appId is empty');
}
// 凭证校验
await authApp({ req, authToken: true, appId, per: permission ? 'owner' : 'w' });
// format nodes data
// 1. dataset search limit, less than model quoteMaxToken
const { nodes: formatNodes } = beforeUpdateAppFormat({ nodes });
// 更新模型
await MongoApp.updateOne(
{
_id: appId
},
{
name,
type,
avatar,
intro,
permission,
version: 'v2',
...(teamTags && teamTags),
...(formatNodes && {
modules: formatNodes
}),
...(edges && {
edges
})
}
);
}
export default NextAPI(handler);

View File

@@ -1,81 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import type { AppUpdateParams } from '@fastgpt/global/core/app/api';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { getLLMModel } from '@fastgpt/service/core/ai/model';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { name, avatar, type, intro, modules, permission, teamTags } =
req.body as AppUpdateParams;
const { appId } = req.query as { appId: string };
if (!appId) {
throw new Error('appId is empty');
}
// 凭证校验
await authApp({ req, authToken: true, appId, per: permission ? 'owner' : 'w' });
// check modules
// 1. dataset search limit, less than model quoteMaxToken
if (modules) {
let maxTokens = 3000;
modules.forEach((item) => {
if (item.flowNodeType === FlowNodeTypeEnum.chatNode) {
const model =
item.inputs.find((item) => item.key === NodeInputKeyEnum.aiModel)?.value || '';
const chatModel = getLLMModel(model);
const quoteMaxToken = chatModel.quoteMaxToken || 3000;
maxTokens = Math.max(maxTokens, quoteMaxToken);
}
});
modules.forEach((item) => {
if (item.flowNodeType === FlowNodeTypeEnum.datasetSearchNode) {
item.inputs.forEach((input) => {
if (input.key === NodeInputKeyEnum.datasetMaxTokens) {
const val = input.value as number;
if (val > maxTokens) {
input.value = maxTokens;
}
}
});
}
});
}
// 更新模型
await MongoApp.findOneAndUpdate(
{
_id: appId
},
{
name,
type,
avatar,
intro,
permission,
teamTags: teamTags,
...(modules && {
modules
})
}
);
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -0,0 +1,54 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { NextAPI } from '@/service/middle/entry';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { MongoAppVersion } from '@fastgpt/service/core/app/versionSchema';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { beforeUpdateAppFormat } from '@fastgpt/service/core/app/controller';
import { getGuideModule, splitGuideModule } from '@fastgpt/global/core/workflow/utils';
import { getNextTimeByCronStringAndTimezone } from '@fastgpt/global/common/string/time';
import { PostPublishAppProps } from '@/global/core/app/api';
type Response = {};
async function handler(req: NextApiRequest, res: NextApiResponse<any>): Promise<{}> {
const { appId } = req.query as { appId: string };
const { nodes = [], edges = [], type } = req.body as PostPublishAppProps;
await authApp({ appId, req, per: 'w', authToken: true });
const { nodes: formatNodes } = beforeUpdateAppFormat({ nodes });
const { scheduledTriggerConfig } = splitGuideModule(getGuideModule(formatNodes || []));
await mongoSessionRun(async (session) => {
// create version histories
await MongoAppVersion.create(
[
{
appId,
nodes: formatNodes,
edges
}
],
{ session }
);
// update app
await MongoApp.findByIdAndUpdate(appId, {
modules: formatNodes,
edges,
updateTime: new Date(),
version: 'v2',
type,
scheduledTriggerConfig,
scheduledTriggerNextTime: scheduledTriggerConfig
? getNextTimeByCronStringAndTimezone(scheduledTriggerConfig)
: null
});
});
return {};
}
export default NextAPI(handler);

View File

@@ -9,6 +9,7 @@ import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { getChatItems } from '@fastgpt/service/core/chat/controller';
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { getAppLatestVersion } from '@fastgpt/service/core/app/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -40,14 +41,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
// get app and history
const { history } = await getChatItems({
appId,
chatId,
limit: 30,
field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${
DispatchNodeResponseKeyEnum.nodeResponse
} ${loadCustomFeedbacks ? 'customFeedbacks' : ''}`
});
const [{ history }, { nodes }] = await Promise.all([
getChatItems({
appId,
chatId,
limit: 30,
field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${
DispatchNodeResponseKeyEnum.nodeResponse
} ${loadCustomFeedbacks ? 'customFeedbacks' : ''}`
}),
getAppLatestVersion(app._id, app)
]);
jsonRes<InitChatResponse>(res, {
data: {
@@ -58,8 +62,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
variables: chat?.variables || {},
history,
app: {
userGuideModule: getGuideModule(app.modules),
chatModels: getChatModelNameListByModules(app.modules),
userGuideModule: getGuideModule(nodes),
chatModels: getChatModelNameListByModules(nodes),
name: app.name,
avatar: app.avatar,
intro: app.intro

View File

@@ -14,6 +14,7 @@ import { AppErrEnum } from '@fastgpt/global/common/error/code/app';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { getAppLatestVersion } from '@fastgpt/service/core/app/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -40,14 +41,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error(ChatErrEnum.unAuthChat);
}
const { history } = await getChatItems({
appId: app._id,
chatId,
limit: 30,
field: `dataId obj value userGoodFeedback userBadFeedback ${
shareChat.responseDetail ? `adminFeedback ${DispatchNodeResponseKeyEnum.nodeResponse}` : ''
} `
});
const [{ history }, { nodes }] = await Promise.all([
getChatItems({
appId: app._id,
chatId,
limit: 30,
field: `dataId obj value userGoodFeedback userBadFeedback ${
shareChat.responseDetail
? `adminFeedback ${DispatchNodeResponseKeyEnum.nodeResponse}`
: ''
} `
}),
getAppLatestVersion(app._id, app)
]);
// pick share response field
history.forEach((item) => {
@@ -66,8 +72,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
variables: chat?.variables || {},
history,
app: {
userGuideModule: getGuideModule(app.modules),
chatModels: getChatModelNameListByModules(app.modules),
userGuideModule: getGuideModule(nodes),
chatModels: getChatModelNameListByModules(nodes),
name: app.name,
avatar: app.avatar,
intro: app.intro

View File

@@ -14,6 +14,7 @@ import { MongoTeam } from '@fastgpt/service/support/user/team/teamSchema';
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
import { filterPublicNodeResponseData } from '@fastgpt/global/core/chat/utils';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { getAppLatestVersion } from '@fastgpt/service/core/app/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -46,12 +47,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
// get app and history
const { history } = await getChatItems({
appId,
chatId,
limit: 30,
field: `dataId obj value userGoodFeedback userBadFeedback adminFeedback ${DispatchNodeResponseKeyEnum.nodeResponse}`
});
const [{ history }, { nodes }] = await Promise.all([
getChatItems({
appId,
chatId,
limit: 30,
field: `dataId obj value userGoodFeedback userBadFeedback adminFeedback ${DispatchNodeResponseKeyEnum.nodeResponse}`
}),
getAppLatestVersion(app._id, app)
]);
// pick share response field
history.forEach((item) => {
@@ -69,8 +73,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
variables: chat?.variables || {},
history,
app: {
userGuideModule: getGuideModule(app.modules),
chatModels: getChatModelNameListByModules(app.modules),
userGuideModule: getGuideModule(nodes),
chatModels: getChatModelNameListByModules(nodes),
name: app.name,
avatar: app.avatar,
intro: app.intro

View File

@@ -1,40 +1,32 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { deleteDatasetData } from '@/service/core/dataset/data/controller';
import { NextAPI } from '@/service/middle/entry';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id: dataId } = req.query as {
id: string;
};
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { id: dataId } = req.query as {
id: string;
};
if (!dataId) {
throw new Error('dataId is required');
}
// 凭证校验
const { teamId, datasetData } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: 'w'
});
await deleteDatasetData(datasetData);
jsonRes(res, {
data: 'success'
});
} catch (err) {
console.log(err);
jsonRes(res, {
code: 500,
error: err
});
if (!dataId) {
throw new Error('dataId is required');
}
});
// 凭证校验
const { teamId, datasetData } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: 'w'
});
await deleteDatasetData(datasetData);
jsonRes(res, {
data: 'success'
});
}
export default NextAPI(handler);

View File

@@ -2,6 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { NextAPI } from '@/service/middle/entry';
export type Response = {
id: string;
@@ -10,29 +11,23 @@ export type Response = {
source: string;
};
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id: dataId } = req.query as {
id: string;
};
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { id: dataId } = req.query as {
id: string;
};
// 凭证校验
const { datasetData } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: 'r'
});
// 凭证校验
const { datasetData } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: 'r'
});
jsonRes(res, {
data: datasetData
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
jsonRes(res, {
data: datasetData
});
}
export default NextAPI(handler);

View File

@@ -5,7 +5,6 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken/index';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
import { hasSameValue } from '@/service/core/dataset/data/utils';
@@ -16,92 +15,87 @@ import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { InsertOneDatasetDataProps } from '@/global/core/dataset/api';
import { simpleText } from '@fastgpt/global/common/string/tools';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
import { NextAPI } from '@/service/middle/entry';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { collectionId, q, a, indexes } = req.body as InsertOneDatasetDataProps;
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { collectionId, q, a, indexes } = req.body as InsertOneDatasetDataProps;
if (!q) {
throw new Error('q is required');
}
if (!collectionId) {
throw new Error('collectionId is required');
}
// 凭证校验
const { teamId, tmbId } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'w'
});
await checkDatasetLimit({
teamId,
insertLen: 1
});
// auth collection and get dataset
const [
{
datasetId: { _id: datasetId, vectorModel }
}
] = await Promise.all([getCollectionWithDataset(collectionId)]);
// format data
const formatQ = simpleText(q);
const formatA = simpleText(a);
const formatIndexes = indexes?.map((item) => ({
...item,
text: simpleText(item.text)
}));
// token check
const token = await countPromptTokens(formatQ + formatA, '');
const vectorModelData = getVectorModel(vectorModel);
if (token > vectorModelData.maxToken) {
return Promise.reject('Q Over Tokens');
}
// Duplicate data check
await hasSameValue({
teamId,
datasetId,
collectionId,
q: formatQ,
a: formatA
});
const { insertId, tokens } = await insertData2Dataset({
teamId,
tmbId,
datasetId,
collectionId,
q: formatQ,
a: formatA,
chunkIndex: 0,
model: vectorModelData.model,
indexes: formatIndexes
});
pushGenerateVectorUsage({
teamId,
tmbId,
tokens,
model: vectorModelData.model
});
jsonRes<string>(res, {
data: insertId
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
if (!q) {
throw new Error('q is required');
}
});
if (!collectionId) {
throw new Error('collectionId is required');
}
// 凭证校验
const { teamId, tmbId } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'w'
});
await checkDatasetLimit({
teamId,
insertLen: 1
});
// auth collection and get dataset
const [
{
datasetId: { _id: datasetId, vectorModel }
}
] = await Promise.all([getCollectionWithDataset(collectionId)]);
// format data
const formatQ = simpleText(q);
const formatA = simpleText(a);
const formatIndexes = indexes?.map((item) => ({
...item,
text: simpleText(item.text)
}));
// token check
const token = await countPromptTokens(formatQ + formatA, '');
const vectorModelData = getVectorModel(vectorModel);
if (token > vectorModelData.maxToken) {
return Promise.reject('Q Over Tokens');
}
// Duplicate data check
await hasSameValue({
teamId,
datasetId,
collectionId,
q: formatQ,
a: formatA
});
const { insertId, tokens } = await insertData2Dataset({
teamId,
tmbId,
datasetId,
collectionId,
q: formatQ,
a: formatA,
chunkIndex: 0,
model: vectorModelData.model,
indexes: formatIndexes
});
pushGenerateVectorUsage({
teamId,
tmbId,
tokens,
model: vectorModelData.model
});
jsonRes<string>(res, {
data: insertId
});
}
export default NextAPI(handler);

View File

@@ -7,62 +7,57 @@ import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { PagingData } from '@/types';
import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import { NextAPI } from '@/service/middle/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
let {
pageNum = 1,
pageSize = 10,
searchText = '',
collectionId
} = req.body as GetDatasetDataListProps;
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let {
pageNum = 1,
pageSize = 10,
searchText = '',
collectionId
} = req.body as GetDatasetDataListProps;
pageSize = Math.min(pageSize, 30);
pageSize = Math.min(pageSize, 30);
// 凭证校验
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'r'
});
// 凭证校验
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'r'
});
searchText = replaceRegChars(searchText).replace(/'/g, '');
searchText = replaceRegChars(searchText).replace(/'/g, '');
const match = {
teamId,
datasetId: collection.datasetId._id,
collectionId,
...(searchText
? {
$or: [{ q: new RegExp(searchText, 'i') }, { a: new RegExp(searchText, 'i') }]
}
: {})
};
const match = {
teamId,
datasetId: collection.datasetId._id,
collectionId,
...(searchText
? {
$or: [{ q: new RegExp(searchText, 'i') }, { a: new RegExp(searchText, 'i') }]
}
: {})
};
const [data, total] = await Promise.all([
MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex')
.sort({ chunkIndex: 1, updateTime: -1 })
.skip((pageNum - 1) * pageSize)
.limit(pageSize)
.lean(),
MongoDatasetData.countDocuments(match)
]);
const [data, total] = await Promise.all([
MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex')
.sort({ chunkIndex: 1, updateTime: -1 })
.skip((pageNum - 1) * pageSize)
.limit(pageSize)
.lean(),
MongoDatasetData.countDocuments(match)
]);
jsonRes<PagingData<DatasetDataListItemType>>(res, {
data: {
pageNum,
pageSize,
data,
total
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
jsonRes<PagingData<DatasetDataListItemType>>(res, {
data: {
pageNum,
pageSize,
data,
total
}
});
}
export default NextAPI(handler);

View File

@@ -2,7 +2,6 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import type {
PushDatasetDataProps,
PushDatasetDataResponse
@@ -11,53 +10,48 @@ import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/
import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { pushDataListToTrainingQueue } from '@fastgpt/service/core/dataset/training/controller';
import { NextAPI } from '@/service/middle/entry';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const body = req.body as PushDatasetDataProps;
const { collectionId, data } = body;
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const body = req.body as PushDatasetDataProps;
const { collectionId, data } = body;
if (!collectionId || !Array.isArray(data)) {
throw new Error('collectionId or data is empty');
}
if (data.length > 200) {
throw new Error('Data is too long, max 200');
}
// 凭证校验
const { teamId, tmbId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'w'
});
// auth dataset limit
await checkDatasetLimit({
teamId,
insertLen: predictDataLimitLength(collection.trainingType, data)
});
jsonRes<PushDatasetDataResponse>(res, {
data: await pushDataListToTrainingQueue({
...body,
teamId,
tmbId,
datasetId: collection.datasetId._id,
agentModel: collection.datasetId.agentModel,
vectorModel: collection.datasetId.vectorModel
})
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
if (!collectionId || !Array.isArray(data)) {
throw new Error('collectionId or data is empty');
}
});
if (data.length > 200) {
throw new Error('Data is too long, max 200');
}
// 凭证校验
const { teamId, tmbId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'w'
});
// auth dataset limit
await checkDatasetLimit({
teamId,
insertLen: predictDataLimitLength(collection.trainingType, data)
});
jsonRes<PushDatasetDataResponse>(res, {
data: await pushDataListToTrainingQueue({
...body,
teamId,
tmbId,
datasetId: collection.datasetId._id,
agentModel: collection.datasetId.agentModel,
vectorModel: collection.datasetId.vectorModel
})
});
}
export default NextAPI(handler);
export const config = {
api: {

View File

@@ -1,59 +1,53 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { connectToDatabase } from '@/service/mongo';
import { updateData2Dataset } from '@/service/core/dataset/data/controller';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { UpdateDatasetDataProps } from '@/global/core/dataset/api';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
import { NextAPI } from '@/service/middle/entry';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, q = '', a, indexes = [] } = req.body as UpdateDatasetDataProps;
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { id, q = '', a, indexes = [] } = req.body as UpdateDatasetDataProps;
// auth data permission
const {
collection: {
datasetId: { vectorModel }
},
teamId,
tmbId
} = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId: id,
per: 'w'
});
// auth data permission
const {
collection: {
datasetId: { vectorModel }
},
teamId,
tmbId
} = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId: id,
per: 'w'
});
// auth team balance
await checkDatasetLimit({
teamId,
insertLen: 1
});
// auth team balance
await checkDatasetLimit({
teamId,
insertLen: 1
});
const { tokens } = await updateData2Dataset({
dataId: id,
q,
a,
indexes,
model: vectorModel
});
const { tokens } = await updateData2Dataset({
dataId: id,
q,
a,
indexes,
model: vectorModel
});
pushGenerateVectorUsage({
teamId,
tmbId,
tokens,
model: vectorModel
});
pushGenerateVectorUsage({
teamId,
tmbId,
tokens,
model: vectorModel
});
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
});
jsonRes(res);
}
export default NextAPI(handler);

View File

@@ -1,94 +1,85 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes, responseWriteController } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { responseWriteController } from '@fastgpt/service/common/response';
import { addLog } from '@fastgpt/service/common/system/log';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import {
checkExportDatasetLimit,
updateExportDatasetLimit
} from '@fastgpt/service/support/user/utils';
import { NextAPI } from '@/service/middle/entry';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
let { datasetId } = req.query as {
datasetId: string;
};
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let { datasetId } = req.query as {
datasetId: string;
};
if (!datasetId || !global.pgClient) {
throw new Error('缺少参数');
}
// 凭证校验
const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'w' });
await checkExportDatasetLimit({
teamId,
limitMinutes: global.feConfigs?.limit?.exportDatasetLimitMinutes
});
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId,
fields: '_id'
});
res.setHeader('Content-Type', 'text/csv; charset=utf-8;');
res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; ');
const cursor = MongoDatasetData.find<{
_id: string;
collectionId: { name: string };
q: string;
a: string;
}>(
{
teamId,
datasetId: { $in: datasets.map((d) => d._id) }
},
'q a'
)
.limit(50000)
.cursor();
const write = responseWriteController({
res,
readStream: cursor
});
write(`\uFEFFindex,content`);
cursor.on('data', (doc) => {
const q = doc.q.replace(/"/g, '""') || '';
const a = doc.a.replace(/"/g, '""') || '';
write(`\n"${q}","${a}"`);
});
cursor.on('end', () => {
cursor.close();
res.end();
});
cursor.on('error', (err) => {
addLog.error(`export dataset error`, err);
res.status(500);
res.end();
});
updateExportDatasetLimit(teamId);
} catch (err) {
res.status(500);
addLog.error(`export dataset error`, err);
jsonRes(res, {
code: 500,
error: err
});
if (!datasetId || !global.pgClient) {
throw new Error('缺少参数');
}
});
// 凭证校验
const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'w' });
await checkExportDatasetLimit({
teamId,
limitMinutes: global.feConfigs?.limit?.exportDatasetLimitMinutes
});
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId,
fields: '_id'
});
res.setHeader('Content-Type', 'text/csv; charset=utf-8;');
res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; ');
const cursor = MongoDatasetData.find<{
_id: string;
collectionId: { name: string };
q: string;
a: string;
}>(
{
teamId,
datasetId: { $in: datasets.map((d) => d._id) }
},
'q a'
)
.limit(50000)
.cursor();
const write = responseWriteController({
res,
readStream: cursor
});
write(`\uFEFFindex,content`);
cursor.on('data', (doc) => {
const q = doc.q.replace(/"/g, '""') || '';
const a = doc.a.replace(/"/g, '""') || '';
write(`\n"${q}","${a}"`);
});
cursor.on('end', () => {
cursor.close();
res.end();
});
cursor.on('error', (err) => {
addLog.error(`export dataset error`, err);
res.status(500);
res.end();
});
updateExportDatasetLimit(teamId);
}
export default NextAPI(handler);
export const config = {
api: {

View File

@@ -1,54 +1,48 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
import { NextAPI } from '@/service/middle/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` };
// 凭证校验
const { teamId, tmbId, teamOwner, role, canWrite } = await authUserRole({
req,
authToken: true,
authApiKey: true
});
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` };
// 凭证校验
const { teamId, tmbId, teamOwner, role, canWrite } = await authUserRole({
req,
authToken: true,
authApiKey: true
});
const datasets = await MongoDataset.find({
...mongoRPermission({ teamId, tmbId, role }),
...(parentId !== undefined && { parentId: parentId || null }),
...(type && { type })
})
.sort({ updateTime: -1 })
.lean();
const datasets = await MongoDataset.find({
...mongoRPermission({ teamId, tmbId, role }),
...(parentId !== undefined && { parentId: parentId || null }),
...(type && { type })
})
.sort({ updateTime: -1 })
.lean();
const data = await Promise.all(
datasets.map<DatasetListItemType>((item) => ({
_id: item._id,
parentId: item.parentId,
avatar: item.avatar,
name: item.name,
intro: item.intro,
type: item.type,
permission: item.permission,
canWrite,
isOwner: teamOwner || String(item.tmbId) === tmbId,
vectorModel: getVectorModel(item.vectorModel)
}))
);
const data = await Promise.all(
datasets.map<DatasetListItemType>((item) => ({
_id: item._id,
parentId: item.parentId,
avatar: item.avatar,
name: item.name,
intro: item.intro,
type: item.type,
permission: item.permission,
canWrite,
isOwner: teamOwner || String(item.tmbId) === tmbId,
vectorModel: getVectorModel(item.vectorModel)
}))
);
jsonRes<DatasetListItemType[]>(res, {
data
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
jsonRes<DatasetListItemType[]>(res, {
data
});
}
export default NextAPI(handler);

View File

@@ -1,8 +1,6 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import type { SearchTestProps, SearchTestResponse } from '@/global/core/dataset/api.d';
import { connectToDatabase } from '@/service/mongo';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { searchDatasetData } from '@fastgpt/service/core/dataset/search/controller';
@@ -14,95 +12,90 @@ import {
checkTeamAIPoints,
checkTeamReRankPermission
} from '@fastgpt/service/support/permission/teamLimit';
import { NextAPI } from '@/service/middle/entry';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
datasetId,
text,
limit = 1500,
similarity,
searchMode,
usingReRank,
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const {
datasetId,
text,
limit = 1500,
similarity,
searchMode,
usingReRank,
datasetSearchUsingExtensionQuery = false,
datasetSearchExtensionModel,
datasetSearchExtensionBg = ''
} = req.body as SearchTestProps;
datasetSearchUsingExtensionQuery = false,
datasetSearchExtensionModel,
datasetSearchExtensionBg = ''
} = req.body as SearchTestProps;
if (!datasetId || !text) {
throw new Error('缺少参数');
}
const start = Date.now();
if (!datasetId || !text) {
throw new Error('缺少参数');
}
const start = Date.now();
// auth dataset role
const { dataset, teamId, tmbId, apikey } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'r'
});
// auth balance
await checkTeamAIPoints(teamId);
// auth dataset role
const { dataset, teamId, tmbId, apikey } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'r'
});
// auth balance
await checkTeamAIPoints(teamId);
// query extension
const extensionModel =
datasetSearchUsingExtensionQuery && datasetSearchExtensionModel
? getLLMModel(datasetSearchExtensionModel)
: undefined;
const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
query: text,
extensionModel,
extensionBg: datasetSearchExtensionBg
});
// query extension
const extensionModel =
datasetSearchUsingExtensionQuery && datasetSearchExtensionModel
? getLLMModel(datasetSearchExtensionModel)
: undefined;
const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
query: text,
extensionModel,
extensionBg: datasetSearchExtensionBg
});
const { searchRes, tokens, ...result } = await searchDatasetData({
teamId,
reRankQuery: rewriteQuery,
queries: concatQueries,
model: dataset.vectorModel,
limit: Math.min(limit, 20000),
similarity,
datasetIds: [datasetId],
searchMode,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId))
});
const { searchRes, tokens, ...result } = await searchDatasetData({
teamId,
reRankQuery: rewriteQuery,
queries: concatQueries,
model: dataset.vectorModel,
limit: Math.min(limit, 20000),
similarity,
datasetIds: [datasetId],
searchMode,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId))
});
// push bill
const { totalPoints } = pushGenerateVectorUsage({
teamId,
tmbId,
tokens,
model: dataset.vectorModel,
source: apikey ? UsageSourceEnum.api : UsageSourceEnum.fastgpt,
// push bill
const { totalPoints } = pushGenerateVectorUsage({
teamId,
tmbId,
tokens,
model: dataset.vectorModel,
source: apikey ? UsageSourceEnum.api : UsageSourceEnum.fastgpt,
...(aiExtensionResult &&
extensionModel && {
extensionModel: extensionModel.name,
extensionTokens: aiExtensionResult.tokens
})
});
if (apikey) {
updateApiKeyUsage({
apikey,
totalPoints: totalPoints
});
}
jsonRes<SearchTestResponse>(res, {
data: {
list: searchRes,
duration: `${((Date.now() - start) / 1000).toFixed(3)}s`,
usingQueryExtension: !!aiExtensionResult,
...result
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
...(aiExtensionResult &&
extensionModel && {
extensionModel: extensionModel.name,
extensionTokens: aiExtensionResult.tokens
})
});
if (apikey) {
updateApiKeyUsage({
apikey,
totalPoints: totalPoints
});
}
});
jsonRes<SearchTestResponse>(res, {
data: {
list: searchRes,
duration: `${((Date.now() - start) / 1000).toFixed(3)}s`,
usingQueryExtension: !!aiExtensionResult,
...result
}
});
}
export default NextAPI(handler);

View File

@@ -1,6 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { getUploadModel } from '@fastgpt/service/common/file/multer';
import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
import fs from 'fs';
@@ -10,12 +9,13 @@ import { authChatCert } from '@/service/support/permission/auth/chat';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { getGuideModule, splitGuideModule } from '@fastgpt/global/core/workflow/utils';
import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
import { NextAPI } from '@/service/middle/entry';
const upload = getUploadModel({
maxSize: 2
});
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let filePaths: string[] = [];
try {
@@ -81,7 +81,9 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
}
removeFilesByPaths(filePaths);
});
}
export default NextAPI(handler);
export const config = {
api: {

View File

@@ -3,7 +3,6 @@ import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { sseErrRes, jsonRes } from '@fastgpt/service/common/response';
import { addLog } from '@fastgpt/service/common/system/log';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
@@ -42,6 +41,9 @@ import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runti
import { dispatchWorkFlowV1 } from '@fastgpt/service/core/workflow/dispatchV1';
import { setEntryEntries } from '@fastgpt/service/core/workflow/dispatchV1/utils';
import { NextAPI } from '@/service/middle/entry';
import { MongoAppVersion } from '@fastgpt/service/core/app/versionSchema';
import { getAppLatestVersion } from '@fastgpt/service/core/app/controller';
type FastGptWebChatProps = {
chatId?: string; // undefined: nonuse history, '': new chat, 'xxxxx': use history
@@ -73,7 +75,7 @@ type AuthResponseType = {
outLinkUserId?: string;
};
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) {
async function handler(req: NextApiRequest, res: NextApiResponse) {
res.on('close', () => {
res.end();
});
@@ -163,13 +165,16 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
})();
// get and concat history
const { history } = await getChatItems({
appId: app._id,
chatId,
limit: 30,
field: `dataId obj value`
});
// 1. get and concat history; 2. get app workflow
const [{ history }, { nodes, edges }] = await Promise.all([
getChatItems({
appId: app._id,
chatId,
limit: 30,
field: `dataId obj value`
}),
getAppLatestVersion(app._id, app)
]);
const concatHistories = history.concat(chatMessages);
const responseChatItemId: string | undefined = messages[messages.length - 1].dataId;
@@ -185,8 +190,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
appId: String(app._id),
chatId,
responseChatItemId,
runtimeNodes: storeNodes2RuntimeNodes(app.modules, getDefaultEntryNodeIds(app.modules)),
runtimeEdges: initWorkflowEdgeStatus(app.edges),
runtimeNodes: storeNodes2RuntimeNodes(nodes, getDefaultEntryNodeIds(nodes)),
runtimeEdges: initWorkflowEdgeStatus(edges),
variables: {
...variables,
userChatInput: text
@@ -349,7 +354,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
}
}
});
}
export default NextAPI(handler);
export const config = {
api: {

View File

@@ -1,7 +1,6 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { connectToDatabase } from '@/service/mongo';
import { getVectorsByText } from '@fastgpt/service/core/ai/embedding';
@@ -19,7 +18,7 @@ type Props = {
type: `${EmbeddingTypeEnm}`;
};
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let { input, model, billId, type } = req.body as Props;
await connectToDatabase();
@@ -80,4 +79,4 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
error: err
});
}
});
}