ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 13:03:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

4.6.4-alpha (#582)

Browse Source
This commit is contained in:
Archer
2023-12-08 15:01:11 +08:00
committed by GitHub
parent 54d52d8d25
commit b58249fc3a
66 changed files with 962 additions and 527 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
files/deploy/fastgpt/docker-compose.yml
View File

@@ -1,4 +1,5 @@
# 非 host 版本, 不使用本机代理
# (不懂 Docker 的，只需要关心 OPENAI_BASE_URL 和 CHAT_API_KEY 即可！)
version: '3.3'
services:
pg:
@@ -47,7 +48,7 @@ services:
environment:
# root 密码，用户名为: root
- DEFAULT_ROOT_PSW=1234
# 中转地址，如果是用官方号，不需要管
# 中转地址，如果是用官方号，不需要管。务必加 /v1
- OPENAI_BASE_URL=https://api.openai.com/v1
- CHAT_API_KEY=sk-xxxx
- DB_MAX_LINK=5 # database max link

24
packages/global/common/error/code/common.ts Normal file
View File

@@ -0,0 +1,24 @@
import { ErrType } from '../errorCode';
/* dataset: 507000 */
const startCode = 507000;
export enum CommonErrEnum {
fileNotFound = 'fileNotFound'
}
const datasetErr = [
{
statusText: CommonErrEnum.fileNotFound,
message: 'error.fileNotFound'
}
];
export default datasetErr.reduce((acc, cur, index) => {
return {
...acc,
[cur.statusText]: {
code: startCode + index,
statusText: cur.statusText,
message: cur.message,
data: null
}
};
}, {} as ErrType<`${CommonErrEnum}`>);

10
packages/global/common/error/code/dataset.ts
View File

@@ -13,23 +13,23 @@ export enum DatasetErrEnum {
const datasetErr = [
{
statusText: DatasetErrEnum.unAuthDataset,
message: '无权操作该知识库'
message: 'core.dataset.error.unAuthDataset'
},
{
statusText: DatasetErrEnum.unAuthDatasetCollection,
message: '无权操作该数据集'
message: 'core.dataset.error.unAuthDatasetCollection'
},
{
statusText: DatasetErrEnum.unAuthDatasetData,
message: '无权操作该数据'
message: 'core.dataset.error.unAuthDatasetData'
},
{
statusText: DatasetErrEnum.unAuthDatasetFile,
message: '无权操作该文件'
message: 'core.dataset.error.unAuthDatasetFile'
},
{
statusText: DatasetErrEnum.unCreateCollection,
message: '无权创建数据集'
message: 'core.dataset.error.unCreateCollection'
},
{
statusText: DatasetErrEnum.unLinkCollection,

4
packages/global/common/error/errorCode.ts
View File

@@ -6,6 +6,7 @@ import pluginErr from './code/plugin';
import outLinkErr from './code/outLink';
import teamErr from './code/team';
import userErr from './code/user';
import commonErr from './code/common';
export const ERROR_CODE: { [key: number]: string } = {
400: '请求失败',
@@ -96,5 +97,6 @@ export const ERROR_RESPONSE: Record<
...outLinkErr,
...teamErr,
...userErr,
...pluginErr
...pluginErr,
...commonErr
};

7
packages/global/common/file/api.d.ts vendored
View File

@@ -1,3 +1,10 @@
export type UploadImgProps = {
base64Img: string;
expiredTime?: Date;
metadata?: Record<string, any>;
shareId?: string;
};
export type UrlFetchParams = {
urlList: string[];
selector?: string;

9
packages/global/common/file/tools.ts
View File

@@ -49,7 +49,14 @@ export const cheerioToHtml = ({
}
});
return $(selector || 'body').html();
const html = $(selector || 'body')
.map((item, dom) => {
return $(dom).html();
})
.get()
.join('\n');
return html;
};
export const urlsFetch = async ({
urlList,

8
packages/global/common/string/markdown.ts
View File

@@ -26,10 +26,14 @@ export const simpleMarkdownText = (rawText: string) => {
rawText = rawText.replace(/\\\\n/g, '\\n');
// Remove headings and code blocks front spaces
['####', '###', '##', '#', '```', '~~~'].forEach((item) => {
['####', '###', '##', '#', '```', '~~~'].forEach((item, i) => {
const isMarkdown = i <= 3;
const reg = new RegExp(`\\n\\s*${item}`, 'g');
if (reg.test(rawText)) {
rawText = rawText.replace(new RegExp(`\\n\\s*(${item})`, 'g'), '\n$1');
rawText = rawText.replace(
new RegExp(`(\\n)\\s*(${item})`, 'g'),
isMarkdown ? '\n$1$2' : '$1$2'
);
}
});

62
packages/global/common/string/textSplitter.ts
View File

@@ -12,12 +12,13 @@ export const splitText2Chunks = (props: {
text: string;
chunkLen: number;
overlapRatio?: number;
customReg?: string[];
}): {
chunks: string[];
tokens: number;
overlapRatio?: number;
} => {
let { text = '', chunkLen, overlapRatio = 0.2 } = props;
let { text = '', chunkLen, overlapRatio = 0.2, customReg = [] } = props;
const splitMarker = 'SPLIT_HERE_SPLIT_HERE';
const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER';
const overlapLen = Math.round(chunkLen * overlapRatio);
@@ -29,22 +30,29 @@ export const splitText2Chunks = (props: {
// The larger maxLen is, the next sentence is less likely to trigger splitting
const stepReges: { reg: RegExp; maxLen: number }[] = [
{ reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
{ reg: /^(##\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
{ reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
{ reg: /^(####\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
...customReg.map((text) => ({ reg: new RegExp(`([${text}])`, 'g'), maxLen: chunkLen * 1.4 })),
{ reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },
{ reg: /^(##\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },
{ reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },
{ reg: /^(####\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },
{ reg: /([\n](`))/g, maxLen: chunkLen * 4 }, // code block
{ reg: /([\n](?![\*\-|>0-9]))/g, maxLen: chunkLen * 1.8 }, // (?![\*\-|>`0-9]): markdown special char
{ reg: /([\n])/g, maxLen: chunkLen * 1.4 },
{ reg: /([\n]([`~]))/g, maxLen: chunkLen * 4 }, // code block
{ reg: /([\n](?!\s*[\*\-|>0-9]))/g, maxLen: chunkLen * 2 }, // (?![\*\-|>`0-9]): markdown special char
{ reg: /([\n])/g, maxLen: chunkLen * 1.2 },
{ reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.4 },
{ reg: /([！]|!\s)/g, maxLen: chunkLen * 1.4 },
{ reg: /([？]|\?\s)/g, maxLen: chunkLen * 1.6 },
{ reg: /([；]|;\s)/g, maxLen: chunkLen * 1.8 },
{ reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.2 },
{ reg: /([！]|!\s)/g, maxLen: chunkLen * 1.2 },
{ reg: /([？]|\?\s)/g, maxLen: chunkLen * 1.4 },
{ reg: /([；]|;\s)/g, maxLen: chunkLen * 1.6 },
{ reg: /([，]|,\s)/g, maxLen: chunkLen * 2 }
];
const customRegLen = customReg.length;
const checkIsCustomStep = (step: number) => step < customRegLen;
const checkIsMarkdownSplit = (step: number) => step >= customRegLen && step <= 3 + customRegLen;
const checkIndependentChunk = (step: number) => step >= customRegLen && step <= 4 + customRegLen;
const checkForbidOverlap = (step: number) => step <= 6 + customRegLen;
// if use markdown title split, Separate record title title
const getSplitTexts = ({ text, step }: { text: string; step: number }) => {
if (step >= stepReges.length) {
@@ -55,11 +63,13 @@ export const splitText2Chunks = (props: {
}
];
}
const isMarkdownSplit = step <= 3;
const isMarkdownSplit = checkIsMarkdownSplit(step);
const independentChunk = checkIndependentChunk(step);
const { reg } = stepReges[step];
const splitTexts = text
.replace(reg, isMarkdownSplit ? `${splitMarker}$1` : `$1${splitMarker}`)
.replace(reg, independentChunk ? `${splitMarker}$1` : `$1${splitMarker}`)
.split(`${splitMarker}`)
.filter((part) => part.trim());
@@ -76,7 +86,7 @@ export const splitText2Chunks = (props: {
};
const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
const forbidOverlap = step <= 6;
const forbidOverlap = checkForbidOverlap(step);
const maxOverlapLen = chunkLen * 0.4;
// step >= stepReges.length: Do not overlap incomplete sentences
@@ -114,7 +124,8 @@ export const splitText2Chunks = (props: {
lastText: string;
mdTitle: string;
}): string[] => {
const isMarkdownSplit = step <= 3;
const independentChunk = checkIndependentChunk(step);
const isCustomStep = checkIsCustomStep(step);
// mini text
if (text.length <= chunkLen) {
@@ -134,12 +145,13 @@ export const splitText2Chunks = (props: {
return chunks;
}
const { maxLen } = stepReges[step];
const minChunkLen = chunkLen * 0.7;
// split text by special char
const splitTexts = getSplitTexts({ text, step });
const maxLen = splitTexts.length > 1 ? stepReges[step].maxLen : chunkLen;
const minChunkLen = chunkLen * 0.7;
const miniChunkLen = 30;
const chunks: string[] = [];
for (let i = 0; i < splitTexts.length; i++) {
const item = splitTexts[i];
@@ -170,8 +182,8 @@ export const splitText2Chunks = (props: {
mdTitle: currentTitle
});
const lastChunk = innerChunks[innerChunks.length - 1];
// last chunk is too small, concat it to lastText
if (!isMarkdownSplit && lastChunk.length < minChunkLen) {
// last chunk is too small, concat it to lastText(next chunk start)
if (!independentChunk && lastChunk.length < minChunkLen) {
chunks.push(...innerChunks.slice(0, -1));
lastText = lastChunk;
} else {
@@ -189,10 +201,14 @@ export const splitText2Chunks = (props: {
lastText = newText;
// markdown paragraph block: Direct addition; If the chunk size reaches, add a chunk
if (isMarkdownSplit || newTextLen >= chunkLen) {
if (
isCustomStep ||
(independentChunk && newTextLen > miniChunkLen) ||
newTextLen >= chunkLen
) {
chunks.push(`${currentTitle}${lastText}`);
lastText = isMarkdownSplit ? '' : getOneTextOverlapText({ text: lastText, step });
lastText = getOneTextOverlapText({ text: lastText, step });
}
}

2
packages/global/core/app/utils.ts
View File

@@ -24,7 +24,7 @@ export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEd
dataset: {
datasets: [],
similarity: 0.4,
limit: 5,
limit: 1500,
searchEmptyText: '',
searchMode: DatasetSearchModeEnum.embedding
},

2
packages/global/core/chat/constants.ts
View File

@@ -55,3 +55,5 @@ export const LOGO_ICON = `/icon/logo.svg`;
export const IMG_BLOCK_KEY = 'img-block';
export const FILE_BLOCK_KEY = 'file-block';
export const MARKDOWN_QUOTE_SIGN = 'QUOTE SIGN';

13
packages/global/core/module/template/system/datasetSearch.ts
View File

@@ -54,17 +54,10 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
{
key: ModuleInputKeyEnum.datasetLimit,
type: FlowNodeInputTypeEnum.hidden,
label: '单次搜索上限',
description: '最多取 n 条记录作为本次问题引用',
value: 5,
label: '引用上限',
description: '单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens',
value: 1500,
valueType: ModuleDataTypeEnum.number,
min: 1,
max: 20,
step: 1,
markList: [
{ label: '1', value: 1 },
{ label: '20', value: 20 }
],
showTargetInApp: false,
showTargetInPlugin: false
},

62
packages/service/common/file/gridfs/controller.ts
View File

@@ -3,6 +3,7 @@ import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import fsp from 'fs/promises';
import fs from 'fs';
import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
import { delImgByFileIdList } from '../image/controller';
export function getGFSCollection(bucket: `${BucketNameEnum}`) {
return connectionMongo.connection.db.collection(`${bucket}.files`);
@@ -69,24 +70,65 @@ export async function getFileById({
_id: new Types.ObjectId(fileId)
});
if (!file) {
return Promise.reject('File not found');
// if (!file) {
// return Promise.reject('File not found');
// }
return file || undefined;
}
return file;
}
export async function delFileById({
export async function delFileByFileIdList({
bucketName,
fileId
fileIdList,
retry = 3
}: {
bucketName: `${BucketNameEnum}`;
fileId: string;
fileIdList: string[];
retry?: number;
}): Promise<any> {
try {
const bucket = getGridBucket(bucketName);
await Promise.all(fileIdList.map((id) => bucket.delete(new Types.ObjectId(id))));
} catch (error) {
if (retry > 0) {
return delFileByFileIdList({ bucketName, fileIdList, retry: retry - 1 });
}
}
}
// delete file by metadata(datasetId)
export async function delFileByMetadata({
bucketName,
datasetId
}: {
bucketName: `${BucketNameEnum}`;
datasetId?: string;
}) {
const bucket = getGridBucket(bucketName);
await bucket.delete(new Types.ObjectId(fileId));
return true;
const files = await bucket
.find(
{
...(datasetId && { 'metadata.datasetId': datasetId })
},
{
projection: {
_id: 1
}
}
)
.toArray();
const idList = files.map((item) => String(item._id));
// delete img
await delImgByFileIdList(idList);
// delete file
await delFileByFileIdList({
bucketName,
fileIdList: idList
});
}
export async function getDownloadStream({

17
packages/service/common/file/image/controller.ts
View File

@@ -1,3 +1,4 @@
import { UploadImgProps } from '@fastgpt/global/common/file/api';
import { imageBaseUrl } from './constant';
import { MongoImage } from './schema';
@@ -9,11 +10,10 @@ export const maxImgSize = 1024 * 1024 * 12;
export async function uploadMongoImg({
base64Img,
teamId,
expiredTime
}: {
base64Img: string;
expiredTime,
metadata
}: UploadImgProps & {
teamId: string;
expiredTime?: Date;
}) {
if (base64Img.length > maxImgSize) {
return Promise.reject('Image too large');
@@ -24,7 +24,8 @@ export async function uploadMongoImg({
const { _id } = await MongoImage.create({
teamId,
binary: Buffer.from(base64Data, 'base64'),
expiredTime
expiredTime: expiredTime,
metadata
});
return getMongoImgUrl(String(_id));
@@ -37,3 +38,9 @@ export async function readMongoImg({ id }: { id: string }) {
}
return data?.binary;
}
export async function delImgByFileIdList(fileIds: string[]) {
return MongoImage.deleteMany({
'metadata.fileId': { $in: fileIds.map((item) => String(item)) }
});
}

8
packages/service/common/file/image/schema.ts
View File

@@ -5,13 +5,17 @@ const { Schema, model, models } = connectionMongo;
const ImageSchema = new Schema({
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName
ref: TeamCollectionName,
required: true
},
binary: {
type: Buffer
},
expiredTime: {
type: Date
},
metadata: {
type: Object
}
});
@@ -21,7 +25,7 @@ try {
console.log(error);
}
export const MongoImage: Model<{ teamId: string; binary: Buffer }> =
export const MongoImage: Model<{ teamId: string; binary: Buffer; metadata?: Record<string, any> }> =
models['image'] || model('image', ImageSchema);
MongoImage.syncIndexes();

2
packages/service/common/response/index.ts
View File

@@ -82,7 +82,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => {
} else if (error?.response?.data?.error?.message) {
msg = error?.response?.data?.error?.message;
} else if (error?.error?.message) {
msg = error?.error?.message;
msg = `${error?.error?.code} ${error?.error?.message}`;
}
addLog.error(`sse error: ${msg}`, error);

25
packages/service/core/dataset/data/controller.ts
View File

@@ -1,11 +1,11 @@
import { MongoDatasetData } from './schema';
import { deletePgDataById } from './pg';
import { MongoDatasetTraining } from '../training/schema';
import { delFileById } from '../../../common/file/gridfs/controller';
import { delFileByFileIdList, delFileByMetadata } from '../../../common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { MongoDatasetCollection } from '../collection/schema';
import { delDatasetFiles } from '../file/controller';
import { delay } from '@fastgpt/global/common/system/utils';
import { delImgByFileIdList } from '../../../common/file/image/controller';
/* delete all data by datasetIds */
export async function delDatasetRelevantData({ datasetIds }: { datasetIds: string[] }) {
@@ -17,9 +17,11 @@ export async function delDatasetRelevantData({ datasetIds }: { datasetIds: strin
});
// delete related files
await Promise.all(datasetIds.map((id) => delDatasetFiles({ datasetId: id })));
await Promise.all(
datasetIds.map((id) => delFileByMetadata({ bucketName: BucketNameEnum.dataset, datasetId: id }))
);
await delay(1000);
await delay(500);
// delete pg data
await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`);
@@ -49,17 +51,16 @@ export async function delCollectionRelevantData({
collectionId: { $in: collectionIds }
});
// delete file
await Promise.all(
filterFileIds.map((fileId) => {
return delFileById({
// delete file and imgs
await Promise.all([
delImgByFileIdList(filterFileIds),
delFileByFileIdList({
bucketName: BucketNameEnum.dataset,
fileId
});
fileIdList: filterFileIds
})
);
]);
await delay(1000);
await delay(500);
// delete pg data
await deletePgDataById(`collection_id IN ('${collectionIds.join("','")}')`);

9
packages/service/core/dataset/file/controller.ts
View File

@@ -1,9 +0,0 @@
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { getGFSCollection } from '../../../common/file/gridfs/controller';
export async function delDatasetFiles({ datasetId }: { datasetId: string }) {
const db = getGFSCollection(BucketNameEnum.dataset);
await db.deleteMany({
'metadata.datasetId': String(datasetId)
});
}

2
packages/service/support/permission/auth/common.ts
View File

@@ -12,7 +12,7 @@ export const authCert = async (props: AuthModeType) => {
canWrite: true
};
};
export async function authCertAndShareId({
export async function authCertOrShareId({
shareId,
...props
}: AuthModeType & { shareId?: string }) {

5
packages/service/support/permission/auth/dataset.ts
View File

@@ -14,6 +14,7 @@ import {
import { getFileById } from '../../../common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { getTeamInfoByTmbId } from '../../user/team/controller';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
export async function authDatasetByTmbId({
teamId,
@@ -167,6 +168,10 @@ export async function authDatasetFile({
const file = await getFileById({ bucketName: BucketNameEnum.dataset, fileId });
if (!file) {
return Promise.reject(CommonErrEnum.fileNotFound);
}
if (file.metadata.teamId !== teamId) {
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
}

24
projects/app/public/locales/en/common.json
View File

@@ -283,6 +283,11 @@
"Speaking": "I'm listening...",
"Stop Speak": "Stop Speak",
"Type a message": "Input problem",
"markdown": {
"Edit Question": "Edit Question",
"Quick Question": "Ask the question immediately",
"Send Question": "Send Question"
},
"quote": {
"Quote Tip": "Only the actual reference content is displayed here. If the data is updated, it will not be updated in real time",
"Read Quote": "Read Quote",
@@ -290,6 +295,9 @@
},
"tts": {
"Stop Speech": "Stop"
},
"error": {
"Messages empty": "Interface content is empty, maybe the text is too long ~"
}
},
"dataset": {
@@ -313,7 +321,6 @@
"Name": "Name",
"Quote Length": "Quote Length",
"Read Dataset": "Read Dataset",
"Search Top K": "Top K",
"Set Empty Result Tip": ",Response empty text",
"Set Website Config": "Configuring Website",
"Similarity": "Similarity",
@@ -372,6 +379,12 @@
"id": "Data ID"
},
"error": {
"unAuthDataset": "No access to this knowledge base ",
"unAuthDatasetCollection": "Not authorized to manipulate this data set ",
"unAuthDatasetData": "Not authorized to manipulate this data ",
"unAuthDatasetFile": "No permission to manipulate this file ",
"unCreateCollection": "No permission to manipulate this data ",
"unLinkCollection": "not a network link collection ",
"Start Sync Failed": "Start Sync Failed"
},
"file": "File",
@@ -403,8 +416,11 @@
},
"link": "Link",
"search": {
"Dataset Search Params": "Dataset Search Params",
"Empty result response": "Empty Response",
"Empty result response Tips": "If you fill in the content, if no suitable content is found, you will directly reply to the content.",
"Max Tokens": "Max Tokens",
"Max Tokens Tips": "The maximum number of Tokens in a single search, about 1 word in Chinese =1.7Tokens, about 1 word in English =1 tokens",
"Min Similarity": "Min Similarity",
"Min Similarity Tips": "The similarity of different index models is different, please use the search test to select the appropriate value",
"Params Setting": "Params Setting",
@@ -517,7 +533,6 @@
}
},
"dataset": {
"Chunk Length": "Chunk Length",
"Confirm move the folder": "Confirm Move",
"Confirm to delete the data": "Confirm to delete the data?",
"Confirm to delete the file": "Are you sure to delete the file and all its data?",
@@ -585,13 +600,10 @@
"import csv tip": "Ensure that the CSV is in UTF-8 format; otherwise, garbled characters will be displayed",
"test": {
"noResult": "Search results are empty"
},
"website": {
"Base Url": "BaseUrl",
"Selector": "Selector"
}
},
"error": {
"fileNotFound": "File not found ~",
"team": {
"overSize": "Team member exceeds limit"
}

26
projects/app/public/locales/zh/common.json
View File

@@ -283,6 +283,11 @@
"Speaking": "我在听，请说...",
"Stop Speak": "停止录音",
"Type a message": "输入问题",
"markdown": {
"Edit Question": "编辑问题",
"Quick Question": "点我立即提问",
"Send Question": "发送问题"
},
"quote": {
"Quote Tip": "此处仅显示实际引用内容，若数据有更新，此处不会实时更新",
"Read Quote": "查看引用",
@@ -290,6 +295,9 @@
},
"tts": {
"Stop Speech": "停止"
},
"error": {
"Messages empty": "接口内容为空，可能文本超长了~"
}
},
"dataset": {
@@ -313,7 +321,6 @@
"Name": "知识库名称",
"Quote Length": "引用内容长度",
"Read Dataset": "查看知识库详情",
"Search Top K": "单次搜索数量",
"Set Empty Result Tip": ",未搜索到内容时回复指定内容",
"Set Website Config": "开始配置网站信息",
"Similarity": "相关度",
@@ -372,7 +379,13 @@
"id": "数据ID"
},
"error": {
"Start Sync Failed": "开始同步失败"
"Start Sync Failed": "开始同步失败",
"unAuthDataset": "无权操作该知识库",
"unAuthDatasetCollection": "无权操作该数据集",
"unAuthDatasetData": "无权操作该数据",
"unAuthDatasetFile": "无权操作该文件",
"unCreateCollection": "无权操作该数据",
"unLinkCollection": "不是网络链接集合"
},
"file": "文件",
"folder": "目录",
@@ -403,8 +416,11 @@
},
"link": "链接",
"search": {
"Dataset Search Params": "搜索参数",
"Empty result response": "空搜索回复",
"Empty result response Tips": "若填写该内容，没有搜索到合适内容时，将直接回复填写的内容。",
"Max Tokens": "引用上限",
"Max Tokens Tips": "单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens",
"Min Similarity": "最低相关度",
"Min Similarity Tips": "不同索引模型的相关度有区别，请通过搜索测试来选择合适的数值，使用 ReRank 时，相关度可能会很低。",
"Params Setting": "搜索参数设置",
@@ -517,7 +533,6 @@
}
},
"dataset": {
"Chunk Length": "数据总量",
"Confirm move the folder": "确认移动到该目录",
"Confirm to delete the data": "确认删除该数据？",
"Confirm to delete the file": "确认删除该文件及其所有数据？",
@@ -585,13 +600,10 @@
"import csv tip": "请确保CSV为UTF-8格式，否则会乱码",
"test": {
"noResult": "搜索结果为空"
},
"website": {
"Base Url": "",
"Selector": ""
}
},
"error": {
"fileNotFound": "文件找不到了~",
"team": {
"overSize": "团队成员超出上限"
}

17
projects/app/src/components/ChatBox/MessageInput.tsx
View File

@@ -69,7 +69,8 @@ const MessageInput = ({
maxCount: 10
});
const uploadFile = async (file: FileItemType) => {
const uploadFile = useCallback(
async (file: FileItemType) => {
if (file.type === FileTypeEnum.image) {
try {
const src = await compressImgFileAndUpload({
@@ -78,7 +79,8 @@ const MessageInput = ({
maxH: 4329,
maxSize: 1024 * 1024 * 5,
// 30 day expired.
expiredTime: addDays(new Date(), 30)
expiredTime: addDays(new Date(), 30),
shareId
});
setFileList((state) =>
state.map((item) =>
@@ -100,8 +102,11 @@ const MessageInput = ({
});
}
}
};
const onSelectFile = useCallback(async (files: File[]) => {
},
[shareId, t, toast]
);
const onSelectFile = useCallback(
async (files: File[]) => {
if (!files || files.length === 0) {
return;
}
@@ -140,7 +145,9 @@ const MessageInput = ({
);
setFileList((state) => [...state, ...loadFiles]);
}, []);
},
[uploadFile]
);
const handleSend = useCallback(async () => {
const textareaValue = TextareaDom.current?.value || '';

44
projects/app/src/components/ChatBox/WholeResponseModal.tsx
View File

@@ -12,12 +12,21 @@ import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import Markdown from '../Markdown';
import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constant';
function Row({ label, value }: { label: string; value?: string | number }) {
function Row({
label,
value,
rawDom
}: {
label: string;
value?: string | number;
rawDom?: React.ReactNode;
}) {
const theme = useTheme();
const val = value || rawDom;
const strValue = `${value}`;
const isCodeBlock = strValue.startsWith('~~~json');
return value !== undefined && value !== '' && value !== 'undefined' ? (
return val !== undefined && val !== '' && val !== 'undefined' ? (
<Box mb={3}>
<Box fontSize={['sm', 'md']} mb={isCodeBlock ? 0 : 1} flex={'0 0 90px'}>
{label}:
@@ -29,7 +38,8 @@ function Row({ label, value }: { label: string; value?: string | number }) {
? { transform: 'translateY(-3px)' }
: { px: 3, py: 1, border: theme.borders.base })}
>
<Markdown source={strValue} />
{value && <Markdown source={strValue} />}
{rawDom}
</Box>
</Box>
) : null;
@@ -113,12 +123,28 @@ const WholeResponseModal = ({
<Row label={t('chat.response.module maxToken')} value={activeModule?.maxToken} />
<Row
label={t('chat.response.module historyPreview')}
value={(() => {
if (!activeModule?.historyPreview) return '';
return activeModule.historyPreview
.map((item, i) => `**${item.obj}**\n${item.value}`)
.join('\n\n---\n\n');
})()}
rawDom={
activeModule.historyPreview ? (
<>
{activeModule.historyPreview?.map((item, i) => (
<Box
key={i}
_notLast={{
borderBottom: '1px solid',
borderBottomColor: 'myWhite.700',
mb: 2
}}
pb={2}
>
<Box fontWeight={'bold'}>{item.obj}</Box>
<Box whiteSpace={'pre-wrap'}>{item.value}</Box>
</Box>
))}
</>
) : (
''
)
}
/>
{activeModule.quoteList && activeModule.quoteList.length > 0 && (
<Row

75
projects/app/src/components/ChatBox/index.tsx
View File

@@ -30,7 +30,7 @@ import {
Textarea
} from '@chakra-ui/react';
import { feConfigs } from '@/web/common/system/staticData';
import { eventBus } from '@/web/common/utils/eventbus';
import { EventNameEnum, eventBus } from '@/web/common/utils/eventbus';
import { adaptChat2GptMessages } from '@fastgpt/global/core/chat/adapt';
import { useMarkdown } from '@/web/common/hooks/useMarkdown';
import { ModuleItemType } from '@fastgpt/global/core/module/type.d';
@@ -48,7 +48,7 @@ import type { AdminMarkType } from './SelectMarkCollection';
import MyIcon from '@/components/Icon';
import Avatar from '@/components/Avatar';
import Markdown from '@/components/Markdown';
import Markdown, { CodeClassName } from '@/components/Markdown';
import MySelect from '@/components/Select';
import MyTooltip from '../MyTooltip';
import ChatBoxDivider from '@/components/core/chat/Divider';
@@ -64,6 +64,7 @@ import { splitGuideModule } from '@fastgpt/global/core/module/utils';
import type { AppTTSConfigType } from '@fastgpt/global/core/module/type.d';
import MessageInput from './MessageInput';
import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
@@ -132,6 +133,7 @@ const ChatBox = (
const ChatBoxRef = useRef<HTMLDivElement>(null);
const theme = useTheme();
const router = useRouter();
const { shareId } = router.query as { shareId?: string };
const { t } = useTranslation();
const { toast } = useToast();
const { isPc, setLoading } = useSystemStore();
@@ -258,7 +260,7 @@ const ChatBox = (
const result = await postQuestionGuide(
{
messages: adaptChat2GptMessages({ messages: history, reserveId: false }).slice(-6),
shareId: router.query.shareId as string
shareId
},
abortSignal
);
@@ -270,7 +272,7 @@ const ChatBox = (
}
} catch (error) {}
},
[questionGuide, scrollToBottom, router.query.shareId]
[questionGuide, scrollToBottom, shareId]
);
/**
@@ -323,7 +325,6 @@ const ChatBox = (
setTimeout(() => {
scrollToBottom();
}, 100);
try {
// create abort obj
const abortSignal = new AbortController();
@@ -518,16 +519,22 @@ const ChatBox = (
}
};
window.addEventListener('message', windowMessage);
eventBus.on('guideClick', ({ text }: { text: string }) => {
eventBus.on(EventNameEnum.sendQuestion, ({ text }: { text: string }) => {
if (!text) return;
handleSubmit((data) => sendPrompt(data, text))();
});
eventBus.on(EventNameEnum.editQuestion, ({ text }: { text: string }) => {
if (!text) return;
resetInputVal(text);
});
return () => {
eventBus.off('guideClick');
eventBus.off(EventNameEnum.sendQuestion);
eventBus.off(EventNameEnum.editQuestion);
window.removeEventListener('message', windowMessage);
};
}, [handleSubmit, sendPrompt]);
}, [handleSubmit, resetInputVal, sendPrompt]);
return (
<Flex flexDirection={'column'} h={'100%'}>
@@ -757,40 +764,30 @@ const ChatBox = (
<Box textAlign={'left'} mt={['6px', 2]}>
<Card bg={'white'} {...MessageCardStyle}>
<Markdown
source={item.value}
source={(() => {
const text = item.value as string;
// replace quote tag: [source1] 标识第一个来源，需要提取数字1，从而去数组里查找来源
const quoteReg = /\[source:(.+)\]/g;
const replaceText = text.replace(quoteReg, `[QUOTE SIGN]($1)`);
// question guide
if (
index === chatHistory.length - 1 &&
!isChatting &&
questionGuides.length > 0
) {
return `${replaceText}\n\`\`\`${
CodeClassName.questionGuide
}\n${JSON.stringify(questionGuides)}`;
}
return replaceText;
})()}
isChatting={index === chatHistory.length - 1 && isChatting}
/>
<ResponseTags responseData={item.responseData} />
{/* question guide */}
{index === chatHistory.length - 1 &&
!isChatting &&
questionGuides.length > 0 && (
<Box mt={2}>
<ChatBoxDivider
icon="core/chat/QGFill"
text={t('chat.Question Guide Tips')}
/>
<Flex alignItems={'center'} flexWrap={'wrap'} gap={2}>
{questionGuides.map((item) => (
<Button
key={item}
borderRadius={'md'}
variant={'outline'}
colorScheme={'gray'}
size={'xs'}
whiteSpace={'pre-wrap'}
h={'auto'}
py={1}
onClick={() => {
resetInputVal(item);
}}
>
{item}
</Button>
))}
</Flex>
</Box>
)}
{/* admin mark content */}
{showMarkIcon && item.adminFeedback && (
<Box>

11
projects/app/src/components/Icon/icons/core/chat/quoteSign.svg Normal file
View File

@@ -0,0 +1,11 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701930523211"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="9398"
xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128">
<path
d="M742.4 179.2H281.6a128 128 0 0 0-128 128v596.7104L298.496 768H742.4a128 128 0 0 0 128-128V307.2a128 128 0 0 0-128-128zM281.6 230.4h460.8a76.8 76.8 0 0 1 76.8 76.8v332.8a76.8 76.8 0 0 1-76.8 76.8H278.272L204.8 785.664V307.2a76.8 76.8 0 0 1 76.8-76.8z"
p-id="9399"></path>
<path
d="M534.016 525.2608v-0.1792c0-66.9952 36.3008-129.1008 95.6416-163.6352a22.3744 22.3744 0 0 1 30.1312 7.2192 20.8384 20.8384 0 0 1-7.424 29.1584 150.7584 150.7584 0 0 0-59.8528 64.0768c27.0848-2.9184 53.248 10.624 65.7664 34.048a62.0288 62.0288 0 0 1-9.3952 71.6032 67.328 67.328 0 0 1-72.4992 17.0752c-25.472-9.3696-42.3424-32.9984-42.368-59.392z m-175.616 0v-0.1792c0-66.9952 36.3008-129.1008 95.6416-163.6352a22.3744 22.3744 0 0 1 30.1568 7.2192 20.8384 20.8384 0 0 1-7.4496 29.1584 150.7584 150.7584 0 0 0-59.8528 64.0768c27.1104-2.9184 53.248 10.624 65.792 34.048a62.0288 62.0288 0 0 1-9.4208 71.6032 67.328 67.328 0 0 1-72.4736 17.0752c-25.4976-9.3696-42.3424-32.9984-42.3936-59.392z"
p-id="9400"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.3 KiB

8
projects/app/src/components/Icon/icons/core/chat/sendLight.svg Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701927696489"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5076"
xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128">
<path
d="M1015.485274 476.463913c-7.599113-15.198226-20.197642-27.796755-35.395868-35.395868L114.790411 8.418547c-18.997782-9.498891-40.495273-10.998716-60.592927-4.299498S17.901721 25.01661 8.40283 43.914404C-1.496015 63.812081-2.695875 87.009374 5.303191 107.806946l155.381863 404.252812L5.303191 916.212582c-7.599113 19.797689-6.999183 41.395168 1.599814 60.692915 8.598996 19.397736 24.297164 34.196008 43.994864 41.795122 9.198926 3.499591 18.797806 5.299381 28.496674 5.299381 12.198576 0 24.397152-2.799673 35.495856-8.299031L980.089406 582.951483c39.095436-19.597712 54.993581-67.392133 35.395868-106.48757zM79.094578 944.509279l151.182353-392.954131h310.363771c21.797456 0 39.49539-17.697934 39.49539-39.49539s-17.697934-39.49539-39.49539-39.49539H230.276931L79.494531 79.210284l865.199007 432.649497L79.094578 944.509279z"
p-id="5077"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

2
projects/app/src/components/Icon/index.tsx
View File

@@ -103,6 +103,8 @@ const iconPaths = {
'core/app/tts': () => import('./icons/core/app/tts.svg'),
'core/app/headphones': () => import('./icons/core/app/headphones.svg'),
'common/playLight': () => import('./icons/common/playLight.svg'),
'core/chat/quoteSign': () => import('./icons/core/chat/quoteSign.svg'),
'core/chat/sendLight': () => import('./icons/core/chat/sendLight.svg'),
'core/chat/sendFill': () => import('./icons/core/chat/sendFill.svg'),
'core/chat/recordFill': () => import('./icons/core/chat/recordFill.svg'),
'core/chat/stopSpeechFill': () => import('./icons/core/chat/stopSpeechFill.svg'),

2
projects/app/src/components/Markdown/CodeLight.tsx
View File

@@ -315,7 +315,7 @@ const CodeLight = ({
</Flex>
</Flex>
<SyntaxHighlighter style={codeLight as any} language={match?.[1]} PreTag="pre">
{String(children)}
{String(children).replace(/&nbsp;/g, ' ')}
</SyntaxHighlighter>
</Box>
);

4
projects/app/src/components/Markdown/chat/Guide.tsx
View File

@@ -5,7 +5,7 @@ import RemarkGfm from 'remark-gfm';
import RemarkMath from 'remark-math';
import RehypeKatex from 'rehype-katex';
import RemarkBreaks from 'remark-breaks';
import { eventBus } from '@/web/common/utils/eventbus';
import { EventNameEnum, eventBus } from '@/web/common/utils/eventbus';
import 'katex/dist/katex.min.css';
import styles from '../index.module.scss';
@@ -27,7 +27,7 @@ function MyLink(e: any) {
textDecoration={'underline'}
cursor={'pointer'}
onClick={() => {
eventBus.emit('guideClick', { text });
eventBus.emit(EventNameEnum.sendQuestion, { text });
}}
>
{text}

92
projects/app/src/components/Markdown/chat/QuestionGuide.tsx Normal file
View File

@@ -0,0 +1,92 @@
import React, { useMemo } from 'react';
import { Box, Flex, useTheme } from '@chakra-ui/react';
import 'katex/dist/katex.min.css';
import ChatBoxDivider from '@/components/core/chat/Divider';
import { useTranslation } from 'next-i18next';
import { EventNameEnum, eventBus } from '@/web/common/utils/eventbus';
import MyTooltip from '@/components/MyTooltip';
import MyIcon from '@/components/Icon';
const QuestionGuide = ({ text }: { text: string }) => {
const theme = useTheme();
const { t } = useTranslation();
const questionGuides = useMemo(() => {
try {
const json = JSON.parse(text);
if (Array.isArray(json) && !json.find((item) => typeof item !== 'string')) {
return json as string[];
}
return [];
} catch (error) {
return [];
}
}, [text]);
return questionGuides.length > 0 ? (
<Box mt={2}>
<ChatBoxDivider icon="core/chat/QGFill" text={t('chat.Question Guide Tips')} />
<Flex alignItems={'center'} flexWrap={'wrap'} gap={2}>
{questionGuides.map((text) => (
<Flex
key={text}
alignItems={'center'}
flexWrap={'wrap'}
fontSize={'sm'}
border={theme.borders.sm}
py={'1px'}
px={3}
borderRadius={'md'}
_hover={{
'.controller': {
display: 'flex'
}
}}
overflow={'hidden'}
position={'relative'}
>
<Box className="textEllipsis" flex={'1 0 0'}>
{text}
</Box>
<Box
className="controller"
display={['flex', 'none']}
pr={2}
position={'absolute'}
right={0}
left={0}
justifyContent={'flex-end'}
alignItems={'center'}
h={'100%'}
lineHeight={0}
bg={`linear-gradient(to left, white,white min(60px,100%),rgba(255,255,255,0) 80%)`}
>
<MyTooltip label={t('core.chat.markdown.Edit Question')}>
<MyIcon
name={'edit'}
w={'14px'}
cursor={'pointer'}
_hover={{
color: 'green.600'
}}
onClick={() => eventBus.emit(EventNameEnum.editQuestion, { text })}
/>
</MyTooltip>
<MyTooltip label={t('core.chat.markdown.Send Question')}>
<MyIcon
ml={4}
name={'core/chat/sendLight'}
w={'14px'}
cursor={'pointer'}
_hover={{ color: 'myBlue.600' }}
onClick={() => eventBus.emit(EventNameEnum.sendQuestion, { text })}
/>
</MyTooltip>
</Box>
</Flex>
))}
</Flex>
</Box>
) : null;
};
export default React.memo(QuestionGuide);

64
projects/app/src/components/Markdown/chat/Quote.tsx
View File

@@ -1,64 +0,0 @@
import React, { useMemo } from 'react';
import { Box, useTheme } from '@chakra-ui/react';
import { getFileAndOpen } from '@/web/core/dataset/utils';
import { useToast } from '@/web/common/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
type QuoteItemType = {
file_id?: string;
filename: string;
};
const QuoteBlock = ({ code }: { code: string }) => {
const theme = useTheme();
const { toast } = useToast();
const quoteList = useMemo(() => {
try {
return JSON.parse(code) as QuoteItemType[];
} catch (error) {
return [];
}
}, [code]);
return (
<Box mt={3} pt={2} borderTop={theme.borders.base}>
{quoteList.length > 0 ? (
<>
<Box>本次回答的引用:</Box>
<Box as={'ol'}>
{quoteList.map((item, i) => (
<Box
key={i}
as={'li'}
{...(item.file_id
? {
textDecoration: 'underline',
color: 'myBlue.800',
cursor: 'pointer'
}
: {})}
onClick={async () => {
if (!item.file_id) return;
try {
await getFileAndOpen(item.file_id);
} catch (error) {
toast({
status: 'warning',
title: getErrText(error, '打开文件失败')
});
}
}}
>
{item.filename}
</Box>
))}
</Box>
</>
) : (
<Box>正在生成引用……</Box>
)}
</Box>
);
};
export default QuoteBlock;

2
projects/app/src/components/Markdown/img/Image.tsx
View File

@@ -46,7 +46,7 @@ const MdImage = ({ src }: { src?: string }) => {
/>
<Modal isOpen={isOpen} onClose={onClose} isCentered>
<ModalOverlay />
<ModalContent maxW={'auto'} w="auto" bg={'transparent'}>
<ModalContent boxShadow={'none'} maxW={'auto'} w="auto" bg={'transparent'}>
<Image
borderRadius={'md'}
src={src}

73
projects/app/src/components/Markdown/index.tsx
View File

@@ -9,17 +9,26 @@ import 'katex/dist/katex.min.css';
import styles from './index.module.scss';
import dynamic from 'next/dynamic';
import CodeLight from './CodeLight';
import { Link, Button } from '@chakra-ui/react';
import MyTooltip from '../MyTooltip';
import { useTranslation } from 'next-i18next';
import { EventNameEnum, eventBus } from '@/web/common/utils/eventbus';
import MyIcon from '../Icon';
import { getFileAndOpen } from '@/web/core/dataset/utils';
import { MARKDOWN_QUOTE_SIGN } from '@fastgpt/global/core/chat/constants';
const CodeLight = dynamic(() => import('./CodeLight'));
const MermaidCodeBlock = dynamic(() => import('./img/MermaidCodeBlock'));
const MdImage = dynamic(() => import('./img/Image'));
const ChatGuide = dynamic(() => import('./chat/Guide'));
const EChartsCodeBlock = dynamic(() => import('./img/EChartsCodeBlock'));
const QuoteBlock = dynamic(() => import('./chat/Quote'));
const ChatGuide = dynamic(() => import('./chat/Guide'));
const QuestionGuide = dynamic(() => import('./chat/QuestionGuide'));
const ImageBlock = dynamic(() => import('./chat/Image'));
export enum CodeClassName {
guide = 'guide',
questionGuide = 'questionGuide',
mermaid = 'mermaid',
echarts = 'echarts',
quote = 'quote',
@@ -37,12 +46,12 @@ function Code({ inline, className, children }: any) {
if (codeType === CodeClassName.guide) {
return <ChatGuide text={String(children)} />;
}
if (codeType === CodeClassName.questionGuide) {
return <QuestionGuide text={String(children)} />;
}
if (codeType === CodeClassName.echarts) {
return <EChartsCodeBlock code={String(children)} />;
}
if (codeType === CodeClassName.quote) {
return <QuoteBlock code={String(children)} />;
}
if (codeType === CodeClassName.img) {
return <ImageBlock images={String(children)} />;
}
@@ -55,6 +64,52 @@ function Code({ inline, className, children }: any) {
function Image({ src }: { src?: string }) {
return <MdImage src={src} />;
}
function A({ children, ...props }: any) {
const { t } = useTranslation();
// empty href link
if (!props.href && typeof children?.[0] === 'string') {
const text = useMemo(() => String(children), [children]);
return (
<MyTooltip label={t('core.chat.markdown.Quick Question')}>
<Button
variant={'base'}
size={'xs'}
borderRadius={'md'}
my={1}
onClick={() => eventBus.emit(EventNameEnum.sendQuestion, { text })}
>
{text}
</Button>
</MyTooltip>
);
}
// quote link
if (children?.length === 1 && typeof children?.[0] === 'string') {
const text = String(children);
if (text === MARKDOWN_QUOTE_SIGN && props.href) {
return (
<MyTooltip label={props.href}>
<MyIcon
name={'core/chat/quoteSign'}
transform={'translateY(-2px)'}
w={'18px'}
color={'myBlue.600'}
cursor={'pointer'}
_hover={{
color: 'myBlue.800'
}}
onClick={() => getFileAndOpen(props.href)}
/>
</MyTooltip>
);
}
}
return <Link {...props}>{children}</Link>;
}
const Markdown = ({ source, isChatting = false }: { source: string; isChatting?: boolean }) => {
const components = useMemo(
@@ -62,14 +117,16 @@ const Markdown = ({ source, isChatting = false }: { source: string; isChatting?:
img: Image,
pre: 'div',
p: 'div',
code: Code
code: Code,
a: A
}),
[]
);
const formatSource = source
.replace(/\\n/g, '\n&nbsp;')
.replace(/(http[s]?:\/\/[^\s，。]+)([。，])/g, '$1 $2');
.replace(/(http[s]?:\/\/[^\s，。]+)([。，])/g, '$1 $2')
.replace(/\n*(\[QUOTE SIGN\]\(.*\))/g, '$1');
return (
<ReactMarkdown

2
projects/app/src/components/common/MyRadio/index.tsx
View File

@@ -82,7 +82,7 @@ const MyRadio = ({
<Box pr={2}>
<Box>{t(item.title)}</Box>
{!!item.desc && (
<Box fontSize={'sm'} color={'myGray.500'}>
<Box fontSize={['xs', 'sm']} color={'myGray.500'}>
{t(item.desc)}
</Box>
)}

26
projects/app/src/components/core/module/DatasetParamsModal.tsx
View File

@@ -18,6 +18,7 @@ type DatasetParamsProps = {
limit?: number;
searchMode: `${DatasetSearchModeEnum}`;
searchEmptyText?: string;
maxTokens?: number;
};
const DatasetParamsModal = ({
@@ -25,6 +26,7 @@ const DatasetParamsModal = ({
limit,
similarity,
searchMode = DatasetSearchModeEnum.embedding,
maxTokens = 3000,
onClose,
onSuccess
}: DatasetParamsProps & { onClose: () => void; onSuccess: (e: DatasetParamsProps) => void }) => {
@@ -52,8 +54,8 @@ const DatasetParamsModal = ({
isOpen={true}
onClose={onClose}
iconSrc="/imgs/modal/params.svg"
title={'搜索参数调整'}
minW={['90vw', '500px']}
title={t('core.dataset.search.Dataset Search Params')}
w={['90vw', '550px']}
h={['90vh', 'auto']}
overflow={'unset'}
isCentered={searchEmptyText !== undefined}
@@ -78,6 +80,7 @@ const DatasetParamsModal = ({
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box flex={1} mx={4}>
<MySlider
markList={[
{ label: '0', value: 0 },
@@ -93,21 +96,26 @@ const DatasetParamsModal = ({
}}
/>
</Box>
</Box>
)}
{limit !== undefined && (
<Box display={['block', 'flex']} py={8}>
<Box flex={'0 0 100px'} mb={[8, 0]}>
{t('core.dataset.search.Top K')}
{t('core.dataset.search.Max Tokens')}
<MyTooltip label={t('core.dataset.search.Max Tokens Tips')} forceShow>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box flex={1}>
<Box flex={1} mx={4}>
<MySlider
markList={[
{ label: '1', value: 1 },
{ label: '30', value: 30 }
{ label: '300', value: 300 },
{ label: maxTokens, value: maxTokens }
]}
min={1}
max={30}
value={getValues(ModuleInputKeyEnum.datasetLimit) ?? 5}
min={300}
max={maxTokens}
step={10}
value={getValues(ModuleInputKeyEnum.datasetLimit) ?? 1000}
onChange={(val) => {
setValue(ModuleInputKeyEnum.datasetLimit, val);
setRefresh(!refresh);

24
projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx
View File

@@ -17,7 +17,7 @@ import {
Grid,
Switch
} from '@chakra-ui/react';
import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import dynamic from 'next/dynamic';
import { onChangeNode, useFlowProviderStore } from '../../FlowProvider';
@@ -37,6 +37,7 @@ import { useQuery } from '@tanstack/react-query';
import type { EditFieldModeType, EditFieldType } from '../modules/FieldEditModal';
import { feConfigs } from '@/web/common/system/staticData';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants';
const FieldEditModal = dynamic(() => import('../modules/FieldEditModal'));
const SelectAppModal = dynamic(() => import('../../SelectAppModal'));
@@ -635,9 +636,12 @@ const SelectAppRender = React.memo(function SelectAppRender({ item, moduleId }:
});
const SelectDatasetParamsRender = React.memo(function SelectDatasetParamsRender({
item,
inputs = [],
moduleId
}: RenderProps) {
const { nodes } = useFlowProviderStore();
const { t } = useTranslation();
const [data, setData] = useState({
searchMode: DatasetSearchModeEnum.embedding,
@@ -645,6 +649,23 @@ const SelectDatasetParamsRender = React.memo(function SelectDatasetParamsRender(
similarity: 0.5
});
const tokenLimit = useMemo(() => {
let maxTokens = 3000;
nodes.forEach((item) => {
if (item.type === FlowNodeTypeEnum.chatNode) {
const model =
item.data.inputs.find((item) => item.key === ModuleInputKeyEnum.aiModel)?.value || '';
const quoteMaxToken =
chatModelList.find((item) => item.model === model)?.quoteMaxToken || 3000;
maxTokens = Math.max(maxTokens, quoteMaxToken);
}
});
return maxTokens;
}, [nodes]);
const { isOpen, onOpen, onClose } = useDisclosure();
useEffect(() => {
@@ -671,6 +692,7 @@ const SelectDatasetParamsRender = React.memo(function SelectDatasetParamsRender(
{isOpen && (
<DatasetParamsModal
{...data}
maxTokens={tokenLimit}
onClose={onClose}
onSuccess={(e) => {
for (let key in e) {

2
projects/app/src/constants/app.ts
View File

@@ -4,7 +4,7 @@ import type { OutLinkEditType } from '@fastgpt/global/support/outLink/type.d';
export const defaultApp: AppDetailType = {
_id: '',
userId: 'userId',
name: '模型加载中',
name: '应用加载中',
type: 'simple',
simpleTemplateId: 'fastgpt-universal',
avatar: '/icon/logo.svg',

2
projects/app/src/global/core/chat/api.d.ts vendored
View File

@@ -65,7 +65,7 @@ export type ClearHistoriesProps = {
/* -------- chat item ---------- */
export type DeleteChatItemProps = {
chatId: string;
contentId: string;
contentId?: string;
shareId?: string;
outLinkUid?: string;
};

106
projects/app/src/global/core/prompt/AIChat.ts
View File

@@ -4,22 +4,42 @@ export const Prompt_QuoteTemplateList: PromptTemplateItem[] = [
{
title: '标准模板',
desc: '标准提示词，用于结构不固定的知识库。',
value: `{{q}}\n{{a}}`
value: `<data>
{{q}}
{{a}}
</data>`
},
{
title: '问答模板',
desc: '适合 QA 问答结构的知识库，或大部分核心介绍位于 a 的知识库。',
value: `{instruction:"{{q}}",output:"{{a}}"}`
desc: '适合 QA 问答结构的知识库，可以让AI较为严格的按预设内容回答',
value: `<QA>
<问题>
{{q}}
</问题>
<答案>
{{a}}
</答案>
</QA>`
},
{
title: '标准严格模板',
desc: '在标准模板基础上，对模型的回答做更严格的要求。',
value: `{{q}}\n{{a}}`
value: `<data>
{{q}}
{{a}}
</data>`
},
{
title: '严格问答模板',
desc: '在问答模板基础上，对模型的回答做更严格的要求。',
value: `{question:"{{q}}",answer:"{{a}}"}`
value: `<QA>
<问题>
{{q}}
</问题>
<答案>
{{a}}
</答案>
</QA>`
}
];
@@ -27,54 +47,70 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [
{
title: '标准模板',
desc: '',
value: `你的知识库:
"""
value: `使用 <data></data> 标记中的内容作为你的知识:
{{quote}}
"""
回答要求：
1. 优先使用知识库内容回答问题。
2. 不要提及你是从知识库获取的知识。
3. 知识库包含 markdown 内容时，按 markdown 格式返回。
我的问题是:"{{question}}"`
- 如果你不清楚答案，你需要澄清。
- 避免提及你是从 data 获取的知识。
- 保持答案与 data 中描述的一致。
- 使用 Markdown 语法优化回答格式。
- 使用与问题相同的语言回答。
问题:"{{question}}"`
},
{
title: '问答模板',
desc: '',
value: `你的知识库:
"""
value: `使用 <QA></QA> 标记中的问答对进行回答。
{{quote}}
"""
回答要求：
1. 优先使用知识库内容回答问题，其中 instruction 是相关介绍，output 是预期回答或补充。
2. 不要提及你是从知识库获取的知识。
3. 知识库包含 markdown 内容时，按 markdown 格式返回。
我的问题是:"{{question}}"`
- 选择其中一个或多个问答对进行回答。
- 回答的内容应尽可能与 <答案></答案> 中的内容一致。
- 如果没有相关的问答对，你需要澄清。
- 避免提及你是从 QA 获取的知识，只需要回复答案。
问题:"{{question}}"`
},
{
title: '标准严格模板',
desc: '',
value: `你的知识库:
"""
value: `忘记你已有的知识，仅使用 <data></data> 标记中的内容作为你的知识:
{{quote}}
"""
思考流程：
1. 判断问题是否与 <data></data> 标记中的内容有关。
2. 如果有关，你按下面的要求回答。
3. 如果无关，你直接拒绝回答本次问题。
回答要求：
1. 仅使用知识库内容回答问题。
2. 与知识库无关的问题，你直接回答我不知道。
3. 不要提及你是从知识库获取的知识。
4. 知识库包含 markdown 内容时，按 markdown 格式返回。
我的问题是:"{{question}}"`
- 避免提及你是从 data 获取的知识。
- 保持答案与 data 中描述的一致。
- 使用 Markdown 语法优化回答格式。
- 使用与问题相同的语言回答。
问题:"{{question}}"`
},
{
title: '严格问答模板',
desc: '',
value: `你的知识库:
"""
value: `忘记你已有的知识，仅使用 <QA></QA> 标记中的问答对进行回答。
{{quote}}
"""
回答要求：
1. 从知识库中选择一个合适的答案进行回答，其中 instruction 是相关问题，answer 是已知答案。
2. 与知识库无关的问题，你直接回答我不知道。
3. 不要提及你是从知识库获取的知识。
我的问题是:"{{question}}"`
思考流程：
1. 判断问题是否与 <QA></QA> 标记中的内容有关。
2. 如果无关，你直接拒绝回答本次问题。
3. 判断是否有相近或相同的问题。
4. 如果有相同的问题，直接输出对应答案。
5. 如果只有相近的问题，请把相近的问题和答案一起输出。
最后，避免提及你是从 QA 获取的知识，只需要回复答案。
问题:"{{question}}"`
}
];

7
projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts
View File

@@ -2,7 +2,10 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { delFileById, getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
import {
delFileByFileIdList,
getGFSCollection
} from '@fastgpt/service/common/file/gridfs/controller';
import { addLog } from '@fastgpt/service/common/mongo/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { delay } from '@fastgpt/global/common/system/utils';
@@ -77,7 +80,7 @@ export async function checkFiles(start: Date, end: Date, limit: number) {
// 3. if not found, delete file
if (hasCollection === 0) {
await delFileById({ bucketName: 'dataset', fileId: String(_id) });
await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(_id)] });
console.log('delete file', _id);
deleteFileAmount++;
}

5
projects/app/src/pages/api/common/file/read.ts
View File

@@ -4,6 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authFileToken } from '@fastgpt/service/support/permission/controller';
import { detect } from 'jschardet';
import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -22,6 +23,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
getDownloadStream({ bucketName, fileId })
]);
if (!file) {
return Promise.reject(CommonErrEnum.fileNotFound);
}
// get encoding
let buffers: Buffer = Buffer.from([]);
for await (const chunk of encodeStream) {

13
projects/app/src/pages/api/common/file/uploadImage.ts
View File

@@ -1,21 +1,22 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common';
import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller';
type Props = { base64Img: string; expiredTime?: Date };
import { UploadImgProps } from '@fastgpt/global/common/file/api';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { teamId } = await authCert({ req, authToken: true });
const { base64Img, expiredTime } = req.body as Props;
const { base64Img, expiredTime, metadata, shareId } = req.body as UploadImgProps;
const { teamId } = await authCertOrShareId({ req, shareId, authToken: true });
const data = await uploadMongoImg({
teamId,
base64Img,
expiredTime
expiredTime,
metadata
});
jsonRes(res, { data });

5
projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts
View File

@@ -4,13 +4,14 @@ import { connectToDatabase } from '@/service/mongo';
import type { CreateQuestionGuideParams } from '@/global/core/ai/api.d';
import { pushQuestionGuideBill } from '@/service/support/wallet/bill/push';
import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
import { authCertAndShareId } from '@fastgpt/service/support/permission/auth/common';
import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { messages, shareId } = req.body as CreateQuestionGuideParams;
const { tmbId, teamId } = await authCertAndShareId({
const { tmbId, teamId } = await authCertOrShareId({
req,
authToken: true,
shareId

33
projects/app/src/pages/api/core/app/update.ts
View File

@@ -4,6 +4,9 @@ import { connectToDatabase } from '@/service/mongo';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import type { AppUpdateParams } from '@fastgpt/global/core/app/api';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants';
import { getChatModel } from '@/service/core/ai/model';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -20,6 +23,36 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
await authApp({ req, authToken: true, appId, per: permission ? 'owner' : 'w' });
// check modules
// 1. dataset search limit, less than model quoteMaxToken
if (modules) {
let maxTokens = 3000;
modules.forEach((item) => {
if (item.flowType === FlowNodeTypeEnum.chatNode) {
const model =
item.inputs.find((item) => item.key === ModuleInputKeyEnum.aiModel)?.value || '';
const chatModel = getChatModel(model);
const quoteMaxToken = chatModel.quoteMaxToken || 3000;
maxTokens = Math.max(maxTokens, quoteMaxToken);
}
});
modules.forEach((item) => {
if (item.flowType === FlowNodeTypeEnum.datasetSearchNode) {
item.inputs.forEach((input) => {
if (input.key === ModuleInputKeyEnum.datasetLimit) {
const val = input.value as number;
if (val > maxTokens) {
input.value = maxTokens;
}
}
});
}
});
}
// 更新模型
await MongoApp.updateOne(
{

4
projects/app/src/pages/api/core/chat/item/delete.ts
View File

@@ -10,6 +10,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await connectToDatabase();
const { chatId, contentId, shareId, outLinkUid } = req.query as DeleteChatItemProps;
if (!contentId || !chatId) {
return jsonRes(res);
}
await autChatCrud({
req,
authToken: true,

4
projects/app/src/pages/api/core/chat/item/getSpeech.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { GetChatSpeechProps } from '@/global/core/chat/api.d';
import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
import { pushAudioSpeechBill } from '@/service/support/wallet/bill/push';
import { authCertAndShareId } from '@fastgpt/service/support/permission/auth/common';
import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common';
import { authType2BillSource } from '@/service/support/wallet/bill/utils';
import { getAudioSpeechModel } from '@/service/core/ai/model';
import { MongoTTSBuffer } from '@fastgpt/service/common/buffer/tts/schema';
@@ -25,7 +25,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('model or voice not found');
}
const { teamId, tmbId, authType } = await authCertAndShareId({ req, authToken: true, shareId });
const { teamId, tmbId, authType } = await authCertOrShareId({ req, authToken: true, shareId });
const ttsModel = getAudioSpeechModel(ttsConfig.model);
const voiceData = ttsModel.voices?.find((item) => item.value === ttsConfig.voice);

4
projects/app/src/pages/api/core/dataset/collection/delete.ts
View File

@@ -24,13 +24,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// find all delete id
const collections = await findCollectionAndChild(collectionId, '_id metadata');
const collections = await findCollectionAndChild(collectionId, '_id fileId');
const delIdList = collections.map((item) => item._id);
// delete
await delCollectionRelevantData({
collectionIds: delIdList,
fileIds: collections.map((item) => item.metadata?.fileId).filter(Boolean)
fileIds: collections.map((item) => item?.fileId || '').filter(Boolean)
});
// delete collection

1
projects/app/src/pages/api/core/dataset/collection/sync/link.ts
View File

@@ -4,7 +4,6 @@ import { connectToDatabase } from '@/service/mongo';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { loadingOneChunkCollection } from '@fastgpt/service/core/dataset/collection/utils';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';

5
projects/app/src/pages/api/core/dataset/update.ts
View File

@@ -15,8 +15,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('缺少参数');
}
// 凭证校验
if (permission) {
await authDataset({ req, authToken: true, datasetId: id, per: 'owner' });
} else {
await authDataset({ req, authToken: true, datasetId: id, per: 'w' });
}
await MongoDataset.findOneAndUpdate(
{

14
projects/app/src/pages/app/detail/components/SimpleEdit/index.tsx
View File

@@ -21,7 +21,6 @@ import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d';
import { chatModelList, simpleModeTemplates } from '@/web/common/system/staticData';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import { chatNodeSystemPromptTip, welcomeTextTip } from '@fastgpt/global/core/module/template/tip';
import type { VariableItemType } from '@fastgpt/global/core/module/type.d';
import type { ModuleItemType } from '@fastgpt/global/core/module/type';
import { useRequest } from '@/web/common/hooks/useRequest';
import { useConfirm } from '@/web/common/hooks/useConfirm';
@@ -67,7 +66,6 @@ function ConfigForm({
}) {
const theme = useTheme();
const router = useRouter();
const { toast } = useToast();
const { t } = useTranslation();
const { appDetail, updateAppDetail } = useAppStore();
const { loadAllDatasets, allDatasets } = useDatasetStore();
@@ -124,6 +122,13 @@ function ConfigForm({
[getValues, refresh]
);
const tokenLimit = useMemo(() => {
return (
chatModelList.find((item) => item.model === getValues('aiSettings.model'))?.quoteMaxToken ||
3000
);
}, [getValues, refresh]);
const { mutate: onSubmitSave, isLoading: isSaving } = useRequest({
mutationFn: async (data: AppSimpleEditFormType) => {
const modules = await postForm2Modules(data, data.templateId);
@@ -361,8 +366,8 @@ function ConfigForm({
)}
</Flex>
<Flex mt={1} color={'myGray.600'} fontSize={['sm', 'md']}>
{t('core.dataset.Similarity')}: {getValues('dataset.similarity')},{' '}
{t('core.dataset.Search Top K')}: {getValues('dataset.limit')}
{t('core.dataset.search.Min Similarity')}: {getValues('dataset.similarity')},{' '}
{t('core.dataset.search.Max Tokens')}: {getValues('dataset.limit')}
{getValues('dataset.searchEmptyText') === ''
? ''
: t('core.dataset.Set Empty Result Tip')}
@@ -458,6 +463,7 @@ function ConfigForm({
{isOpenDatasetParams && (
<DatasetParamsModal
{...getValues('dataset')}
maxTokens={tokenLimit}
onClose={onCloseKbParams}
onSuccess={(e) => {
setValue('dataset', {

6
projects/app/src/pages/app/detail/index.tsx
View File

@@ -15,6 +15,7 @@ import Loading from '@/components/Loading';
import SimpleEdit from './components/SimpleEdit';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { useAppStore } from '@/web/core/app/store/useAppStore';
import Head from 'next/head';
const AdEdit = dynamic(() => import('./components/AdEdit'), {
loading: () => <Loading />
@@ -92,6 +93,10 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
});
return (
<>
<Head>
<title>{appDetail.name}</title>
</Head>
<PageContainer>
<Flex flexDirection={['column', 'row']} h={'100%'}>
{/* pc tab */}
@@ -176,6 +181,7 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
</Box>
</Flex>
</PageContainer>
</>
);
};

2
projects/app/src/pages/chat/index.tsx
View File

@@ -1,7 +1,7 @@
import React, { useCallback, useRef } from 'react';
import Head from 'next/head';
import { useRouter } from 'next/router';
import { getInitChatInfo, putChatHistory } from '@/web/core/chat/api';
import { getInitChatInfo } from '@/web/core/chat/api';
import {
Box,
Flex,

18
projects/app/src/pages/chat/share.tsx
View File

@@ -116,7 +116,9 @@ const OutLink = ({
updateHistory({
...currentChat,
updateTime: new Date(),
title: newTitle
title: newTitle,
shareId,
outLinkUid
});
}
@@ -148,7 +150,7 @@ const OutLink = ({
return { responseText, responseData, isNewChat: forbidRefresh.current };
},
[chatId, shareId, outLinkUid, setChatData, appId, updateHistory, router, histories]
[chatId, shareId, outLinkUid, setChatData, appId, pushHistory, router, histories, updateHistory]
);
const loadChatInfo = useCallback(
@@ -309,13 +311,19 @@ const OutLink = ({
});
}}
onSetHistoryTop={(e) => {
updateHistory(e);
updateHistory({
...e,
shareId,
outLinkUid
});
}}
onSetCustomTitle={async (e) => {
updateHistory({
chatId: e.chatId,
title: e.title,
customTitle: e.title
customTitle: e.title,
shareId,
outLinkUid
});
}}
/>
@@ -349,7 +357,7 @@ const OutLink = ({
feedbackType={'user'}
onUpdateVariable={(e) => {}}
onStartChat={startChat}
onDelMessage={(e) => delOneHistoryItem({ ...e, chatId })}
onDelMessage={(e) => delOneHistoryItem({ ...e, chatId, shareId, outLinkUid })}
/>
</Box>
</Flex>

4
projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
View File

@@ -173,7 +173,9 @@ const FileSelect = ({
case 'pdf':
return readPdfContent(file);
case 'docx':
return readDocContent(file);
return readDocContent(file, {
fileId
});
}
return '';
})();

2
projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
View File

@@ -408,7 +408,7 @@ export function RawSourceText({
await getFileAndOpen(sourceId as string);
} catch (error) {
toast({
title: getErrText(error, '获取文件地址失败'),
title: t(getErrText(error, 'error.fileNotFound')),
status: 'error'
});
}

1
projects/app/src/pages/login/provider.tsx
View File

@@ -9,7 +9,6 @@ import { oauthLogin } from '@/web/support/user/api';
import { useToast } from '@/web/common/hooks/useToast';
import Loading from '@/components/Loading';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { useQuery } from '@tanstack/react-query';
import { getErrText } from '@fastgpt/global/common/error/utils';
const provider = ({ code, state }: { code: string; state: string }) => {

62
projects/app/src/service/core/dataset/data/pg.ts
View File

@@ -11,6 +11,7 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { jiebaSplit } from '../utils';
import { reRankRecall } from '../../ai/rerank';
import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
export async function insertData2Pg({
mongoDataId,
@@ -108,38 +109,51 @@ type SearchProps = {
text: string;
model: string;
similarity?: number; // min distance
limit: number;
limit: number; // max Token limit
datasetIds: string[];
searchMode?: `${DatasetSearchModeEnum}`;
};
export async function searchDatasetData(props: SearchProps) {
let { text, similarity = 0, limit, searchMode = DatasetSearchModeEnum.embedding } = props;
let {
text,
similarity = 0,
limit: maxTokens,
searchMode = DatasetSearchModeEnum.embedding
} = props;
searchMode = global.systemEnv.pluginBaseUrl ? searchMode : DatasetSearchModeEnum.embedding;
// Compatible with topk limit
if (maxTokens < 50) {
maxTokens = 1500;
}
const rerank =
searchMode === DatasetSearchModeEnum.embeddingReRank ||
searchMode === DatasetSearchModeEnum.embFullTextReRank;
const oneChunkToken = 50;
const { embeddingLimit, fullTextLimit } = (() => {
// Increase search range, reduce hnsw loss
const estimatedLen = Math.max(20, Math.ceil(maxTokens / oneChunkToken));
// Increase search range, reduce hnsw loss. 20 ~ 100
if (searchMode === DatasetSearchModeEnum.embedding) {
return {
embeddingLimit: limit * 2,
embeddingLimit: Math.min(estimatedLen, 100),
fullTextLimit: 0
};
}
// 50 < 2*limit < value < 100
if (searchMode === DatasetSearchModeEnum.embeddingReRank) {
return {
embeddingLimit: Math.min(100, Math.max(50, limit * 2)),
embeddingLimit: Math.min(100, Math.max(50, estimatedLen * 2)),
fullTextLimit: 0
};
}
// 50 < 3*limit < embedding < 80
// 50 < 2*limit < embedding < 80
// 20 < limit < fullTextLimit < 40
return {
embeddingLimit: Math.min(80, Math.max(50, limit * 2)),
fullTextLimit: Math.min(40, Math.max(20, limit))
embeddingLimit: Math.min(80, Math.max(50, estimatedLen * 2)),
fullTextLimit: Math.min(40, Math.max(20, estimatedLen))
};
})();
@@ -174,9 +188,14 @@ export async function searchDatasetData(props: SearchProps) {
return true;
});
// token slice
if (!rerank) {
return {
searchRes: filterSameDataResults.filter((item) => item.score >= similarity).slice(0, limit),
searchRes: filterResultsByMaxTokens(
filterSameDataResults.filter((item) => item.score >= similarity),
maxTokens
),
tokenLen
};
}
@@ -190,7 +209,10 @@ export async function searchDatasetData(props: SearchProps) {
).filter((item) => item.score > similarity);
return {
searchRes: reRankResults.slice(0, limit),
searchRes: filterResultsByMaxTokens(
reRankResults.filter((item) => item.score >= similarity),
maxTokens
),
tokenLen
};
}
@@ -357,6 +379,8 @@ export async function reRankSearchResult({
}))
});
if (!Array.isArray(results)) return data;
// add new score to data
const mergeResult = results
.map((item) => {
@@ -376,4 +400,22 @@ export async function reRankSearchResult({
return data;
}
}
export function filterResultsByMaxTokens(list: SearchDataResponseItemType[], maxTokens: number) {
const results: SearchDataResponseItemType[] = [];
let totalTokens = 0;
for (let i = 0; i < list.length; i++) {
const item = list[i];
totalTokens += countPromptTokens(item.q + item.a);
if (totalTokens > maxTokens + 200) {
break;
}
results.push(item);
if (totalTokens > maxTokens) {
break;
}
}
return results;
}
// ------------------ search end ------------------

4
projects/app/src/service/moduleDispatch/chat/oneapi.ts
View File

@@ -124,6 +124,10 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
))
];
if (concatMessages.length === 0) {
return Promise.reject('core.chat.error.Messages empty');
}
const response = await ai.chat.completions.create(
{
model,

1
projects/app/src/utils/service/core/chat/index.ts
View File

@@ -10,7 +10,6 @@ export function selectShareResponse({
'moduleName',
'moduleLogo',
'runningTime',
'historyPreview',
'quoteList',
'question'
];

4
projects/app/src/web/common/file/api.ts
View File

@@ -1,8 +1,8 @@
import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
import type { UploadImgProps } from '@fastgpt/global/common/file/api.d';
import { AxiosProgressEvent } from 'axios';
export const postUploadImg = (base64Img: string, expiredTime?: Date) =>
POST<string>('/common/file/uploadImage', { base64Img, expiredTime });
export const postUploadImg = (e: UploadImgProps) => POST<string>('/common/file/uploadImage', e);
export const postUploadFiles = (
data: FormData,

34
projects/app/src/web/common/file/controller.ts
View File

@@ -1,4 +1,5 @@
import { postUploadImg, postUploadFiles } from '@/web/common/file/api';
import { UploadImgProps } from '@fastgpt/global/common/file/api';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
/**
@@ -34,23 +35,24 @@ export const uploadFiles = ({
* @param maxSize The max size of the compressed image
*/
export const compressBase64ImgAndUpload = ({
base64,
base64Img,
maxW = 1080,
maxH = 1080,
maxSize = 1024 * 500, // 300kb
expiredTime
}: {
base64: string;
expiredTime,
metadata,
shareId
}: UploadImgProps & {
maxW?: number;
maxH?: number;
maxSize?: number;
expiredTime?: Date;
}) => {
return new Promise<string>((resolve, reject) => {
const fileType = /^data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,/.exec(base64)?.[1] || 'image/jpeg';
const fileType =
/^data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,/.exec(base64Img)?.[1] || 'image/jpeg';
const img = new Image();
img.src = base64;
img.src = base64Img;
img.onload = async () => {
let width = img.width;
let height = img.height;
@@ -86,7 +88,12 @@ export const compressBase64ImgAndUpload = ({
}
try {
const src = await postUploadImg(compressedDataUrl, expiredTime);
const src = await postUploadImg({
shareId,
base64Img: compressedDataUrl,
expiredTime,
metadata
});
resolve(src);
} catch (error) {
reject(error);
@@ -100,18 +107,20 @@ export const compressImgFileAndUpload = async ({
maxW,
maxH,
maxSize,
expiredTime
expiredTime,
shareId
}: {
file: File;
maxW?: number;
maxH?: number;
maxSize?: number;
expiredTime?: Date;
shareId?: string;
}) => {
const reader = new FileReader();
reader.readAsDataURL(file);
const base64 = await new Promise<string>((resolve, reject) => {
const base64Img = await new Promise<string>((resolve, reject) => {
reader.onload = async () => {
resolve(reader.result as string);
};
@@ -122,10 +131,11 @@ export const compressImgFileAndUpload = async ({
});
return compressBase64ImgAndUpload({
base64,
base64Img,
maxW,
maxH,
maxSize,
expiredTime
expiredTime,
shareId
});
};

17
projects/app/src/web/common/file/utils.ts
View File

@@ -107,7 +107,7 @@ export const readPdfContent = (file: File) =>
/**
* read docx to markdown
*/
export const readDocContent = (file: File) =>
export const readDocContent = (file: File, metadata: Record<string, any>) =>
new Promise<string>((resolve, reject) => {
try {
const reader = new FileReader();
@@ -120,7 +120,7 @@ export const readDocContent = (file: File) =>
arrayBuffer: target.result as ArrayBuffer
});
const rawText = await formatMarkdown(res?.value);
const rawText = await formatMarkdown(res?.value, metadata);
resolve(rawText);
} catch (error) {
@@ -173,24 +173,25 @@ export const readCsvContent = async (file: File) => {
* 1. upload base64
* 2. replace \
*/
export const formatMarkdown = async (rawText: string = '') => {
export const formatMarkdown = async (rawText: string = '', metadata: Record<string, any>) => {
// match base64, upload and replace it
const base64Regex = /data:image\/.*;base64,([^\)]+)/g;
const base64Arr = rawText.match(base64Regex) || [];
// upload base64 and replace it
await Promise.all(
base64Arr.map(async (base64) => {
base64Arr.map(async (base64Img) => {
try {
const str = await compressBase64ImgAndUpload({
base64,
base64Img,
maxW: 4329,
maxH: 4329,
maxSize: 1024 * 1024 * 5
maxSize: 1024 * 1024 * 5,
metadata
});
rawText = rawText.replace(base64, str);
rawText = rawText.replace(base64Img, str);
} catch (error) {
rawText = rawText.replace(base64, '');
rawText = rawText.replace(base64Img, '');
rawText = rawText.replace(/!\[.*\]\(\)/g, '');
}
})

2
projects/app/src/web/common/hooks/useToast.ts
View File

@@ -4,7 +4,7 @@ export const useToast = (props?: UseToastOptions) => {
const toast = uToast({
position: 'top',
duration: 2000,
...props
...(props && props)
});
return {

3
projects/app/src/web/common/utils/eventbus.ts
View File

@@ -1,5 +1,6 @@
export enum EventNameEnum {
guideClick = 'guideClick',
sendQuestion = 'sendQuestion',
editQuestion = 'editQuestion',
updaterNode = 'updaterNode'
}
type EventNameType = `${EventNameEnum}`;

34
projects/app/src/web/core/app/templates.ts
View File

@@ -368,20 +368,7 @@ export const appTemplates: (AppItemType & {
type: 'slider',
label: '单次搜索上限',
description: '最多取 n 条记录作为本次问题引用',
value: 5,
min: 1,
max: 20,
step: 1,
markList: [
{
label: '1',
value: 1
},
{
label: '20',
value: 20
}
],
value: 1500,
connected: true
},
{
@@ -1418,22 +1405,9 @@ export const appTemplates: (AppItemType & {
{
key: 'limit',
type: 'slider',
label: '单次搜索上限',
description: '最多取 n 条记录作为本次问题引用',
value: 5,
min: 1,
max: 20,
step: 1,
markList: [
{
label: '1',
value: 1
},
{
label: '20',
value: 20
}
],
label: '引用上限',
description: '单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens',
value: 1500,
connected: true
},
{

10
projects/app/src/web/core/chat/storeChat.ts
View File

@@ -7,7 +7,8 @@ import type {
getHistoriesProps,
ClearHistoriesProps,
DelHistoryProps,
UpdateHistoryProps
UpdateHistoryProps,
DeleteChatItemProps
} from '@/global/core/chat/api';
import {
delChatHistoryById,
@@ -31,7 +32,7 @@ type State = {
setLastChatAppId: (id: string) => void;
lastChatId: string;
setLastChatId: (id: string) => void;
delOneHistoryItem: (e: { chatId: string; contentId?: string; index: number }) => Promise<any>;
delOneHistoryItem: (e: DeleteChatItemProps & { index: number }) => Promise<any>;
};
export const useChatStore = create<State>()(
@@ -119,7 +120,8 @@ export const useChatStore = create<State>()(
});
}
},
async delOneHistoryItem({ chatId, contentId, index }) {
async delOneHistoryItem({ index, ...props }) {
const { chatId, contentId } = props;
if (!chatId || !contentId) return;
try {
@@ -127,7 +129,7 @@ export const useChatStore = create<State>()(
...state,
history: state.history.filter((_, i) => i !== index)
}));
await delChatRecordById({ chatId, contentId });
await delChatRecordById(props);
} catch (err) {
console.log(err);
}