mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-25 06:14:06 +00:00
@@ -38,22 +38,22 @@ export const insertDatasetDataVector = async ({
|
||||
};
|
||||
};
|
||||
|
||||
// export const updateDatasetDataVector = async ({
|
||||
// id,
|
||||
// ...props
|
||||
// }: InsertVectorProps & {
|
||||
// id: string;
|
||||
// query: string;
|
||||
// model: VectorModelItemType;
|
||||
// }) => {
|
||||
// // insert new vector
|
||||
// const { charsLength, insertId } = await insertDatasetDataVector(props);
|
||||
export const updateDatasetDataVector = async ({
|
||||
id,
|
||||
...props
|
||||
}: InsertVectorProps & {
|
||||
id: string;
|
||||
query: string;
|
||||
model: VectorModelItemType;
|
||||
}) => {
|
||||
// insert new vector
|
||||
const { charsLength, insertId } = await insertDatasetDataVector(props);
|
||||
|
||||
// // delete old vector
|
||||
// await deleteDatasetDataVector({
|
||||
// teamId: props.teamId,
|
||||
// id
|
||||
// });
|
||||
// delete old vector
|
||||
await deleteDatasetDataVector({
|
||||
teamId: props.teamId,
|
||||
id
|
||||
});
|
||||
|
||||
// return { charsLength, insertId };
|
||||
// };
|
||||
return { charsLength, insertId };
|
||||
};
|
||||
|
@@ -25,18 +25,6 @@ export async function initPg() {
|
||||
await PgClient.query(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);`
|
||||
);
|
||||
await PgClient.query(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id);`
|
||||
);
|
||||
await PgClient.query(
|
||||
` CREATE INDEX CONCURRENTLY IF NOT EXISTS team_collection_index ON ${PgDatasetTableName} USING btree(team_id, collection_id);`
|
||||
);
|
||||
await PgClient.query(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS team_id_index ON ${PgDatasetTableName} USING btree(team_id, id);`
|
||||
);
|
||||
await PgClient.query(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${PgDatasetTableName} USING btree(createtime);`
|
||||
);
|
||||
|
||||
console.log('init pg successful');
|
||||
} catch (error) {
|
||||
|
@@ -11,7 +11,6 @@ export const getAIApi = (props?: {
|
||||
timeout?: number;
|
||||
}) => {
|
||||
const { userKey, timeout } = props || {};
|
||||
|
||||
return new OpenAI({
|
||||
apiKey: userKey?.key || systemAIChatKey,
|
||||
baseURL: userKey?.baseUrl || baseUrl,
|
||||
|
159
packages/service/core/ai/functions/cfr.ts
Normal file
159
packages/service/core/ai/functions/cfr.ts
Normal file
@@ -0,0 +1,159 @@
|
||||
import { replaceVariable } from '@fastgpt/global/common/string/tools';
|
||||
import { getAIApi } from '../config';
|
||||
import { ChatItemType } from '@fastgpt/global/core/chat/type';
|
||||
|
||||
/*
|
||||
cfr: coreference resolution - 指代消除
|
||||
可以根据上下文,完事当前问题指代内容,利于检索。
|
||||
*/
|
||||
|
||||
const defaultPrompt = `请不要回答任何问题。
|
||||
你的任务是结合历史记录,为当前问题,实现代词替换,确保问题描述的对象清晰明确。例如:
|
||||
历史记录:
|
||||
"""
|
||||
Q: 对话背景。
|
||||
A: 关于 FatGPT 的介绍和使用等问题。
|
||||
"""
|
||||
当前问题: 怎么下载
|
||||
输出: FastGPT 怎么下载?
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
Q: 报错 "no connection"
|
||||
A: FastGPT 报错"no connection"可能是因为……
|
||||
"""
|
||||
当前问题: 怎么解决
|
||||
输出: FastGPT 报错"no connection"如何解决?
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
Q: 作者是谁?
|
||||
A: FastGPT 的作者是 labring。
|
||||
"""
|
||||
当前问题: 介绍下他
|
||||
输出: 介绍下 FastGPT 的作者 labring。
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
Q: 作者是谁?
|
||||
A: FastGPT 的作者是 labring。
|
||||
"""
|
||||
当前问题: 我想购买商业版。
|
||||
输出: FastGPT 商业版如何购买?
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
Q: 对话背景。
|
||||
A: 关于 FatGPT 的介绍和使用等问题。
|
||||
"""
|
||||
当前问题: nh
|
||||
输出: nh
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
Q: FastGPT 如何收费?
|
||||
A: FastGPT 收费可以参考……
|
||||
"""
|
||||
当前问题: 你知道 laf 么?
|
||||
输出: 你知道 laf 么?
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
Q: FastGPT 的优势
|
||||
A: 1. 开源
|
||||
2. 简便
|
||||
3. 扩展性强
|
||||
"""
|
||||
当前问题: 介绍下第2点。
|
||||
输出: 介绍下 FastGPT 简便的优势。
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
Q: 什么是 FastGPT?
|
||||
A: FastGPT 是一个 RAG 平台。
|
||||
Q: 什么是 Sealos?
|
||||
A: Sealos 是一个云操作系统。
|
||||
"""
|
||||
当前问题: 它们有什么关系?
|
||||
输出: FastGPT 和 Sealos 有什么关系?
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
{{histories}}
|
||||
"""
|
||||
当前问题: {{query}}
|
||||
输出: `;
|
||||
|
||||
export const queryCfr = async ({
|
||||
chatBg,
|
||||
query,
|
||||
histories = [],
|
||||
model
|
||||
}: {
|
||||
chatBg?: string;
|
||||
query: string;
|
||||
histories: ChatItemType[];
|
||||
model: string;
|
||||
}) => {
|
||||
if (histories.length === 0 && !chatBg) {
|
||||
return {
|
||||
rawQuery: query,
|
||||
cfrQuery: query,
|
||||
model,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0
|
||||
};
|
||||
}
|
||||
|
||||
const systemFewShot = chatBg
|
||||
? `Q: 对话背景。
|
||||
A: ${chatBg}
|
||||
`
|
||||
: '';
|
||||
const historyFewShot = histories
|
||||
.map((item) => {
|
||||
const role = item.obj === 'Human' ? 'Q' : 'A';
|
||||
return `${role}: ${item.value}`;
|
||||
})
|
||||
.join('\n');
|
||||
const concatFewShot = `${systemFewShot}${historyFewShot}`.trim();
|
||||
|
||||
const ai = getAIApi({
|
||||
timeout: 480000
|
||||
});
|
||||
|
||||
const result = await ai.chat.completions.create({
|
||||
model: model,
|
||||
temperature: 0.01,
|
||||
max_tokens: 150,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: replaceVariable(defaultPrompt, {
|
||||
query: `${query}`,
|
||||
histories: concatFewShot
|
||||
})
|
||||
}
|
||||
],
|
||||
stream: false
|
||||
});
|
||||
|
||||
const answer = result.choices?.[0]?.message?.content || '';
|
||||
if (!answer) {
|
||||
return {
|
||||
rawQuery: query,
|
||||
cfrQuery: query,
|
||||
model,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
rawQuery: query,
|
||||
cfrQuery: answer,
|
||||
model,
|
||||
inputTokens: result.usage?.prompt_tokens || 0,
|
||||
outputTokens: result.usage?.completion_tokens || 0
|
||||
};
|
||||
};
|
@@ -1,6 +1,5 @@
|
||||
import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d';
|
||||
import { getAIApi } from '../config';
|
||||
import { countGptMessagesChars } from '../../chat/utils';
|
||||
|
||||
export const Prompt_QuestionGuide = `我不太清楚问你什么问题,请帮我生成 3 个问题,引导我继续提问。问题的长度应小于20个字符,按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
|
||||
|
||||
@@ -11,13 +10,6 @@ export async function createQuestionGuide({
|
||||
messages: ChatMessageItemType[];
|
||||
model: string;
|
||||
}) {
|
||||
const concatMessages: ChatMessageItemType[] = [
|
||||
...messages,
|
||||
{
|
||||
role: 'user',
|
||||
content: Prompt_QuestionGuide
|
||||
}
|
||||
];
|
||||
const ai = getAIApi({
|
||||
timeout: 480000
|
||||
});
|
||||
@@ -25,21 +17,28 @@ export async function createQuestionGuide({
|
||||
model: model,
|
||||
temperature: 0.1,
|
||||
max_tokens: 200,
|
||||
messages: concatMessages,
|
||||
messages: [
|
||||
...messages,
|
||||
{
|
||||
role: 'user',
|
||||
content: Prompt_QuestionGuide
|
||||
}
|
||||
],
|
||||
stream: false
|
||||
});
|
||||
|
||||
const answer = data.choices?.[0]?.message?.content || '';
|
||||
const inputTokens = data.usage?.prompt_tokens || 0;
|
||||
const outputTokens = data.usage?.completion_tokens || 0;
|
||||
|
||||
const start = answer.indexOf('[');
|
||||
const end = answer.lastIndexOf(']');
|
||||
|
||||
const charsLength = countGptMessagesChars(concatMessages);
|
||||
|
||||
if (start === -1 || end === -1) {
|
||||
return {
|
||||
result: [],
|
||||
charsLength: 0
|
||||
inputTokens,
|
||||
outputTokens
|
||||
};
|
||||
}
|
||||
|
||||
@@ -51,12 +50,14 @@ export async function createQuestionGuide({
|
||||
try {
|
||||
return {
|
||||
result: JSON.parse(jsonStr),
|
||||
charsLength
|
||||
inputTokens,
|
||||
outputTokens
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
result: [],
|
||||
charsLength: 0
|
||||
inputTokens,
|
||||
outputTokens
|
||||
};
|
||||
}
|
||||
}
|
||||
|
@@ -1,19 +1,18 @@
|
||||
import { replaceVariable } from '@fastgpt/global/common/string/tools';
|
||||
import { getAIApi } from '../config';
|
||||
import { ChatItemType } from '@fastgpt/global/core/chat/type';
|
||||
import { countGptMessagesChars } from '../../chat/utils';
|
||||
|
||||
/*
|
||||
query extension - 问题扩展
|
||||
可以根据上下文,消除指代性问题以及扩展问题,利于检索。
|
||||
*/
|
||||
|
||||
const defaultPrompt = `作为一个向量检索助手,你的任务是结合历史记录,从不同角度,为“原问题”生成个不同版本的“检索词”,从而提高向量检索的语义丰富度,提高向量检索的精度。生成的问题要求指向对象清晰明确,并与原问题语言相同。例如:
|
||||
const defaultPrompt = `作为一个向量检索助手,你的任务是结合历史记录,从不同角度,为“原问题”生成个不同版本的“检索词”,从而提高向量检索的语义丰富度,提高向量检索的精度。生成的问题要求指向对象清晰明确。例如:
|
||||
历史记录:
|
||||
"""
|
||||
"""
|
||||
原问题: 介绍下剧情。
|
||||
检索词: ["介绍下故事的背景和主要人物。","故事的主题是什么?","剧情是是如何发展的?"]
|
||||
检索词: ["发生了什么故事?","故事梗概是什么?","讲述了什么故事?"]
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
@@ -21,7 +20,7 @@ Q: 对话背景。
|
||||
A: 当前对话是关于 FatGPT 的介绍和使用等。
|
||||
"""
|
||||
原问题: 怎么下载
|
||||
检索词: ["FastGPT 如何下载?","下载 FastGPT 需要什么条件?","有哪些渠道可以下载 FastGPT?"]
|
||||
检索词: ["FastGPT 怎么下载?","下载 FastGPT 需要什么条件?","有哪些渠道可以下载 FastGPT?"]
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
@@ -31,15 +30,15 @@ Q: 报错 "no connection"
|
||||
A: 报错"no connection"可能是因为……
|
||||
"""
|
||||
原问题: 怎么解决
|
||||
检索词: ["FastGPT 报错"no connection"如何解决?", "造成 'no connection' 报错的原因。", "FastGPT提示'no connection',要怎么办?"]
|
||||
检索词: ["FastGPT 报错"no connection"如何解决?", "报错 'no connection' 是什么原因?", "FastGPT提示'no connection',要怎么办?"]
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
Q: 作者是谁?
|
||||
A: FastGPT 的作者是 labring。
|
||||
"""
|
||||
原问题: Tell me about him
|
||||
检索词: ["Introduce labring, the author of FastGPT." ," Background information on author labring." "," Why does labring do FastGPT?"]
|
||||
原问题: 介绍下他
|
||||
检索词: ["介绍下 FastGPT 的作者 labring。","作者 labring 的背景信息。","labring 为什么要做 FastGPT?"]
|
||||
----------------
|
||||
历史记录:
|
||||
"""
|
||||
@@ -106,7 +105,8 @@ export const queryExtension = async ({
|
||||
rawQuery: string;
|
||||
extensionQueries: string[];
|
||||
model: string;
|
||||
charsLength: number;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
}> => {
|
||||
const systemFewShot = chatBg
|
||||
? `Q: 对话背景。
|
||||
@@ -125,20 +125,18 @@ A: ${chatBg}
|
||||
timeout: 480000
|
||||
});
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: 'user',
|
||||
content: replaceVariable(defaultPrompt, {
|
||||
query: `${query}`,
|
||||
histories: concatFewShot
|
||||
})
|
||||
}
|
||||
];
|
||||
const result = await ai.chat.completions.create({
|
||||
model: model,
|
||||
temperature: 0.01,
|
||||
// @ts-ignore
|
||||
messages,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: replaceVariable(defaultPrompt, {
|
||||
query: `${query}`,
|
||||
histories: concatFewShot
|
||||
})
|
||||
}
|
||||
],
|
||||
stream: false
|
||||
});
|
||||
|
||||
@@ -148,7 +146,8 @@ A: ${chatBg}
|
||||
rawQuery: query,
|
||||
extensionQueries: [],
|
||||
model,
|
||||
charsLength: 0
|
||||
inputTokens: 0,
|
||||
outputTokens: 0
|
||||
};
|
||||
}
|
||||
|
||||
@@ -161,7 +160,8 @@ A: ${chatBg}
|
||||
rawQuery: query,
|
||||
extensionQueries: queries,
|
||||
model,
|
||||
charsLength: countGptMessagesChars(messages)
|
||||
inputTokens: result.usage?.prompt_tokens || 0,
|
||||
outputTokens: result.usage?.completion_tokens || 0
|
||||
};
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
@@ -169,7 +169,8 @@ A: ${chatBg}
|
||||
rawQuery: query,
|
||||
extensionQueries: [],
|
||||
model,
|
||||
charsLength: 0
|
||||
inputTokens: 0,
|
||||
outputTokens: 0
|
||||
};
|
||||
}
|
||||
};
|
||||
|
@@ -61,9 +61,6 @@ const AppSchema = new Schema({
|
||||
type: String,
|
||||
enum: Object.keys(PermissionTypeMap),
|
||||
default: PermissionTypeEnum.private
|
||||
},
|
||||
teamTags: {
|
||||
type: [String]
|
||||
}
|
||||
});
|
||||
|
||||
|
@@ -92,8 +92,6 @@ try {
|
||||
ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }, { background: true });
|
||||
// admin charts
|
||||
ChatItemSchema.index({ time: -1, obj: 1 }, { background: true });
|
||||
// timer, clear history
|
||||
ChatItemSchema.index({ teamId: 1, time: -1 }, { background: true });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
@@ -83,9 +83,6 @@ try {
|
||||
ChatSchema.index({ teamId: 1, appId: 1, updateTime: -1 }, { background: true });
|
||||
// get share chat history
|
||||
ChatSchema.index({ shareId: 1, outLinkUid: 1, updateTime: -1, source: 1 }, { background: true });
|
||||
|
||||
// timer, clear history
|
||||
ChatSchema.index({ teamId: 1, updateTime: -1 }, { background: true });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
@@ -2,10 +2,7 @@ import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
|
||||
import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
|
||||
import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
|
||||
import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
|
||||
import type {
|
||||
ChatCompletionContentPart,
|
||||
ChatMessageItemType
|
||||
} from '@fastgpt/global/core/ai/type.d';
|
||||
import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
|
||||
import axios from 'axios';
|
||||
|
||||
/* slice chat context by tokens */
|
||||
@@ -59,12 +56,6 @@ export function ChatContextFilter({
|
||||
return [...systemPrompts, ...chats];
|
||||
}
|
||||
|
||||
export const countMessagesChars = (messages: ChatItemType[]) => {
|
||||
return messages.reduce((sum, item) => sum + item.value.length, 0);
|
||||
};
|
||||
export const countGptMessagesChars = (messages: ChatMessageItemType[]) =>
|
||||
messages.reduce((sum, item) => sum + item.content.length, 0);
|
||||
|
||||
/**
|
||||
string to vision model. Follow the markdown code block rule for interception:
|
||||
|
||||
|
@@ -147,6 +147,8 @@ export async function delCollectionAndRelatedSources({
|
||||
collectionId: { $in: collectionIds }
|
||||
});
|
||||
|
||||
await delay(2000);
|
||||
|
||||
// delete dataset.datas
|
||||
await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } }, { session });
|
||||
// delete imgs
|
||||
|
@@ -66,11 +66,6 @@ export async function delDatasetRelevantData({
|
||||
if (!datasets.length) return;
|
||||
|
||||
const teamId = datasets[0].teamId;
|
||||
|
||||
if (!teamId) {
|
||||
return Promise.reject('teamId is required');
|
||||
}
|
||||
|
||||
const datasetIds = datasets.map((item) => String(item._id));
|
||||
|
||||
// Get _id, teamId, fileId, metadata.relatedImgId for all collections
|
||||
|
@@ -7,6 +7,10 @@ import {
|
||||
} from '@fastgpt/global/support/user/team/constant';
|
||||
import { DatasetCollectionName } from '../schema';
|
||||
import { DatasetColCollectionName } from '../collection/schema';
|
||||
import {
|
||||
DatasetDataIndexTypeEnum,
|
||||
DatasetDataIndexTypeMap
|
||||
} from '@fastgpt/global/core/dataset/constants';
|
||||
|
||||
export const DatasetDataCollectionName = 'dataset.datas';
|
||||
|
||||
@@ -50,6 +54,11 @@ const DatasetDataSchema = new Schema({
|
||||
type: Boolean,
|
||||
default: false
|
||||
},
|
||||
type: {
|
||||
type: String,
|
||||
enum: Object.keys(DatasetDataIndexTypeMap),
|
||||
default: DatasetDataIndexTypeEnum.custom
|
||||
},
|
||||
dataId: {
|
||||
type: String,
|
||||
required: true
|
||||
|
@@ -14,54 +14,22 @@ export const datasetSearchQueryExtension = async ({
|
||||
extensionBg?: string;
|
||||
histories?: ChatItemType[];
|
||||
}) => {
|
||||
const filterSamQuery = (queries: string[]) => {
|
||||
const set = new Set<string>();
|
||||
const filterSameQueries = queries.filter((item) => {
|
||||
// 删除所有的标点符号与空格等,只对文本进行比较
|
||||
const str = hashStr(item.replace(/[^\p{L}\p{N}]/gu, ''));
|
||||
if (set.has(str)) return false;
|
||||
set.add(str);
|
||||
return true;
|
||||
});
|
||||
|
||||
return filterSameQueries;
|
||||
};
|
||||
|
||||
let { queries, rewriteQuery, alreadyExtension } = (() => {
|
||||
// concat query
|
||||
let rewriteQuery =
|
||||
histories.length > 0
|
||||
? `${histories
|
||||
.map((item) => {
|
||||
return `${item.obj}: ${item.value}`;
|
||||
})
|
||||
.join('\n')}
|
||||
Human: ${query}
|
||||
`
|
||||
: query;
|
||||
|
||||
/* if query already extension, direct parse */
|
||||
try {
|
||||
const jsonParse = JSON.parse(query);
|
||||
const queries: string[] = Array.isArray(jsonParse) ? filterSamQuery(jsonParse) : [query];
|
||||
const alreadyExtension = Array.isArray(jsonParse);
|
||||
return {
|
||||
queries,
|
||||
rewriteQuery: alreadyExtension ? queries.join('\n') : rewriteQuery,
|
||||
alreadyExtension: alreadyExtension
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
queries: [query],
|
||||
rewriteQuery,
|
||||
alreadyExtension: false
|
||||
};
|
||||
}
|
||||
})();
|
||||
// concat query
|
||||
let queries = [query];
|
||||
let rewriteQuery =
|
||||
histories.length > 0
|
||||
? `${histories
|
||||
.map((item) => {
|
||||
return `${item.obj}: ${item.value}`;
|
||||
})
|
||||
.join('\n')}
|
||||
Human: ${query}
|
||||
`
|
||||
: query;
|
||||
|
||||
// ai extension
|
||||
const aiExtensionResult = await (async () => {
|
||||
if (!extensionModel || alreadyExtension) return;
|
||||
if (!extensionModel) return;
|
||||
const result = await queryExtension({
|
||||
chatBg: extensionBg,
|
||||
query,
|
||||
@@ -71,13 +39,23 @@ export const datasetSearchQueryExtension = async ({
|
||||
if (result.extensionQueries?.length === 0) return;
|
||||
return result;
|
||||
})();
|
||||
|
||||
if (aiExtensionResult) {
|
||||
queries = filterSamQuery(queries.concat(aiExtensionResult.extensionQueries));
|
||||
queries = queries.concat(aiExtensionResult.extensionQueries);
|
||||
rewriteQuery = queries.join('\n');
|
||||
}
|
||||
|
||||
const set = new Set<string>();
|
||||
const filterSameQueries = queries.filter((item) => {
|
||||
// 删除所有的标点符号与空格等,只对文本进行比较
|
||||
const str = hashStr(item.replace(/[^\p{L}\p{N}]/gu, ''));
|
||||
if (set.has(str)) return false;
|
||||
set.add(str);
|
||||
return true;
|
||||
});
|
||||
|
||||
return {
|
||||
concatQueries: queries,
|
||||
concatQueries: filterSameQueries,
|
||||
rewriteQuery,
|
||||
aiExtensionResult
|
||||
};
|
||||
|
@@ -57,7 +57,7 @@ export async function pushDataListToTrainingQueue({
|
||||
if (trainingMode === TrainingModeEnum.chunk) {
|
||||
const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
|
||||
if (!vectorModelData) {
|
||||
return Promise.reject(`File model ${vectorModel} is inValid`);
|
||||
return Promise.reject(`Model ${vectorModel} is inValid`);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -70,7 +70,7 @@ export async function pushDataListToTrainingQueue({
|
||||
if (trainingMode === TrainingModeEnum.qa) {
|
||||
const qaModelData = datasetModelList?.find((item) => item.model === agentModel);
|
||||
if (!qaModelData) {
|
||||
return Promise.reject(`Vector model ${agentModel} is inValid`);
|
||||
return Promise.reject(`Model ${agentModel} is inValid`);
|
||||
}
|
||||
return {
|
||||
maxToken: qaModelData.maxContext * 0.8,
|
||||
|
@@ -2,7 +2,7 @@
|
||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
|
||||
import { DatasetDataIndexTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
|
||||
import { DatasetColCollectionName } from '../collection/schema';
|
||||
import { DatasetCollectionName } from '../schema';
|
||||
import {
|
||||
@@ -86,6 +86,11 @@ const TrainingDataSchema = new Schema({
|
||||
indexes: {
|
||||
type: [
|
||||
{
|
||||
type: {
|
||||
type: String,
|
||||
enum: Object.keys(DatasetDataIndexTypeMap),
|
||||
required: true
|
||||
},
|
||||
text: {
|
||||
type: String,
|
||||
required: true
|
||||
|
@@ -15,7 +15,7 @@
|
||||
"nextjs-cors": "^2.1.2",
|
||||
"node-cron": "^3.0.3",
|
||||
"pg": "^8.10.0",
|
||||
"date-fns": "2.30.0",
|
||||
"date-fns": "^2.30.0",
|
||||
"tunnel": "^0.0.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
@@ -19,15 +19,14 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
|
||||
// auth limit
|
||||
// @ts-ignore
|
||||
if (global.feConfigs?.isPlus) {
|
||||
await POST('/support/openapi/authLimit', {
|
||||
openApi: openApi.toObject()
|
||||
} as AuthOpenApiLimitProps);
|
||||
await POST('/support/openapi/authLimit', { openApi } as AuthOpenApiLimitProps);
|
||||
}
|
||||
|
||||
updateApiKeyUsedTime(openApi._id);
|
||||
|
||||
return {
|
||||
apikey,
|
||||
userId: String(openApi.userId),
|
||||
teamId: String(openApi.teamId),
|
||||
tmbId: String(openApi.tmbId),
|
||||
appId: openApi.appId || ''
|
||||
|
@@ -1,6 +1,8 @@
|
||||
import { connectionMongo, type Model } from '../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
|
||||
import { PRICE_SCALE } from '@fastgpt/global/support/wallet/bill/constants';
|
||||
import { formatStorePrice2Read } from '@fastgpt/global/support/wallet/bill/tools';
|
||||
import {
|
||||
TeamCollectionName,
|
||||
TeamMemberCollectionName
|
||||
@@ -8,6 +10,10 @@ import {
|
||||
|
||||
const OpenApiSchema = new Schema(
|
||||
{
|
||||
userId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'user'
|
||||
},
|
||||
teamId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: TeamCollectionName,
|
||||
@@ -38,17 +44,22 @@ const OpenApiSchema = new Schema(
|
||||
type: String,
|
||||
default: 'Api Key'
|
||||
},
|
||||
usagePoints: {
|
||||
usage: {
|
||||
// total usage. value from bill total
|
||||
type: Number,
|
||||
default: 0
|
||||
default: 0,
|
||||
get: (val: number) => formatStorePrice2Read(val)
|
||||
},
|
||||
limit: {
|
||||
expiredTime: {
|
||||
type: Date
|
||||
},
|
||||
maxUsagePoints: {
|
||||
credit: {
|
||||
// value from user settings
|
||||
type: Number,
|
||||
default: -1
|
||||
default: -1,
|
||||
set: (val: number) => val * PRICE_SCALE,
|
||||
get: (val: number) => formatStorePrice2Read(val)
|
||||
}
|
||||
}
|
||||
},
|
||||
|
@@ -8,21 +8,15 @@ export function updateApiKeyUsedTime(id: string) {
|
||||
});
|
||||
}
|
||||
|
||||
export function updateApiKeyUsage({
|
||||
apikey,
|
||||
totalPoints
|
||||
}: {
|
||||
apikey: string;
|
||||
totalPoints: number;
|
||||
}) {
|
||||
export function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) {
|
||||
MongoOpenApi.findOneAndUpdate(
|
||||
{ apiKey: apikey },
|
||||
{
|
||||
$inc: {
|
||||
usagePoints: totalPoints
|
||||
usage
|
||||
}
|
||||
}
|
||||
).catch((err) => {
|
||||
console.log('update apiKey totalPoints error', err);
|
||||
console.log('update apiKey usage error', err);
|
||||
});
|
||||
}
|
||||
|
@@ -35,7 +35,8 @@ const OutLinkSchema = new Schema({
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
usagePoints: {
|
||||
total: {
|
||||
// total amount
|
||||
type: Number,
|
||||
default: 0
|
||||
},
|
||||
@@ -47,10 +48,6 @@ const OutLinkSchema = new Schema({
|
||||
default: false
|
||||
},
|
||||
limit: {
|
||||
maxUsagePoints: {
|
||||
type: Number,
|
||||
default: -1
|
||||
},
|
||||
expiredTime: {
|
||||
type: Date
|
||||
},
|
||||
@@ -58,18 +55,16 @@ const OutLinkSchema = new Schema({
|
||||
type: Number,
|
||||
default: 1000
|
||||
},
|
||||
credit: {
|
||||
type: Number,
|
||||
default: -1
|
||||
},
|
||||
hookUrl: {
|
||||
type: String
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
OutLinkSchema.index({ shareId: -1 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
export const MongoOutLink: Model<SchemaType> =
|
||||
models['outlinks'] || model('outlinks', OutLinkSchema);
|
||||
|
||||
|
@@ -1,19 +1,18 @@
|
||||
import axios from 'axios';
|
||||
import { MongoOutLink } from './schema';
|
||||
import { FastGPTProUrl } from '../../common/system/constants';
|
||||
import { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type';
|
||||
|
||||
export const addOutLinkUsage = async ({
|
||||
export const updateOutLinkUsage = async ({
|
||||
shareId,
|
||||
totalPoints
|
||||
total
|
||||
}: {
|
||||
shareId: string;
|
||||
totalPoints: number;
|
||||
total: number;
|
||||
}) => {
|
||||
MongoOutLink.findOneAndUpdate(
|
||||
{ shareId },
|
||||
{
|
||||
$inc: { usagePoints: totalPoints },
|
||||
$inc: { total },
|
||||
lastTime: new Date()
|
||||
}
|
||||
).catch((err) => {
|
||||
@@ -24,13 +23,11 @@ export const addOutLinkUsage = async ({
|
||||
export const pushResult2Remote = async ({
|
||||
outLinkUid,
|
||||
shareId,
|
||||
appName,
|
||||
responseData
|
||||
}: {
|
||||
outLinkUid?: string; // raw id, not parse
|
||||
shareId?: string;
|
||||
appName: string;
|
||||
responseData?: ChatHistoryItemResType[];
|
||||
responseData?: any[];
|
||||
}) => {
|
||||
if (!shareId || !outLinkUid || !FastGPTProUrl) return;
|
||||
try {
|
||||
@@ -45,7 +42,6 @@ export const pushResult2Remote = async ({
|
||||
url: '/shareAuth/finish',
|
||||
data: {
|
||||
token: outLinkUid,
|
||||
appName,
|
||||
responseData
|
||||
}
|
||||
});
|
||||
|
@@ -107,7 +107,7 @@ export async function authDatasetCollection({
|
||||
collection: CollectionWithDatasetType;
|
||||
}
|
||||
> {
|
||||
const { teamId, tmbId } = await parseHeaderCert(props);
|
||||
const { userId, teamId, tmbId } = await parseHeaderCert(props);
|
||||
const { role } = await getTmbInfoByTmbId({ tmbId });
|
||||
|
||||
const { collection, isOwner, canWrite } = await (async () => {
|
||||
@@ -143,6 +143,7 @@ export async function authDatasetCollection({
|
||||
})();
|
||||
|
||||
return {
|
||||
userId,
|
||||
teamId,
|
||||
tmbId,
|
||||
collection,
|
||||
@@ -162,7 +163,7 @@ export async function authDatasetFile({
|
||||
file: DatasetFileSchema;
|
||||
}
|
||||
> {
|
||||
const { teamId, tmbId } = await parseHeaderCert(props);
|
||||
const { userId, teamId, tmbId } = await parseHeaderCert(props);
|
||||
|
||||
const [file, collection] = await Promise.all([
|
||||
getFileById({ bucketName: BucketNameEnum.dataset, fileId }),
|
||||
@@ -189,6 +190,7 @@ export async function authDatasetFile({
|
||||
});
|
||||
|
||||
return {
|
||||
userId,
|
||||
teamId,
|
||||
tmbId,
|
||||
file,
|
||||
@@ -198,4 +200,4 @@ export async function authDatasetFile({
|
||||
} catch (error) {
|
||||
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@@ -12,7 +12,7 @@ export async function authUserNotVisitor(props: AuthModeType): Promise<
|
||||
role: `${TeamMemberRoleEnum}`;
|
||||
}
|
||||
> {
|
||||
const { teamId, tmbId } = await parseHeaderCert(props);
|
||||
const { userId, teamId, tmbId } = await parseHeaderCert(props);
|
||||
const team = await getTmbInfoByTmbId({ tmbId });
|
||||
|
||||
if (team.role === TeamMemberRoleEnum.visitor) {
|
||||
@@ -20,6 +20,7 @@ export async function authUserNotVisitor(props: AuthModeType): Promise<
|
||||
}
|
||||
|
||||
return {
|
||||
userId,
|
||||
teamId,
|
||||
tmbId,
|
||||
team,
|
||||
|
@@ -94,10 +94,10 @@ export async function parseHeaderCert({
|
||||
})();
|
||||
|
||||
// auth apikey
|
||||
const { teamId, tmbId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
|
||||
const { userId, teamId, tmbId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
|
||||
|
||||
return {
|
||||
uid: '',
|
||||
uid: userId,
|
||||
teamId,
|
||||
tmbId,
|
||||
apikey,
|
||||
@@ -217,4 +217,4 @@ export const authFileToken = (token?: string) =>
|
||||
fileId: decoded.fileId
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
23
packages/service/support/permission/limit/dataset.ts
Normal file
23
packages/service/support/permission/limit/dataset.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import { StandSubPlanLevelMapType } from '@fastgpt/global/support/wallet/sub/type';
|
||||
import { getVectorCountByTeamId } from '../../../common/vectorStore/controller';
|
||||
import { getTeamDatasetMaxSize } from '../../wallet/sub/utils';
|
||||
|
||||
export const checkDatasetLimit = async ({
|
||||
teamId,
|
||||
insertLen = 0,
|
||||
standardPlans
|
||||
}: {
|
||||
teamId: string;
|
||||
insertLen?: number;
|
||||
standardPlans?: StandSubPlanLevelMapType;
|
||||
}) => {
|
||||
const [{ maxSize }, usedSize] = await Promise.all([
|
||||
getTeamDatasetMaxSize({ teamId, standardPlans }),
|
||||
getVectorCountByTeamId(teamId)
|
||||
]);
|
||||
|
||||
if (usedSize + insertLen >= maxSize) {
|
||||
return Promise.reject(`数据库容量不足,无法继续添加。可以在账号页面进行扩容。`);
|
||||
}
|
||||
return;
|
||||
};
|
@@ -2,6 +2,7 @@ import { UserType } from '@fastgpt/global/support/user/type';
|
||||
import { MongoUser } from './schema';
|
||||
import { getTmbInfoByTmbId, getUserDefaultTeam } from './team/controller';
|
||||
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
|
||||
import { UserErrEnum } from '@fastgpt/global/common/error/code/user';
|
||||
|
||||
export async function authUserExist({ userId, username }: { userId?: string; username?: string }) {
|
||||
if (userId) {
|
||||
@@ -46,3 +47,22 @@ export async function getUserDetail({
|
||||
team: tmb
|
||||
};
|
||||
}
|
||||
|
||||
export async function getUserAndAuthBalance({
|
||||
tmbId,
|
||||
minBalance
|
||||
}: {
|
||||
tmbId: string;
|
||||
minBalance?: number;
|
||||
}) {
|
||||
const user = await getUserDetail({ tmbId });
|
||||
|
||||
if (!user) {
|
||||
return Promise.reject(UserErrEnum.unAuthUser);
|
||||
}
|
||||
if (minBalance !== undefined && user.team.balance < minBalance) {
|
||||
return Promise.reject(UserErrEnum.balanceNotEnough);
|
||||
}
|
||||
|
||||
return user;
|
||||
}
|
||||
|
@@ -1,7 +1,7 @@
|
||||
import { connectionMongo, type Model } from '../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { hashStr } from '@fastgpt/global/common/string/tools';
|
||||
import { PRICE_SCALE } from '@fastgpt/global/support/wallet/constants';
|
||||
import { PRICE_SCALE } from '@fastgpt/global/support/wallet/bill/constants';
|
||||
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
|
||||
import { UserStatusEnum, userStatusMap } from '@fastgpt/global/support/user/constant';
|
||||
|
||||
@@ -63,8 +63,6 @@ const UserSchema = new Schema({
|
||||
});
|
||||
|
||||
try {
|
||||
// login
|
||||
UserSchema.index({ username: 1, password: 1 });
|
||||
UserSchema.index({ createTime: -1 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
@@ -36,7 +36,7 @@ export async function getTmbInfoByTmbId({ tmbId }: { tmbId: string }) {
|
||||
return Promise.reject('tmbId or userId is required');
|
||||
}
|
||||
return getTeamMember({
|
||||
_id: new Types.ObjectId(String(tmbId)),
|
||||
_id: new Types.ObjectId(tmbId),
|
||||
status: notLeaveStatus
|
||||
});
|
||||
}
|
||||
|
@@ -27,10 +27,7 @@ const TeamSchema = new Schema({
|
||||
},
|
||||
maxSize: {
|
||||
type: Number,
|
||||
default: 1
|
||||
},
|
||||
tagsUrl: {
|
||||
type: String
|
||||
default: 3
|
||||
},
|
||||
limit: {
|
||||
lastExportDatasetTime: {
|
||||
|
@@ -1,35 +0,0 @@
|
||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { TeamTagsSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
|
||||
import {
|
||||
TeamCollectionName,
|
||||
TeamTagsCollectionName
|
||||
} from '@fastgpt/global/support/user/team/constant';
|
||||
|
||||
const TeamTagsSchema = new Schema({
|
||||
label: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
teamId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: TeamCollectionName,
|
||||
required: true
|
||||
},
|
||||
key: {
|
||||
type: String
|
||||
},
|
||||
createTime: {
|
||||
type: Date,
|
||||
default: () => new Date()
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
TeamTagsSchema.index({ teamId: 1 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
export const MongoTeamTags: Model<TeamTagsSchemaType> =
|
||||
models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagsSchema);
|
@@ -1,8 +1,8 @@
|
||||
import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
|
||||
import { MongoUsage } from './schema';
|
||||
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
|
||||
import { MongoBill } from './schema';
|
||||
import { ClientSession } from '../../../common/mongo';
|
||||
|
||||
export const createTrainingUsage = async ({
|
||||
export const createTrainingBill = async ({
|
||||
teamId,
|
||||
tmbId,
|
||||
appName,
|
||||
@@ -14,33 +14,33 @@ export const createTrainingUsage = async ({
|
||||
teamId: string;
|
||||
tmbId: string;
|
||||
appName: string;
|
||||
billSource: `${UsageSourceEnum}`;
|
||||
billSource: `${BillSourceEnum}`;
|
||||
vectorModel: string;
|
||||
agentModel: string;
|
||||
session?: ClientSession;
|
||||
}) => {
|
||||
const [{ _id }] = await MongoUsage.create(
|
||||
const [{ _id }] = await MongoBill.create(
|
||||
[
|
||||
{
|
||||
teamId,
|
||||
tmbId,
|
||||
appName,
|
||||
source: billSource,
|
||||
totalPoints: 0,
|
||||
list: [
|
||||
{
|
||||
moduleName: 'support.wallet.moduleName.index',
|
||||
moduleName: 'wallet.moduleName.index',
|
||||
model: vectorModel,
|
||||
charsLength: 0,
|
||||
amount: 0
|
||||
},
|
||||
{
|
||||
moduleName: 'support.wallet.moduleName.qa',
|
||||
moduleName: 'wallet.moduleName.qa',
|
||||
model: agentModel,
|
||||
charsLength: 0,
|
||||
amount: 0
|
||||
}
|
||||
]
|
||||
],
|
||||
total: 0
|
||||
}
|
||||
],
|
||||
{ session }
|
@@ -1,15 +1,13 @@
|
||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { UsageSchemaType } from '@fastgpt/global/support/wallet/usage/type';
|
||||
import { UsageSourceMap } from '@fastgpt/global/support/wallet/usage/constants';
|
||||
import { BillSchema as BillType } from '@fastgpt/global/support/wallet/bill/type';
|
||||
import { BillSourceMap } from '@fastgpt/global/support/wallet/bill/constants';
|
||||
import {
|
||||
TeamCollectionName,
|
||||
TeamMemberCollectionName
|
||||
} from '@fastgpt/global/support/user/team/constant';
|
||||
|
||||
export const UsageCollectionName = 'usages';
|
||||
|
||||
const UsageSchema = new Schema({
|
||||
const BillSchema = new Schema({
|
||||
teamId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: TeamCollectionName,
|
||||
@@ -20,11 +18,6 @@ const UsageSchema = new Schema({
|
||||
ref: TeamMemberCollectionName,
|
||||
required: true
|
||||
},
|
||||
source: {
|
||||
type: String,
|
||||
enum: Object.keys(UsageSourceMap),
|
||||
required: true
|
||||
},
|
||||
appName: {
|
||||
type: String,
|
||||
default: ''
|
||||
@@ -38,16 +31,16 @@ const UsageSchema = new Schema({
|
||||
type: Date,
|
||||
default: () => new Date()
|
||||
},
|
||||
totalPoints: {
|
||||
// total points
|
||||
total: {
|
||||
// 1 * PRICE_SCALE
|
||||
type: Number,
|
||||
required: true
|
||||
},
|
||||
// total: {
|
||||
// // total points
|
||||
// type: Number,
|
||||
// required: true
|
||||
// },
|
||||
source: {
|
||||
type: String,
|
||||
enum: Object.keys(BillSourceMap),
|
||||
required: true
|
||||
},
|
||||
list: {
|
||||
type: Array,
|
||||
default: []
|
||||
@@ -55,15 +48,11 @@ const UsageSchema = new Schema({
|
||||
});
|
||||
|
||||
try {
|
||||
UsageSchema.index({ teamId: 1, tmbId: 1, source: 1, time: -1 }, { background: true });
|
||||
// timer task. clear dead team
|
||||
UsageSchema.index({ teamId: 1, time: -1 }, { background: true });
|
||||
|
||||
UsageSchema.index({ time: 1 }, { expireAfterSeconds: 180 * 24 * 60 * 60 });
|
||||
BillSchema.index({ teamId: 1, tmbId: 1, source: 1, time: -1 }, { background: true });
|
||||
BillSchema.index({ time: 1 }, { expireAfterSeconds: 180 * 24 * 60 * 60 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
export const MongoUsage: Model<UsageSchemaType> =
|
||||
models[UsageCollectionName] || model(UsageCollectionName, UsageSchema);
|
||||
MongoUsage.syncIndexes();
|
||||
export const MongoBill: Model<BillType> = models['bill'] || model('bill', BillSchema);
|
||||
MongoBill.syncIndexes();
|
@@ -1,8 +1,3 @@
|
||||
/*
|
||||
user sub plan
|
||||
1. type=standard: There will only be 1, and each team will have one
|
||||
2. type=extraDatasetSize/extraPoints: Can buy multiple
|
||||
*/
|
||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
|
||||
@@ -28,10 +23,25 @@ const SubSchema = new Schema({
|
||||
required: true
|
||||
},
|
||||
status: {
|
||||
// active: continue sub; canceled: canceled sub;
|
||||
type: String,
|
||||
enum: Object.keys(subStatusMap),
|
||||
required: true
|
||||
},
|
||||
mode: {
|
||||
type: String,
|
||||
enum: Object.keys(subModeMap)
|
||||
},
|
||||
currentMode: {
|
||||
type: String,
|
||||
enum: Object.keys(subModeMap),
|
||||
required: true
|
||||
},
|
||||
nextMode: {
|
||||
type: String,
|
||||
enum: Object.keys(subModeMap),
|
||||
required: true
|
||||
},
|
||||
startTime: {
|
||||
type: Date,
|
||||
default: () => new Date()
|
||||
@@ -45,16 +55,12 @@ const SubSchema = new Schema({
|
||||
type: Number,
|
||||
required: true
|
||||
},
|
||||
pointPrice: {
|
||||
// stand level point total price
|
||||
type: Number
|
||||
},
|
||||
|
||||
// standard sub
|
||||
currentMode: {
|
||||
type: String,
|
||||
enum: Object.keys(subModeMap)
|
||||
},
|
||||
nextMode: {
|
||||
type: String,
|
||||
enum: Object.keys(subModeMap)
|
||||
},
|
||||
// sub content
|
||||
currentSubLevel: {
|
||||
type: String,
|
||||
enum: Object.keys(standardSubLevelMap)
|
||||
@@ -63,34 +69,79 @@ const SubSchema = new Schema({
|
||||
type: String,
|
||||
enum: Object.keys(standardSubLevelMap)
|
||||
},
|
||||
|
||||
// stand sub and extra points sub. Plan total points
|
||||
totalPoints: {
|
||||
type: Number
|
||||
},
|
||||
pointPrice: {
|
||||
// stand level point total price
|
||||
|
||||
currentExtraDatasetSize: {
|
||||
type: Number
|
||||
},
|
||||
surplusPoints: {
|
||||
// plan surplus points
|
||||
nextExtraDatasetSize: {
|
||||
type: Number
|
||||
},
|
||||
|
||||
// extra dataset size
|
||||
currentExtraDatasetSize: {
|
||||
currentExtraPoints: {
|
||||
type: Number
|
||||
}
|
||||
},
|
||||
nextExtraPoints: {
|
||||
type: Number
|
||||
},
|
||||
|
||||
// standard sub limit
|
||||
// maxTeamMember: {
|
||||
// type: Number
|
||||
// },
|
||||
// maxAppAmount: {
|
||||
// type: Number
|
||||
// },
|
||||
// maxDatasetAmount: {
|
||||
// type: Number
|
||||
// },
|
||||
// chatHistoryStoreDuration: {
|
||||
// // n day
|
||||
// type: Number
|
||||
// },
|
||||
// maxDatasetSize: {
|
||||
// type: Number
|
||||
// },
|
||||
// trainingWeight: {
|
||||
// // 0 1 2 3
|
||||
// type: Number
|
||||
// },
|
||||
// customApiKey: {
|
||||
// type: Boolean
|
||||
// },
|
||||
// customCopyright: {
|
||||
// type: Boolean
|
||||
// },
|
||||
// websiteSyncInterval: {
|
||||
// // hours
|
||||
// type: Number
|
||||
// },
|
||||
// reRankWeight: {
|
||||
// // 0 1 2 3
|
||||
// type: Number
|
||||
// },
|
||||
// totalPoints: {
|
||||
// // record standard sub points
|
||||
// type: Number
|
||||
// },
|
||||
|
||||
surplusPoints: {
|
||||
// standard sub / extra points sub
|
||||
type: Number
|
||||
},
|
||||
|
||||
// abandon
|
||||
renew: Boolean, //决定是否续费
|
||||
datasetStoreAmount: Number
|
||||
});
|
||||
|
||||
try {
|
||||
// get team plan
|
||||
SubSchema.index({ teamId: 1, type: 1, expiredTime: -1 });
|
||||
|
||||
// timer task. check expired plan; update standard plan;
|
||||
SubSchema.index({ type: 1, expiredTime: -1 });
|
||||
// timer task. clear dead team
|
||||
SubSchema.index({ type: 1, currentSubLevel: 1, nextSubLevel: 1 });
|
||||
SubSchema.index({ teamId: 1 });
|
||||
SubSchema.index({ status: 1 });
|
||||
SubSchema.index({ type: 1 });
|
||||
SubSchema.index({ expiredTime: -1 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
@@ -1,70 +1,87 @@
|
||||
import { SubTypeEnum } from '@fastgpt/global/support/wallet/sub/constants';
|
||||
import { MongoTeamSub } from './schema';
|
||||
import {
|
||||
FeTeamPlanStatusType,
|
||||
StandSubPlanLevelMapType
|
||||
} from '@fastgpt/global/support/wallet/sub/type.d';
|
||||
import { addHours } from 'date-fns';
|
||||
import { FeTeamSubType, StandSubPlanLevelMapType } from '@fastgpt/global/support/wallet/sub/type.d';
|
||||
import { getVectorCountByTeamId } from '../../../common/vectorStore/controller';
|
||||
|
||||
export const getTeamSubPlans = async ({
|
||||
teamId,
|
||||
standardPlans
|
||||
}: {
|
||||
teamId: string;
|
||||
standardPlans?: StandSubPlanLevelMapType;
|
||||
}): Promise<FeTeamPlanStatusType> => {
|
||||
const [plans, usedDatasetSize] = await Promise.all([
|
||||
MongoTeamSub.find({ teamId }).lean(),
|
||||
getVectorCountByTeamId(teamId)
|
||||
]);
|
||||
|
||||
const standard = plans.find((plan) => plan.type === SubTypeEnum.standard);
|
||||
const extraDatasetSize = plans.filter((plan) => plan.type === SubTypeEnum.extraDatasetSize);
|
||||
const extraPoints = plans.filter((plan) => plan.type === SubTypeEnum.extraPoints);
|
||||
|
||||
const totalPoints =
|
||||
(standard?.totalPoints || 0) +
|
||||
extraPoints.reduce((acc, cur) => acc + (cur.totalPoints || 0), 0);
|
||||
const surplusPoints =
|
||||
(standard?.surplusPoints || 0) +
|
||||
extraPoints.reduce((acc, cur) => acc + (cur.surplusPoints || 0), 0);
|
||||
|
||||
const standardMaxDatasetSize =
|
||||
standard?.currentSubLevel && standardPlans
|
||||
? standardPlans[standard.currentSubLevel]?.maxDatasetSize || Infinity
|
||||
: Infinity;
|
||||
const totalDatasetSize =
|
||||
standardMaxDatasetSize +
|
||||
extraDatasetSize.reduce((acc, cur) => acc + (cur.currentExtraDatasetSize || 0), 0);
|
||||
|
||||
return {
|
||||
[SubTypeEnum.standard]: standard,
|
||||
standardConstants:
|
||||
standard?.currentSubLevel && standardPlans
|
||||
? standardPlans[standard.currentSubLevel]
|
||||
: undefined,
|
||||
|
||||
totalPoints,
|
||||
usedPoints: totalPoints - surplusPoints,
|
||||
|
||||
datasetMaxSize: totalDatasetSize,
|
||||
usedDatasetSize
|
||||
};
|
||||
};
|
||||
export const getTeamStandPlan = async ({
|
||||
/* get team dataset max size */
|
||||
export const getTeamDatasetMaxSize = async ({
|
||||
teamId,
|
||||
standardPlans
|
||||
}: {
|
||||
teamId: string;
|
||||
standardPlans?: StandSubPlanLevelMapType;
|
||||
}) => {
|
||||
const standard = await MongoTeamSub.findOne({ teamId, type: SubTypeEnum.standard }).lean();
|
||||
if (!standardPlans) {
|
||||
return {
|
||||
maxSize: Infinity,
|
||||
sub: null
|
||||
};
|
||||
}
|
||||
|
||||
const plans = await MongoTeamSub.find({
|
||||
teamId,
|
||||
expiredTime: { $gte: addHours(new Date(), -3) }
|
||||
}).lean();
|
||||
|
||||
const standard = plans.find((plan) => plan.type === SubTypeEnum.standard);
|
||||
const extraDatasetSize = plans.find((plan) => plan.type === SubTypeEnum.extraDatasetSize);
|
||||
|
||||
const standardMaxDatasetSize =
|
||||
standard?.currentSubLevel && standardPlans
|
||||
? standardPlans[standard.currentSubLevel]?.maxDatasetSize || Infinity
|
||||
: Infinity;
|
||||
const totalDatasetSize =
|
||||
standardMaxDatasetSize + (extraDatasetSize?.currentExtraDatasetSize || 0);
|
||||
|
||||
return {
|
||||
maxSize: totalDatasetSize,
|
||||
sub: extraDatasetSize
|
||||
};
|
||||
};
|
||||
|
||||
export const getTeamSubPlanStatus = async ({
|
||||
teamId,
|
||||
standardPlans
|
||||
}: {
|
||||
teamId: string;
|
||||
standardPlans?: StandSubPlanLevelMapType;
|
||||
}): Promise<FeTeamSubType> => {
|
||||
const [plans, usedDatasetSize] = await Promise.all([
|
||||
MongoTeamSub.find({ teamId }).lean(),
|
||||
getVectorCountByTeamId(teamId)
|
||||
]);
|
||||
|
||||
const standard = plans.find((plan) => plan.type === SubTypeEnum.standard);
|
||||
const extraDatasetSize = plans.find((plan) => plan.type === SubTypeEnum.extraDatasetSize);
|
||||
const extraPoints = plans.find((plan) => plan.type === SubTypeEnum.extraPoints);
|
||||
|
||||
const standardMaxDatasetSize =
|
||||
standard?.currentSubLevel && standardPlans
|
||||
? standardPlans[standard.currentSubLevel]?.maxDatasetSize || Infinity
|
||||
: Infinity;
|
||||
const totalDatasetSize =
|
||||
standardMaxDatasetSize + (extraDatasetSize?.currentExtraDatasetSize || 0);
|
||||
|
||||
const standardMaxPoints =
|
||||
standard?.currentSubLevel && standardPlans
|
||||
? standardPlans[standard.currentSubLevel]?.totalPoints || Infinity
|
||||
: Infinity;
|
||||
const totalPoints = standardMaxPoints + (extraPoints?.currentExtraPoints || 0);
|
||||
|
||||
const surplusPoints = (standard?.surplusPoints || 0) + (extraPoints?.surplusPoints || 0);
|
||||
|
||||
return {
|
||||
[SubTypeEnum.standard]: standard,
|
||||
standardConstants:
|
||||
standard?.currentSubLevel && standardPlans
|
||||
? standardPlans[standard.currentSubLevel]
|
||||
: undefined
|
||||
[SubTypeEnum.extraDatasetSize]: extraDatasetSize,
|
||||
[SubTypeEnum.extraPoints]: extraPoints,
|
||||
|
||||
standardMaxDatasetSize,
|
||||
datasetMaxSize: totalDatasetSize,
|
||||
usedDatasetSize,
|
||||
|
||||
standardMaxPoints,
|
||||
totalPoints,
|
||||
usedPoints: totalPoints - surplusPoints
|
||||
};
|
||||
};
|
||||
|
Reference in New Issue
Block a user