mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-22 20:37:48 +00:00
4.8.19-feature (#3636)
* feat: sync org from wecom, pref: member list pagination (#3549) * feat: sync org * chore: fe * chore: loading * chore: type * pref: team member list change to pagination. Edit a sort of list apis. * feat: member update avatar * chore: user avatar move to tmb * chore: init scripts move user avatar * chore: sourceMember * fix: list api sourceMember * fix: member sync * fix: pagination * chore: adjust code * chore: move changeOwner to pro * chore: init v4819 script * chore: adjust code * chore: UserBox * perf: scroll page code * perf: list data * docs:更新用户答疑 (#3576) * docs: add custom uid docs (#3572) * fix: pagination bug (#3577) * 4.8.19 test (#3584) * faet: dataset search filter * fix: scroll page * fix: collection list api old version (#3591) * fix: collection list api format * fix: type error of addSourceMemeber * fix: scroll fetch (#3592) * fix: yuque dataset file folder can enter (#3593) * perf: load members;perf: yuque load;fix: workflow llm params cannot close (#3594) * chat openapi doc * feat: dataset openapi doc * perf: load members * perf: member load code * perf: yuque load * fix: workflow llm params cannot close * fix: api dataset reference tag preview (#3600) * perf: doc * feat: chat page config * fix: http parse (#3634) * update doc * fix: http parse * fix code run node reset template (#3633) Co-authored-by: Archer <545436317@qq.com> * docs:faq (#3627) * docs:faq * docsFix * perf: sleep plugin * fix: selector --------- Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com> Co-authored-by: heheer <heheer@sealos.io>
This commit is contained in:
@@ -99,7 +99,13 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
|
||||
if (files.some((file) => !file.id || !file.name || typeof file.type === 'undefined')) {
|
||||
return Promise.reject('Invalid file data format');
|
||||
}
|
||||
return files;
|
||||
|
||||
const formattedFiles = files.map((file) => ({
|
||||
...file,
|
||||
hasChild: file.type === 'folder'
|
||||
}));
|
||||
|
||||
return formattedFiles;
|
||||
};
|
||||
|
||||
const getFileContent = async ({ teamId, apiFileId }: { teamId: string; apiFileId: string }) => {
|
||||
|
@@ -284,7 +284,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
|
||||
{
|
||||
_id: { $in: collectionIdList }
|
||||
},
|
||||
'_id name fileId rawLink externalFileId externalFileUrl',
|
||||
'_id name fileId rawLink apiFileId externalFileId externalFileUrl',
|
||||
{ ...readFromSecondary }
|
||||
).lean()
|
||||
]);
|
||||
@@ -525,7 +525,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
|
||||
{
|
||||
_id: { $in: searchResults.map((item) => item.collectionId) }
|
||||
},
|
||||
'_id name fileId rawLink externalFileId externalFileUrl',
|
||||
'_id name fileId rawLink apiFileId externalFileId externalFileUrl',
|
||||
{ ...readFromSecondary }
|
||||
).lean()
|
||||
]);
|
||||
|
39
packages/service/core/dataset/utils.ts
Normal file
39
packages/service/core/dataset/utils.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { getTmbInfoByTmbId } from '../../support/user/team/controller';
|
||||
import { getResourcePermission } from '../../support/permission/controller';
|
||||
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
|
||||
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
|
||||
|
||||
// TODO: 需要优化成批量获取权限
|
||||
export const filterDatasetsByTmbId = async ({
|
||||
datasetIds,
|
||||
tmbId
|
||||
}: {
|
||||
datasetIds: string[];
|
||||
tmbId: string;
|
||||
}) => {
|
||||
const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
|
||||
|
||||
// First get all permissions
|
||||
const permissions = await Promise.all(
|
||||
datasetIds.map(async (datasetId) => {
|
||||
const per = await getResourcePermission({
|
||||
teamId,
|
||||
tmbId,
|
||||
resourceId: datasetId,
|
||||
resourceType: PerResourceTypeEnum.dataset
|
||||
});
|
||||
|
||||
if (per === undefined) return false;
|
||||
|
||||
const datasetPer = new DatasetPermission({
|
||||
per,
|
||||
isOwner: tmbPer.isOwner
|
||||
});
|
||||
|
||||
return datasetPer.hasReadPer;
|
||||
})
|
||||
);
|
||||
|
||||
// Then filter datasetIds based on permissions
|
||||
return datasetIds.filter((_, index) => permissions[index]);
|
||||
};
|
@@ -17,6 +17,7 @@ import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
|
||||
import { checkTeamReRankPermission } from '../../../../support/permission/teamLimit';
|
||||
import { MongoDataset } from '../../../dataset/schema';
|
||||
import { i18nT } from '../../../../../web/i18n/utils';
|
||||
import { filterDatasetsByTmbId } from '../../../dataset/utils';
|
||||
|
||||
type DatasetSearchProps = ModuleDispatchProps<{
|
||||
[NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
|
||||
@@ -29,6 +30,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
|
||||
[NodeInputKeyEnum.datasetSearchExtensionModel]: string;
|
||||
[NodeInputKeyEnum.datasetSearchExtensionBg]: string;
|
||||
[NodeInputKeyEnum.collectionFilterMatch]: string;
|
||||
[NodeInputKeyEnum.authTmbId]: boolean;
|
||||
}>;
|
||||
export type DatasetSearchResponse = DispatchNodeResultType<{
|
||||
[NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[];
|
||||
@@ -39,6 +41,7 @@ export async function dispatchDatasetSearch(
|
||||
): Promise<DatasetSearchResponse> {
|
||||
const {
|
||||
runningAppInfo: { teamId },
|
||||
runningUserInfo: { tmbId },
|
||||
histories,
|
||||
node,
|
||||
params: {
|
||||
@@ -52,7 +55,8 @@ export async function dispatchDatasetSearch(
|
||||
datasetSearchUsingExtensionQuery,
|
||||
datasetSearchExtensionModel,
|
||||
datasetSearchExtensionBg,
|
||||
collectionFilterMatch
|
||||
collectionFilterMatch,
|
||||
authTmbId = false
|
||||
}
|
||||
} = props as DatasetSearchProps;
|
||||
|
||||
@@ -64,18 +68,20 @@ export async function dispatchDatasetSearch(
|
||||
return Promise.reject(i18nT('common:core.chat.error.Select dataset empty'));
|
||||
}
|
||||
|
||||
const emptyResult = {
|
||||
quoteQA: [],
|
||||
[DispatchNodeResponseKeyEnum.nodeResponse]: {
|
||||
totalPoints: 0,
|
||||
query: '',
|
||||
limit,
|
||||
searchMode
|
||||
},
|
||||
nodeDispatchUsages: [],
|
||||
[DispatchNodeResponseKeyEnum.toolResponses]: []
|
||||
};
|
||||
|
||||
if (!userChatInput) {
|
||||
return {
|
||||
quoteQA: [],
|
||||
[DispatchNodeResponseKeyEnum.nodeResponse]: {
|
||||
totalPoints: 0,
|
||||
query: '',
|
||||
limit,
|
||||
searchMode
|
||||
},
|
||||
nodeDispatchUsages: [],
|
||||
[DispatchNodeResponseKeyEnum.toolResponses]: []
|
||||
};
|
||||
return emptyResult;
|
||||
}
|
||||
|
||||
// query extension
|
||||
@@ -83,13 +89,24 @@ export async function dispatchDatasetSearch(
|
||||
? getLLMModel(datasetSearchExtensionModel)
|
||||
: undefined;
|
||||
|
||||
const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
|
||||
query: userChatInput,
|
||||
extensionModel,
|
||||
extensionBg: datasetSearchExtensionBg,
|
||||
histories: getHistories(6, histories)
|
||||
});
|
||||
const [{ concatQueries, rewriteQuery, aiExtensionResult }, datasetIds] = await Promise.all([
|
||||
datasetSearchQueryExtension({
|
||||
query: userChatInput,
|
||||
extensionModel,
|
||||
extensionBg: datasetSearchExtensionBg,
|
||||
histories: getHistories(6, histories)
|
||||
}),
|
||||
authTmbId
|
||||
? filterDatasetsByTmbId({
|
||||
datasetIds: datasets.map((item) => item.datasetId),
|
||||
tmbId
|
||||
})
|
||||
: Promise.resolve(datasets.map((item) => item.datasetId))
|
||||
]);
|
||||
|
||||
if (datasetIds.length === 0) {
|
||||
return emptyResult;
|
||||
}
|
||||
// console.log(concatQueries, rewriteQuery, aiExtensionResult);
|
||||
|
||||
// get vector
|
||||
@@ -110,7 +127,7 @@ export async function dispatchDatasetSearch(
|
||||
model: vectorModel.model,
|
||||
similarity,
|
||||
limit,
|
||||
datasetIds: datasets.map((item) => item.datasetId),
|
||||
datasetIds,
|
||||
searchMode,
|
||||
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
|
||||
collectionFilterMatch
|
||||
|
@@ -72,6 +72,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
|
||||
showStatus: false
|
||||
};
|
||||
});
|
||||
|
||||
const runtimeVariables = {
|
||||
...filterSystemVariables(props.variables),
|
||||
appId: String(plugin.id)
|
||||
|
@@ -127,8 +127,16 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
|
||||
if (typeof val === 'object') return JSON.stringify(val);
|
||||
|
||||
if (typeof val === 'string') {
|
||||
const str = JSON.stringify(val);
|
||||
return str.startsWith('"') && str.endsWith('"') ? str.slice(1, -1) : str;
|
||||
try {
|
||||
const parsed = JSON.parse(val);
|
||||
if (typeof parsed === 'object') {
|
||||
return JSON.stringify(parsed);
|
||||
}
|
||||
return val;
|
||||
} catch (error) {
|
||||
const str = JSON.stringify(val);
|
||||
return str.startsWith('"') && str.endsWith('"') ? str.slice(1, -1) : str;
|
||||
}
|
||||
}
|
||||
|
||||
return String(val);
|
||||
@@ -235,7 +243,9 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
|
||||
}
|
||||
if (!httpJsonBody) return {};
|
||||
if (httpContentType === ContentTypes.json) {
|
||||
return json5.parse(replaceJsonBodyString(httpJsonBody));
|
||||
httpJsonBody = replaceJsonBodyString(httpJsonBody);
|
||||
console.log(httpJsonBody);
|
||||
return json5.parse(httpJsonBody);
|
||||
}
|
||||
|
||||
// Raw text, xml
|
||||
|
Reference in New Issue
Block a user