4.8.19-feature (#3636)

* feat: sync org from wecom, pref: member list pagination (#3549)

* feat: sync org

* chore: fe

* chore: loading

* chore: type

* pref: team member list change to pagination. Edit a sort of list apis.

* feat: member update avatar

* chore: user avatar move to tmb

* chore: init scripts move user avatar

* chore: sourceMember

* fix: list api sourceMember

* fix: member sync

* fix: pagination

* chore: adjust code

* chore: move changeOwner to pro

* chore: init v4819 script

* chore: adjust code

* chore: UserBox

* perf: scroll page code

* perf: list data

* docs:更新用户答疑 (#3576)

* docs: add custom uid docs (#3572)

* fix: pagination bug (#3577)

* 4.8.19 test (#3584)

* faet: dataset search filter

* fix: scroll page

* fix: collection list api old version (#3591)

* fix: collection list api format

* fix: type error of addSourceMemeber

* fix: scroll fetch (#3592)

* fix: yuque dataset file folder can enter (#3593)

* perf: load members;perf: yuque load;fix: workflow llm params cannot close (#3594)

* chat openapi doc

* feat: dataset openapi doc

* perf: load members

* perf: member load code

* perf: yuque load

* fix: workflow llm params cannot close

* fix: api dataset reference tag preview (#3600)

* perf: doc

* feat: chat page config

* fix: http parse (#3634)

* update doc

* fix: http parse

* fix code run node reset template (#3633)

Co-authored-by: Archer <545436317@qq.com>

* docs:faq (#3627)

* docs:faq

* docsFix

* perf: sleep plugin

* fix: selector

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
Co-authored-by: heheer <heheer@sealos.io>
This commit is contained in:
Archer
2025-01-20 19:42:33 +08:00
committed by GitHub
parent 9f33729ca9
commit 3c97757e4d
170 changed files with 2317 additions and 1615 deletions

View File

@@ -99,7 +99,13 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
if (files.some((file) => !file.id || !file.name || typeof file.type === 'undefined')) {
return Promise.reject('Invalid file data format');
}
return files;
const formattedFiles = files.map((file) => ({
...file,
hasChild: file.type === 'folder'
}));
return formattedFiles;
};
const getFileContent = async ({ teamId, apiFileId }: { teamId: string; apiFileId: string }) => {

View File

@@ -284,7 +284,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
{
_id: { $in: collectionIdList }
},
'_id name fileId rawLink externalFileId externalFileUrl',
'_id name fileId rawLink apiFileId externalFileId externalFileUrl',
{ ...readFromSecondary }
).lean()
]);
@@ -525,7 +525,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
{
_id: { $in: searchResults.map((item) => item.collectionId) }
},
'_id name fileId rawLink externalFileId externalFileUrl',
'_id name fileId rawLink apiFileId externalFileId externalFileUrl',
{ ...readFromSecondary }
).lean()
]);

View File

@@ -0,0 +1,39 @@
import { getTmbInfoByTmbId } from '../../support/user/team/controller';
import { getResourcePermission } from '../../support/permission/controller';
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
// TODO: 需要优化成批量获取权限
export const filterDatasetsByTmbId = async ({
datasetIds,
tmbId
}: {
datasetIds: string[];
tmbId: string;
}) => {
const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
// First get all permissions
const permissions = await Promise.all(
datasetIds.map(async (datasetId) => {
const per = await getResourcePermission({
teamId,
tmbId,
resourceId: datasetId,
resourceType: PerResourceTypeEnum.dataset
});
if (per === undefined) return false;
const datasetPer = new DatasetPermission({
per,
isOwner: tmbPer.isOwner
});
return datasetPer.hasReadPer;
})
);
// Then filter datasetIds based on permissions
return datasetIds.filter((_, index) => permissions[index]);
};

View File

@@ -17,6 +17,7 @@ import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
import { checkTeamReRankPermission } from '../../../../support/permission/teamLimit';
import { MongoDataset } from '../../../dataset/schema';
import { i18nT } from '../../../../../web/i18n/utils';
import { filterDatasetsByTmbId } from '../../../dataset/utils';
type DatasetSearchProps = ModuleDispatchProps<{
[NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
@@ -29,6 +30,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
[NodeInputKeyEnum.datasetSearchExtensionModel]: string;
[NodeInputKeyEnum.datasetSearchExtensionBg]: string;
[NodeInputKeyEnum.collectionFilterMatch]: string;
[NodeInputKeyEnum.authTmbId]: boolean;
}>;
export type DatasetSearchResponse = DispatchNodeResultType<{
[NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[];
@@ -39,6 +41,7 @@ export async function dispatchDatasetSearch(
): Promise<DatasetSearchResponse> {
const {
runningAppInfo: { teamId },
runningUserInfo: { tmbId },
histories,
node,
params: {
@@ -52,7 +55,8 @@ export async function dispatchDatasetSearch(
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel,
datasetSearchExtensionBg,
collectionFilterMatch
collectionFilterMatch,
authTmbId = false
}
} = props as DatasetSearchProps;
@@ -64,18 +68,20 @@ export async function dispatchDatasetSearch(
return Promise.reject(i18nT('common:core.chat.error.Select dataset empty'));
}
const emptyResult = {
quoteQA: [],
[DispatchNodeResponseKeyEnum.nodeResponse]: {
totalPoints: 0,
query: '',
limit,
searchMode
},
nodeDispatchUsages: [],
[DispatchNodeResponseKeyEnum.toolResponses]: []
};
if (!userChatInput) {
return {
quoteQA: [],
[DispatchNodeResponseKeyEnum.nodeResponse]: {
totalPoints: 0,
query: '',
limit,
searchMode
},
nodeDispatchUsages: [],
[DispatchNodeResponseKeyEnum.toolResponses]: []
};
return emptyResult;
}
// query extension
@@ -83,13 +89,24 @@ export async function dispatchDatasetSearch(
? getLLMModel(datasetSearchExtensionModel)
: undefined;
const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
query: userChatInput,
extensionModel,
extensionBg: datasetSearchExtensionBg,
histories: getHistories(6, histories)
});
const [{ concatQueries, rewriteQuery, aiExtensionResult }, datasetIds] = await Promise.all([
datasetSearchQueryExtension({
query: userChatInput,
extensionModel,
extensionBg: datasetSearchExtensionBg,
histories: getHistories(6, histories)
}),
authTmbId
? filterDatasetsByTmbId({
datasetIds: datasets.map((item) => item.datasetId),
tmbId
})
: Promise.resolve(datasets.map((item) => item.datasetId))
]);
if (datasetIds.length === 0) {
return emptyResult;
}
// console.log(concatQueries, rewriteQuery, aiExtensionResult);
// get vector
@@ -110,7 +127,7 @@ export async function dispatchDatasetSearch(
model: vectorModel.model,
similarity,
limit,
datasetIds: datasets.map((item) => item.datasetId),
datasetIds,
searchMode,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
collectionFilterMatch

View File

@@ -72,6 +72,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
showStatus: false
};
});
const runtimeVariables = {
...filterSystemVariables(props.variables),
appId: String(plugin.id)

View File

@@ -127,8 +127,16 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
if (typeof val === 'object') return JSON.stringify(val);
if (typeof val === 'string') {
const str = JSON.stringify(val);
return str.startsWith('"') && str.endsWith('"') ? str.slice(1, -1) : str;
try {
const parsed = JSON.parse(val);
if (typeof parsed === 'object') {
return JSON.stringify(parsed);
}
return val;
} catch (error) {
const str = JSON.stringify(val);
return str.startsWith('"') && str.endsWith('"') ? str.slice(1, -1) : str;
}
}
return String(val);
@@ -235,7 +243,9 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
}
if (!httpJsonBody) return {};
if (httpContentType === ContentTypes.json) {
return json5.parse(replaceJsonBodyString(httpJsonBody));
httpJsonBody = replaceJsonBodyString(httpJsonBody);
console.log(httpJsonBody);
return json5.parse(httpJsonBody);
}
// Raw text, xml