ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-24 22:03:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Sync collection (#3368)

Browse Source 
* feat: sync collection

* feat: sync collection

* perf: website selector

* update doc
This commit is contained in:
Archer
2024-12-11 15:03:41 +08:00
committed by GitHub
parent 048f5a2d53
commit d5752ddbaa
40 changed files with 365 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
packages/service/core/dataset/collection/utils.ts
View File

@@ -163,6 +163,10 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
...sourceReadType
});
if (!rawText) {
return DatasetCollectionSyncResultEnum.failed;
}
// Check if the original text is the same: skip if same
const hashRawText = hashStr(rawText);
if (collection.hashRawText && hashRawText === collection.hashRawText) {
@@ -178,28 +182,30 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
createCollectionParams: {
teamId: collection.teamId,
tmbId: collection.tmbId,
datasetId: collection.datasetId._id,
name: collection.name,
datasetId: collection.datasetId._id,
parentId: collection.parentId,
type: collection.type,
trainingType: collection.trainingType,
chunkSize: collection.chunkSize,
chunkSplitter: collection.chunkSplitter,
qaPrompt: collection.qaPrompt,
fileId: collection.fileId,
rawLink: collection.rawLink,
externalFileId: collection.externalFileId,
externalFileUrl: collection.externalFileUrl,
apiFileId: collection.apiFileId,
rawTextLength: rawText.length,
hashRawText,
rawTextLength: rawText.length,
metadata: collection.metadata,
tags: collection.tags,
createTime: collection.createTime,
parentId: collection.parentId,
trainingType: collection.trainingType,
chunkSize: collection.chunkSize,
chunkSplitter: collection.chunkSplitter,
qaPrompt: collection.qaPrompt,
metadata: collection.metadata
updateTime: new Date()
}
});