ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-08-10 01:52:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

4.6.3-website dataset (#532)

Browse Source
This commit is contained in:
Archer
2023-12-03 20:45:57 +08:00
committed by GitHub
parent b916183848
commit a9ae270335
122 changed files with 3793 additions and 1360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
projects/app/src/pages/api/admin/initv462.ts
View File

@@ -1,7 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { jiebaSplit } from '@/service/core/dataset/utils';
@@ -17,10 +17,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
console.log(
'total',
await MongoDatasetData.countDocuments({ fullTextToken: { $exists: false } })
await MongoDatasetData.countDocuments({
fullTextToken: { $exists: false },
updateTime: { $lt: new Date() }
})
);
await initFullTextToken(limit);
await initFullTextToken(limit, new Date());
jsonRes(res, {
message: 'success'
@@ -34,9 +37,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
}
}
export async function initFullTextToken(limit = 50): Promise<any> {
export async function initFullTextToken(limit = 50, endDate: Date): Promise<any> {
try {
const dataList = await MongoDatasetData.find({ fullTextToken: { $exists: false } }, '_id q a')
const dataList = await MongoDatasetData.find(
{ fullTextToken: { $exists: false }, updateTime: { $lt: endDate } },
'_id q a'
)
.limit(limit)
.lean();
if (dataList.length === 0) return;
@@ -56,9 +62,9 @@ export async function initFullTextToken(limit = 50): Promise<any> {
success += result.filter((item) => item.status === 'fulfilled').length;
console.log(`success: ${success}`);
return initFullTextToken(limit);
return initFullTextToken(limit, endDate);
} catch (error) {
await delay(1000);
return initFullTextToken(limit);
return initFullTextToken(limit, endDate);
}
}