mirror of
https://github.com/labring/FastGPT.git
synced 2025-08-05 22:55:27 +00:00
4.6.7-alpha commit (#743)
Co-authored-by: Archer <545436317@qq.com> Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
@@ -1,184 +0,0 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { delay } from '@fastgpt/global/common/system/utils';
|
||||
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
|
||||
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
|
||||
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
|
||||
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller';
|
||||
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
|
||||
import { defaultQAModels } from '@fastgpt/global/core/ai/model';
|
||||
|
||||
let success = 0;
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { limit = 50 } = req.body as { limit: number };
|
||||
await authCert({ req, authRoot: true });
|
||||
await connectToDatabase();
|
||||
success = 0;
|
||||
|
||||
try {
|
||||
await Promise.allSettled([
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN data_id VARCHAR(50);`),
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN q DROP NOT NULL;`), // q can null
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN a DROP NOT NULL;`), // a can null
|
||||
PgClient.query(
|
||||
`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id TYPE VARCHAR(50) USING team_id::VARCHAR(50);`
|
||||
), // team_id varchar
|
||||
PgClient.query(
|
||||
`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id TYPE VARCHAR(50) USING tmb_id::VARCHAR(50);`
|
||||
), // tmb_id varchar
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id SET NOT NULL;`), // team_id not null
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id SET NOT NULL;`), // tmb_id not null
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN dataset_id SET NOT NULL;`), // dataset_id not null
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN collection_id SET NOT NULL;`) // collection_id not null
|
||||
]);
|
||||
} catch (error) {}
|
||||
|
||||
try {
|
||||
await initPgData();
|
||||
} catch (error) {}
|
||||
|
||||
await MongoDataset.updateMany(
|
||||
{},
|
||||
{
|
||||
agentModel: defaultQAModels[0].model
|
||||
}
|
||||
);
|
||||
|
||||
jsonRes(res, {
|
||||
data: await init(limit),
|
||||
message:
|
||||
'初始化任务已开始，请注意日志进度。可通过 select count(id) from modeldata where data_id is null; 检查是否完全初始化，如果结果为 0 ，则完全初始化。'
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
type PgItemType = {
|
||||
id: string;
|
||||
q: string;
|
||||
a: string;
|
||||
dataset_id: string;
|
||||
collection_id: string;
|
||||
team_id: string;
|
||||
tmb_id: string;
|
||||
};
|
||||
|
||||
async function initPgData() {
|
||||
const limit = 10;
|
||||
const { rows } = await PgClient.query<{ user_id: string }>(`
|
||||
SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id='null';
|
||||
`);
|
||||
console.log('init pg', rows.length);
|
||||
let success = 0;
|
||||
for (let i = 0; i < limit; i++) {
|
||||
init(i);
|
||||
}
|
||||
|
||||
async function init(index: number): Promise<any> {
|
||||
const userId = rows[index]?.user_id;
|
||||
if (!userId) return;
|
||||
try {
|
||||
const tmb = await getUserDefaultTeam({ userId });
|
||||
console.log(tmb);
|
||||
|
||||
// update pg
|
||||
await PgClient.query(
|
||||
`Update ${PgDatasetTableName} set team_id = '${String(tmb.teamId)}', tmb_id = '${String(
|
||||
tmb.tmbId
|
||||
)}' where user_id = '${userId}' AND team_id='null';`
|
||||
);
|
||||
console.log(++success);
|
||||
init(index + limit);
|
||||
} catch (error) {
|
||||
if (error === 'default team not exist') {
|
||||
return;
|
||||
}
|
||||
console.log(error);
|
||||
await delay(1000);
|
||||
return init(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function init(limit: number): Promise<any> {
|
||||
const { rows: idList } = await PgClient.query<{ id: string }>(
|
||||
`SELECT id FROM ${PgDatasetTableName} WHERE data_id IS NULL`
|
||||
);
|
||||
|
||||
console.log('totalCount', idList.length);
|
||||
if (idList.length === 0) return;
|
||||
|
||||
for (let i = 0; i < limit; i++) {
|
||||
initData(i);
|
||||
}
|
||||
|
||||
async function initData(index: number): Promise<any> {
|
||||
const dataId = idList[index]?.id;
|
||||
if (!dataId) {
|
||||
console.log('done');
|
||||
return;
|
||||
}
|
||||
// get limit data where data_id is null
|
||||
const { rows } = await PgClient.query<PgItemType>(
|
||||
`SELECT id,q,a,dataset_id,collection_id,team_id,tmb_id FROM ${PgDatasetTableName} WHERE id=${dataId};`
|
||||
);
|
||||
const data = rows[0];
|
||||
if (!data) {
|
||||
console.log('done');
|
||||
return;
|
||||
}
|
||||
|
||||
let id = '';
|
||||
try {
|
||||
// create mongo data and update data_id
|
||||
const { _id } = await MongoDatasetData.create({
|
||||
teamId: data.team_id.trim(),
|
||||
tmbId: data.tmb_id.trim(),
|
||||
datasetId: data.dataset_id,
|
||||
collectionId: data.collection_id,
|
||||
q: data.q,
|
||||
a: data.a,
|
||||
fullTextToken: '',
|
||||
indexes: [
|
||||
{
|
||||
defaultIndex: !data.a,
|
||||
type: data.a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk,
|
||||
dataId: data.id,
|
||||
text: data.q
|
||||
}
|
||||
]
|
||||
});
|
||||
id = _id;
|
||||
// update pg data_id
|
||||
await PgClient.query(
|
||||
`UPDATE ${PgDatasetTableName} SET data_id='${String(_id)}' WHERE id=${dataId};`
|
||||
);
|
||||
|
||||
console.log(++success);
|
||||
return initData(index + limit);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
console.log(data);
|
||||
|
||||
try {
|
||||
if (id) {
|
||||
await MongoDatasetData.findByIdAndDelete(id);
|
||||
}
|
||||
} catch (error) {}
|
||||
await delay(500);
|
||||
return initData(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { delay } from '@fastgpt/global/common/system/utils';
|
||||
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
|
||||
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
|
||||
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { Types, connectionMongo } from '@fastgpt/service/common/mongo';
|
||||
import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant';
|
||||
import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller';
|
||||
import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
|
||||
|
||||
let success = 0;
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { limit = 50 } = req.body as { limit: number };
|
||||
await authCert({ req, authRoot: true });
|
||||
await connectToDatabase();
|
||||
success = 0;
|
||||
|
||||
await init(limit);
|
||||
await initCollectionFileTeam(limit);
|
||||
|
||||
jsonRes(res, {});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
type PgItemType = {
|
||||
id: string;
|
||||
q: string;
|
||||
a: string;
|
||||
dataset_id: string;
|
||||
collection_id: string;
|
||||
data_id: string;
|
||||
};
|
||||
|
||||
async function init(limit: number): Promise<any> {
|
||||
const { rows } = await PgClient.query<{ id: string; data_id: string }>(
|
||||
`SELECT id,data_id FROM ${PgDatasetTableName} WHERE team_id = tmb_id`
|
||||
);
|
||||
|
||||
console.log('totalCount', rows.length);
|
||||
|
||||
await delay(2000);
|
||||
|
||||
if (rows.length === 0) return;
|
||||
|
||||
for (let i = 0; i < limit; i++) {
|
||||
initData(i);
|
||||
}
|
||||
|
||||
async function initData(index: number): Promise<any> {
|
||||
const item = rows[index];
|
||||
if (!item) {
|
||||
console.log('done');
|
||||
return;
|
||||
}
|
||||
// get mongo
|
||||
const mongoData = await MongoDatasetData.findById(item.data_id, '_id teamId tmbId');
|
||||
if (!mongoData) {
|
||||
return initData(index + limit);
|
||||
}
|
||||
|
||||
try {
|
||||
// find team owner
|
||||
const db = connectionMongo?.connection?.db;
|
||||
const TeamMember = db.collection(TeamMemberCollectionName);
|
||||
|
||||
const tmb = await TeamMember.findOne({
|
||||
teamId: new Types.ObjectId(mongoData.teamId),
|
||||
role: 'owner'
|
||||
});
|
||||
|
||||
if (!tmb) {
|
||||
return initData(index + limit);
|
||||
}
|
||||
|
||||
// update mongo and pg tmb_id
|
||||
await MongoDatasetData.findByIdAndUpdate(item.data_id, {
|
||||
tmbId: tmb._id
|
||||
});
|
||||
await PgClient.query(
|
||||
`UPDATE ${PgDatasetTableName} SET tmb_id = '${String(tmb._id)}' WHERE id = '${item.id}'`
|
||||
);
|
||||
|
||||
console.log(++success);
|
||||
|
||||
return initData(index + limit);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
await delay(500);
|
||||
return initData(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function initCollectionFileTeam(limit: number) {
|
||||
/* init user default Team */
|
||||
const DatasetFile = getGFSCollection('dataset');
|
||||
const matchWhere = {
|
||||
$or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }]
|
||||
};
|
||||
const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([
|
||||
{
|
||||
$match: matchWhere
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$metadata.userId', // 按 metadata.userId 分组以去重
|
||||
userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId
|
||||
}
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
_id: 0, // 不显示 _id 字段
|
||||
userId: 1 // 只显示 userId 字段
|
||||
}
|
||||
}
|
||||
]).toArray();
|
||||
const users = uniqueUsersWithNoTeamId;
|
||||
|
||||
console.log('un init total', users.length);
|
||||
// limit 组一次
|
||||
const userArr: any[][] = [];
|
||||
for (let i = 0; i < users.length; i += limit) {
|
||||
userArr.push(users.slice(i, i + limit));
|
||||
}
|
||||
|
||||
let success = 0;
|
||||
for await (const item of userArr) {
|
||||
await Promise.all(item.map((item) => init(item.userId)));
|
||||
success += limit;
|
||||
console.log(success);
|
||||
}
|
||||
|
||||
async function init(userId: string): Promise<any> {
|
||||
try {
|
||||
const tmb = await getUserDefaultTeam({
|
||||
userId
|
||||
});
|
||||
|
||||
await DatasetFile.updateMany(
|
||||
{
|
||||
'metadata.userId': String(userId),
|
||||
...matchWhere
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
'metadata.teamId': String(tmb.teamId),
|
||||
'metadata.tmbId': String(tmb.tmbId)
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
if (error === 'team not exist' || error === 'tmbId or userId is required') {
|
||||
return;
|
||||
}
|
||||
console.log(error);
|
||||
await delay(1000);
|
||||
return init(userId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,330 +0,0 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema';
|
||||
import {
|
||||
createDefaultTeam,
|
||||
getUserDefaultTeam
|
||||
} from '@fastgpt/service/support/user/team/controller';
|
||||
import { MongoUser } from '@fastgpt/service/support/user/schema';
|
||||
import { UserModelSchema } from '@fastgpt/global/support/user/type';
|
||||
import { delay } from '@fastgpt/global/common/system/utils';
|
||||
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
|
||||
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
|
||||
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
|
||||
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
|
||||
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
|
||||
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
|
||||
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
|
||||
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
|
||||
import { MongoApp } from '@fastgpt/service/core/app/schema';
|
||||
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
|
||||
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
|
||||
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
|
||||
import { POST } from '@fastgpt/service/common/api/plusRequest';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
|
||||
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { limit = 50, maxSize = 3 } = req.body as { limit: number; maxSize: number };
|
||||
await authCert({ req, authRoot: true });
|
||||
await connectToDatabase();
|
||||
|
||||
await initDefaultTeam(limit, maxSize);
|
||||
await initMongoTeamId(limit);
|
||||
await initDatasetAndApp();
|
||||
await initCollectionFileTeam(limit);
|
||||
|
||||
if (FastGPTProUrl) {
|
||||
POST('/admin/init46');
|
||||
}
|
||||
|
||||
await initPgData();
|
||||
|
||||
jsonRes(res, {
|
||||
data: {}
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function initDefaultTeam(limit: number, maxSize: number) {
|
||||
/* init user default Team */
|
||||
const users = await MongoUser.find({}, '_id balance');
|
||||
console.log('init user default team', users.length);
|
||||
// 100 组一次
|
||||
const userArr: UserModelSchema[][] = [];
|
||||
for (let i = 0; i < users.length; i += limit) {
|
||||
userArr.push(users.slice(i, i + limit));
|
||||
}
|
||||
let success = 0;
|
||||
for await (const users of userArr) {
|
||||
await Promise.all(users.map(init));
|
||||
success += limit;
|
||||
console.log(success);
|
||||
}
|
||||
|
||||
async function init(user: UserModelSchema): Promise<any> {
|
||||
try {
|
||||
await createDefaultTeam({
|
||||
userId: user._id,
|
||||
balance: user.balance,
|
||||
maxSize
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
await delay(1000);
|
||||
return init(user);
|
||||
}
|
||||
}
|
||||
}
|
||||
async function initMongoTeamId(limit: number) {
|
||||
const mongoSchema = [
|
||||
{
|
||||
label: 'MongoPlugin',
|
||||
schema: MongoPlugin
|
||||
},
|
||||
{
|
||||
label: 'MongoChat',
|
||||
schema: MongoChat
|
||||
},
|
||||
{
|
||||
label: 'MongoChatItem',
|
||||
schema: MongoChatItem
|
||||
},
|
||||
{
|
||||
label: 'MongoApp',
|
||||
schema: MongoApp
|
||||
},
|
||||
{
|
||||
label: 'MongoDataset',
|
||||
schema: MongoDataset
|
||||
},
|
||||
{
|
||||
label: 'MongoDatasetCollection',
|
||||
schema: MongoDatasetCollection
|
||||
},
|
||||
{
|
||||
label: 'MongoDatasetTraining',
|
||||
schema: MongoDatasetTraining
|
||||
},
|
||||
{
|
||||
label: 'MongoBill',
|
||||
schema: MongoBill
|
||||
},
|
||||
{
|
||||
label: 'MongoOutLink',
|
||||
schema: MongoOutLink
|
||||
},
|
||||
{
|
||||
label: 'MongoOpenApi',
|
||||
schema: MongoOpenApi
|
||||
}
|
||||
];
|
||||
/* init user default Team */
|
||||
|
||||
for await (const item of mongoSchema) {
|
||||
console.log('start init', item.label);
|
||||
await initTeamTmbId(item.schema);
|
||||
console.log('finish init', item.label);
|
||||
}
|
||||
|
||||
async function initTeamTmbId(schema: any) {
|
||||
const emptyWhere = {
|
||||
$or: [{ teamId: { $exists: false } }, { teamId: null }]
|
||||
};
|
||||
const uniqueUsersWithNoTeamId = await schema.aggregate([
|
||||
{
|
||||
$match: emptyWhere
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$userId', // 按 userId 分组以去重
|
||||
userId: { $first: '$userId' } // 保留第一个出现的 userId
|
||||
}
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
_id: 0, // 不显示 _id 字段
|
||||
userId: 1 // 只显示 userId 字段
|
||||
}
|
||||
}
|
||||
]);
|
||||
const users = uniqueUsersWithNoTeamId;
|
||||
|
||||
console.log('un init total', users.length);
|
||||
// limit 组一次
|
||||
const userArr: any[][] = [];
|
||||
for (let i = 0; i < users.length; i += limit) {
|
||||
userArr.push(users.slice(i, i + limit));
|
||||
}
|
||||
|
||||
let success = 0;
|
||||
for await (const users of userArr) {
|
||||
await Promise.all(users.map((item) => init(item.userId)));
|
||||
success += limit;
|
||||
console.log(success);
|
||||
}
|
||||
|
||||
async function init(userId: string): Promise<any> {
|
||||
try {
|
||||
const tmb = await getUserDefaultTeam({ userId });
|
||||
|
||||
await schema.updateMany(
|
||||
{
|
||||
userId,
|
||||
...emptyWhere
|
||||
},
|
||||
{
|
||||
teamId: tmb.teamId,
|
||||
tmbId: tmb.tmbId
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
if (error === 'team not exist' || error === 'tmbId or userId is required') {
|
||||
return;
|
||||
}
|
||||
console.log(error);
|
||||
await delay(1000);
|
||||
return init(userId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
async function initDatasetAndApp() {
|
||||
await MongoDataset.updateMany(
|
||||
{},
|
||||
{
|
||||
$set: {
|
||||
permission: PermissionTypeEnum.private
|
||||
}
|
||||
}
|
||||
);
|
||||
await MongoApp.updateMany(
|
||||
{},
|
||||
{
|
||||
$set: {
|
||||
permission: PermissionTypeEnum.private
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
async function initCollectionFileTeam(limit: number) {
|
||||
/* init user default Team */
|
||||
const DatasetFile = getGFSCollection('dataset');
|
||||
const matchWhere = {
|
||||
$or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }]
|
||||
};
|
||||
const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([
|
||||
{
|
||||
$match: matchWhere
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$metadata.userId', // 按 metadata.userId 分组以去重
|
||||
userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId
|
||||
}
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
_id: 0, // 不显示 _id 字段
|
||||
userId: 1 // 只显示 userId 字段
|
||||
}
|
||||
}
|
||||
]).toArray();
|
||||
const users = uniqueUsersWithNoTeamId;
|
||||
|
||||
console.log('un init total', users.length);
|
||||
// limit 组一次
|
||||
const userArr: any[][] = [];
|
||||
for (let i = 0; i < users.length; i += limit) {
|
||||
userArr.push(users.slice(i, i + limit));
|
||||
}
|
||||
|
||||
let success = 0;
|
||||
for await (const item of userArr) {
|
||||
await Promise.all(item.map((item) => init(item.userId)));
|
||||
success += limit;
|
||||
console.log(success);
|
||||
}
|
||||
|
||||
async function init(userId: string): Promise<any> {
|
||||
try {
|
||||
const tmb = await getUserDefaultTeam({
|
||||
userId
|
||||
});
|
||||
|
||||
await DatasetFile.updateMany(
|
||||
{
|
||||
'metadata.userId': String(userId),
|
||||
...matchWhere
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
'metadata.teamId': String(tmb.teamId),
|
||||
'metadata.tmbId': String(tmb.tmbId)
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
if (error === 'team not exist' || error === 'tmbId or userId is required') {
|
||||
return;
|
||||
}
|
||||
console.log(error);
|
||||
await delay(1000);
|
||||
return init(userId);
|
||||
}
|
||||
}
|
||||
}
|
||||
async function initPgData() {
|
||||
const limit = 10;
|
||||
// add column
|
||||
try {
|
||||
await Promise.allSettled([
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id VARCHAR(50);`),
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id VARCHAR(50);`),
|
||||
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN user_id DROP NOT NULL;`)
|
||||
]);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
console.log('column exists');
|
||||
}
|
||||
|
||||
const { rows } = await PgClient.query<{ user_id: string }>(`
|
||||
SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id IS NULL;
|
||||
`);
|
||||
console.log('init pg', rows.length);
|
||||
let success = 0;
|
||||
for (let i = 0; i < limit; i++) {
|
||||
init(i);
|
||||
}
|
||||
async function init(index: number): Promise<any> {
|
||||
const userId = rows[index]?.user_id;
|
||||
if (!userId) return;
|
||||
try {
|
||||
const tmb = await getUserDefaultTeam({ userId });
|
||||
// update pg
|
||||
await PgClient.query(
|
||||
`Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id IS NULL;`
|
||||
);
|
||||
console.log(++success);
|
||||
init(index + limit);
|
||||
} catch (error) {
|
||||
if (error === 'default team not exist') {
|
||||
return;
|
||||
}
|
||||
console.log(error);
|
||||
await delay(1000);
|
||||
return init(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
|
||||
import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
|
||||
import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
|
||||
|
||||
let success = 0;
|
||||
|
||||
106
projects/app/src/pages/api/admin/initv467.ts
Normal file
106
projects/app/src/pages/api/admin/initv467.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
|
||||
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
|
||||
import { MongoImage } from '@fastgpt/service/common/file/image/schema';
|
||||
import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type';
|
||||
import { delay } from '@fastgpt/global/common/system/utils';
|
||||
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
|
||||
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
||||
|
||||
let success = 0;
|
||||
let deleteImg = 0;
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { test = false } = req.body as { test: boolean };
|
||||
await authCert({ req, authRoot: true });
|
||||
await connectToDatabase();
|
||||
success = 0;
|
||||
deleteImg = 0;
|
||||
|
||||
// 取消 pg tmb_id 和 data_id 的null
|
||||
await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id DROP NOT NULL;`);
|
||||
await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN data_id DROP NOT NULL;`);
|
||||
|
||||
// 重新绑定 images 和 collections
|
||||
const images = await MongoImage.find(
|
||||
{ 'metadata.fileId': { $exists: true } },
|
||||
'_id metadata'
|
||||
).lean();
|
||||
|
||||
// 去除 fileId 相同的数据
|
||||
const fileIdMap = new Map<string, MongoImageSchemaType>();
|
||||
images.forEach((image) => {
|
||||
// @ts-ignore
|
||||
const fileId = image.metadata?.fileId;
|
||||
if (!fileIdMap.has(fileId) && fileId) {
|
||||
fileIdMap.set(fileId, image);
|
||||
}
|
||||
});
|
||||
const images2 = Array.from(fileIdMap.values());
|
||||
|
||||
console.log('total image list', images2.length);
|
||||
|
||||
for await (const image of images2) {
|
||||
await initImages(image, test);
|
||||
}
|
||||
|
||||
jsonRes(res, {
|
||||
data: success,
|
||||
message: 'success'
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error
|
||||
});
|
||||
}
|
||||
}
|
||||
export const initImages = async (image: MongoImageSchemaType, test: boolean): Promise<any> => {
|
||||
try {
|
||||
//@ts-ignore
|
||||
const fileId = image.metadata.fileId as string;
|
||||
if (!fileId) return;
|
||||
|
||||
// 找到集合
|
||||
const collection = await MongoDatasetCollection.findOne({ fileId }, '_id metadata').lean();
|
||||
|
||||
if (!collection) {
|
||||
deleteImg++;
|
||||
console.log('deleteImg', deleteImg);
|
||||
|
||||
if (test) return;
|
||||
return MongoImage.deleteOne({ _id: image._id });
|
||||
}
|
||||
|
||||
const relatedImageId = getNanoid(24);
|
||||
|
||||
// update image
|
||||
if (!test) {
|
||||
await Promise.all([
|
||||
MongoImage.updateMany(
|
||||
{ 'metadata.fileId': fileId },
|
||||
{ $set: { 'metadata.relatedId': relatedImageId } }
|
||||
),
|
||||
MongoDatasetCollection.findByIdAndUpdate(collection._id, {
|
||||
$set: {
|
||||
'metadata.relatedImgId': relatedImageId
|
||||
}
|
||||
})
|
||||
]);
|
||||
}
|
||||
|
||||
success++;
|
||||
console.log('success', success);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
await delay(1000);
|
||||
return initImages(image, test);
|
||||
}
|
||||
};
|
||||
@@ -1,95 +0,0 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import {
|
||||
delFileByFileIdList,
|
||||
getGFSCollection
|
||||
} from '@fastgpt/service/common/file/gridfs/controller';
|
||||
import { addLog } from '@fastgpt/service/common/system/log';
|
||||
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
|
||||
import { delay } from '@fastgpt/global/common/system/utils';
|
||||
|
||||
/*
|
||||
check dataset.files data. If there is no match in dataset.collections, delete it
|
||||
*/
|
||||
let deleteFileAmount = 0;
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const {
|
||||
startDay = 10,
|
||||
endDay = 3,
|
||||
limit = 30
|
||||
} = req.body as { startDay?: number; endDay?: number; limit?: number };
|
||||
await authCert({ req, authRoot: true });
|
||||
await connectToDatabase();
|
||||
|
||||
// start: now - maxDay, end: now - 3 day
|
||||
const start = new Date(Date.now() - startDay * 24 * 60 * 60 * 1000);
|
||||
const end = new Date(Date.now() - endDay * 24 * 60 * 60 * 1000);
|
||||
deleteFileAmount = 0;
|
||||
|
||||
checkFiles(start, end, limit);
|
||||
|
||||
jsonRes(res, {
|
||||
message: 'success'
|
||||
});
|
||||
} catch (error) {
|
||||
addLog.error(`check valid dataset files error`, error);
|
||||
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function checkFiles(start: Date, end: Date, limit: number) {
|
||||
const collection = getGFSCollection('dataset');
|
||||
const where = {
|
||||
uploadDate: { $gte: start, $lte: end }
|
||||
};
|
||||
|
||||
// 1. get all _id
|
||||
const ids = await collection
|
||||
.find(where, {
|
||||
projection: {
|
||||
_id: 1
|
||||
}
|
||||
})
|
||||
.toArray();
|
||||
console.log('total files', ids.length);
|
||||
|
||||
for (let i = 0; i < limit; i++) {
|
||||
check(i);
|
||||
}
|
||||
|
||||
async function check(index: number): Promise<any> {
|
||||
const id = ids[index];
|
||||
if (!id) {
|
||||
console.log(`检测完成，共删除 ${deleteFileAmount} 个无效文件`);
|
||||
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const { _id } = id;
|
||||
|
||||
// 2. find fileId in dataset.collections
|
||||
const hasCollection = await MongoDatasetCollection.countDocuments({ fileId: _id });
|
||||
|
||||
// 3. if not found, delete file
|
||||
if (hasCollection === 0) {
|
||||
await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(_id)] });
|
||||
console.log('delete file', _id);
|
||||
deleteFileAmount++;
|
||||
}
|
||||
index % 100 === 0 && console.log(index);
|
||||
return check(index + limit);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
await delay(2000);
|
||||
return check(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user