This commit is contained in:
Archer
2023-09-18 13:37:25 +08:00
committed by GitHub
parent 0bb31b985d
commit b8ea546b3f
25 changed files with 288 additions and 225 deletions

View File

@@ -0,0 +1,27 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase, Bill } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
try {
await Bill.collection.dropIndex('time_1');
} catch (error) {}
try {
await Bill.collection.createIndex({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 });
} catch (error) {}
jsonRes(res, {
data: {}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}

View File

@@ -1,6 +1,6 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
@@ -21,6 +21,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
// other data. Delete only vector data
if (fileId === OtherFileId) {
await PgClient.delete(PgDatasetTableName, {
where: [
@@ -31,6 +32,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
]
});
} else {
// auth file
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
@@ -40,6 +42,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await PgClient.delete(PgDatasetTableName, {
where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]]
});
// delete all training data
await TrainingData.deleteMany({
userId,
file_id: fileId
});
// delete file
await bucket.delete(new Types.ObjectId(fileId));

View File

@@ -18,7 +18,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('最多 10 组 API 秘钥');
}
const apiKey = `fastgpt-${nanoid()}`;
const apiKey = `${global.systemEnv?.openapiPrefix || 'fastgpt'}-${nanoid()}`;
await OpenApi.create({
userId,

View File

@@ -116,7 +116,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
appId,
userId
}),
getChatHistory({ chatId, userId })
getChatHistory({ chatId, appId, userId })
]);
const isOwner = !shareId && userId === String(app.userId);

View File

@@ -7,6 +7,7 @@ import { Types } from 'mongoose';
import type { ChatItemType } from '@/types/chat';
export type Props = {
appId?: string;
chatId?: string;
limit?: number;
};
@@ -36,9 +37,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
export async function getChatHistory({
chatId,
userId,
appId,
limit = 30
}: Props & { userId: string }): Promise<Response> {
if (!chatId) {
if (!chatId || !appId) {
return { history: [] };
}
@@ -46,6 +48,7 @@ export async function getChatHistory({
{
$match: {
chatId,
appId: new Types.ObjectId(appId),
userId: new Types.ObjectId(userId)
}
},

View File

@@ -2,9 +2,10 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { findAllChildrenIds } from '../delete';
import QueryStream from 'pg-query-stream';
import Papa from 'papaparse';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -12,7 +13,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
kbId: string;
};
if (!kbId) {
if (!kbId || !global.pgClient) {
throw new Error('缺少参数');
}
@@ -22,7 +23,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { userId } = await authUser({ req, authToken: true });
const exportIds = [kbId, ...(await findAllChildrenIds(kbId))];
console.log(exportIds);
const thirtyMinutesAgo = new Date(
Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000
@@ -45,37 +45,50 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error(`上次导出未到 ${minutes},每 ${minutes}仅可导出一次。`);
}
const where: any = [
['user_id', userId],
'AND',
`kb_id IN (${exportIds.map((id) => `'${id}'`).join(',')})`
];
// 从 pg 中获取所有数据
const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
PgDatasetTableName,
{
where,
fields: ['q', 'a', 'source'],
order: [{ field: 'id', mode: 'DESC' }],
limit: 1000000
// connect pg
global.pgClient.connect((err, client, done) => {
if (err) {
console.error(err);
res.end('Error connecting to database');
return;
}
);
// create pg select stream
const query = new QueryStream(
`SELECT q, a, source FROM ${PgDatasetTableName} where user_id='${userId}' AND kb_id IN (${exportIds
.map((id) => `'${id}'`)
.join(',')})`
);
const stream = client.query(query);
const data: [string, string, string][] = pgData.rows.map((item) => [
item.q.replace(/\n/g, '\\n'),
item.a.replace(/\n/g, '\\n'),
item.source
]);
res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv');
res.setHeader('Content-Type', 'text/csv');
// update export time
await User.findByIdAndUpdate(userId, {
'limit.exportKbTime': new Date()
});
res.write('index,content,source');
jsonRes(res, {
data
// parse data every row
stream.on('data', (row: { q: string; a: string; source?: string }) => {
const csv = Papa.unparse([row], { header: false });
res.write(`\n${csv}`);
});
stream.on('end', async () => {
try {
// update export time
await User.findByIdAndUpdate(userId, {
'limit.exportKbTime': new Date()
});
} catch (error) {}
// close response
done();
res.end();
});
stream.on('error', (err) => {
done(err);
res.end('Error exporting data');
});
});
} catch (err) {
res.status(500);
jsonRes(res, {
code: 500,
error: err