From f6aea484ced8cfb03b22b4c58277717c79f3c085 Mon Sep 17 00:00:00 2001
From: Archer <545436317@qq.com>
Date: Thu, 16 Nov 2023 17:10:04 +0800
Subject: [PATCH] fix: 46 tmbId empty (#480)
* mongo init
* perf: mongo connect
* perf: favicon
* fix: member id
* 46fix sh
* doc
---
.../content/docs/installation/upgrading/46.md | 12 ++++
projects/app/src/pages/_app.tsx | 2 +-
.../admin/{initv46-3.ts => initv46-fix.ts} | 62 ++++++++++---------
.../service/core/dataset/data/controller.ts | 3 +
.../app/src/service/events/generateVector.ts | 4 +-
5 files changed, 51 insertions(+), 32 deletions(-)
rename projects/app/src/pages/api/admin/{initv46-3.ts => initv46-fix.ts} (56%)
diff --git a/docSite/content/docs/installation/upgrading/46.md b/docSite/content/docs/installation/upgrading/46.md
index 552326b10..5867d42c7 100644
--- a/docSite/content/docs/installation/upgrading/46.md
+++ b/docSite/content/docs/installation/upgrading/46.md
@@ -53,3 +53,15 @@ curl --location --request POST 'https://{{host}}/api/admin/initv46-2' \
4. 新增 - 支持知识库配置文本预处理模型
5. 线上环境新增 - ReRank向量召回，提高召回精度
6. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈
+
+## 4.6缺陷修复
+
+旧的 4.6 版本由于缺少一个字段，导致文件导入时知识库数据无法显示，可执行下面的脚本：
+
+https://xxxxx/api/admin/initv46-fix
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv46-fix' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
\ No newline at end of file
diff --git a/projects/app/src/pages/_app.tsx b/projects/app/src/pages/_app.tsx
index b80ad7bbd..d0b379540 100644
--- a/projects/app/src/pages/_app.tsx
+++ b/projects/app/src/pages/_app.tsx
@@ -101,7 +101,7 @@ function App({ Component, pageProps }: AppProps) {
name="viewport"
content="width=device-width,initial-scale=1.0,maximum-scale=1.0,minimum-scale=1.0,user-scalable=no, viewport-fit=cover"
/>
-
+
{scripts?.map((item, i) => )}
diff --git a/projects/app/src/pages/api/admin/initv46-3.ts b/projects/app/src/pages/api/admin/initv46-fix.ts
similarity index 56%
rename from projects/app/src/pages/api/admin/initv46-3.ts
rename to projects/app/src/pages/api/admin/initv46-fix.ts
index a564e069b..0a639b9ab 100644
--- a/projects/app/src/pages/api/admin/initv46-3.ts
+++ b/projects/app/src/pages/api/admin/initv46-fix.ts
@@ -10,6 +10,8 @@ import {
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { Types, connectionMongo } from '@fastgpt/service/common/mongo';
+import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
@@ -43,57 +45,59 @@ type PgItemType = {
};
async function init(limit: number): Promise {
- const { rows: idList } = await PgClient.query<{ id: string }>(
- `SELECT id FROM ${PgDatasetTableName} WHERE inited=1`
+ const { rows } = await PgClient.query<{ id: string; data_id: string }>(
+ `SELECT id,data_id FROM ${PgDatasetTableName} WHERE team_id = tmb_id`
);
- console.log('totalCount', idList.length);
+ console.log('totalCount', rows.length);
await delay(2000);
- if (idList.length === 0) return;
+ if (rows.length === 0) return;
for (let i = 0; i < limit; i++) {
initData(i);
}
async function initData(index: number): Promise {
- const dataId = idList[index]?.id;
- if (!dataId) {
+ const item = rows[index];
+ if (!item) {
console.log('done');
return;
}
- // get limit data where data_id is null
- const { rows } = await PgClient.query(
- `SELECT id,q,a,dataset_id,collection_id,data_id FROM ${PgDatasetTableName} WHERE id=${dataId};`
- );
- const data = rows[0];
- if (!data) {
- console.log('done');
- return;
+ // get mongo
+ const mongoData = await MongoDatasetData.findById(item.data_id, '_id teamId tmbId');
+ if (!mongoData) {
+ return initData(index + limit);
}
try {
- // update mongo data and update inited
- await MongoDatasetData.findByIdAndUpdate(data.data_id, {
- q: data.q,
- a: data.a,
- indexes: [
- {
- defaultIndex: !data.a,
- type: data.a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk,
- dataId: data.id,
- text: data.q
- }
- ]
+ // find team owner
+ const db = connectionMongo?.connection?.db;
+ const TeamMember = db.collection(TeamMemberCollectionName);
+
+ const tmb = await TeamMember.findOne({
+ teamId: new Types.ObjectId(mongoData.teamId),
+ role: 'owner'
});
- // update pg data_id
- await PgClient.query(`UPDATE ${PgDatasetTableName} SET inited=0 WHERE id=${dataId};`);
+
+ if (!tmb) {
+ return initData(index + limit);
+ }
+
+ // update mongo and pg tmb_id
+ await MongoDatasetData.findByIdAndUpdate(item.data_id, {
+ tmbId: tmb._id
+ });
+ await PgClient.query(
+ `UPDATE ${PgDatasetTableName} SET tmb_id = '${String(tmb._id)}' WHERE id = '${item.id}'`
+ );
+
+ console.log(++success);
return initData(index + limit);
} catch (error) {
console.log(error);
- console.log(data);
await delay(500);
return initData(index);
}
diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts
index 4d2cf2326..4a7281ba4 100644
--- a/projects/app/src/service/core/dataset/data/controller.ts
+++ b/projects/app/src/service/core/dataset/data/controller.ts
@@ -29,6 +29,9 @@ export async function insertData2Dataset({
if (!q || !datasetId || !collectionId || !model) {
return Promise.reject('q, datasetId, collectionId, model is required');
}
+ if (String(teamId) === String(tmbId)) {
+ return Promise.reject("teamId and tmbId can't be the same");
+ }
q = q.trim();
a = a.trim();
diff --git a/projects/app/src/service/events/generateVector.ts b/projects/app/src/service/events/generateVector.ts
index fd2471b5f..f333eead3 100644
--- a/projects/app/src/service/events/generateVector.ts
+++ b/projects/app/src/service/events/generateVector.ts
@@ -113,7 +113,7 @@ export async function generateVector(): Promise {
// insert data to pg
const { tokenLen } = await insertData2Dataset({
teamId: data.teamId,
- tmbId: data.teamId,
+ tmbId: data.tmbId,
datasetId: data.datasetId,
collectionId: data.collectionId,
q: dataItem.q,
@@ -124,7 +124,7 @@ export async function generateVector(): Promise {
// push bill
pushGenerateVectorBill({
teamId: data.teamId,
- tmbId: data.teamId,
+ tmbId: data.tmbId,
tokenLen: tokenLen,
model: data.model,
billId: data.billId