mirror of
https://github.com/labring/FastGPT.git
synced 2025-10-19 01:54:04 +00:00
4.8.6 merge (#1943)
* Dataset collection forbid (#1885) * perf: tool call support same id * feat: collection forbid * feat: collection forbid * Inheritance Permission for apps (#1897) * feat: app schema define chore: references of authapp * feat: authApp method inheritance * feat: create and update api * feat: update * feat: inheritance Permission controller for app. * feat: abstract version of inheritPermission * feat: ancestorId for apps * chore: update app * fix: inheritPermission abstract version * feat: update folder defaultPermission * feat: app update api * chore: inheritance frontend * chore: app list api * feat: update defaultPermission in app deatil * feat: backend api finished * feat: app inheritance permission fe * fix: app update defaultpermission causes collaborator miss * fix: ts error * chore: adjust the codes * chore: i18n chore: i18n * chore: fe adjust and i18n * chore: adjust the code * feat: resume api; chore: rewrite update api and inheritPermission methods * chore: something * chore: fe code adjusting * feat: frontend adjusting * chore: fe code adjusting * chore: adjusting the code * perf: fe loading * format * Inheritance fix (#1908) * fix: SlideCard * fix: authapp did not return parent app for inheritance app * fix: fe adjusting * feat: fe adjusing * perf: inherit per ux * doc * fix: ts errors (#1916) * perf: inherit permission * fix: permission inherit * Workflow type (#1938) * perf: workflow type tmp workflow perf: workflow type feat: custom field config * perf: dynamic input * perf: node classify * perf: node classify * perf: node classify * perf: node classify * fix: workflow custom input * feat: text editor and customFeedback move to basic nodes * feat: community system plugin * fix: ts * feat: exprEval plugin * perf: workflow type * perf: plugin important * fix: default templates * perf: markdown hr css * lock * perf: fetch url * perf: new plugin version * fix: chat histories update * fix: collection paths invalid * perf: app card ui --------- Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
This commit is contained in:
Archer
@@ -1,4 +1,4 @@
|
||||
import { Types, connectionMongo } from '../../mongo';
|
||||
import { Types, connectionMongo, ReadPreference } from '../../mongo';
|
||||
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
|
||||
import fsp from 'fs/promises';
|
||||
import fs from 'fs';
|
||||
@@ -17,7 +17,9 @@ export function getGFSCollection(bucket: `${BucketNameEnum}`) {
|
||||
}
|
||||
export function getGridBucket(bucket: `${BucketNameEnum}`) {
|
||||
return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db, {
|
||||
bucketName: bucket
|
||||
bucketName: bucket,
|
||||
// @ts-ignore
|
||||
readPreference: ReadPreference.SECONDARY_PREFERRED // Read from secondary node
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -10,3 +10,5 @@ export const connectionMongo = (() => {
|
||||
|
||||
return global.mongodb;
|
||||
})();
|
||||
|
||||
export const ReadPreference = mongoose.mongo.ReadPreference;
|
||||
|
||||
@@ -43,6 +43,10 @@ export async function connectMongo({
|
||||
maxIdleTimeMS: 300000,
|
||||
retryWrites: true,
|
||||
retryReads: true
|
||||
|
||||
// readPreference: 'secondaryPreferred',
|
||||
// readConcern: { level: 'local' },
|
||||
// writeConcern: { w: 'majority', j: true }
|
||||
});
|
||||
|
||||
console.log('mongo connected');
|
||||
|
||||
@@ -1,20 +1,22 @@
|
||||
import { addLog } from '../system/log';
|
||||
import { connectionMongo, ClientSession } from './index';
|
||||
|
||||
const timeout = 60000;
|
||||
|
||||
export const mongoSessionRun = async <T = unknown>(fn: (session: ClientSession) => Promise<T>) => {
|
||||
const session = await connectionMongo.startSession();
|
||||
let committed = false;
|
||||
|
||||
try {
|
||||
session.startTransaction();
|
||||
session.startTransaction({
|
||||
maxCommitTimeMS: timeout
|
||||
});
|
||||
const result = await fn(session);
|
||||
|
||||
await session.commitTransaction();
|
||||
committed = true;
|
||||
|
||||
return result as T;
|
||||
} catch (error) {
|
||||
if (!committed) {
|
||||
if (!session.transaction.isCommitted) {
|
||||
await session.abortTransaction();
|
||||
} else {
|
||||
addLog.warn('Un catch mongo session error', { error });
|
||||
|
||||
4
packages/service/common/mongo/utils.ts
Normal file
4
packages/service/common/mongo/utils.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
export const readFromSecondary = {
|
||||
readPreference: 'secondaryPreferred',
|
||||
readConcern: 'local'
|
||||
};
|
||||
@@ -24,6 +24,9 @@ export type InsertVectorControllerProps = InsertVectorProps & {
|
||||
export type EmbeddingRecallProps = {
|
||||
teamId: string;
|
||||
datasetIds: string[];
|
||||
|
||||
forbidCollectionIdList: string[];
|
||||
// forbidEmbIndexIdList: string[];
|
||||
// similarity?: number;
|
||||
// efSearch?: number;
|
||||
};
|
||||
|
||||
@@ -213,14 +213,19 @@ export class MilvusCtrl {
|
||||
};
|
||||
embRecall = async (props: EmbeddingRecallCtrlProps): Promise<EmbeddingRecallResponse> => {
|
||||
const client = await this.getClient();
|
||||
const { teamId, datasetIds, vector, limit, retry = 2 } = props;
|
||||
const { teamId, datasetIds, vector, limit, forbidCollectionIdList, retry = 2 } = props;
|
||||
|
||||
const forbidColQuery =
|
||||
forbidCollectionIdList.length > 0
|
||||
? `and (collectionId not in [${forbidCollectionIdList.map((id) => `"${String(id)}"`).join(',')}])`
|
||||
: '';
|
||||
|
||||
try {
|
||||
const { results } = await client.search({
|
||||
collection_name: DatasetVectorTableName,
|
||||
data: vector,
|
||||
limit,
|
||||
filter: `(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${String(id)}"`).join(',')}])`,
|
||||
filter: `(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${String(id)}"`).join(',')}]) ${forbidColQuery}`,
|
||||
output_fields: ['collectionId']
|
||||
});
|
||||
|
||||
|
||||
@@ -118,9 +118,29 @@ export class PgVectorCtrl {
|
||||
}
|
||||
};
|
||||
embRecall = async (props: EmbeddingRecallCtrlProps): Promise<EmbeddingRecallResponse> => {
|
||||
const { teamId, datasetIds, vector, limit, retry = 2 } = props;
|
||||
const { teamId, datasetIds, vector, limit, forbidCollectionIdList, retry = 2 } = props;
|
||||
|
||||
const forbidCollectionSql =
|
||||
forbidCollectionIdList.length > 0
|
||||
? `AND collection_id NOT IN (${forbidCollectionIdList.map((id) => `'${String(id)}'`).join(',')})`
|
||||
: 'AND collection_id IS NOT NULL';
|
||||
// const forbidDataSql =
|
||||
// forbidEmbIndexIdList.length > 0 ? `AND id NOT IN (${forbidEmbIndexIdList.join(',')})` : '';
|
||||
|
||||
try {
|
||||
// const explan: any = await PgClient.query(
|
||||
// `BEGIN;
|
||||
// SET LOCAL hnsw.ef_search = ${global.systemEnv?.pgHNSWEfSearch || 100};
|
||||
// EXPLAIN ANALYZE select id, collection_id, vector <#> '[${vector}]' AS score
|
||||
// from ${DatasetVectorTableName}
|
||||
// where team_id='${teamId}'
|
||||
// AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
|
||||
// ${forbidCollectionSql}
|
||||
// order by score limit ${limit};
|
||||
// COMMIT;`
|
||||
// );
|
||||
// console.log(explan[2].rows);
|
||||
|
||||
const results: any = await PgClient.query(
|
||||
`
|
||||
BEGIN;
|
||||
@@ -129,6 +149,7 @@ export class PgVectorCtrl {
|
||||
from ${DatasetVectorTableName}
|
||||
where team_id='${teamId}'
|
||||
AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
|
||||
${forbidCollectionSql}
|
||||
order by score limit ${limit};
|
||||
COMMIT;`
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user