mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-21 11:43:56 +00:00
v4.5.1 (#417)
This commit is contained in:
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"name": "@fastgpt/common",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"mongoose": "^7.0.2",
|
||||
"winston": "^3.10.0",
|
||||
"winston-mongodb": "^5.1.1",
|
||||
"axios": "^1.5.1",
|
||||
"nextjs-cors": "^2.1.2",
|
||||
"next": "13.5.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.8.5"
|
||||
}
|
||||
}
|
@@ -1,15 +0,0 @@
|
||||
import { POST } from './request';
|
||||
|
||||
export const postTextCensor = (data: { text: string }) =>
|
||||
POST<{ code?: number; message: string }>('/common/censor/text_baidu', data)
|
||||
.then((res) => {
|
||||
if (res?.code === 5000) {
|
||||
return Promise.reject(res);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
if (err?.code === 5000) {
|
||||
return Promise.reject(err.message);
|
||||
}
|
||||
return Promise.resolve('');
|
||||
});
|
0
packages/common/type/chat.d.ts
vendored
0
packages/common/type/chat.d.ts
vendored
0
packages/core/chat/type.d.ts
vendored
0
packages/core/chat/type.d.ts
vendored
13
packages/core/dataset/type.d.ts
vendored
13
packages/core/dataset/type.d.ts
vendored
@@ -1,13 +0,0 @@
|
||||
import { DatasetTypeEnum } from './constant';
|
||||
|
||||
export type DatasetSchemaType = {
|
||||
_id: string;
|
||||
userId: string;
|
||||
parentId: string;
|
||||
updateTime: Date;
|
||||
avatar: string;
|
||||
name: string;
|
||||
vectorModel: string;
|
||||
tags: string[];
|
||||
type: `${DatasetTypeEnum}`;
|
||||
};
|
@@ -1,8 +0,0 @@
|
||||
import { datasetSpecialIds } from './constant';
|
||||
import { strIsLink } from '@fastgpt/common/tools/str';
|
||||
|
||||
export function isSpecialFileId(id: string) {
|
||||
if (datasetSpecialIds.includes(id)) return true;
|
||||
if (strIsLink(id)) return true;
|
||||
return false;
|
||||
}
|
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"name": "@fastgpt/core",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@fastgpt/common": "workspace:*",
|
||||
"@fastgpt/support": "workspace:*",
|
||||
"encoding": "^0.1.13",
|
||||
"openai": "^4.12.1",
|
||||
"tunnel": "^0.0.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/tunnel": "^0.0.4"
|
||||
}
|
||||
}
|
3
packages/global/common/bill/types/billReq.d.ts
vendored
Normal file
3
packages/global/common/bill/types/billReq.d.ts
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
export type CreateTrainingBillType = {
|
||||
name: string;
|
||||
};
|
@@ -27,7 +27,8 @@ export enum ERROR_ENUM {
|
||||
insufficientQuota = 'insufficientQuota',
|
||||
unAuthModel = 'unAuthModel',
|
||||
unAuthApiKey = 'unAuthApiKey',
|
||||
unAuthKb = 'unAuthKb',
|
||||
unAuthDataset = 'unAuthDataset',
|
||||
unAuthDatasetCollection = 'unAuthDatasetCollection',
|
||||
unAuthFile = 'unAuthFile'
|
||||
}
|
||||
export const ERROR_RESPONSE: Record<
|
||||
@@ -57,9 +58,9 @@ export const ERROR_RESPONSE: Record<
|
||||
message: '无权使用该模型',
|
||||
data: null
|
||||
},
|
||||
[ERROR_ENUM.unAuthKb]: {
|
||||
[ERROR_ENUM.unAuthDataset]: {
|
||||
code: 512,
|
||||
statusText: ERROR_ENUM.unAuthKb,
|
||||
statusText: ERROR_ENUM.unAuthDataset,
|
||||
message: '无权使用该知识库',
|
||||
data: null
|
||||
},
|
||||
@@ -74,5 +75,11 @@ export const ERROR_RESPONSE: Record<
|
||||
statusText: ERROR_ENUM.unAuthApiKey,
|
||||
message: 'Api Key 不合法',
|
||||
data: null
|
||||
},
|
||||
[ERROR_ENUM.unAuthDatasetCollection]: {
|
||||
code: 515,
|
||||
statusText: ERROR_ENUM.unAuthDatasetCollection,
|
||||
message: '无权使用该知识库文件',
|
||||
data: null
|
||||
}
|
||||
};
|
5
packages/global/common/error/utils.ts
Normal file
5
packages/global/common/error/utils.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export const getErrText = (err: any, def = '') => {
|
||||
const msg: string = typeof err === 'string' ? err : err?.message || def || '';
|
||||
msg && console.log('error =>', msg);
|
||||
return msg;
|
||||
};
|
@@ -1,4 +1,4 @@
|
||||
import { strIsLink } from './str';
|
||||
import { strIsLink } from '../string/tools';
|
||||
|
||||
export const fileImgs = [
|
||||
{ suffix: 'pdf', src: '/imgs/files/pdf.svg' },
|
||||
@@ -10,14 +10,7 @@ export const fileImgs = [
|
||||
];
|
||||
|
||||
export function getFileIcon(name = '') {
|
||||
return fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src;
|
||||
}
|
||||
export function getSpecialFileIcon(name = '') {
|
||||
if (name === 'manual') {
|
||||
return '/imgs/files/manual.svg';
|
||||
} else if (name === 'mark') {
|
||||
return '/imgs/files/mark.svg';
|
||||
} else if (strIsLink(name)) {
|
||||
return '/imgs/files/link.svg';
|
||||
}
|
||||
return (
|
||||
fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src || '/imgs/files/file.svg'
|
||||
);
|
||||
}
|
9
packages/global/common/file/tools.ts
Normal file
9
packages/global/common/file/tools.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
export const formatFileSize = (bytes: number): string => {
|
||||
if (bytes === 0) return '0 B';
|
||||
|
||||
const k = 1024;
|
||||
const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
};
|
4
packages/global/common/parentFolder/type.d.ts
vendored
Normal file
4
packages/global/common/parentFolder/type.d.ts
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
export type ParentTreePathItemType = {
|
||||
parentId: string;
|
||||
parentName: string;
|
||||
};
|
4
packages/global/common/plugin/types/pluginRes.d.ts
vendored
Normal file
4
packages/global/common/plugin/types/pluginRes.d.ts
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
export type FetchResultItem = {
|
||||
url: string;
|
||||
content: string;
|
||||
};
|
@@ -13,10 +13,10 @@ export const hashStr = (psw: string) => {
|
||||
/* simple text, remove chinese space and extra \n */
|
||||
export const simpleText = (text: string) => {
|
||||
text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
|
||||
text = text.replace(/\r\n|\r/g, '\n');
|
||||
text = text.replace(/\n{3,}/g, '\n\n');
|
||||
text = text.replace(/[\s&&[^\n]]{2,}/g, ' ');
|
||||
text = text.replace(/[\x00-\x08]/g, ' ');
|
||||
text = text.replace(/\r\n|\r/g, '\n');
|
||||
|
||||
return text;
|
||||
};
|
@@ -1,6 +1,3 @@
|
||||
import type { Mongoose } from '../mongo';
|
||||
import type { Logger } from 'winston';
|
||||
|
||||
export type FeConfigsType = {
|
||||
show_emptyChat?: boolean;
|
||||
show_register?: boolean;
|
||||
@@ -36,8 +33,6 @@ export type SystemEnvType = {
|
||||
};
|
||||
|
||||
declare global {
|
||||
var mongodb: Mongoose | undefined;
|
||||
var logger: Logger;
|
||||
var feConfigs: FeConfigsType;
|
||||
var systemEnv: SystemEnvType;
|
||||
}
|
80
packages/global/common/time/timezone.ts
Normal file
80
packages/global/common/time/timezone.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import timezones from 'timezones-list';
|
||||
import dayjs from 'dayjs';
|
||||
import utc from 'dayjs/plugin/utc';
|
||||
import timezone from 'dayjs/plugin/timezone';
|
||||
|
||||
dayjs.extend(utc);
|
||||
dayjs.extend(timezone);
|
||||
|
||||
/**
|
||||
* Returns the offset from UTC in hours for the current locale.
|
||||
* @param {string} timeZone Timezone to get offset for
|
||||
* @returns {number} The offset from UTC in hours.
|
||||
*
|
||||
* Generated by Trelent
|
||||
*/
|
||||
export const getTimezoneOffset = (timeZone: string): number => {
|
||||
const now = new Date();
|
||||
const tzString = now.toLocaleString('en-US', {
|
||||
timeZone
|
||||
});
|
||||
const localString = now.toLocaleString('en-US');
|
||||
const diff = (Date.parse(localString) - Date.parse(tzString)) / 3600000;
|
||||
const offset = diff + now.getTimezoneOffset() / 60;
|
||||
return -offset;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns a list of timezones sorted by their offset from UTC.
|
||||
* @returns {object[]} A list of the given timezones sorted by their offset from UTC.
|
||||
*
|
||||
* Generated by Trelent
|
||||
*/
|
||||
export const timezoneList = () => {
|
||||
const result = timezones
|
||||
.map((timezone) => {
|
||||
try {
|
||||
let display = dayjs().tz(timezone.tzCode).format('Z');
|
||||
|
||||
return {
|
||||
name: `(UTC${display}) ${timezone.tzCode}`,
|
||||
value: timezone.tzCode,
|
||||
time: getTimezoneOffset(timezone.tzCode)
|
||||
};
|
||||
} catch (e) {}
|
||||
})
|
||||
.filter((item) => item);
|
||||
|
||||
result.sort((a, b) => {
|
||||
if (!a || !b) return 0;
|
||||
if (a.time > b.time) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (b.time > a.time) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
});
|
||||
|
||||
return [
|
||||
{
|
||||
name: 'UTC',
|
||||
time: 0,
|
||||
value: 'UTC'
|
||||
},
|
||||
...result
|
||||
] as {
|
||||
name: string;
|
||||
value: string;
|
||||
time: number;
|
||||
}[];
|
||||
};
|
||||
|
||||
export const getSystemTime = (timeZone: string) => {
|
||||
const timezoneDiff = getTimezoneOffset(timeZone);
|
||||
const now = Date.now();
|
||||
const targetTime = now + timezoneDiff * 60 * 60 * 1000;
|
||||
return dayjs(targetTime).format('YYYY-MM-DD HH:mm:ss');
|
||||
};
|
@@ -12,11 +12,37 @@ export const DatasetTypeMap = {
|
||||
}
|
||||
};
|
||||
|
||||
export enum FileStatusEnum {
|
||||
embedding = 'embedding',
|
||||
ready = 'ready'
|
||||
export enum DatasetCollectionTypeEnum {
|
||||
file = 'file',
|
||||
folder = 'folder',
|
||||
link = 'link',
|
||||
virtual = 'virtual'
|
||||
}
|
||||
|
||||
export const DatasetCollectionTypeMap = {
|
||||
[DatasetCollectionTypeEnum.file]: {
|
||||
name: 'dataset.file'
|
||||
},
|
||||
[DatasetCollectionTypeEnum.folder]: {
|
||||
name: 'dataset.folder'
|
||||
},
|
||||
[DatasetCollectionTypeEnum.link]: {
|
||||
name: 'dataset.link'
|
||||
},
|
||||
[DatasetCollectionTypeEnum.virtual]: {
|
||||
name: 'dataset.Virtual File'
|
||||
}
|
||||
};
|
||||
|
||||
export enum TrainingModeEnum {
|
||||
'qa' = 'qa',
|
||||
'index' = 'index'
|
||||
}
|
||||
export const TrainingTypeMap = {
|
||||
[TrainingModeEnum.qa]: 'qa',
|
||||
[TrainingModeEnum.index]: 'index'
|
||||
};
|
||||
|
||||
export enum DatasetSpecialIdEnum {
|
||||
manual = 'manual',
|
||||
mark = 'mark'
|
75
packages/global/core/dataset/type.d.ts
vendored
Normal file
75
packages/global/core/dataset/type.d.ts
vendored
Normal file
@@ -0,0 +1,75 @@
|
||||
import { DatasetCollectionTypeEnum, DatasetTypeEnum, TrainingModeEnum } from './constant';
|
||||
|
||||
export type DatasetSchemaType = {
|
||||
_id: string;
|
||||
userId: string;
|
||||
parentId: string;
|
||||
updateTime: Date;
|
||||
avatar: string;
|
||||
name: string;
|
||||
vectorModel: string;
|
||||
tags: string[];
|
||||
type: `${DatasetTypeEnum}`;
|
||||
};
|
||||
|
||||
export type DatasetCollectionSchemaType = {
|
||||
_id: string;
|
||||
userId: string;
|
||||
datasetId: string;
|
||||
parentId?: string;
|
||||
name: string;
|
||||
type: `${DatasetCollectionTypeEnum}`;
|
||||
updateTime: Date;
|
||||
metadata: {
|
||||
fileId?: string;
|
||||
rawLink?: string;
|
||||
pgCollectionId?: string;
|
||||
};
|
||||
};
|
||||
|
||||
export type DatasetTrainingSchemaType = {
|
||||
_id: string;
|
||||
userId: string;
|
||||
datasetId: string;
|
||||
datasetCollectionId: string;
|
||||
billId: string;
|
||||
expireAt: Date;
|
||||
lockTime: Date;
|
||||
mode: `${TrainingModeEnum}`;
|
||||
model: string;
|
||||
prompt: string;
|
||||
q: string;
|
||||
a: string;
|
||||
};
|
||||
|
||||
/* ================= dataset ===================== */
|
||||
|
||||
/* ================= collection ===================== */
|
||||
|
||||
/* ================= data ===================== */
|
||||
export type PgDataItemType = {
|
||||
id: string;
|
||||
q: string;
|
||||
a: string;
|
||||
dataset_id: string;
|
||||
collection_id: string;
|
||||
};
|
||||
export type DatasetChunkItemType = {
|
||||
q: string;
|
||||
a: string;
|
||||
};
|
||||
export type DatasetDataItemType = DatasetChunkItemType & {
|
||||
id: string;
|
||||
datasetId: string;
|
||||
collectionId: string;
|
||||
sourceName: string;
|
||||
sourceId?: string;
|
||||
};
|
||||
|
||||
/* ============= search =============== */
|
||||
export type SearchDataResultItemType = PgDataItemType & {
|
||||
score: number;
|
||||
};
|
||||
export type SearchDataResponseItemType = DatasetDataItemType & {
|
||||
score: number;
|
||||
};
|
21
packages/global/core/dataset/utils.ts
Normal file
21
packages/global/core/dataset/utils.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import { DatasetCollectionTypeEnum } from './constant';
|
||||
import { getFileIcon } from '../../common/file/icon';
|
||||
|
||||
export function getCollectionIcon(
|
||||
type: `${DatasetCollectionTypeEnum}` = DatasetCollectionTypeEnum.file,
|
||||
name = ''
|
||||
) {
|
||||
if (type === DatasetCollectionTypeEnum.folder) {
|
||||
return '/imgs/files/folder.svg';
|
||||
} else if (type === DatasetCollectionTypeEnum.link) {
|
||||
return '/imgs/files/link.svg';
|
||||
} else if (type === DatasetCollectionTypeEnum.virtual) {
|
||||
if (name === '手动录入') {
|
||||
return '/imgs/files/manual.svg';
|
||||
} else if (name === '手动标注') {
|
||||
return '/imgs/files/mark.svg';
|
||||
}
|
||||
return '/imgs/files/collection.svg';
|
||||
}
|
||||
return getFileIcon(name);
|
||||
}
|
14
packages/global/package.json
Normal file
14
packages/global/package.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "@fastgpt/global",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"axios": "^1.5.1",
|
||||
"timezones-list": "^3.0.2",
|
||||
"dayjs": "^1.11.7",
|
||||
"encoding": "^0.1.13",
|
||||
"openai": "^4.12.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.8.5"
|
||||
}
|
||||
}
|
@@ -1,5 +1,5 @@
|
||||
export enum OutLinkTypeEnum {
|
||||
'share' = 'share',
|
||||
'iframe' = 'iframe',
|
||||
share = 'share',
|
||||
iframe = 'iframe',
|
||||
apikey = 'apikey'
|
||||
}
|
39
packages/service/common/mongo/sessionRun.ts
Normal file
39
packages/service/common/mongo/sessionRun.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import mongoose from './index';
|
||||
|
||||
export class MongoSession {
|
||||
tasks: (() => Promise<any>)[] = [];
|
||||
session: mongoose.mongo.ClientSession | null = null;
|
||||
opts: {
|
||||
session: mongoose.mongo.ClientSession;
|
||||
new: boolean;
|
||||
} | null = null;
|
||||
|
||||
constructor() {}
|
||||
async init() {
|
||||
this.session = await mongoose.startSession();
|
||||
this.opts = { session: this.session, new: true };
|
||||
}
|
||||
push(
|
||||
tasks: ((opts: {
|
||||
session: mongoose.mongo.ClientSession;
|
||||
new: boolean;
|
||||
}) => () => Promise<any>)[] = []
|
||||
) {
|
||||
if (!this.opts) return;
|
||||
// this.tasks = this.tasks.concat(tasks.map((item) => item(this.opts)));
|
||||
}
|
||||
async run() {
|
||||
if (!this.session || !this.opts) return;
|
||||
try {
|
||||
this.session.startTransaction();
|
||||
|
||||
const opts = { session: this.session, new: true };
|
||||
|
||||
await this.session.commitTransaction();
|
||||
} catch (error) {
|
||||
await this.session.abortTransaction();
|
||||
console.error(error);
|
||||
}
|
||||
this.session.endSession();
|
||||
}
|
||||
}
|
7
packages/service/common/mongo/type.d.ts
vendored
Normal file
7
packages/service/common/mongo/type.d.ts
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
import type { Mongoose } from 'mongoose';
|
||||
import type { Logger } from 'winston';
|
||||
|
||||
declare global {
|
||||
var mongodb: Mongoose | undefined;
|
||||
var logger: Logger;
|
||||
}
|
@@ -14,7 +14,7 @@ export function responseWriteController({
|
||||
return (text: string | Buffer) => {
|
||||
const writeResult = res.write(text);
|
||||
if (!writeResult) {
|
||||
readStream.pause();
|
||||
readStream?.pause();
|
||||
}
|
||||
};
|
||||
}
|
@@ -1,4 +1,4 @@
|
||||
import type { UserModelSchema } from '@fastgpt/support/user/type.d';
|
||||
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
|
||||
import OpenAI from 'openai';
|
||||
|
||||
export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
|
@@ -1,4 +1,4 @@
|
||||
import { ChatCompletionRequestMessage } from '../type';
|
||||
import type { ChatCompletionRequestMessage } from '@fastgpt/global/core/ai/type.d';
|
||||
import { getAIApi } from '../config';
|
||||
|
||||
export const Prompt_QuestionGuide = `我不太清楚问你什么问题,请帮我生成 3 个问题,引导我继续提问。问题的长度应小于20个字符,按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
|
26
packages/service/core/dataset/auth.ts
Normal file
26
packages/service/core/dataset/auth.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
|
||||
import { MongoDatasetCollection } from './collection/schema';
|
||||
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
|
||||
export async function authCollection({
|
||||
collectionId,
|
||||
userId
|
||||
}: {
|
||||
collectionId: string;
|
||||
userId: string;
|
||||
}) {
|
||||
const collection = await MongoDatasetCollection.findOne({
|
||||
_id: collectionId,
|
||||
userId
|
||||
})
|
||||
.populate('datasetId')
|
||||
.lean();
|
||||
|
||||
if (collection) {
|
||||
return {
|
||||
...collection,
|
||||
dataset: collection.datasetId as unknown as DatasetSchemaType
|
||||
};
|
||||
}
|
||||
return Promise.reject(ERROR_ENUM.unAuthDataset);
|
||||
}
|
66
packages/service/core/dataset/collection/schema.ts
Normal file
66
packages/service/core/dataset/collection/schema.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
||||
import { DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant';
|
||||
import { DatasetCollectionName } from '../schema';
|
||||
|
||||
export const DatasetColCollectionName = 'dataset.collections';
|
||||
|
||||
const DatasetCollectionSchema = new Schema({
|
||||
parentId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: DatasetColCollectionName,
|
||||
default: null
|
||||
},
|
||||
userId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'user',
|
||||
required: true
|
||||
},
|
||||
datasetId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: DatasetCollectionName,
|
||||
required: true
|
||||
},
|
||||
name: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
type: {
|
||||
type: String,
|
||||
enum: Object.keys(DatasetCollectionTypeMap),
|
||||
required: true
|
||||
},
|
||||
updateTime: {
|
||||
type: Date,
|
||||
default: () => new Date()
|
||||
},
|
||||
metadata: {
|
||||
type: {
|
||||
fileId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'dataset.files'
|
||||
},
|
||||
rawLink: {
|
||||
type: String,
|
||||
default: ''
|
||||
},
|
||||
// 451 初始化
|
||||
pgCollectionId: {
|
||||
type: String
|
||||
}
|
||||
},
|
||||
default: {}
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
DatasetCollectionSchema.index({ datasetId: 1 });
|
||||
DatasetCollectionSchema.index({ userId: 1 });
|
||||
DatasetCollectionSchema.index({ updateTime: -1 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
export const MongoDatasetCollection: Model<DatasetCollectionSchemaType> =
|
||||
models[DatasetColCollectionName] || model(DatasetColCollectionName, DatasetCollectionSchema);
|
62
packages/service/core/dataset/collection/utils.ts
Normal file
62
packages/service/core/dataset/collection/utils.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import { MongoDatasetCollection } from './schema';
|
||||
import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
|
||||
|
||||
/**
|
||||
* get all collection by top collectionId
|
||||
*/
|
||||
export async function findCollectionAndChild(id: string, fields = '_id parentId name metadata') {
|
||||
async function find(id: string) {
|
||||
// find children
|
||||
const children = await MongoDatasetCollection.find({ parentId: id }, fields);
|
||||
|
||||
let collections = children;
|
||||
|
||||
for (const child of children) {
|
||||
const grandChildrenIds = await find(child._id);
|
||||
collections = collections.concat(grandChildrenIds);
|
||||
}
|
||||
|
||||
return collections;
|
||||
}
|
||||
const [collection, childCollections] = await Promise.all([
|
||||
MongoDatasetCollection.findById(id, fields),
|
||||
find(id)
|
||||
]);
|
||||
|
||||
if (!collection) {
|
||||
return Promise.reject('Collection not found');
|
||||
}
|
||||
|
||||
return [collection, ...childCollections];
|
||||
}
|
||||
|
||||
export async function getDatasetCollectionPaths({
|
||||
parentId = '',
|
||||
userId
|
||||
}: {
|
||||
parentId?: string;
|
||||
userId: string;
|
||||
}): Promise<ParentTreePathItemType[]> {
|
||||
async function find(parentId?: string): Promise<ParentTreePathItemType[]> {
|
||||
if (!parentId) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const parent = await MongoDatasetCollection.findOne({ _id: parentId, userId }, 'name parentId');
|
||||
|
||||
if (!parent) return [];
|
||||
|
||||
const paths = await find(parent.parentId);
|
||||
paths.push({ parentId, parentName: parent.name });
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
return await find(parentId);
|
||||
}
|
||||
|
||||
export function getCollectionUpdateTime({ name, time }: { time?: Date; name: string }) {
|
||||
if (time) return time;
|
||||
if (name.startsWith('手动') || ['manual', 'mark'].includes(name)) return new Date('2999/9/9');
|
||||
return new Date();
|
||||
}
|
@@ -1,12 +1,14 @@
|
||||
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
|
||||
import { connectionMongo, type Model } from '../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { DatasetSchemaType } from './type';
|
||||
import { DatasetTypeMap } from './constant';
|
||||
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
||||
import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constant';
|
||||
|
||||
export const DatasetCollectionName = 'datasets';
|
||||
|
||||
const DatasetSchema = new Schema({
|
||||
parentId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'kb',
|
||||
ref: DatasetCollectionName,
|
||||
default: null
|
||||
},
|
||||
userId: {
|
||||
@@ -43,4 +45,11 @@ const DatasetSchema = new Schema({
|
||||
}
|
||||
});
|
||||
|
||||
export const MongoDataset: Model<DatasetSchemaType> = models['kb'] || model('kb', DatasetSchema);
|
||||
try {
|
||||
DatasetSchema.index({ userId: 1 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
export const MongoDataset: Model<DatasetSchemaType> =
|
||||
models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema);
|
72
packages/service/core/dataset/training/schema.ts
Normal file
72
packages/service/core/dataset/training/schema.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
/* 模型的知识库 */
|
||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
|
||||
import { DatasetColCollectionName } from '../collection/schema';
|
||||
import { DatasetCollectionName } from '../schema';
|
||||
|
||||
export const DatasetTrainingCollectionName = 'dataset.trainings';
|
||||
|
||||
const TrainingDataSchema = new Schema({
|
||||
userId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'user',
|
||||
required: true
|
||||
},
|
||||
datasetId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: DatasetCollectionName,
|
||||
required: true
|
||||
},
|
||||
datasetCollectionId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: DatasetColCollectionName,
|
||||
required: true
|
||||
},
|
||||
billId: {
|
||||
type: String,
|
||||
default: ''
|
||||
},
|
||||
mode: {
|
||||
type: String,
|
||||
enum: Object.keys(TrainingTypeMap),
|
||||
required: true
|
||||
},
|
||||
expireAt: {
|
||||
type: Date,
|
||||
default: () => new Date()
|
||||
},
|
||||
lockTime: {
|
||||
type: Date,
|
||||
default: () => new Date('2000/1/1')
|
||||
},
|
||||
model: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
prompt: {
|
||||
// qa split prompt
|
||||
type: String,
|
||||
default: ''
|
||||
},
|
||||
q: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
a: {
|
||||
type: String,
|
||||
default: ''
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
TrainingDataSchema.index({ lockTime: 1 });
|
||||
TrainingDataSchema.index({ userId: 1 });
|
||||
TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
export const MongoDatasetTraining: Model<DatasetTrainingSchemaType> =
|
||||
models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema);
|
24
packages/service/package.json
Normal file
24
packages/service/package.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "@fastgpt/service",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@fastgpt/global": "workspace:*",
|
||||
"axios": "^1.5.1",
|
||||
"nextjs-cors": "^2.1.2",
|
||||
"next": "13.5.2",
|
||||
"cookie": "^0.5.0",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"mongoose": "^7.0.2",
|
||||
"winston": "^3.10.0",
|
||||
"winston-mongodb": "^5.1.1",
|
||||
"tunnel": "^0.0.6",
|
||||
"encoding": "^0.1.13",
|
||||
"openai": "^4.12.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/tunnel": "^0.0.4",
|
||||
"@types/node": "^20.8.5",
|
||||
"@types/cookie": "^0.5.2",
|
||||
"@types/jsonwebtoken": "^9.0.3"
|
||||
}
|
||||
}
|
@@ -1,8 +1,8 @@
|
||||
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
|
||||
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
|
||||
import { updateApiKeyUsedTime } from './tools';
|
||||
import { MongoOpenApi } from './schema';
|
||||
import { POST } from '@fastgpt/common/plusApi/request';
|
||||
import { OpenApiSchema } from './type.d';
|
||||
import { POST } from '../../common/api/plusRequest';
|
||||
import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
|
||||
|
||||
export type AuthOpenApiLimitProps = { openApi: OpenApiSchema };
|
||||
|
@@ -1,8 +1,8 @@
|
||||
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
|
||||
import { connectionMongo, type Model } from '../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { OpenApiSchema } from './type.d';
|
||||
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
|
||||
import { formatPrice } from '@fastgpt/common/bill/index';
|
||||
import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
|
||||
import { PRICE_SCALE } from '@fastgpt/global/common/bill/constants';
|
||||
import { formatPrice } from '@fastgpt/global/common/bill/tools';
|
||||
|
||||
const OpenApiSchema = new Schema(
|
||||
{
|
@@ -1,7 +1,7 @@
|
||||
import { AuthUserTypeEnum, authBalanceByUid } from '../user/auth';
|
||||
import { MongoOutLink } from './schema';
|
||||
import { POST } from '@fastgpt/common/plusApi/request';
|
||||
import { OutLinkSchema } from './type.d';
|
||||
import { POST } from '../../common/api/plusRequest';
|
||||
import { OutLinkSchema } from '@fastgpt/global/support/outLink/type';
|
||||
|
||||
export type AuthLinkProps = { ip?: string | null; authToken?: string; question: string };
|
||||
export type AuthLinkLimitProps = AuthLinkProps & { outLink: OutLinkSchema };
|
@@ -1,7 +1,7 @@
|
||||
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
|
||||
import { connectionMongo, type Model } from '../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { OutLinkSchema as SchemaType } from './type.d';
|
||||
import { OutLinkTypeEnum } from './constant';
|
||||
import { OutLinkSchema as SchemaType } from '@fastgpt/global/support/outLink/type';
|
||||
import { OutLinkTypeEnum } from '@fastgpt/global/support/outLink/constant';
|
||||
|
||||
const OutLinkSchema = new Schema({
|
||||
shareId: {
|
@@ -4,8 +4,8 @@ import jwt from 'jsonwebtoken';
|
||||
import { authOpenApiKey } from '../openapi/auth';
|
||||
import { authOutLinkId } from '../outLink/auth';
|
||||
import { MongoUser } from './schema';
|
||||
import type { UserModelSchema } from './type.d';
|
||||
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
|
||||
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
|
||||
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
|
||||
|
||||
export enum AuthUserTypeEnum {
|
||||
token = 'token',
|
@@ -1,8 +1,8 @@
|
||||
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
|
||||
import { connectionMongo, type Model } from '../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { hashStr } from '@fastgpt/common/tools/str';
|
||||
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
|
||||
import type { UserModelSchema } from './type.d';
|
||||
import { hashStr } from '@fastgpt/global/common/string/tools';
|
||||
import { PRICE_SCALE } from '@fastgpt/global/common/bill/constants';
|
||||
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
|
||||
|
||||
const UserSchema = new Schema({
|
||||
username: {
|
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"name": "@fastgpt/support",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@fastgpt/common": "workspace:*",
|
||||
"cookie": "^0.5.0",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"axios": "^1.5.1",
|
||||
"next": "13.5.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cookie": "^0.5.2",
|
||||
"@types/jsonwebtoken": "^9.0.3"
|
||||
}
|
||||
}
|
8
packages/web/package.json
Normal file
8
packages/web/package.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"name": "@fastgpt/web",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"axios": "^1.5.1"
|
||||
},
|
||||
"devDependencies": {}
|
||||
}
|
@@ -14,10 +14,7 @@
|
||||
"isolatedModules": true,
|
||||
"jsx": "preserve",
|
||||
"incremental": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./*"]
|
||||
}
|
||||
"baseUrl": "."
|
||||
},
|
||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"],
|
||||
"exclude": ["node_modules"]
|
Reference in New Issue
Block a user