This commit is contained in:
Archer
2023-10-22 23:54:04 +08:00
committed by GitHub
parent 3091a90df6
commit a3534407bf
365 changed files with 7266 additions and 6055 deletions

View File

@@ -0,0 +1 @@
export const PRICE_SCALE = 100000;

View File

@@ -0,0 +1,9 @@
/* bill common */
import { PRICE_SCALE } from './constants';
/**
* dataset price / PRICE_SCALE = real price
*/
export const formatPrice = (val = 0, multiple = 1) => {
return Number(((val / PRICE_SCALE) * multiple).toFixed(10));
};

View File

@@ -0,0 +1,3 @@
export type CreateTrainingBillType = {
name: string;
};

View File

@@ -0,0 +1,85 @@
export const ERROR_CODE: { [key: number]: string } = {
400: '请求失败',
401: '无权访问',
403: '紧张访问',
404: '请求不存在',
405: '请求方法错误',
406: '请求的格式错误',
410: '资源已删除',
422: '验证错误',
500: '服务器发生错误',
502: '网关错误',
503: '服务器暂时过载或维护',
504: '网关超时'
};
export const TOKEN_ERROR_CODE: Record<number, string> = {
403: '登录状态无效,请重新登录'
};
export const proxyError: Record<string, boolean> = {
ECONNABORTED: true,
ECONNRESET: true
};
export enum ERROR_ENUM {
unAuthorization = 'unAuthorization',
insufficientQuota = 'insufficientQuota',
unAuthModel = 'unAuthModel',
unAuthApiKey = 'unAuthApiKey',
unAuthDataset = 'unAuthDataset',
unAuthDatasetCollection = 'unAuthDatasetCollection',
unAuthFile = 'unAuthFile'
}
export const ERROR_RESPONSE: Record<
any,
{
code: number;
statusText: string;
message: string;
data?: any;
}
> = {
[ERROR_ENUM.unAuthorization]: {
code: 403,
statusText: ERROR_ENUM.unAuthorization,
message: '凭证错误',
data: null
},
[ERROR_ENUM.insufficientQuota]: {
code: 510,
statusText: ERROR_ENUM.insufficientQuota,
message: '账号余额不足',
data: null
},
[ERROR_ENUM.unAuthModel]: {
code: 511,
statusText: ERROR_ENUM.unAuthModel,
message: '无权使用该模型',
data: null
},
[ERROR_ENUM.unAuthDataset]: {
code: 512,
statusText: ERROR_ENUM.unAuthDataset,
message: '无权使用该知识库',
data: null
},
[ERROR_ENUM.unAuthFile]: {
code: 513,
statusText: ERROR_ENUM.unAuthFile,
message: '无权阅读该文件',
data: null
},
[ERROR_ENUM.unAuthApiKey]: {
code: 514,
statusText: ERROR_ENUM.unAuthApiKey,
message: 'Api Key 不合法',
data: null
},
[ERROR_ENUM.unAuthDatasetCollection]: {
code: 515,
statusText: ERROR_ENUM.unAuthDatasetCollection,
message: '无权使用该知识库文件',
data: null
}
};

View File

@@ -0,0 +1,5 @@
export const getErrText = (err: any, def = '') => {
const msg: string = typeof err === 'string' ? err : err?.message || def || '';
msg && console.log('error =>', msg);
return msg;
};

View File

@@ -0,0 +1,16 @@
import { strIsLink } from '../string/tools';
export const fileImgs = [
{ suffix: 'pdf', src: '/imgs/files/pdf.svg' },
{ suffix: 'csv', src: '/imgs/files/csv.svg' },
{ suffix: '(doc|docs)', src: '/imgs/files/doc.svg' },
{ suffix: 'txt', src: '/imgs/files/txt.svg' },
{ suffix: 'md', src: '/imgs/files/markdown.svg' },
{ suffix: '.', src: '/imgs/files/file.svg' }
];
export function getFileIcon(name = '') {
return (
fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src || '/imgs/files/file.svg'
);
}

View File

@@ -0,0 +1,9 @@
export const formatFileSize = (bytes: number): string => {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
};

View File

@@ -0,0 +1,4 @@
export type ParentTreePathItemType = {
parentId: string;
parentName: string;
};

View File

@@ -0,0 +1,4 @@
export type FetchResultItem = {
url: string;
content: string;
};

View File

@@ -0,0 +1,22 @@
import crypto from 'crypto';
export function strIsLink(str?: string) {
if (!str) return false;
if (/^((http|https)?:\/\/|www\.|\/)[^\s/$.?#].[^\s]*$/i.test(str)) return true;
return false;
}
export const hashStr = (psw: string) => {
return crypto.createHash('sha256').update(psw).digest('hex');
};
/* simple text, remove chinese space and extra \n */
export const simpleText = (text: string) => {
text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
text = text.replace(/\r\n|\r/g, '\n');
text = text.replace(/\n{3,}/g, '\n\n');
text = text.replace(/[\s&&[^\n]]{2,}/g, ' ');
text = text.replace(/[\x00-\x08]/g, ' ');
return text;
};

View File

@@ -0,0 +1,38 @@
export type FeConfigsType = {
show_emptyChat?: boolean;
show_register?: boolean;
show_appStore?: boolean;
show_contact?: boolean;
show_git?: boolean;
show_doc?: boolean;
show_pay?: boolean;
show_openai_account?: boolean;
show_promotion?: boolean;
hide_app_flow?: boolean;
openAPIUrl?: string;
systemTitle?: string;
authorText?: string;
googleClientVerKey?: string;
isPlus?: boolean;
oauth?: {
github?: string;
google?: string;
};
limit?: {
exportLimitMinutes?: number;
};
scripts?: { [key: string]: string }[];
};
export type SystemEnvType = {
pluginBaseUrl?: string;
openapiPrefix?: string;
vectorMaxProcess: number;
qaMaxProcess: number;
pgHNSWEfSearch: number;
};
declare global {
var feConfigs: FeConfigsType;
var systemEnv: SystemEnvType;
}

View File

@@ -0,0 +1,80 @@
import timezones from 'timezones-list';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc';
import timezone from 'dayjs/plugin/timezone';
dayjs.extend(utc);
dayjs.extend(timezone);
/**
* Returns the offset from UTC in hours for the current locale.
* @param {string} timeZone Timezone to get offset for
* @returns {number} The offset from UTC in hours.
*
* Generated by Trelent
*/
export const getTimezoneOffset = (timeZone: string): number => {
const now = new Date();
const tzString = now.toLocaleString('en-US', {
timeZone
});
const localString = now.toLocaleString('en-US');
const diff = (Date.parse(localString) - Date.parse(tzString)) / 3600000;
const offset = diff + now.getTimezoneOffset() / 60;
return -offset;
};
/**
* Returns a list of timezones sorted by their offset from UTC.
* @returns {object[]} A list of the given timezones sorted by their offset from UTC.
*
* Generated by Trelent
*/
export const timezoneList = () => {
const result = timezones
.map((timezone) => {
try {
let display = dayjs().tz(timezone.tzCode).format('Z');
return {
name: `(UTC${display}) ${timezone.tzCode}`,
value: timezone.tzCode,
time: getTimezoneOffset(timezone.tzCode)
};
} catch (e) {}
})
.filter((item) => item);
result.sort((a, b) => {
if (!a || !b) return 0;
if (a.time > b.time) {
return 1;
}
if (b.time > a.time) {
return -1;
}
return 0;
});
return [
{
name: 'UTC',
time: 0,
value: 'UTC'
},
...result
] as {
name: string;
value: string;
time: number;
}[];
};
export const getSystemTime = (timeZone: string) => {
const timezoneDiff = getTimezoneOffset(timeZone);
const now = Date.now();
const targetTime = now + timezoneDiff * 60 * 60 * 1000;
return dayjs(targetTime).format('YYYY-MM-DD HH:mm:ss');
};

View File

@@ -0,0 +1,6 @@
export enum ChatCompletionRequestMessageRoleEnum {
'System' = 'system',
'User' = 'user',
'Assistant' = 'assistant',
'Function' = 'function'
}

12
packages/global/core/ai/type.d.ts vendored Normal file
View File

@@ -0,0 +1,12 @@
import OpenAI from 'openai';
export type ChatCompletionRequestMessage = OpenAI.Chat.CreateChatCompletionRequestMessage;
export type ChatCompletion = OpenAI.Chat.ChatCompletion;
export type CreateChatCompletionRequest = OpenAI.Chat.ChatCompletionCreateParams;
export type StreamChatType = Stream<OpenAI.Chat.ChatCompletionChunk>;
export type PromptTemplateItem = {
title: string;
desc: string;
value: string;
};

View File

@@ -0,0 +1,62 @@
export enum DatasetTypeEnum {
folder = 'folder',
dataset = 'dataset'
}
export const DatasetTypeMap = {
[DatasetTypeEnum.folder]: {
name: 'folder'
},
[DatasetTypeEnum.dataset]: {
name: 'dataset'
}
};
export enum DatasetCollectionTypeEnum {
file = 'file',
folder = 'folder',
link = 'link',
virtual = 'virtual'
}
export const DatasetCollectionTypeMap = {
[DatasetCollectionTypeEnum.file]: {
name: 'dataset.file'
},
[DatasetCollectionTypeEnum.folder]: {
name: 'dataset.folder'
},
[DatasetCollectionTypeEnum.link]: {
name: 'dataset.link'
},
[DatasetCollectionTypeEnum.virtual]: {
name: 'dataset.Virtual File'
}
};
export enum TrainingModeEnum {
'qa' = 'qa',
'index' = 'index'
}
export const TrainingTypeMap = {
[TrainingModeEnum.qa]: 'qa',
[TrainingModeEnum.index]: 'index'
};
export enum DatasetSpecialIdEnum {
manual = 'manual',
mark = 'mark'
}
export const datasetSpecialIdMap = {
[DatasetSpecialIdEnum.manual]: {
name: 'kb.Manual Data',
sourceName: 'kb.Manual Input'
},
[DatasetSpecialIdEnum.mark]: {
name: 'kb.Mark Data',
sourceName: 'kb.Manual Mark'
}
};
export const datasetSpecialIds: string[] = [DatasetSpecialIdEnum.manual, DatasetSpecialIdEnum.mark];
export const FolderAvatarSrc = '/imgs/files/folder.svg';

75
packages/global/core/dataset/type.d.ts vendored Normal file
View File

@@ -0,0 +1,75 @@
import { DatasetCollectionTypeEnum, DatasetTypeEnum, TrainingModeEnum } from './constant';
export type DatasetSchemaType = {
_id: string;
userId: string;
parentId: string;
updateTime: Date;
avatar: string;
name: string;
vectorModel: string;
tags: string[];
type: `${DatasetTypeEnum}`;
};
export type DatasetCollectionSchemaType = {
_id: string;
userId: string;
datasetId: string;
parentId?: string;
name: string;
type: `${DatasetCollectionTypeEnum}`;
updateTime: Date;
metadata: {
fileId?: string;
rawLink?: string;
pgCollectionId?: string;
};
};
export type DatasetTrainingSchemaType = {
_id: string;
userId: string;
datasetId: string;
datasetCollectionId: string;
billId: string;
expireAt: Date;
lockTime: Date;
mode: `${TrainingModeEnum}`;
model: string;
prompt: string;
q: string;
a: string;
};
/* ================= dataset ===================== */
/* ================= collection ===================== */
/* ================= data ===================== */
export type PgDataItemType = {
id: string;
q: string;
a: string;
dataset_id: string;
collection_id: string;
};
export type DatasetChunkItemType = {
q: string;
a: string;
};
export type DatasetDataItemType = DatasetChunkItemType & {
id: string;
datasetId: string;
collectionId: string;
sourceName: string;
sourceId?: string;
};
/* ============= search =============== */
export type SearchDataResultItemType = PgDataItemType & {
score: number;
};
export type SearchDataResponseItemType = DatasetDataItemType & {
score: number;
};

View File

@@ -0,0 +1,21 @@
import { DatasetCollectionTypeEnum } from './constant';
import { getFileIcon } from '../../common/file/icon';
export function getCollectionIcon(
type: `${DatasetCollectionTypeEnum}` = DatasetCollectionTypeEnum.file,
name = ''
) {
if (type === DatasetCollectionTypeEnum.folder) {
return '/imgs/files/folder.svg';
} else if (type === DatasetCollectionTypeEnum.link) {
return '/imgs/files/link.svg';
} else if (type === DatasetCollectionTypeEnum.virtual) {
if (name === '手动录入') {
return '/imgs/files/manual.svg';
} else if (name === '手动标注') {
return '/imgs/files/mark.svg';
}
return '/imgs/files/collection.svg';
}
return getFileIcon(name);
}

5
packages/global/core/type.d.ts vendored Normal file
View File

@@ -0,0 +1,5 @@
import type { Agent } from 'http';
declare global {
var httpsAgent: Agent;
}

View File

@@ -0,0 +1,14 @@
{
"name": "@fastgpt/global",
"version": "1.0.0",
"dependencies": {
"axios": "^1.5.1",
"timezones-list": "^3.0.2",
"dayjs": "^1.11.7",
"encoding": "^0.1.13",
"openai": "^4.12.1"
},
"devDependencies": {
"@types/node": "^20.8.5"
}
}

View File

@@ -0,0 +1,14 @@
export type OpenApiSchema = {
_id: string;
userId: string;
createTime: Date;
lastUsedTime?: Date;
apiKey: string;
appId?: string;
name: string;
usage: number;
limit?: {
expiredTime?: Date;
credit?: number;
};
};

View File

@@ -0,0 +1,5 @@
export enum OutLinkTypeEnum {
share = 'share',
iframe = 'iframe',
apikey = 'apikey'
}

View File

@@ -0,0 +1,26 @@
import { OutLinkTypeEnum } from './constant';
export type OutLinkSchema = {
_id: string;
shareId: string;
userId: string;
appId: string;
name: string;
total: number;
lastTime: Date;
type: `${OutLinkTypeEnum}`;
responseDetail: boolean;
limit?: {
expiredTime?: Date;
QPM: number;
credit: number;
hookUrl?: string;
};
};
export type OutLinkEditType = {
_id?: string;
name: string;
responseDetail: OutLinkSchema['responseDetail'];
limit: OutLinkSchema['limit'];
};

20
packages/global/support/user/type.d.ts vendored Normal file
View File

@@ -0,0 +1,20 @@
export type UserModelSchema = {
_id: string;
username: string;
password: string;
avatar: string;
balance: number;
promotionRate: number;
inviterId?: string;
openaiKey: string;
createTime: number;
timezone: string;
openaiAccount?: {
key: string;
baseUrl: string;
};
limit: {
exportKbTime?: Date;
datasetMaxCount?: number;
};
};

View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"target": "es2015",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"forceConsistentCasingInFileNames": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "node",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"baseUrl": "."
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts"],
"exclude": ["node_modules"]
}