new framwork

This commit is contained in:
archer
2023-06-09 12:57:42 +08:00
parent d9450bd7ee
commit ba9d9c3d5f
263 changed files with 12269 additions and 11599 deletions

16
client/src/utils/adapt.ts Normal file
View File

@@ -0,0 +1,16 @@
import { formatPrice } from './user';
import dayjs from 'dayjs';
import type { BillSchema } from '../types/mongoSchema';
import type { UserBillType } from '@/types/user';
export const adaptBill = (bill: BillSchema): UserBillType => {
return {
id: bill._id,
type: bill.type,
modelName: bill.modelName,
time: bill.time,
textLen: bill.textLen,
tokenLen: bill.tokenLen,
price: formatPrice(bill.price)
};
};

264
client/src/utils/file.ts Normal file
View File

@@ -0,0 +1,264 @@
import mammoth from 'mammoth';
import Papa from 'papaparse';
import { getOpenAiEncMap } from './plugin/openai';
import { getErrText } from './tools';
/**
* 读取 txt 文件内容
*/
export const readTxtContent = (file: File) => {
return new Promise((resolve: (_: string) => void, reject) => {
try {
const reader = new FileReader();
reader.onload = () => {
resolve(reader.result as string);
};
reader.onerror = (err) => {
console.log('error txt read:', err);
reject('读取 txt 文件失败');
};
reader.readAsText(file);
} catch (error) {
reject('浏览器不支持文件内容读取');
}
});
};
/**
* 读取 pdf 内容
*/
export const readPdfContent = (file: File) =>
new Promise<string>((resolve, reject) => {
try {
const pdfjsLib = window['pdfjs-dist/build/pdf'];
pdfjsLib.workerSrc = '/js/pdf.worker.js';
const readPDFPage = async (doc: any, pageNo: number) => {
const page = await doc.getPage(pageNo);
const tokenizedText = await page.getTextContent();
const pageText = tokenizedText.items.map((token: any) => token.str).join(' ');
return pageText;
};
let reader = new FileReader();
reader.readAsArrayBuffer(file);
reader.onload = async (event) => {
if (!event?.target?.result) return reject('解析 PDF 失败');
try {
const doc = await pdfjsLib.getDocument(event.target.result).promise;
const pageTextPromises = [];
for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
pageTextPromises.push(readPDFPage(doc, pageNo));
}
const pageTexts = await Promise.all(pageTextPromises);
resolve(pageTexts.join('\n'));
} catch (err) {
console.log(err, 'pdfjs error');
reject('解析 PDF 失败');
}
};
reader.onerror = (err) => {
console.log(err, 'reader error');
reject('解析 PDF 失败');
};
} catch (error) {
reject('浏览器不支持文件内容读取');
}
});
/**
* 读取doc
*/
export const readDocContent = (file: File) =>
new Promise<string>((resolve, reject) => {
try {
const reader = new FileReader();
reader.readAsArrayBuffer(file);
reader.onload = async ({ target }) => {
if (!target?.result) return reject('读取 doc 文件失败');
try {
const res = await mammoth.extractRawText({
arrayBuffer: target.result as ArrayBuffer
});
resolve(res?.value);
} catch (error) {
reject('读取 doc 文件失败, 请转换成 PDF');
}
};
reader.onerror = (err) => {
console.log('error doc read:', err);
reject('读取 doc 文件失败');
};
} catch (error) {
reject('浏览器不支持文件内容读取');
}
});
/**
* 读取csv
*/
export const readCsvContent = async (file: File) => {
try {
const textArr = await readTxtContent(file);
const json = Papa.parse(textArr).data as string[][];
if (json.length === 0) {
throw new Error('csv 解析失败');
}
return {
header: json.shift()?.filter((item) => item) as string[],
data: json.map((item) => item?.filter((item) => item))
};
} catch (error) {
return Promise.reject('解析 csv 文件失败');
}
};
/**
* file download
*/
export const fileDownload = ({
text,
type,
filename
}: {
text: string;
type: string;
filename: string;
}) => {
// 导出为文件
const blob = new Blob([`\uFEFF${text}`], { type: `${type};charset=utf-8;` });
// 创建下载链接
const downloadLink = document.createElement('a');
downloadLink.href = window.URL.createObjectURL(blob);
downloadLink.download = filename;
// 添加链接到页面并触发下载
document.body.appendChild(downloadLink);
downloadLink.click();
document.body.removeChild(downloadLink);
};
/**
* text split into chunks
* maxLen - one chunk len. max: 3500
* slideLen - The size of the before and after Text
* maxLen > slideLen
*/
export const splitText_token = ({
text,
maxLen,
slideLen
}: {
text: string;
maxLen: number;
slideLen: number;
}) => {
try {
const enc = getOpenAiEncMap()['gpt-3.5-turbo'];
// filter empty text. encode sentence
const encodeText = enc.encode(text);
const chunks: string[] = [];
let tokens = 0;
let startIndex = 0;
let endIndex = Math.min(startIndex + maxLen, encodeText.length);
let chunkEncodeArr = encodeText.slice(startIndex, endIndex);
const decoder = new TextDecoder();
while (startIndex < encodeText.length) {
tokens += chunkEncodeArr.length;
chunks.push(decoder.decode(enc.decode(chunkEncodeArr)));
startIndex += maxLen - slideLen;
endIndex = Math.min(startIndex + maxLen, encodeText.length);
chunkEncodeArr = encodeText.slice(
Math.min(encodeText.length - slideLen, startIndex),
endIndex
);
}
return {
chunks,
tokens
};
} catch (err) {
throw new Error(getErrText(err));
}
};
export const fileToBase64 = (file: File) => {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result);
reader.onerror = (error) => reject(error);
});
};
/**
* compress image. response base64
* @param maxSize The max size of the compressed image
*/
export const compressImg = ({
file,
maxW = 200,
maxH = 200,
maxSize = 1024 * 100
}: {
file: File;
maxW?: number;
maxH?: number;
maxSize?: number;
}) =>
new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = () => {
const img = new Image();
// @ts-ignore
img.src = reader.result;
img.onload = () => {
let width = img.width;
let height = img.height;
if (width > height) {
if (width > maxW) {
height *= maxW / width;
width = maxW;
}
} else {
if (height > maxH) {
width *= maxH / height;
height = maxH;
}
}
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d');
if (!ctx) {
return reject('压缩图片异常');
}
ctx.drawImage(img, 0, 0, width, height);
const compressedDataUrl = canvas.toDataURL(file.type, 1);
// 移除 canvas 元素
canvas.remove();
if (compressedDataUrl.length > maxSize) {
return reject('图片太大了');
}
resolve(compressedDataUrl);
};
};
reader.onerror = (err) => {
console.log(err);
reject('压缩图片异常');
};
});

View File

@@ -0,0 +1,36 @@
import axios from 'axios';
import { Obj2Query } from '../tools';
export const getClientToken = (googleVerKey: string) => {
if (typeof grecaptcha === 'undefined' || !grecaptcha?.ready) return '';
return new Promise<string>((resolve, reject) => {
grecaptcha.ready(async () => {
try {
const token = await grecaptcha.execute(googleVerKey, {
action: 'submit'
});
resolve(token);
} catch (error) {
reject(error);
}
});
});
};
// service run
export const authGoogleToken = async (data: {
secret: string;
response: string;
remoteip?: string;
}) => {
const res = await axios.post<{
score?: number;
success: boolean;
'error-codes': string[];
}>(`https://www.recaptcha.net/recaptcha/api/siteverify?${Obj2Query(data)}`);
if (res.data.success) {
return Promise.resolve('');
}
return Promise.reject(res?.data?.['error-codes']?.[0] || '非法环境');
};

View File

@@ -0,0 +1,38 @@
import { ClaudeEnum, OpenAiChatEnum } from '@/constants/model';
import type { ChatModelType } from '@/constants/model';
import type { ChatItemSimpleType } from '@/types/chat';
import { countOpenAIToken, openAiSliceTextByToken } from './openai';
import { gpt_chatItemTokenSlice } from '@/pages/api/openapi/text/gptMessagesSlice';
export const modelToolMap: Record<
ChatModelType,
{
countTokens: (data: { messages: ChatItemSimpleType[] }) => number;
sliceText: (data: { text: string; length: number }) => string;
tokenSlice: (data: {
messages: ChatItemSimpleType[];
maxToken: number;
}) => ChatItemSimpleType[];
}
> = {
[OpenAiChatEnum.GPT35]: {
countTokens: ({ messages }) => countOpenAIToken({ model: OpenAiChatEnum.GPT35, messages }),
sliceText: (data) => openAiSliceTextByToken({ model: OpenAiChatEnum.GPT35, ...data }),
tokenSlice: (data) => gpt_chatItemTokenSlice({ model: OpenAiChatEnum.GPT35, ...data })
},
[OpenAiChatEnum.GPT4]: {
countTokens: ({ messages }) => countOpenAIToken({ model: OpenAiChatEnum.GPT4, messages }),
sliceText: (data) => openAiSliceTextByToken({ model: OpenAiChatEnum.GPT4, ...data }),
tokenSlice: (data) => gpt_chatItemTokenSlice({ model: OpenAiChatEnum.GPT4, ...data })
},
[OpenAiChatEnum.GPT432k]: {
countTokens: ({ messages }) => countOpenAIToken({ model: OpenAiChatEnum.GPT432k, messages }),
sliceText: (data) => openAiSliceTextByToken({ model: OpenAiChatEnum.GPT432k, ...data }),
tokenSlice: (data) => gpt_chatItemTokenSlice({ model: OpenAiChatEnum.GPT432k, ...data })
},
[ClaudeEnum.Claude]: {
countTokens: ({ messages }) => countOpenAIToken({ model: OpenAiChatEnum.GPT35, messages }),
sliceText: (data) => openAiSliceTextByToken({ model: OpenAiChatEnum.GPT35, ...data }),
tokenSlice: (data) => gpt_chatItemTokenSlice({ model: OpenAiChatEnum.GPT35, ...data })
}
};

View File

@@ -0,0 +1,162 @@
import { encoding_for_model, type Tiktoken } from '@dqbd/tiktoken';
import type { ChatItemSimpleType } from '@/types/chat';
import { ChatRoleEnum } from '@/constants/chat';
import { ChatCompletionRequestMessage, ChatCompletionRequestMessageRoleEnum } from 'openai';
import { OpenAiChatEnum } from '@/constants/model';
import Graphemer from 'graphemer';
const textDecoder = new TextDecoder();
const graphemer = new Graphemer();
export const getOpenAiEncMap = () => {
if (typeof window !== 'undefined') {
window.OpenAiEncMap = window.OpenAiEncMap || {
'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4': encoding_for_model('gpt-4', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4-32k': encoding_for_model('gpt-4-32k', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
})
};
return window.OpenAiEncMap;
}
if (typeof global !== 'undefined') {
global.OpenAiEncMap = global.OpenAiEncMap || {
'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4': encoding_for_model('gpt-4', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4-32k': encoding_for_model('gpt-4-32k', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
})
};
return global.OpenAiEncMap;
}
return {
'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4': encoding_for_model('gpt-4', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4-32k': encoding_for_model('gpt-4-32k', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
})
};
};
export const adaptChatItem_openAI = ({
messages
}: {
messages: ChatItemSimpleType[];
}): ChatCompletionRequestMessage[] => {
const map = {
[ChatRoleEnum.AI]: ChatCompletionRequestMessageRoleEnum.Assistant,
[ChatRoleEnum.Human]: ChatCompletionRequestMessageRoleEnum.User,
[ChatRoleEnum.System]: ChatCompletionRequestMessageRoleEnum.System
};
return messages.map((item) => ({
role: map[item.obj] || ChatCompletionRequestMessageRoleEnum.System,
content: item.value || ''
}));
};
export function countOpenAIToken({
messages,
model
}: {
messages: ChatItemSimpleType[];
model: `${OpenAiChatEnum}`;
}) {
function getChatGPTEncodingText(
messages: {
role: 'system' | 'user' | 'assistant';
content: string;
name?: string;
}[],
model: 'gpt-3.5-turbo' | 'gpt-4' | 'gpt-4-32k'
) {
const isGpt3 = model === 'gpt-3.5-turbo';
const msgSep = isGpt3 ? '\n' : '';
const roleSep = isGpt3 ? '\n' : '<|im_sep|>';
return [
messages
.map(({ name = '', role, content }) => {
return `<|im_start|>${name || role}${roleSep}${content}<|im_end|>`;
})
.join(msgSep),
`<|im_start|>assistant${roleSep}`
].join(msgSep);
}
function text2TokensLen(encoder: Tiktoken, inputText: string) {
const encoding = encoder.encode(inputText, 'all');
const segments: { text: string; tokens: { id: number; idx: number }[] }[] = [];
let byteAcc: number[] = [];
let tokenAcc: { id: number; idx: number }[] = [];
let inputGraphemes = graphemer.splitGraphemes(inputText);
for (let idx = 0; idx < encoding.length; idx++) {
const token = encoding[idx]!;
byteAcc.push(...encoder.decode_single_token_bytes(token));
tokenAcc.push({ id: token, idx });
const segmentText = textDecoder.decode(new Uint8Array(byteAcc));
const graphemes = graphemer.splitGraphemes(segmentText);
if (graphemes.every((item, idx) => inputGraphemes[idx] === item)) {
segments.push({ text: segmentText, tokens: tokenAcc });
byteAcc = [];
tokenAcc = [];
inputGraphemes = inputGraphemes.slice(graphemes.length);
}
}
return segments.reduce((memo, i) => memo + i.tokens.length, 0) ?? 0;
}
const adaptMessages = adaptChatItem_openAI({ messages });
return text2TokensLen(getOpenAiEncMap()[model], getChatGPTEncodingText(adaptMessages, model));
}
export const openAiSliceTextByToken = ({
model = 'gpt-3.5-turbo',
text,
length
}: {
model: `${OpenAiChatEnum}`;
text: string;
length: number;
}) => {
const enc = getOpenAiEncMap()[model];
const encodeText = enc.encode(text);
const decoder = new TextDecoder();
return decoder.decode(enc.decode(encodeText.slice(0, length)));
};

135
client/src/utils/tools.ts Normal file
View File

@@ -0,0 +1,135 @@
import crypto from 'crypto';
import { useToast } from '@/hooks/useToast';
import dayjs from 'dayjs';
/**
* copy text data
*/
export const useCopyData = () => {
const { toast } = useToast();
return {
copyData: async (data: string, title: string = '复制成功') => {
try {
if (navigator.clipboard) {
await navigator.clipboard.writeText(data);
} else {
throw new Error('');
}
} catch (error) {
const textarea = document.createElement('textarea');
textarea.value = data;
document.body.appendChild(textarea);
textarea.select();
document.execCommand('copy');
document.body.removeChild(textarea);
}
toast({
title,
status: 'success',
duration: 1000
});
}
};
};
/**
* 密码加密
*/
export const createHashPassword = (text: string) => {
const hash = crypto.createHash('sha256').update(text).digest('hex');
return hash;
};
/**
* 对象转成 query 字符串
*/
export const Obj2Query = (obj: Record<string, string | number>) => {
const queryParams = new URLSearchParams();
for (const key in obj) {
queryParams.append(key, `${obj[key]}`);
}
return queryParams.toString();
};
/**
* 格式化时间成聊天格式
*/
export const formatTimeToChatTime = (time: Date) => {
const now = dayjs();
const target = dayjs(time);
// 如果传入时间小于60秒返回刚刚
if (now.diff(target, 'second') < 60) {
return '刚刚';
}
// 如果时间是今天,展示几时:几秒
if (now.isSame(target, 'day')) {
return target.format('HH:mm');
}
// 如果是昨天,展示昨天
if (now.subtract(1, 'day').isSame(target, 'day')) {
return '昨天';
}
// 如果是前天,展示前天
if (now.subtract(2, 'day').isSame(target, 'day')) {
return '前天';
}
// 如果是今年,展示某月某日
if (now.isSame(target, 'year')) {
return target.format('M月D日');
}
// 如果是更久之前,展示某年某月某日
return target.format('YYYY/M/D');
};
export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window;
/**
* voice broadcast
*/
export const voiceBroadcast = ({ text }: { text: string }) => {
window.speechSynthesis?.cancel();
const msg = new SpeechSynthesisUtterance(text);
const voices = window.speechSynthesis?.getVoices?.(); // 获取语言包
const voice = voices.find((item) => {
return item.name === 'Microsoft Yaoyao - Chinese (Simplified, PRC)';
});
if (voice) {
msg.voice = voice;
}
window.speechSynthesis?.speak(msg);
msg.onerror = (e) => {
console.log(e);
};
return {
cancel: () => window.speechSynthesis?.cancel()
};
};
export const formatLinkText = (text: string) => {
const httpReg =
/(http|https|ftp):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[\w\-\@?^=%&amp;/~\+#])?/gi;
return text.replace(httpReg, ` $& `);
};
export const getErrText = (err: any, def = '') => {
const msg = typeof err === 'string' ? err : err?.message || def || '';
msg && console.log('error =>', msg);
return msg;
};
export const delay = (ms: number) =>
new Promise((resolve) => {
setTimeout(() => {
resolve('');
}, ms);
});

17
client/src/utils/user.ts Normal file
View File

@@ -0,0 +1,17 @@
import { PRICE_SCALE } from '@/constants/common';
import { loginOut } from '@/api/user';
export const clearCookie = () => {
try {
loginOut();
} catch (error) {
error;
}
};
/**
* 把数据库读取到的price转化成元
*/
export const formatPrice = (val = 0, multiple = 1) => {
return Number(((val / PRICE_SCALE) * multiple).toFixed(10));
};