This commit is contained in:
Archer
2023-12-18 16:24:50 +08:00
committed by GitHub
parent d33c99f564
commit 703583fff7
130 changed files with 3418 additions and 2579 deletions

View File

@@ -71,7 +71,7 @@ instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err)
export function request(url: string, data: any, config: ConfigType, method: Method): any {
if (!global.systemEnv || !global.systemEnv?.pluginBaseUrl) {
console.log('未部署商业版接口');
console.log('未部署商业版接口', url);
return Promise.reject('The The request was denied...');
}

View File

@@ -0,0 +1,95 @@
import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api';
import * as cheerio from 'cheerio';
import axios from 'axios';
import { htmlToMarkdown } from './markdown';
export const cheerioToHtml = ({
fetchUrl,
$,
selector
}: {
fetchUrl: string;
$: cheerio.CheerioAPI;
selector?: string;
}) => {
// get origin url
const originUrl = new URL(fetchUrl).origin;
const selectDom = $(selector || 'body');
// remove i element
selectDom.find('i,script').remove();
// remove empty a element
selectDom
.find('a')
.filter((i, el) => {
return $(el).text().trim() === '' && $(el).children().length === 0;
})
.remove();
// if link,img startWith /, add origin url
selectDom.find('a').each((i, el) => {
const href = $(el).attr('href');
if (href && href.startsWith('/')) {
$(el).attr('href', originUrl + href);
}
});
selectDom.find('img').each((i, el) => {
const src = $(el).attr('src');
if (src && src.startsWith('/')) {
$(el).attr('src', originUrl + src);
}
});
const html = selectDom
.map((item, dom) => {
return $(dom).html();
})
.get()
.join('\n');
return html;
};
export const urlsFetch = async ({
urlList,
selector
}: UrlFetchParams): Promise<UrlFetchResponse> => {
urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
const response = (
await Promise.all(
urlList.map(async (url) => {
try {
const fetchRes = await axios.get(url, {
timeout: 30000
});
const $ = cheerio.load(fetchRes.data);
const md = await htmlToMarkdown(
cheerioToHtml({
fetchUrl: url,
$,
selector
})
);
return {
url,
content: md
};
} catch (error) {
console.log(error, 'fetch error');
return {
url,
content: ''
};
}
})
)
).filter((item) => item.content);
return response;
};

View File

@@ -0,0 +1,23 @@
import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
import { Worker } from 'worker_threads';
import { getWorkerPath } from './utils';
/* html string to markdown */
export const htmlToMarkdown = (html?: string | null) =>
new Promise<string>((resolve, reject) => {
if (!html) return resolve('');
const start = Date.now();
// worker
const worker = new Worker(getWorkerPath('html2md'));
worker.on('message', (md: string) => {
resolve(simpleMarkdownText(md));
});
worker.on('error', (err) => {
reject(err);
});
worker.postMessage(html);
});

View File

@@ -0,0 +1,9 @@
export const getWorkerPath = (name: string) => {
// @ts-ignore
const isSubModule = !!global?.systemConfig;
const isProd = process.env.NODE_ENV === 'production';
return isProd
? `/app/worker/${name}.js`
: `../../${isSubModule ? 'FastGPT/' : ''}/worker/${name}.js`;
};

View File

@@ -61,8 +61,8 @@ const ChatItemSchema = new Schema({
userBadFeedback: {
type: String
},
robotBadFeedback: {
type: String
customFeedbacks: {
type: [String]
},
adminFeedback: {
type: {
@@ -86,7 +86,7 @@ try {
ChatItemSchema.index({ chatId: 1 });
ChatItemSchema.index({ userGoodFeedback: 1 });
ChatItemSchema.index({ userBadFeedback: 1 });
ChatItemSchema.index({ robotBadFeedback: 1 });
ChatItemSchema.index({ customFeedbacks: 1 });
ChatItemSchema.index({ adminFeedback: 1 });
} catch (error) {
console.log(error);

View File

@@ -1,5 +1,6 @@
import type { ChatItemType } from '@fastgpt/global/core/chat/type';
import { MongoChatItem } from './chatItemSchema';
import { addLog } from '../../common/system/log';
export async function getChatItems({
chatId,
@@ -20,3 +21,29 @@ export async function getChatItems({
return { history };
}
export const addCustomFeedbacks = async ({
chatId,
chatItemId,
feedbacks
}: {
chatId?: string;
chatItemId?: string;
feedbacks: string[];
}) => {
if (!chatId || !chatItemId) return;
try {
await MongoChatItem.findOneAndUpdate(
{
chatId,
dataId: chatItemId
},
{
$push: { customFeedbacks: { $each: feedbacks } }
}
);
} catch (error) {
addLog.error('addCustomFeedbacks error', error);
}
};

View File

@@ -4,7 +4,7 @@ import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { MongoDatasetTraining } from '../training/schema';
import { urlsFetch } from '@fastgpt/global/common/file/tools';
import { urlsFetch } from '../../../common/string/cheerio';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
/**
@@ -105,7 +105,8 @@ export const loadingOneChunkCollection = async ({
// split data
const { chunks } = splitText2Chunks({
text: newRawText,
chunkLen: collection.chunkSize || 512
chunkLen: collection.chunkSize || 512,
countTokens: false
});
// insert to training queue

View File

@@ -44,6 +44,7 @@ const getPluginTemplateById = async (id: string): Promise<PluginTemplateType> =>
name: item.name,
avatar: item.avatar,
intro: item.intro,
showStatus: true,
source: PluginSourceEnum.personal,
modules: item.modules,
templateType: ModuleTemplateTypeEnum.personalPlugin
@@ -67,7 +68,7 @@ export async function getPluginPreviewModule({
avatar: plugin.avatar,
name: plugin.name,
intro: plugin.intro,
showStatus: true,
showStatus: plugin.showStatus,
...plugin2ModuleIO(plugin.id, plugin.modules)
};
}

View File

@@ -3,22 +3,24 @@
"version": "1.0.0",
"dependencies": {
"@fastgpt/global": "workspace:*",
"axios": "^1.5.1",
"cookie": "^0.5.0",
"encoding": "^0.1.13",
"jsonwebtoken": "^9.0.2",
"mongoose": "^7.0.2",
"nanoid": "^4.0.1",
"dayjs": "^1.11.7",
"next": "13.5.2",
"multer": "1.4.5-lts.1",
"axios": "^1.5.1",
"cheerio": "1.0.0-rc.12",
"nextjs-cors": "^2.1.2",
"pg": "^8.10.0",
"tunnel": "^0.0.6",
"dayjs": "^1.11.7"
"tunnel": "^0.0.6"
},
"devDependencies": {
"@types/cookie": "^0.5.2",
"@types/jsonwebtoken": "^9.0.3",
"@types/node": "^20.8.5",
"@types/multer": "^1.4.10",
"@types/pg": "^8.6.6",
"@types/tunnel": "^0.0.4"
}

View File

@@ -31,3 +31,11 @@ export async function authCertOrShareId({
canWrite: false
};
}
/* auth the request from local service */
export const authRequestFromLocal = ({ req }: AuthModeType) => {
const host = `${process.env.HOSTNAME || 'localhost'}:${process.env.PORT || 3000}`;
if (host !== req.headers.host) {
return Promise.reject('Invalid request');
}
};