mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 13:03:50 +00:00
fix: add order:true to all create transactions (#3948)
This commit is contained in:
@@ -216,7 +216,7 @@ export async function createOneCollection({
|
||||
nextSyncTime
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
return collection;
|
||||
|
@@ -97,7 +97,7 @@ export const createOrGetCollectionTags = async ({
|
||||
datasetId,
|
||||
tag: tagContent
|
||||
})),
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
return [...existingTags.map((tag) => tag._id), ...newTags.map((tag) => tag._id)];
|
||||
|
@@ -196,7 +196,8 @@ export async function syncCollaborators({
|
||||
permission: item.permission
|
||||
})),
|
||||
{
|
||||
session
|
||||
session,
|
||||
ordered: true
|
||||
}
|
||||
);
|
||||
}
|
||||
|
@@ -100,7 +100,7 @@ export const initTeamFreePlan = async ({
|
||||
surplusPoints: freePoints
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
};
|
||||
|
||||
|
@@ -160,7 +160,7 @@ export const createTrainingUsage = async ({
|
||||
]
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
return { billId: String(_id) };
|
||||
|
@@ -5,56 +5,56 @@ import dotenv from 'dotenv';
|
||||
dotenv.config();
|
||||
|
||||
const userAgents = [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0'
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0'
|
||||
];
|
||||
|
||||
export const quickFetch = async (req: Request, res: Response): Promise<void> => {
|
||||
const { url } = req.query;
|
||||
const { url } = req.query;
|
||||
|
||||
if (!url) {
|
||||
res.status(400).json({
|
||||
status: 400,
|
||||
error: {
|
||||
code: "MISSING_PARAM",
|
||||
message: "缺少必要参数: url"
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!url) {
|
||||
res.status(400).json({
|
||||
status: 400,
|
||||
error: {
|
||||
code: 'MISSING_PARAM',
|
||||
message: '缺少必要参数: url'
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url as string, {
|
||||
headers: {
|
||||
'User-Agent': userAgents[Math.floor(Math.random() * userAgents.length)],
|
||||
'Referer': 'https://www.google.com/',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Connection': 'keep-alive',
|
||||
'Cache-Control': 'no-cache'
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
const data = await response.text();
|
||||
res.status(200).json({
|
||||
status: 200,
|
||||
data: {
|
||||
content: data
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching the page:', error);
|
||||
res.status(500).json({
|
||||
status: 500,
|
||||
error: {
|
||||
code: "INTERNAL_SERVER_ERROR",
|
||||
message: "发生错误"
|
||||
}
|
||||
});
|
||||
try {
|
||||
const response = await fetch(url as string, {
|
||||
headers: {
|
||||
'User-Agent': userAgents[Math.floor(Math.random() * userAgents.length)],
|
||||
Referer: 'https://www.google.com/',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
Connection: 'keep-alive',
|
||||
'Cache-Control': 'no-cache'
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
const data = await response.text();
|
||||
res.status(200).json({
|
||||
status: 200,
|
||||
data: {
|
||||
content: data
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching the page:', error);
|
||||
res.status(500).json({
|
||||
status: 500,
|
||||
error: {
|
||||
code: 'INTERNAL_SERVER_ERROR',
|
||||
message: '发生错误'
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export default { quickFetch };
|
@@ -16,16 +16,16 @@ const blacklistDomains = process.env.BLACKLIST ? JSON.parse(process.env.BLACKLIS
|
||||
|
||||
export const readPage = async (req: Request, res: Response): Promise<void> => {
|
||||
const { queryUrl } = req.query;
|
||||
console.log("-------");
|
||||
console.log('-------');
|
||||
console.log(queryUrl);
|
||||
console.log("-------");
|
||||
console.log('-------');
|
||||
|
||||
if (!queryUrl) {
|
||||
res.status(400).json({
|
||||
status: 400,
|
||||
error: {
|
||||
code: "MISSING_PARAM",
|
||||
message: "缺少必要参数: queryUrl"
|
||||
code: 'MISSING_PARAM',
|
||||
message: '缺少必要参数: queryUrl'
|
||||
}
|
||||
});
|
||||
return;
|
||||
@@ -36,8 +36,8 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
|
||||
res.status(403).json({
|
||||
status: 403,
|
||||
error: {
|
||||
code: "BLACKLISTED_DOMAIN",
|
||||
message: "该域名受到保护中"
|
||||
code: 'BLACKLISTED_DOMAIN',
|
||||
message: '该域名受到保护中'
|
||||
}
|
||||
});
|
||||
return;
|
||||
@@ -46,11 +46,14 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
|
||||
try {
|
||||
const response = await fetch(queryUrl as string, {
|
||||
headers: {
|
||||
'User-Agent': new UserAgent({ deviceCategory: 'desktop', platform: 'Linux x86_64' }).toString(),
|
||||
'Referer': 'https://www.google.com/',
|
||||
'User-Agent': new UserAgent({
|
||||
deviceCategory: 'desktop',
|
||||
platform: 'Linux x86_64'
|
||||
}).toString(),
|
||||
Referer: 'https://www.google.com/',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Connection': 'keep-alive',
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
Connection: 'keep-alive',
|
||||
'Cache-Control': 'no-cache'
|
||||
}
|
||||
});
|
||||
@@ -69,7 +72,7 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
|
||||
});
|
||||
|
||||
await updateCacheAsync(queryUrl as string, cleanedContent || '');
|
||||
console.log("Page read successfully");
|
||||
console.log('Page read successfully');
|
||||
return;
|
||||
} else {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
@@ -80,22 +83,25 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
|
||||
|
||||
try {
|
||||
const browser = await puppeteer.launch({
|
||||
ignoreDefaultArgs: ["--enable-automation"],
|
||||
ignoreDefaultArgs: ['--enable-automation'],
|
||||
headless: true,
|
||||
executablePath: "/usr/bin/chromium", // 明确指定 Chromium 路径
|
||||
executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
|
||||
pipe: true,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu',
|
||||
// '--single-process'
|
||||
'--disable-gpu'
|
||||
// '--single-process'
|
||||
]
|
||||
});
|
||||
const page = await browser.newPage();
|
||||
|
||||
// 检测是否需要特殊处理
|
||||
if (typeof queryUrl === 'string' && detectWebsites.some(website => queryUrl.includes(website))) {
|
||||
if (
|
||||
typeof queryUrl === 'string' &&
|
||||
detectWebsites.some((website) => queryUrl.includes(website))
|
||||
) {
|
||||
await setupPage(page);
|
||||
} else {
|
||||
const userAgent = new UserAgent({ deviceCategory: 'desktop', platform: 'Linux x86_64' });
|
||||
@@ -128,14 +134,14 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
|
||||
});
|
||||
|
||||
await updateCacheAsync(queryUrl as string, cleanedContent || '');
|
||||
console.log("Page read successfully");
|
||||
console.log('Page read successfully');
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
res.status(500).json({
|
||||
status: 500,
|
||||
error: {
|
||||
code: "INTERNAL_SERVER_ERROR",
|
||||
message: "读取页面时发生内部服务器错误"
|
||||
code: 'INTERNAL_SERVER_ERROR',
|
||||
message: '读取页面时发生内部服务器错误'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
@@ -12,15 +12,21 @@ const detectWebsites = process.env.DETECT_WEBSITES?.split(',') || [];
|
||||
const maxConcurrency = parseInt(process.env.MAX_CONCURRENCY || '10', 10);
|
||||
|
||||
export const search = async (req: Request, res: Response): Promise<void> => {
|
||||
const { query, pageCount = 10, needDetails = 'false', engine = 'baidu', categories = 'general' } = req.query;
|
||||
const needDetailsBool = (needDetails === 'true');
|
||||
const {
|
||||
query,
|
||||
pageCount = 10,
|
||||
needDetails = 'false',
|
||||
engine = 'baidu',
|
||||
categories = 'general'
|
||||
} = req.query;
|
||||
const needDetailsBool = needDetails === 'true';
|
||||
|
||||
if (!query) {
|
||||
res.status(400).json({
|
||||
status: 400,
|
||||
error: {
|
||||
code: "MISSING_PARAM",
|
||||
message: "缺少必要参数: query"
|
||||
code: 'MISSING_PARAM',
|
||||
message: '缺少必要参数: query'
|
||||
}
|
||||
});
|
||||
return;
|
||||
@@ -28,24 +34,29 @@ export const search = async (req: Request, res: Response): Promise<void> => {
|
||||
let fetchSearchResults;
|
||||
let searchUrlBase;
|
||||
try {
|
||||
if (engine === 'baidu') {
|
||||
fetchSearchResults = fetchBaiduResults;
|
||||
searchUrlBase = process.env.ENGINE_BAIDUURL;
|
||||
} else if (engine === 'searchxng') {
|
||||
fetchSearchResults = fetchSearchxngResults;
|
||||
searchUrlBase = process.env.ENGINE_SEARCHXNGURL;
|
||||
} else {
|
||||
res.status(400).json({
|
||||
status: 400,
|
||||
error: {
|
||||
code: "INVALID_ENGINE",
|
||||
message: "无效的搜索引擎"
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (engine === 'baidu') {
|
||||
fetchSearchResults = fetchBaiduResults;
|
||||
searchUrlBase = process.env.ENGINE_BAIDUURL;
|
||||
} else if (engine === 'searchxng') {
|
||||
fetchSearchResults = fetchSearchxngResults;
|
||||
searchUrlBase = process.env.ENGINE_SEARCHXNGURL;
|
||||
} else {
|
||||
res.status(400).json({
|
||||
status: 400,
|
||||
error: {
|
||||
code: 'INVALID_ENGINE',
|
||||
message: '无效的搜索引擎'
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const { resultUrls, results } = await fetchSearchResults(query as string, Number(pageCount), searchUrlBase || '', categories as string);
|
||||
const { resultUrls, results } = await fetchSearchResults(
|
||||
query as string,
|
||||
Number(pageCount),
|
||||
searchUrlBase || '',
|
||||
categories as string
|
||||
);
|
||||
|
||||
//如果返回值为空,返回空数组
|
||||
if (results.size === 0) {
|
||||
@@ -79,20 +90,27 @@ export const search = async (req: Request, res: Response): Promise<void> => {
|
||||
concurrency: Cluster.CONCURRENCY_CONTEXT,
|
||||
maxConcurrency: maxConcurrency,
|
||||
puppeteerOptions: {
|
||||
ignoreDefaultArgs: ["--enable-automation"],
|
||||
headless: "true",
|
||||
executablePath: "/usr/bin/chromium", // 明确指定 Chromium 路径
|
||||
ignoreDefaultArgs: ['--enable-automation'],
|
||||
headless: 'true',
|
||||
executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
|
||||
pipe: true,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu',
|
||||
'--disable-gpu'
|
||||
]
|
||||
}
|
||||
});
|
||||
|
||||
const sortedResults = await performDeepSearch(clusterInstance, resultUrls, results, strategies, detectWebsites, Number(pageCount));
|
||||
const sortedResults = await performDeepSearch(
|
||||
clusterInstance,
|
||||
resultUrls,
|
||||
results,
|
||||
strategies,
|
||||
detectWebsites,
|
||||
Number(pageCount)
|
||||
);
|
||||
res.status(200).json({
|
||||
status: 200,
|
||||
data: {
|
||||
@@ -104,8 +122,8 @@ export const search = async (req: Request, res: Response): Promise<void> => {
|
||||
res.status(500).json({
|
||||
status: 500,
|
||||
error: {
|
||||
code: "INTERNAL_SERVER_ERROR",
|
||||
message: "发生错误"
|
||||
code: 'INTERNAL_SERVER_ERROR',
|
||||
message: '发生错误'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
@@ -5,200 +5,203 @@ import { setupPage } from '../utils/setupPage';
|
||||
import { Cluster } from 'puppeteer-cluster';
|
||||
|
||||
async function randomWait(min: number, max: number) {
|
||||
// 随机等待时间
|
||||
const delay = Math.floor(Math.random() * (max - min + 1)) + min;
|
||||
return new Promise(resolve => setTimeout(resolve, delay));
|
||||
// 随机等待时间
|
||||
const delay = Math.floor(Math.random() * (max - min + 1)) + min;
|
||||
return new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
|
||||
export const fetchSearchResults = async (query: string, pageCount: number, searchUrlBase: string, categories: string) => {
|
||||
console.log(`Fetching Baidu search results for query: ${query}`);
|
||||
// 如果 searchUrlBase 为空,返回空数组
|
||||
if (!searchUrlBase) {
|
||||
return { resultUrls: [], results: new Map() };
|
||||
export const fetchSearchResults = async (
|
||||
query: string,
|
||||
pageCount: number,
|
||||
searchUrlBase: string,
|
||||
categories: string
|
||||
) => {
|
||||
console.log(`Fetching Baidu search results for query: ${query}`);
|
||||
// 如果 searchUrlBase 为空,返回空数组
|
||||
if (!searchUrlBase) {
|
||||
return { resultUrls: [], results: new Map() };
|
||||
}
|
||||
const resultUrls: string[] = [];
|
||||
const results = new Map<string, any>();
|
||||
|
||||
const pagesToFetch = Math.ceil(pageCount / 10);
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
ignoreDefaultArgs: ['--enable-automation'],
|
||||
headless: true,
|
||||
executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
|
||||
pipe: true,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu'
|
||||
// '--single-process'
|
||||
]
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await setupPage(page);
|
||||
|
||||
for (let i = 0; i < pagesToFetch; i++) {
|
||||
const searchUrl = new URL(`${searchUrlBase}?wd=${encodeURIComponent(query)}&pn=${i * 10}`);
|
||||
console.log(`Fetching page ${i + 1} from Baidu: ${searchUrl.toString()}`);
|
||||
let retryCount = 0;
|
||||
let success = false;
|
||||
|
||||
while (retryCount < 5 && !success) {
|
||||
try {
|
||||
console.time(`Page Load Time for page ${i + 1}`);
|
||||
await page.goto(searchUrl.toString(), { waitUntil: 'load' });
|
||||
console.timeEnd(`Page Load Time for page ${i + 1}`);
|
||||
|
||||
let content = await page.content();
|
||||
let dom = new JSDOM(content);
|
||||
let document = dom.window.document;
|
||||
console.log(document.title);
|
||||
|
||||
// 如果是百度安全验证页面,重新设置页面并重新访问
|
||||
if (document.title.includes('百度安全验证')) {
|
||||
console.log('Detected Baidu security verification, retrying...');
|
||||
await setupPage(page);
|
||||
retryCount++;
|
||||
//随机等待时间
|
||||
await randomWait(1000, 3000);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 解析搜索结果
|
||||
console.time(`Link Retrieval Time for page ${i + 1}`);
|
||||
|
||||
const resultContainers = document.querySelectorAll('.result.c-container');
|
||||
for (const result of resultContainers) {
|
||||
if (resultUrls.length > pageCount + 5) {
|
||||
break;
|
||||
}
|
||||
const titleElement = result.querySelector('h3 a');
|
||||
const title = titleElement ? titleElement.textContent : '';
|
||||
const url = titleElement ? titleElement.getAttribute('href') : '';
|
||||
const contentElement = result.querySelector('[class^="content"]');
|
||||
const content = contentElement ? contentElement.textContent : '';
|
||||
|
||||
if (url) {
|
||||
resultUrls.push(url);
|
||||
results.set(url, {
|
||||
title,
|
||||
url,
|
||||
snippet: content,
|
||||
source: 'baidu',
|
||||
crawlStatus: 'Pending',
|
||||
score: 0
|
||||
});
|
||||
}
|
||||
}
|
||||
console.timeEnd(`Link Retrieval Time for page ${i + 1}`);
|
||||
success = true;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching page ${i + 1}:`, error);
|
||||
retryCount++;
|
||||
}
|
||||
}
|
||||
const resultUrls: string[] = [];
|
||||
const results = new Map<string, any>();
|
||||
}
|
||||
|
||||
const pagesToFetch = Math.ceil(pageCount / 10);
|
||||
await browser.close();
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
ignoreDefaultArgs: ["--enable-automation"],
|
||||
headless: true,
|
||||
executablePath: "/usr/bin/chromium", // 明确指定 Chromium 路径
|
||||
pipe: true,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu',
|
||||
// '--single-process'
|
||||
]
|
||||
console.log('fetch all fake urls');
|
||||
|
||||
// 快速检索真实 URL
|
||||
const urlsToProcessWithPuppeteer = [];
|
||||
for (const url of resultUrls) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
|
||||
Referer: 'https://www.google.com/',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
Connection: 'keep-alive',
|
||||
'Cache-Control': 'no-cache'
|
||||
}
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await setupPage(page);
|
||||
|
||||
for (let i = 0; i < pagesToFetch; i++) {
|
||||
const searchUrl = new URL(`${searchUrlBase}?wd=${encodeURIComponent(query)}&pn=${i * 10}`);
|
||||
console.log(`Fetching page ${i + 1} from Baidu: ${searchUrl.toString()}`);
|
||||
let retryCount = 0;
|
||||
let success = false;
|
||||
|
||||
while (retryCount < 5 && !success) {
|
||||
try {
|
||||
console.time(`Page Load Time for page ${i + 1}`);
|
||||
await page.goto(searchUrl.toString(), { waitUntil: 'load' });
|
||||
console.timeEnd(`Page Load Time for page ${i + 1}`);
|
||||
|
||||
let content = await page.content();
|
||||
let dom = new JSDOM(content);
|
||||
let document = dom.window.document;
|
||||
console.log(document.title);
|
||||
|
||||
// 如果是百度安全验证页面,重新设置页面并重新访问
|
||||
if (document.title.includes('百度安全验证')) {
|
||||
console.log('Detected Baidu security verification, retrying...');
|
||||
await setupPage(page);
|
||||
retryCount++;
|
||||
//随机等待时间
|
||||
await randomWait(1000, 3000);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 解析搜索结果
|
||||
console.time(`Link Retrieval Time for page ${i + 1}`);
|
||||
|
||||
|
||||
const resultContainers = document.querySelectorAll('.result.c-container');
|
||||
for (const result of resultContainers) {
|
||||
if (resultUrls.length > pageCount + 5) {
|
||||
break;
|
||||
}
|
||||
const titleElement = result.querySelector('h3 a');
|
||||
const title = titleElement ? titleElement.textContent : '';
|
||||
const url = titleElement ? titleElement.getAttribute('href') : '';
|
||||
const contentElement = result.querySelector('[class^="content"]');
|
||||
const content = contentElement ? contentElement.textContent : '';
|
||||
|
||||
if (url) {
|
||||
resultUrls.push(url);
|
||||
results.set(url, {
|
||||
title,
|
||||
url,
|
||||
snippet: content,
|
||||
source: 'baidu',
|
||||
crawlStatus: 'Pending',
|
||||
score: 0
|
||||
});
|
||||
}
|
||||
}
|
||||
console.timeEnd(`Link Retrieval Time for page ${i + 1}`);
|
||||
success = true;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching page ${i + 1}:`, error);
|
||||
retryCount++;
|
||||
}
|
||||
if (response.ok) {
|
||||
const realUrl = response.url;
|
||||
console.log('realurl:', realUrl);
|
||||
const result = results.get(url);
|
||||
if (result) {
|
||||
result.url = realUrl;
|
||||
result.crawlStatus = 'Success';
|
||||
}
|
||||
} else {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error fetching original URL for ${url}:`, error);
|
||||
urlsToProcessWithPuppeteer.push(url);
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
console.log('pass quickfetch');
|
||||
|
||||
console.log('fetch all fake urls');
|
||||
|
||||
// 快速检索真实 URL
|
||||
const urlsToProcessWithPuppeteer = [];
|
||||
for (const url of resultUrls) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
|
||||
'Referer': 'https://www.google.com/',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Connection': 'keep-alive',
|
||||
'Cache-Control': 'no-cache'
|
||||
}
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const realUrl = response.url;
|
||||
console.log('realurl:', realUrl);
|
||||
const result = results.get(url);
|
||||
if (result) {
|
||||
result.url = realUrl;
|
||||
result.crawlStatus = 'Success';
|
||||
}
|
||||
} else {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error fetching original URL for ${url}:`, error);
|
||||
urlsToProcessWithPuppeteer.push(url);
|
||||
}
|
||||
// 并发处理真实 URL
|
||||
const cluster = await Cluster.launch({
|
||||
concurrency: Cluster.CONCURRENCY_CONTEXT,
|
||||
maxConcurrency: 10,
|
||||
puppeteerOptions: {
|
||||
ignoreDefaultArgs: ['--enable-automation'],
|
||||
headless: 'true',
|
||||
executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
|
||||
pipe: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
|
||||
}
|
||||
});
|
||||
|
||||
console.log('pass quickfetch');
|
||||
let failedUrlCount = 0;
|
||||
|
||||
// 并发处理真实 URL
|
||||
const cluster = await Cluster.launch({
|
||||
concurrency: Cluster.CONCURRENCY_CONTEXT,
|
||||
maxConcurrency: 10,
|
||||
puppeteerOptions: {
|
||||
ignoreDefaultArgs: ["--enable-automation"],
|
||||
headless: "true",
|
||||
executablePath: "/usr/bin/chromium", // 明确指定 Chromium 路径
|
||||
pipe: true,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu',
|
||||
]
|
||||
await cluster.task(async ({ page, data: url }) => {
|
||||
let retryUrlCount = 0;
|
||||
let urlSuccess = false;
|
||||
while (retryUrlCount < 3 && !urlSuccess) {
|
||||
console.log(`Fetching original URL for ${url}, attempt ${retryUrlCount + 1}`);
|
||||
try {
|
||||
await page.goto(url, { waitUntil: 'load' });
|
||||
// 检查页面是否被分离
|
||||
if (page.isClosed()) {
|
||||
throw new Error('Page has been closed');
|
||||
}
|
||||
});
|
||||
|
||||
let failedUrlCount = 0;
|
||||
|
||||
await cluster.task(async ({ page, data: url }) => {
|
||||
let retryUrlCount = 0;
|
||||
let urlSuccess = false;
|
||||
while (retryUrlCount < 3 && !urlSuccess) {
|
||||
console.log(`Fetching original URL for ${url}, attempt ${retryUrlCount + 1}`);
|
||||
try {
|
||||
await page.goto(url, { waitUntil: 'load' });
|
||||
// 检查页面是否被分离
|
||||
if (page.isClosed()) {
|
||||
throw new Error('Page has been closed');
|
||||
}
|
||||
const realUrl = page.url(); // 获取真实 URL
|
||||
const result = results.get(url);
|
||||
if (result) {
|
||||
result.url = realUrl;
|
||||
result.crawlStatus = 'Success';
|
||||
}
|
||||
urlSuccess = true;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching original URL, retrying...`, error);
|
||||
retryUrlCount++;
|
||||
await randomWait(1000, 3000);
|
||||
}
|
||||
const realUrl = page.url(); // 获取真实 URL
|
||||
const result = results.get(url);
|
||||
if (result) {
|
||||
result.url = realUrl;
|
||||
result.crawlStatus = 'Success';
|
||||
}
|
||||
if (!urlSuccess) {
|
||||
failedUrlCount++;
|
||||
}
|
||||
});
|
||||
|
||||
for (const url of urlsToProcessWithPuppeteer) {
|
||||
cluster.queue(url);
|
||||
urlSuccess = true;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching original URL, retrying...`, error);
|
||||
retryUrlCount++;
|
||||
await randomWait(1000, 3000);
|
||||
}
|
||||
}
|
||||
if (!urlSuccess) {
|
||||
failedUrlCount++;
|
||||
}
|
||||
});
|
||||
|
||||
await cluster.idle();
|
||||
await cluster.close();
|
||||
for (const url of urlsToProcessWithPuppeteer) {
|
||||
cluster.queue(url);
|
||||
}
|
||||
|
||||
console.log(`Number of URLs that failed to return a real URL: ${failedUrlCount}`);
|
||||
await cluster.idle();
|
||||
await cluster.close();
|
||||
|
||||
// 过滤并返回前 pageCount 个结果
|
||||
const filteredResults = Array.from(results.values()).slice(0, pageCount);
|
||||
console.log(`Number of URLs that failed to return a real URL: ${failedUrlCount}`);
|
||||
|
||||
return { resultUrls: filteredResults.map(result => result.url), results: new Map(filteredResults.map(result => [result.url, result])) };
|
||||
// 过滤并返回前 pageCount 个结果
|
||||
const filteredResults = Array.from(results.values()).slice(0, pageCount);
|
||||
|
||||
return {
|
||||
resultUrls: filteredResults.map((result) => result.url),
|
||||
results: new Map(filteredResults.map((result) => [result.url, result]))
|
||||
};
|
||||
};
|
@@ -6,9 +6,13 @@ dotenv.config();
|
||||
|
||||
const blacklistDomains = process.env.BLACKLIST ? JSON.parse(process.env.BLACKLIST) : [];
|
||||
|
||||
export const fetchSearchResults = async (query: string, pageCount: number, searchUrlBase: string, categories: string) => {
|
||||
|
||||
const MAX_PAGES = (pageCount / 10 +1) * 2+1; // 最多搜索的页面数
|
||||
export const fetchSearchResults = async (
|
||||
query: string,
|
||||
pageCount: number,
|
||||
searchUrlBase: string,
|
||||
categories: string
|
||||
) => {
|
||||
const MAX_PAGES = (pageCount / 10 + 1) * 2 + 1; // 最多搜索的页面数
|
||||
//如果searchUrlBase为空,返回空数组,pagecount是需要搜索结果的数量
|
||||
if (!searchUrlBase) {
|
||||
return { resultUrls: [], results: new Map() };
|
||||
@@ -20,7 +24,9 @@ export const fetchSearchResults = async (query: string, pageCount: number, searc
|
||||
let pageIndex = 0;
|
||||
|
||||
while (fetchedResultsCount < pageCount && pageIndex < MAX_PAGES) {
|
||||
const searchUrl = new URL(`${searchUrlBase}?q=${encodeURIComponent(query)}&pageno=${pageIndex + 1}&format=json&categories=${categories}`);
|
||||
const searchUrl = new URL(
|
||||
`${searchUrlBase}?q=${encodeURIComponent(query)}&pageno=${pageIndex + 1}&format=json&categories=${categories}`
|
||||
);
|
||||
console.log(`Fetching page ${pageIndex + 1} from SearchXNG: ${searchUrl.toString()}`);
|
||||
const response = await axios.get(searchUrl.toString());
|
||||
const jsonResults = response.data.results;
|
||||
@@ -28,7 +34,10 @@ export const fetchSearchResults = async (query: string, pageCount: number, searc
|
||||
for (let index = 0; index < jsonResults.length; index++) {
|
||||
const result = jsonResults[index];
|
||||
const resultDomain = new URL(result.url).hostname;
|
||||
if (blacklistDomains.some((domain: string) => resultDomain.endsWith(domain)) || resultDomain.includes('zhihu')) {
|
||||
if (
|
||||
blacklistDomains.some((domain: string) => resultDomain.endsWith(domain)) ||
|
||||
resultDomain.includes('zhihu')
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
resultUrls.push(result.url);
|
||||
|
@@ -4,7 +4,7 @@ const authMiddleware = (req: Request, res: Response, next: NextFunction) => {
|
||||
const bearerHeader = req.headers['authorization'];
|
||||
|
||||
if (bearerHeader) {
|
||||
console.log("bearerHeader:" + bearerHeader);
|
||||
console.log('bearerHeader:' + bearerHeader);
|
||||
const bearer = bearerHeader.split(' ');
|
||||
const bearerToken = bearer[1];
|
||||
|
||||
|
@@ -3,17 +3,22 @@ import { Page } from 'puppeteer';
|
||||
export const handleSpecialWebsite = async (page: Page, url: string): Promise<string | null> => {
|
||||
if (url.includes('blog.csdn.net')) {
|
||||
await page.waitForSelector('article');
|
||||
const content = await page.$eval('article', el => el.innerHTML);
|
||||
const content = await page.$eval('article', (el) => el.innerHTML);
|
||||
return content;
|
||||
}
|
||||
if (url.includes('zhuanlan.zhihu.com')) {
|
||||
console.log('是知乎,需要点击按掉!');
|
||||
console.log(await page.content());
|
||||
if((await page.content()).includes('{"error":{"message":"您当前请求存在异常,暂时限制本次访问。如有疑问,您可以通过手机摇一摇或登录后私信知乎小管家反馈。","code":40362}}')) return null;
|
||||
if (
|
||||
(await page.content()).includes(
|
||||
'{"error":{"message":"您当前请求存在异常,暂时限制本次访问。如有疑问,您可以通过手机摇一摇或登录后私信知乎小管家反馈。","code":40362}}'
|
||||
)
|
||||
)
|
||||
return null;
|
||||
await page.waitForSelector('button[aria-label="关闭"]');
|
||||
await page.click('button[aria-label="关闭"]'); // 使用 aria-label 选择按钮
|
||||
await page.waitForSelector('article');
|
||||
const content = await page.$eval('article', el => el.innerHTML);
|
||||
const content = await page.$eval('article', (el) => el.innerHTML);
|
||||
return content;
|
||||
}
|
||||
// 可以添加更多特殊网站的处理逻辑
|
||||
|
@@ -1,4 +1,3 @@
|
||||
|
||||
import NodeCache from 'node-cache';
|
||||
import { MongoClient } from 'mongodb';
|
||||
import crypto from 'crypto';
|
||||
@@ -19,10 +18,15 @@ const connectToMongo = async () => {
|
||||
const createTTLIndex = async () => {
|
||||
try {
|
||||
const db = await connectToMongo();
|
||||
await db.collection(collectionName).createIndex({ "updatedAt": 1 }, { expireAfterSeconds: parseInt(process.env.EXPIRE_AFTER_SECONDS || '9000') });
|
||||
console.log("TTL index created successfully");
|
||||
await db
|
||||
.collection(collectionName)
|
||||
.createIndex(
|
||||
{ updatedAt: 1 },
|
||||
{ expireAfterSeconds: parseInt(process.env.EXPIRE_AFTER_SECONDS || '9000') }
|
||||
);
|
||||
console.log('TTL index created successfully');
|
||||
} catch (error) {
|
||||
console.error("Error creating TTL index:", error);
|
||||
console.error('Error creating TTL index:', error);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -53,11 +57,7 @@ const savePageToCache = async (url: string, content: string) => {
|
||||
|
||||
try {
|
||||
const db = await connectToMongo();
|
||||
await db.collection(collectionName).updateOne(
|
||||
{ url },
|
||||
{ $set: page },
|
||||
{ upsert: true }
|
||||
); // 更新持久化缓存
|
||||
await db.collection(collectionName).updateOne({ url }, { $set: page }, { upsert: true }); // 更新持久化缓存
|
||||
} catch (error) {
|
||||
console.error('Error saving page to cache:', error);
|
||||
throw error;
|
||||
|
@@ -13,12 +13,19 @@ interface CachedPage {
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
export const performDeepSearch = async (clusterInstance: Cluster, resultUrls: string[], results: Map<string, any>, strategies: any[], detectWebsites: string[], pageCount: number) => {
|
||||
export const performDeepSearch = async (
|
||||
clusterInstance: Cluster,
|
||||
resultUrls: string[],
|
||||
results: Map<string, any>,
|
||||
strategies: any[],
|
||||
detectWebsites: string[],
|
||||
pageCount: number
|
||||
) => {
|
||||
const tasks = [];
|
||||
|
||||
await clusterInstance.task(async ({ page, data: { searchUrl } }) => {
|
||||
try {
|
||||
const cachedPage = await getCachedPage(searchUrl) as CachedPage | null;
|
||||
const cachedPage = (await getCachedPage(searchUrl)) as CachedPage | null;
|
||||
if (cachedPage) {
|
||||
const result = results.get(searchUrl);
|
||||
if (result) {
|
||||
@@ -29,18 +36,25 @@ export const performDeepSearch = async (clusterInstance: Cluster, resultUrls: st
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`从缓存获取页面 ${searchUrl} 时发生错误:`, error);
|
||||
results.set(searchUrl, { url: searchUrl, error: (error as Error).message, crawlStatus: 'Failed' });
|
||||
results.set(searchUrl, {
|
||||
url: searchUrl,
|
||||
error: (error as Error).message,
|
||||
crawlStatus: 'Failed'
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(searchUrl, {
|
||||
headers: {
|
||||
'User-Agent': new UserAgent({ deviceCategory: 'desktop', platform: 'Linux x86_64' }).toString(),
|
||||
'Referer': 'https://www.google.com/',
|
||||
'User-Agent': new UserAgent({
|
||||
deviceCategory: 'desktop',
|
||||
platform: 'Linux x86_64'
|
||||
}).toString(),
|
||||
Referer: 'https://www.google.com/',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Connection': 'keep-alive',
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
Connection: 'keep-alive',
|
||||
'Cache-Control': 'no-cache'
|
||||
}
|
||||
});
|
||||
@@ -66,7 +80,7 @@ export const performDeepSearch = async (clusterInstance: Cluster, resultUrls: st
|
||||
}
|
||||
|
||||
try {
|
||||
if (detectWebsites.some(website => searchUrl.includes(website))) {
|
||||
if (detectWebsites.some((website) => searchUrl.includes(website))) {
|
||||
await setupPage(page);
|
||||
} else {
|
||||
const userAgent = new UserAgent({ deviceCategory: 'desktop', platform: 'Linux x86_64' });
|
||||
@@ -118,7 +132,11 @@ export const performDeepSearch = async (clusterInstance: Cluster, resultUrls: st
|
||||
|
||||
await updateCacheAsync(searchUrl, cleanedContent || '');
|
||||
} catch (error) {
|
||||
results.set(searchUrl, { url: searchUrl, error: (error as Error).message, crawlStatus: 'Failed' });
|
||||
results.set(searchUrl, {
|
||||
url: searchUrl,
|
||||
error: (error as Error).message,
|
||||
crawlStatus: 'Failed'
|
||||
});
|
||||
} finally {
|
||||
await page.close().catch(() => {});
|
||||
}
|
||||
|
@@ -8,7 +8,7 @@ const getRandomUserAgent = () => {
|
||||
};
|
||||
|
||||
const getRandomPlatform = () => {
|
||||
const platforms = ["Win32", "MacIntel", "Linux x86_64"];
|
||||
const platforms = ['Win32', 'MacIntel', 'Linux x86_64'];
|
||||
return platforms[Math.floor(Math.random() * platforms.length)];
|
||||
};
|
||||
|
||||
@@ -16,14 +16,16 @@ const getRandomPlatform = () => {
|
||||
const validateproxy = process.env.VALIDATE_PROXY ? JSON.parse(process.env.VALIDATE_PROXY) : [];
|
||||
|
||||
const getRandomProxy = () => {
|
||||
return validateproxy.length > 0 ? validateproxy[Math.floor(Math.random() * validateproxy.length)] : null;
|
||||
return validateproxy.length > 0
|
||||
? validateproxy[Math.floor(Math.random() * validateproxy.length)]
|
||||
: null;
|
||||
};
|
||||
|
||||
const getRandomLanguages = () => {
|
||||
const languages = [
|
||||
["zh-CN", "zh", "en"],
|
||||
["en-US", "en", "fr"],
|
||||
["es-ES", "es", "en"]
|
||||
['zh-CN', 'zh', 'en'],
|
||||
['en-US', 'en', 'fr'],
|
||||
['es-ES', 'es', 'en']
|
||||
];
|
||||
return languages[Math.floor(Math.random() * languages.length)];
|
||||
};
|
||||
@@ -42,30 +44,38 @@ export const setupPage = async (page: Page): Promise<void> => {
|
||||
delete newProto.webdriver;
|
||||
(navigator as any).__proto__ = newProto;
|
||||
(window as any).chrome = {};
|
||||
(window as any).chrome.app = {"InstallState":"testt", "RunningState":"estt", "getDetails":"stte", "getIsInstalled":"ttes"};
|
||||
(window as any).chrome.csi = function(){};
|
||||
(window as any).chrome.loadTimes = function(){};
|
||||
(window as any).chrome.runtime = function(){};
|
||||
(window as any).chrome.app = {
|
||||
InstallState: 'testt',
|
||||
RunningState: 'estt',
|
||||
getDetails: 'stte',
|
||||
getIsInstalled: 'ttes'
|
||||
};
|
||||
(window as any).chrome.csi = function () {};
|
||||
(window as any).chrome.loadTimes = function () {};
|
||||
(window as any).chrome.runtime = function () {};
|
||||
Object.defineProperty(navigator, 'userAgent', {
|
||||
get: () => getRandomUserAgent(),
|
||||
get: () => getRandomUserAgent()
|
||||
});
|
||||
Object.defineProperty(navigator, 'platform', {
|
||||
get: () => getRandomPlatform(),
|
||||
get: () => getRandomPlatform()
|
||||
});
|
||||
Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => [{"description": "Shockwave Flash",
|
||||
"filename": "pepflashplayer.dll",
|
||||
"length": 1,
|
||||
"name": "Shockwave Flash"}]
|
||||
get: () => [
|
||||
{
|
||||
description: 'Shockwave Flash',
|
||||
filename: 'pepflashplayer.dll',
|
||||
length: 1,
|
||||
name: 'Shockwave Flash'
|
||||
}
|
||||
]
|
||||
});
|
||||
Object.defineProperty(navigator, 'languages', {
|
||||
get: () => getRandomLanguages(),
|
||||
get: () => getRandomLanguages()
|
||||
});
|
||||
const originalQuery = (window.navigator.permissions as any).query;
|
||||
(window.navigator.permissions as any).query = (parameters: any) => (
|
||||
parameters.name === 'notifications' ?
|
||||
Promise.resolve({ state: Notification.permission } as PermissionStatus) :
|
||||
originalQuery(parameters)
|
||||
);
|
||||
(window.navigator.permissions as any).query = (parameters: any) =>
|
||||
parameters.name === 'notifications'
|
||||
? Promise.resolve({ state: Notification.permission } as PermissionStatus)
|
||||
: originalQuery(parameters);
|
||||
});
|
||||
};
|
@@ -63,7 +63,7 @@ async function initHttp(teamId?: string): Promise<any> {
|
||||
}
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
/* 批量创建子插件 */
|
||||
@@ -88,7 +88,7 @@ async function initHttp(teamId?: string): Promise<any> {
|
||||
}
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
if (item.version === 'v2') {
|
||||
await MongoAppVersion.create(
|
||||
@@ -100,7 +100,7 @@ async function initHttp(teamId?: string): Promise<any> {
|
||||
edges: item.edges
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -160,7 +160,7 @@ async function initPlugin(teamId?: string): Promise<any> {
|
||||
}
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
if (plugin.version === 'v2') {
|
||||
@@ -173,7 +173,7 @@ async function initPlugin(teamId?: string): Promise<any> {
|
||||
edges: plugin.edges
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
}
|
||||
|
||||
|
@@ -98,7 +98,8 @@ async function handler(
|
||||
}
|
||||
],
|
||||
{
|
||||
session
|
||||
session,
|
||||
ordered: true
|
||||
}
|
||||
);
|
||||
}
|
||||
|
@@ -126,7 +126,7 @@ export const onCreateApp = async ({
|
||||
'pluginData.nodeVersion': defaultNodeVersion
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
if (!AppFolderTypeList.includes(type!)) {
|
||||
@@ -144,7 +144,7 @@ export const onCreateApp = async ({
|
||||
isPublish: true
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
}
|
||||
|
||||
|
@@ -89,7 +89,8 @@ async function handler(req: ApiRequestProps<CreateAppFolderBody>) {
|
||||
}
|
||||
],
|
||||
{
|
||||
session
|
||||
session,
|
||||
ordered: true
|
||||
}
|
||||
);
|
||||
}
|
||||
|
@@ -45,7 +45,7 @@ async function handler(req: ApiRequestProps<PostPublishAppProps>, res: NextApiRe
|
||||
tmbId
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
// update app
|
||||
|
@@ -88,7 +88,7 @@ async function handler(
|
||||
yuqueServer
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
await refreshSourceAvatar(avatar, undefined, session);
|
||||
|
||||
|
@@ -87,7 +87,7 @@ async function handler(
|
||||
permission: OwnerPermissionVal
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
}
|
||||
});
|
||||
|
@@ -122,7 +122,8 @@ async function handler(req: ApiRequestProps<rebuildEmbeddingBody>): Promise<Resp
|
||||
}
|
||||
],
|
||||
{
|
||||
session
|
||||
session,
|
||||
ordered: true
|
||||
}
|
||||
);
|
||||
}
|
||||
|
@@ -98,7 +98,7 @@ export async function insertData2Dataset({
|
||||
}))
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
// 3. Create mongo data text
|
||||
@@ -112,7 +112,7 @@ export async function insertData2Dataset({
|
||||
fullTextToken: jiebaSplit({ text: qaStr })
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
return {
|
||||
|
@@ -192,7 +192,7 @@ const rebuildData = async ({
|
||||
retryCount: 50
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
}
|
||||
});
|
||||
|
@@ -37,7 +37,7 @@ export async function initRootUser(retry = 3): Promise<any> {
|
||||
password: hashStr(psw)
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
rootId = _id;
|
||||
}
|
||||
|
Reference in New Issue
Block a user