From 6c37776de1149173bc871f4ba73163bdb3a30992 Mon Sep 17 00:00:00 2001 From: dreamer6680 <1468683855@qq.com> Date: Wed, 30 Jul 2025 15:38:30 +0800 Subject: [PATCH] fix: resole crawl cannot get docs (#5344) --- document/app/[lang]/robots.txt/route.ts | 71 ++-- document/app/api/meta/route.ts | 86 ++++ document/components/docs/not-found.tsx | 54 ++- document/content/docs/api/index.mdx | 4 +- document/content/docs/index.mdx | 8 - .../content/docs/introduction/FAQ/index.mdx | 8 - .../development/custom-models/index.mdx | 8 - .../introduction/development/design/index.mdx | 8 - .../docs/introduction/development/index.mdx | 8 - .../development/migration/index.mdx | 8 - .../development/modelConfig/index.mdx | 8 - .../development/openapi/index.mdx | 8 - .../introduction/development/proxy/index.mdx | 8 - .../development/upgrading/index.mdx | 8 - .../introduction/guide/DialogBoxes/index.mdx | 8 - .../docs/introduction/guide/admin/index.mdx | 8 - .../docs/introduction/guide/course/index.mdx | 8 - .../introduction/guide/dashboard/index.mdx | 8 - .../guide/dashboard/workflow/index.mdx | 8 - .../content/docs/introduction/guide/index.mdx | 8 - .../guide/knowledge_base/index.mdx | 8 - .../content/docs/introduction/guide/meta.json | 12 +- .../docs/introduction/guide/plugins/index.mdx | 8 - .../guide/team_permissions/index.mdx | 8 - .../docs/introduction/shopping_cart/index.mdx | 8 - document/content/docs/protocol/index.mdx | 3 +- .../docs/use-cases/app-cases/index.mdx | 8 - .../use-cases/external-integration/index.mdx | 8 - document/data/doc-last-modified.json | 373 +++++++++--------- 29 files changed, 358 insertions(+), 421 deletions(-) create mode 100644 document/app/api/meta/route.ts delete mode 100644 document/content/docs/index.mdx delete mode 100644 document/content/docs/introduction/FAQ/index.mdx delete mode 100644 document/content/docs/introduction/development/custom-models/index.mdx delete mode 100644 document/content/docs/introduction/development/design/index.mdx delete mode 100644 document/content/docs/introduction/development/index.mdx delete mode 100644 document/content/docs/introduction/development/migration/index.mdx delete mode 100644 document/content/docs/introduction/development/modelConfig/index.mdx delete mode 100644 document/content/docs/introduction/development/openapi/index.mdx delete mode 100644 document/content/docs/introduction/development/proxy/index.mdx delete mode 100644 document/content/docs/introduction/development/upgrading/index.mdx delete mode 100644 document/content/docs/introduction/guide/DialogBoxes/index.mdx delete mode 100644 document/content/docs/introduction/guide/admin/index.mdx delete mode 100644 document/content/docs/introduction/guide/course/index.mdx delete mode 100644 document/content/docs/introduction/guide/dashboard/index.mdx delete mode 100644 document/content/docs/introduction/guide/dashboard/workflow/index.mdx delete mode 100644 document/content/docs/introduction/guide/index.mdx delete mode 100644 document/content/docs/introduction/guide/knowledge_base/index.mdx delete mode 100644 document/content/docs/introduction/guide/plugins/index.mdx delete mode 100644 document/content/docs/introduction/guide/team_permissions/index.mdx delete mode 100644 document/content/docs/introduction/shopping_cart/index.mdx delete mode 100644 document/content/docs/use-cases/app-cases/index.mdx delete mode 100644 document/content/docs/use-cases/external-integration/index.mdx diff --git a/document/app/[lang]/robots.txt/route.ts b/document/app/[lang]/robots.txt/route.ts index a6d208315..ee05b476a 100644 --- a/document/app/[lang]/robots.txt/route.ts +++ b/document/app/[lang]/robots.txt/route.ts @@ -1,71 +1,64 @@ import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; import fg from 'fast-glob'; import matter from 'gray-matter'; import { i18n } from '@/lib/i18n'; export const revalidate = false; -// 将文件路径转换为URL路径 +// 黑名单路径(不带语言前缀) +const blacklist = ['use-cases/index', 'protocol/index', 'api/index']; + +// 将文件路径转换为 URL 路径(包括文件名) function filePathToUrl(filePath: string, defaultLanguage: string): string { - // 移除 ./content/docs/ 前缀 - let urlPath = filePath.replace('./content/docs/', ''); - - // 确定基础路径 + let relativePath = filePath.replace('./content/docs/', ''); + const basePath = defaultLanguage === 'zh-CN' ? '/docs' : '/en/docs'; - - // 如果是英文文件,移除 .en 后缀 - if (defaultLanguage !== 'zh-CN' && urlPath.endsWith('.en.mdx')) { - urlPath = urlPath.replace('.en.mdx', ''); - } else if (urlPath.endsWith('.mdx')) { - urlPath = urlPath.replace('.mdx', ''); + + if (defaultLanguage !== 'zh-CN' && relativePath.endsWith('.en.mdx')) { + relativePath = relativePath.replace(/\.en\.mdx$/, ''); + } else if (relativePath.endsWith('.mdx')) { + relativePath = relativePath.replace(/\.mdx$/, ''); } - - // 处理 index 文件 - if (urlPath.endsWith('/index')) { - urlPath = urlPath.replace('/index', ''); - } - - // 拼接完整路径 - return `${basePath}/${urlPath}`.replace(/\/\/+/g, '/'); + + return `${basePath}/${relativePath}`.replace(/\/\/+/g, '/'); +} + +// 判断是否为黑名单路径 +function isBlacklisted(url: string): boolean { + return blacklist.some( + (item) => url.endsWith(`/docs/${item}`) || url.endsWith(`/en/docs/${item}`) + ); } export async function GET(request: Request) { const defaultLanguage = i18n.defaultLanguage; - - // 检查请求路径是否为 /en/robots + const requestUrl = new URL(request.url); const isEnRobotsRoute = requestUrl.pathname === '/en/robots'; let globPattern; - if (isEnRobotsRoute) { - // 如果是 /en/robots 路由,只选择 .en.mdx 文件 globPattern = ['./content/docs/**/*.en.mdx']; } else if (defaultLanguage === 'zh-CN') { - // 中文环境下的普通路由 globPattern = ['./content/docs/**/*.mdx']; } else { - // 英文环境下的普通路由 globPattern = ['./content/docs/**/*.en.mdx']; } - const files = await fg(globPattern); + const files = await fg(globPattern, { caseSensitiveMatch: true }); - const urls = await Promise.all( - files.map(async (file: string) => { - const urlPath = filePathToUrl(file, defaultLanguage); - return `${urlPath}`; - }) - ); + // 转换文件路径为 URL,并过滤黑名单 + const urls = files + .map((file) => filePathToUrl(file, defaultLanguage)) + .filter((url) => !isBlacklisted(url)); - // 按URL排序 urls.sort((a, b) => a.localeCompare(b)); - // 生成HTML链接列表 const html = ` - FastGPT Documentation Links + FastGPT 文档目录