diff --git a/document/app/[lang]/robots.txt/route.ts b/document/app/[lang]/robots.txt/route.ts
index a6d208315..ee05b476a 100644
--- a/document/app/[lang]/robots.txt/route.ts
+++ b/document/app/[lang]/robots.txt/route.ts
@@ -1,71 +1,64 @@
import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
import fg from 'fast-glob';
import matter from 'gray-matter';
import { i18n } from '@/lib/i18n';
export const revalidate = false;
-// 将文件路径转换为URL路径
+// 黑名单路径（不带语言前缀）
+const blacklist = ['use-cases/index', 'protocol/index', 'api/index'];
+
+// 将文件路径转换为 URL 路径（包括文件名）
function filePathToUrl(filePath: string, defaultLanguage: string): string {
- // 移除 ./content/docs/ 前缀
- let urlPath = filePath.replace('./content/docs/', '');
-
- // 确定基础路径
+ let relativePath = filePath.replace('./content/docs/', '');
+
const basePath = defaultLanguage === 'zh-CN' ? '/docs' : '/en/docs';
-
- // 如果是英文文件，移除 .en 后缀
- if (defaultLanguage !== 'zh-CN' && urlPath.endsWith('.en.mdx')) {
- urlPath = urlPath.replace('.en.mdx', '');
- } else if (urlPath.endsWith('.mdx')) {
- urlPath = urlPath.replace('.mdx', '');
+
+ if (defaultLanguage !== 'zh-CN' && relativePath.endsWith('.en.mdx')) {
+ relativePath = relativePath.replace(/\.en\.mdx$/, '');
+ } else if (relativePath.endsWith('.mdx')) {
+ relativePath = relativePath.replace(/\.mdx$/, '');
}
-
- // 处理 index 文件
- if (urlPath.endsWith('/index')) {
- urlPath = urlPath.replace('/index', '');
- }
-
- // 拼接完整路径
- return `${basePath}/${urlPath}`.replace(/\/\/+/g, '/');
+
+ return `${basePath}/${relativePath}`.replace(/\/\/+/g, '/');
+}
+
+// 判断是否为黑名单路径
+function isBlacklisted(url: string): boolean {
+ return blacklist.some(
+ (item) => url.endsWith(`/docs/${item}`) || url.endsWith(`/en/docs/${item}`)
+ );
}
export async function GET(request: Request) {
const defaultLanguage = i18n.defaultLanguage;
-
- // 检查请求路径是否为 /en/robots
+
const requestUrl = new URL(request.url);
const isEnRobotsRoute = requestUrl.pathname === '/en/robots';
let globPattern;
-
if (isEnRobotsRoute) {
- // 如果是 /en/robots 路由，只选择 .en.mdx 文件
globPattern = ['./content/docs/**/*.en.mdx'];
} else if (defaultLanguage === 'zh-CN') {
- // 中文环境下的普通路由
globPattern = ['./content/docs/**/*.mdx'];
} else {
- // 英文环境下的普通路由
globPattern = ['./content/docs/**/*.en.mdx'];
}
- const files = await fg(globPattern);
+ const files = await fg(globPattern, { caseSensitiveMatch: true });
- const urls = await Promise.all(
- files.map(async (file: string) => {
- const urlPath = filePathToUrl(file, defaultLanguage);
- return `${urlPath}`;
- })
- );
+ // 转换文件路径为 URL，并过滤黑名单
+ const urls = files
+ .map((file) => filePathToUrl(file, defaultLanguage))
+ .filter((url) => !isBlacklisted(url));
- // 按URL排序
urls.sort((a, b) => a.localeCompare(b));
- // 生成HTML链接列表
const html = `
- FastGPT Documentation Links
+ FastGPT 文档目录