diff --git a/docSite/content/zh-cn/docs/development/openapi/dataset.md b/docSite/content/zh-cn/docs/development/openapi/dataset.md index d7d143034..8f059053b 100644 --- a/docSite/content/zh-cn/docs/development/openapi/dataset.md +++ b/docSite/content/zh-cn/docs/development/openapi/dataset.md @@ -1093,6 +1093,22 @@ A2: {{< tab tabName="请求示例" >}} {{< markdownify >}} +**4.8.11+** + +```bash +curl --location --request POST 'http://localhost:3000/api/core/dataset/data/v2/list' \ +--header 'Authorization: Bearer {{authorization}}' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "offset": 0, + "pageSize": 10, + "collectionId":"65abd4ac9d1448617cba6171", + "searchText":"" +}' +``` + +**4.6.7+** + ```bash curl --location --request POST 'http://localhost:3000/api/core/dataset/data/list' \ --header 'Authorization: Bearer {{authorization}}' \ @@ -1112,10 +1128,13 @@ curl --location --request POST 'http://localhost:3000/api/core/dataset/data/list {{< markdownify >}} {{% alert icon=" " context="success" %}} + +- pageNum: 偏移量(选填) - pageNum: 页码(选填) - pageSize: 每页数量,最大30(选填) - collectionId: 集合的ID(必填) - searchText: 模糊搜索词(选填) + {{% /alert %}} {{< /markdownify >}} diff --git a/docSite/content/zh-cn/docs/development/upgrading/4812.md b/docSite/content/zh-cn/docs/development/upgrading/4812.md index 2d3d7523a..1235ea6e4 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4812.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4812.md @@ -10,4 +10,5 @@ weight: 812 ## 更新说明 1. 新增 - 全局变量支持更多数据类型 -2. 修复 - 文件后缀判断,去除 query 影响。 \ No newline at end of file +2. 新增 - FE_DOMAIN 环境变量,配置该环境变量后,上传文件/图片会补全后缀后得到完整地址。(可解决 docx 文件图片链接,有时会无法被模型识别问题) +3. 修复 - 文件后缀判断,去除 query 影响。 \ No newline at end of file diff --git a/files/docker/docker-compose-milvus.yml b/files/docker/docker-compose-milvus.yml index 7828316e0..9c466d24f 100644 --- a/files/docker/docker-compose-milvus.yml +++ b/files/docker/docker-compose-milvus.yml @@ -154,6 +154,8 @@ services: - MILVUS_TOKEN=none # sandbox 地址 - SANDBOX_URL=http://sandbox:3000 + # 前端地址 + - FE_DOMAIN= # 日志等级: debug, info, warn, error - LOG_LEVEL=info - STORE_LOG_LEVEL=warn diff --git a/files/docker/docker-compose-pgvector.yml b/files/docker/docker-compose-pgvector.yml index b09e3490a..fa3ff257c 100644 --- a/files/docker/docker-compose-pgvector.yml +++ b/files/docker/docker-compose-pgvector.yml @@ -111,6 +111,8 @@ services: - PG_URL=postgresql://username:password@pg:5432/postgres # sandbox 地址 - SANDBOX_URL=http://sandbox:3000 + # 前端地址 + - FE_DOMAIN= # 日志等级: debug, info, warn, error - LOG_LEVEL=info - STORE_LOG_LEVEL=warn diff --git a/files/docker/docker-compose-zilliz.yml b/files/docker/docker-compose-zilliz.yml index 6e1e09dc5..e641a0bfb 100644 --- a/files/docker/docker-compose-zilliz.yml +++ b/files/docker/docker-compose-zilliz.yml @@ -92,6 +92,8 @@ services: - MILVUS_TOKEN=zilliz_cloud_token # sandbox 地址 - SANDBOX_URL=http://sandbox:3000 + # 前端地址 + - FE_DOMAIN= # 日志等级: debug, info, warn, error - LOG_LEVEL=info - STORE_LOG_LEVEL=warn diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index 5c37a6165..0c9110882 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -5,10 +5,6 @@ import { ClientSession } from '../../../common/mongo'; import { guessBase64ImageType } from '../utils'; import { readFromSecondary } from '../../mongo/utils'; -export function getMongoImgUrl(id: string, extension: string) { - return `${imageBaseUrl}${id}.${extension}`; -} - export const maxImgSize = 1024 * 1024 * 12; const base64MimeRegex = /data:image\/([^\)]+);base64/; export async function uploadMongoImg({ @@ -39,7 +35,7 @@ export async function uploadMongoImg({ shareId }); - return getMongoImgUrl(String(_id), extension); + return `${process.env.FE_DOMAIN || ''}${imageBaseUrl}${String(_id)}.${extension}`; } export async function readMongoImg({ id }: { id: string }) { diff --git a/projects/app/.env.template b/projects/app/.env.template index 090bad8cb..81cecfa93 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -31,7 +31,7 @@ SANDBOX_URL=http://localhost:3001 # 商业版地址 PRO_URL= # 页面的地址,用于自动补全相对路径资源的 domain -# FE_DOMAIN=http://localhost:3000 +FE_DOMAIN=http://localhost:3000 # 日志等级: debug, info, warn, error LOG_LEVEL=debug diff --git a/projects/app/src/pages/api/core/dataset/data/list.ts b/projects/app/src/pages/api/core/dataset/data/list.ts index df85d107d..53ec061b4 100644 --- a/projects/app/src/pages/api/core/dataset/data/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/list.ts @@ -3,20 +3,19 @@ import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { replaceRegChars } from '@fastgpt/global/common/string/tools'; import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; +import { PagingData, RequestPaging } from '@/types'; import { ApiRequestProps } from '@fastgpt/service/type/next'; import { DatasetDataListItemType } from '@/global/core/dataset/type'; -import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type'; -export type GetDatasetDataListProps = PaginationProps & { +export type GetDatasetDataListProps = RequestPaging & { searchText?: string; collectionId: string; }; -export type GetDatasetDataListRes = PaginationResponse; async function handler( req: ApiRequestProps -): Promise { - let { offset, pageSize = 10, searchText = '', collectionId } = req.body; +): Promise> { + let { pageNum = 1, pageSize = 10, searchText = '', collectionId } = req.body; pageSize = Math.min(pageSize, 30); @@ -41,17 +40,19 @@ async function handler( : {}) }; - const [list, total] = await Promise.all([ + const [data, total] = await Promise.all([ MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex') .sort({ chunkIndex: 1, updateTime: -1 }) - .skip(offset) + .skip((pageNum - 1) * pageSize) .limit(pageSize) .lean(), MongoDatasetData.countDocuments(match) ]); return { - list, + pageNum, + pageSize, + data, total }; } diff --git a/projects/app/src/pages/api/core/dataset/data/v2/list.ts b/projects/app/src/pages/api/core/dataset/data/v2/list.ts new file mode 100644 index 000000000..df85d107d --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/data/v2/list.ts @@ -0,0 +1,59 @@ +import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { replaceRegChars } from '@fastgpt/global/common/string/tools'; +import { NextAPI } from '@/service/middleware/entry'; +import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; +import { ApiRequestProps } from '@fastgpt/service/type/next'; +import { DatasetDataListItemType } from '@/global/core/dataset/type'; +import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type'; + +export type GetDatasetDataListProps = PaginationProps & { + searchText?: string; + collectionId: string; +}; +export type GetDatasetDataListRes = PaginationResponse; + +async function handler( + req: ApiRequestProps +): Promise { + let { offset, pageSize = 10, searchText = '', collectionId } = req.body; + + pageSize = Math.min(pageSize, 30); + + // 凭证校验 + const { teamId, collection } = await authDatasetCollection({ + req, + authToken: true, + authApiKey: true, + collectionId, + per: ReadPermissionVal + }); + + const queryReg = new RegExp(`${replaceRegChars(searchText)}`, 'i'); + const match = { + teamId, + datasetId: collection.datasetId._id, + collectionId, + ...(searchText.trim() + ? { + $or: [{ q: queryReg }, { a: queryReg }] + } + : {}) + }; + + const [list, total] = await Promise.all([ + MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex') + .sort({ chunkIndex: 1, updateTime: -1 }) + .skip(offset) + .limit(pageSize) + .lean(), + MongoDatasetData.countDocuments(match) + ]); + + return { + list, + total + }; +} + +export default NextAPI(handler); diff --git a/projects/app/src/web/core/dataset/api.ts b/projects/app/src/web/core/dataset/api.ts index 8aee08c46..08b192941 100644 --- a/projects/app/src/web/core/dataset/api.ts +++ b/projects/app/src/web/core/dataset/api.ts @@ -51,7 +51,7 @@ import type { UpdateDatasetCollectionParams } from '@/pages/api/core/dataset/col import type { GetDatasetDataListProps, GetDatasetDataListRes -} from '@/pages/api/core/dataset/data/list'; +} from '@/pages/api/core/dataset/data/v2/list'; import type { UpdateDatasetDataProps } from '@fastgpt/global/core/dataset/controller'; import type { DatasetFolderCreateBody } from '@/pages/api/core/dataset/folder/create'; import type { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type'; @@ -159,7 +159,7 @@ export const getScrollCollectionList = (data: GetScrollCollectionsProps) => /* =============================== data ==================================== */ /* get dataset list */ export const getDatasetDataList = (data: GetDatasetDataListProps) => - POST(`/core/dataset/data/list`, data); + POST(`/core/dataset/data/v2/list`, data); export const getDatasetDataItemById = (id: string) => GET(`/core/dataset/data/detail`, { id });