fix: delete dataset field error (#3925)

* fix: collection list count

* fix: collection list count

* update doc

* perf: tts selector ui

* fix: delete dataset field error

* doc

packages/service/core/ai/utils.ts

@@ -95,11 +95,145 @@ export const llmCompletionsBodyFormat = <T extends CompletionsBodyType>(
   return requestBody as unknown as InferCompletionsBody<T>;
 };
 
-export const llmStreamResponseToText = async (response: StreamChatType) => {
+export const llmStreamResponseToAnswerText = async (response: StreamChatType) => {
   let answer = '';
   for await (const part of response) {
     const content = part.choices?.[0]?.delta?.content || '';
     answer += content;
   }
-  return answer;
+  return parseReasoningContent(answer)[1];
+};
+
+// Parse <think></think> tags to think and answer - unstream response
+export const parseReasoningContent = (text: string): [string, string] => {
+  const regex = /<think>([\s\S]*?)<\/think>/;
+  const match = text.match(regex);
+
+  if (!match) {
+    return ['', text];
+  }
+
+  const thinkContent = match[1].trim();
+
+  // Add answer (remaining text after think tag)
+  const answerContent = text.slice(match.index! + match[0].length);
+
+  return [thinkContent, answerContent];
+};
+
+// Parse <think></think> tags to think and answer - stream response
+export const parseReasoningStreamContent = () => {
+  let isInThinkTag: boolean | undefined;
+
+  const startTag = '<think>';
+  let startTagBuffer = '';
+
+  const endTag = '</think>';
+  let endTagBuffer = '';
+
+  /* 
+    parseReasoning - 只控制是否主动解析 <think></think>，如果接口已经解析了，仍然会返回 think 内容。
+  */
+  const parsePart = (
+    part: {
+      choices: {
+        delta: {
+          content?: string;
+          reasoning_content?: string;
+        };
+      }[];
+    },
+    parseReasoning = false
+  ): [string, string] => {
+    const content = part.choices?.[0]?.delta?.content || '';
+
+    // @ts-ignore
+    const reasoningContent = part.choices?.[0]?.delta?.reasoning_content || '';
+    if (reasoningContent || !parseReasoning) {
+      isInThinkTag = false;
+      return [reasoningContent, content];
+    }
+
+    if (!content) {
+      return ['', ''];
+    }
+
+    // 如果不在 think 标签中，或者有 reasoningContent(接口已解析），则返回 reasoningContent 和 content
+    if (isInThinkTag === false) {
+      return ['', content];
+    }
+
+    // 检测是否为 think 标签开头的数据
+    if (isInThinkTag === undefined) {
+      // Parse content think and answer
+      startTagBuffer += content;
+      // 太少内容时候，暂时不解析
+      if (startTagBuffer.length < startTag.length) {
+        return ['', ''];
+      }
+
+      if (startTagBuffer.startsWith(startTag)) {
+        isInThinkTag = true;
+        return [startTagBuffer.slice(startTag.length), ''];
+      }
+
+      // 如果未命中 think 标签，则认为不在 think 标签中，返回 buffer 内容作为 content
+      isInThinkTag = false;
+      return ['', startTagBuffer];
+    }
+
+    // 确认是 think 标签内容，开始返回 think 内容，并实时检测 </think>
+    /* 
+      检测 </think> 方案。
+      存储所有疑似 </think> 的内容，直到检测到完整的 </think> 标签或超出 </think> 长度。
+      content 返回值包含以下几种情况:
+        abc - 完全未命中尾标签
+        abc<th - 命中一部分尾标签
+        abc</think> - 完全命中尾标签
+        abc</think>abc - 完全命中尾标签
+        </think>abc - 完全命中尾标签
+        k>abc - 命中一部分尾标签
+    */
+    // endTagBuffer 专门用来记录疑似尾标签的内容
+    if (endTagBuffer) {
+      endTagBuffer += content;
+      if (endTagBuffer.includes(endTag)) {
+        isInThinkTag = false;
+        const answer = endTagBuffer.slice(endTag.length);
+        return ['', answer];
+      } else if (endTagBuffer.length >= endTag.length) {
+        // 缓存内容超出尾标签长度，且仍未命中 </think>，则认为本次猜测 </think> 失败，仍处于 think 阶段。
+        const tmp = endTagBuffer;
+        endTagBuffer = '';
+        return [tmp, ''];
+      }
+      return ['', ''];
+    } else if (content.includes(endTag)) {
+      // 返回内容，完整命中</think>，直接结束
+      isInThinkTag = false;
+      const [think, answer] = content.split(endTag);
+      return [think, answer];
+    } else {
+      // 无 buffer，且未命中 </think>，开始疑似 </think> 检测。
+      for (let i = 1; i < endTag.length; i++) {
+        const partialEndTag = endTag.slice(0, i);
+        // 命中一部分尾标签
+        if (content.endsWith(partialEndTag)) {
+          const think = content.slice(0, -partialEndTag.length);
+          endTagBuffer += partialEndTag;
+          return [think, ''];
+        }
+      }
+    }
+
+    // 完全未命中尾标签，还是 think 阶段。
+    return [content, ''];
+  };
+
+  const getStartTagBuffer = () => startTagBuffer;
+
+  return {
+    parsePart,
+    getStartTagBuffer
+  };
 };

packages/service/core/dataset/collection/controller.ts

@@ -277,7 +277,7 @@ export async function delCollection({
   // Delete training data
   await MongoDatasetTraining.deleteMany({
     teamId,
-    datasetIds: { $in: datasetIds },
+    datasetId: { $in: datasetIds },
     collectionId: { $in: collectionIds }
   });
 
@@ -288,12 +288,12 @@ export async function delCollection({
 
   // Delete dataset_datas
   await MongoDatasetData.deleteMany(
-    { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
+    { teamId, datasetId: { $in: datasetIds }, collectionId: { $in: collectionIds } },
     { session }
   );
   // Delete dataset_data_texts
   await MongoDatasetDataText.deleteMany(
-    { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
+    { teamId, datasetId: { $in: datasetIds }, collectionId: { $in: collectionIds } },
     { session }
   );
 
@@ -332,13 +332,13 @@ export async function delOnlyCollection({
   // delete training data
   await MongoDatasetTraining.deleteMany({
     teamId,
-    datasetIds: { $in: datasetIds },
+    datasetId: { $in: datasetIds },
     collectionId: { $in: collectionIds }
   });
 
   // delete dataset.datas
   await MongoDatasetData.deleteMany(
-    { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
+    { teamId, datasetId: { $in: datasetIds }, collectionId: { $in: collectionIds } },
     { session }
   );
 

packages/service/core/dataset/controller.ts

@@ -7,6 +7,7 @@ import { MongoDatasetTraining } from './training/schema';
 import { MongoDatasetData } from './data/schema';
 import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
 import { MongoDatasetDataText } from './data/dataTextSchema';
+import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 
 /* ============= dataset ========== */
 /* find all datasetId by top datasetId */
@@ -54,7 +55,7 @@ export async function getCollectionWithDataset(collectionId: string) {
     .populate<{ dataset: DatasetSchemaType }>('dataset')
     .lean();
   if (!data) {
-    return Promise.reject('Collection is not exist');
+    return Promise.reject(DatasetErrEnum.unExistCollection);
   }
   return data;
 }

packages/service/core/workflow/dispatch/chat/oneapi.ts

@@ -3,11 +3,8 @@ import { filterGPTMessageByMaxContext, loadRequestMessages } from '../../../chat
 import type { ChatItemType, UserChatItemValueItemType } from '@fastgpt/global/core/chat/type.d';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
-import {
-  parseReasoningContent,
-  parseReasoningStreamContent,
-  textAdaptGptResponse
-} from '@fastgpt/global/core/workflow/runtime/utils';
+import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils';
+import { parseReasoningContent, parseReasoningStreamContent } from '../../../ai/utils';
 import { createChatCompletion } from '../../../ai/config';
 import type { ChatCompletionMessageParam, StreamChatType } from '@fastgpt/global/core/ai/type.d';
 import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';

packages/service/test/utils.ts

@@ -0,0 +1,94 @@
+import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
+
+export type TestTokenType = {
+  userId: string;
+  teamId: string;
+  tmbId: string;
+  isRoot: boolean;
+};
+
+export type TestRequest = {
+  headers: {
+    cookie?: {
+      token?: TestTokenType;
+    };
+    authorization?: string; // testkey
+    rootkey?: string; // rootkey
+  };
+  query: {
+    [key: string]: string;
+  };
+  body: {
+    [key: string]: string;
+  };
+};
+
+export function getTestRequest<Q = any, B = any>({
+  query = {},
+  body = {},
+  authToken = true,
+  // authRoot = false,
+  // authApiKey = false,
+  user
+}: {
+  query?: Partial<Q>;
+  body?: Partial<B>;
+  authToken?: boolean;
+  authRoot?: boolean;
+  authApiKey?: boolean;
+  user?: {
+    uid: string;
+    tmbId: string;
+    teamId: string;
+    isRoot: boolean;
+  };
+}): [any, any] {
+  const headers: TestRequest['headers'] = {};
+  if (authToken) {
+    headers.cookie = {
+      token: {
+        userId: String(user?.uid || ''),
+        teamId: String(user?.teamId || ''),
+        tmbId: String(user?.tmbId || ''),
+        isRoot: user?.isRoot || false
+      }
+    };
+  }
+  return [
+    {
+      headers,
+      query,
+      body
+    },
+    {}
+  ];
+}
+
+export const parseHeaderCertMock = async ({
+  req,
+  authToken = true,
+  authRoot = false,
+  authApiKey = false
+}: {
+  req: TestRequest;
+  authToken?: boolean;
+  authRoot?: boolean;
+  authApiKey?: boolean;
+}): Promise<TestTokenType> => {
+  if (authToken) {
+    const token = req.headers?.cookie?.token;
+    if (!token) {
+      return Promise.reject(ERROR_ENUM.unAuthorization);
+    }
+    return token;
+  }
+  // if (authRoot) {
+  //   // TODO: unfinished
+  //   return req.headers.rootkey;
+  // }
+  // if (authApiKey) {
+  //   // TODO: unfinished
+  //   return req.headers.authorization;
+  // }
+  return {} as any;
+};