perf: 文本

2025-07-22 12:20:34 +00:00 · 2023-03-26 23:52:37 +08:00
parent 249ed18d15
commit 42e12d7db1
6 changed files with 186 additions and 7 deletions
--- a/package.json
+++ b/package.json
@@ -11,6 +11,7 @@
    "format": "prettier --config \"./.prettierrc.js\" --write \"./src/**/*.{ts,tsx,scss}\""
  },
  "dependencies": {
+    "@alicloud/alinlp20200629": "^2.0.2",
    "@chakra-ui/icons": "^2.0.17",
    "@chakra-ui/react": "^2.5.1",
    "@emotion/react": "^11.10.6",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1,6 +1,7 @@
 lockfileVersion: 5.4

 specifiers:
+  '@alicloud/alinlp20200629': ^2.0.2
  '@chakra-ui/icons': ^2.0.17
  '@chakra-ui/react': ^2.5.1
  '@emotion/react': ^11.10.6
@@ -57,6 +58,7 @@ specifiers:
  zustand: ^4.3.5

 dependencies:
+  '@alicloud/alinlp20200629': registry.npmmirror.com/@alicloud/alinlp20200629/2.0.2
  '@chakra-ui/icons': registry.npmmirror.com/@chakra-ui/icons/2.0.17_react@18.2.0
  '@chakra-ui/react': registry.npmmirror.com/@chakra-ui/react/2.5.1_e6pzu3hsaqmql4fl7jx73ckiym
  '@emotion/react': registry.npmmirror.com/@emotion/react/11.10.6_pmekkgnqduwlme35zpnqhenc34
@@ -116,6 +118,117 @@ devDependencies:

 packages:

+  registry.npmmirror.com/@alicloud/alinlp20200629/2.0.2:
+    resolution: {integrity: sha512-V8XMLBEQlB+6f5JCrSDYW+UM1fK1D+iWC6n8BFhf7hlHKUZBsSjsT3e9VgpdrqPJWhrIy0HIR4k+kMgAHVLVaQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@alicloud/alinlp20200629/-/alinlp20200629-2.0.2.tgz}
+    name: '@alicloud/alinlp20200629'
+    version: 2.0.2
+    dependencies:
+      '@alicloud/endpoint-util': registry.npmmirror.com/@alicloud/endpoint-util/0.0.1
+      '@alicloud/openapi-client': registry.npmmirror.com/@alicloud/openapi-client/0.4.5
+      '@alicloud/openapi-util': registry.npmmirror.com/@alicloud/openapi-util/0.3.1
+      '@alicloud/tea-typescript': registry.npmmirror.com/@alicloud/tea-typescript/1.8.0
+      '@alicloud/tea-util': registry.npmmirror.com/@alicloud/tea-util/1.4.5
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  registry.npmmirror.com/@alicloud/credentials/2.2.6:
+    resolution: {integrity: sha512-jG+msY77dHmAF3x+8VTy7fEgORyXLHmDci8t92HeipBdCHsPptDegA++GEwKgR7f6G4wvafYt+aqMZ1iligdrQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@alicloud/credentials/-/credentials-2.2.6.tgz}
+    name: '@alicloud/credentials'
+    version: 2.2.6
+    dependencies:
+      '@alicloud/tea-typescript': registry.npmmirror.com/@alicloud/tea-typescript/1.8.0
+      httpx: registry.npmmirror.com/httpx/2.2.7
+      ini: registry.npmmirror.com/ini/1.3.8
+      kitx: registry.npmmirror.com/kitx/2.1.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  registry.npmmirror.com/@alicloud/endpoint-util/0.0.1:
+    resolution: {integrity: sha512-+pH7/KEXup84cHzIL6UJAaPqETvln4yXlD9JzlrqioyCSaWxbug5FUobsiI6fuUOpw5WwoB3fWAtGbFnJ1K3Yg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@alicloud/endpoint-util/-/endpoint-util-0.0.1.tgz}
+    name: '@alicloud/endpoint-util'
+    version: 0.0.1
+    dependencies:
+      '@alicloud/tea-typescript': registry.npmmirror.com/@alicloud/tea-typescript/1.8.0
+      kitx: registry.npmmirror.com/kitx/2.1.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  registry.npmmirror.com/@alicloud/gateway-spi/0.0.8:
+    resolution: {integrity: sha512-KM7fu5asjxZPmrz9sJGHJeSU+cNQNOxW+SFmgmAIrITui5hXL2LB+KNRuzWmlwPjnuA2X3/keq9h6++S9jcV5g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@alicloud/gateway-spi/-/gateway-spi-0.0.8.tgz}
+    name: '@alicloud/gateway-spi'
+    version: 0.0.8
+    dependencies:
+      '@alicloud/credentials': registry.npmmirror.com/@alicloud/credentials/2.2.6
+      '@alicloud/tea-typescript': registry.npmmirror.com/@alicloud/tea-typescript/1.8.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  registry.npmmirror.com/@alicloud/openapi-client/0.4.5:
+    resolution: {integrity: sha512-x1blwhfPOVkH/JCLWFssFRWDL0C75RToun9AwhNV+84gqJB2/GUipm3quHGLon8JiQ0DQ9YBUho2rukSoAvhJQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@alicloud/openapi-client/-/openapi-client-0.4.5.tgz}
+    name: '@alicloud/openapi-client'
+    version: 0.4.5
+    dependencies:
+      '@alicloud/credentials': registry.npmmirror.com/@alicloud/credentials/2.2.6
+      '@alicloud/gateway-spi': registry.npmmirror.com/@alicloud/gateway-spi/0.0.8
+      '@alicloud/openapi-util': registry.npmmirror.com/@alicloud/openapi-util/0.3.1
+      '@alicloud/tea-typescript': registry.npmmirror.com/@alicloud/tea-typescript/1.8.0
+      '@alicloud/tea-util': registry.npmmirror.com/@alicloud/tea-util/1.4.5
+      '@alicloud/tea-xml': registry.npmmirror.com/@alicloud/tea-xml/0.0.2
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  registry.npmmirror.com/@alicloud/openapi-util/0.3.1:
+    resolution: {integrity: sha512-6mGT+hs+SXismZi/CEkjPhhbn2U3qTT/Qv/RXAYFA1DC3Jk4/YaX3N7RtpgdzOhdD7uI8XtNkaULKHZY3BrtxQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@alicloud/openapi-util/-/openapi-util-0.3.1.tgz}
+    name: '@alicloud/openapi-util'
+    version: 0.3.1
+    dependencies:
+      '@alicloud/tea-typescript': registry.npmmirror.com/@alicloud/tea-typescript/1.8.0
+      '@alicloud/tea-util': registry.npmmirror.com/@alicloud/tea-util/1.4.5
+      kitx: registry.npmmirror.com/kitx/2.1.0
+      sm3: registry.npmmirror.com/sm3/1.0.3
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  registry.npmmirror.com/@alicloud/tea-typescript/1.8.0:
+    resolution: {integrity: sha512-CWXWaquauJf0sW30mgJRVu9aaXyBth5uMBCUc+5vKTK1zlgf3hIqRUjJZbjlwHwQ5y9anwcu18r48nOZb7l2QQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@alicloud/tea-typescript/-/tea-typescript-1.8.0.tgz}
+    name: '@alicloud/tea-typescript'
+    version: 1.8.0
+    dependencies:
+      '@types/node': registry.npmmirror.com/@types/node/12.20.55
+      httpx: registry.npmmirror.com/httpx/2.2.7
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  registry.npmmirror.com/@alicloud/tea-util/1.4.5:
+    resolution: {integrity: sha512-7NuThYUi90/ivT/ORKusm0NVKlc1khPTtlzTR77xEqSBt7d24Ee/Lo70hx9PWP28nHpIZ1gM0NKYBtpq7HUDlg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@alicloud/tea-util/-/tea-util-1.4.5.tgz}
+    name: '@alicloud/tea-util'
+    version: 1.4.5
+    dependencies:
+      '@alicloud/tea-typescript': registry.npmmirror.com/@alicloud/tea-typescript/1.8.0
+      kitx: registry.npmmirror.com/kitx/2.1.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  registry.npmmirror.com/@alicloud/tea-xml/0.0.2:
+    resolution: {integrity: sha512-Xs7v5y7YSNSDDYmiDWAC0/013VWPjS3dQU4KezSLva9VGiTVPaL3S7Nk4NrTmAYCG6MKcrRj/nGEDIWL5KRoPg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@alicloud/tea-xml/-/tea-xml-0.0.2.tgz}
+    name: '@alicloud/tea-xml'
+    version: 0.0.2
+    dependencies:
+      '@alicloud/tea-typescript': registry.npmmirror.com/@alicloud/tea-typescript/1.8.0
+      '@types/xml2js': registry.npmmirror.com/@types/xml2js/0.4.11
+      xml2js: registry.npmmirror.com/xml2js/0.4.23
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
  registry.npmmirror.com/@ampproject/remapping/2.2.0:
    resolution: {integrity: sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@ampproject/remapping/-/remapping-2.2.0.tgz}
    name: '@ampproject/remapping'
@@ -4815,6 +4928,18 @@ packages:
    version: 0.7.31
    dev: false

+  registry.npmmirror.com/@types/node/12.20.55:
+    resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node/-/node-12.20.55.tgz}
+    name: '@types/node'
+    version: 12.20.55
+    dev: false
+
+  registry.npmmirror.com/@types/node/14.18.41:
+    resolution: {integrity: sha512-2cfHr8AsUjKx6u4Q+d2eqK51z8+HueoumCQGCKVt95y/yGG4uajOuCANSnE20mbLw94h3tMcddIJ8nYkTu2mFw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node/-/node-14.18.41.tgz}
+    name: '@types/node'
+    version: 14.18.41
+    dev: false
+
  registry.npmmirror.com/@types/node/18.14.0:
    resolution: {integrity: sha512-5EWrvLmglK+imbCJY0+INViFWUHg1AHel1sq4ZVSfdcNqGy9Edv3UB9IIzzg+xPaUcAgZYcfVs2fBcwDeZzU0A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node/-/node-18.14.0.tgz}
    name: '@types/node'
@@ -4909,6 +5034,14 @@ packages:
      '@types/webidl-conversions': registry.npmmirror.com/@types/webidl-conversions/7.0.0
    dev: false

+  registry.npmmirror.com/@types/xml2js/0.4.11:
+    resolution: {integrity: sha512-JdigeAKmCyoJUiQljjr7tQG3if9NkqGUgwEUqBvV0N7LM4HyQk7UXCnusRa1lnvXAEYJ8mw8GtZWioagNztOwA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/xml2js/-/xml2js-0.4.11.tgz}
+    name: '@types/xml2js'
+    version: 0.4.11
+    dependencies:
+      '@types/node': registry.npmmirror.com/@types/node/18.14.0
+    dev: false
+
  registry.npmmirror.com/@typescript-eslint/parser/5.52.0_7kw3g6rralp5ps6mg3uyzz6azm:
    resolution: {integrity: sha512-e2KiLQOZRo4Y0D/b+3y08i3jsekoSkOYStROYmPUnGMEoA0h+k2qOH5H6tcjIc68WDvGwH+PaOrP1XRzLJ6QlA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@typescript-eslint/parser/-/parser-5.52.0.tgz}
    id: registry.npmmirror.com/@typescript-eslint/parser/5.52.0
@@ -7146,6 +7279,17 @@ packages:
      react-is: registry.npmmirror.com/react-is/16.13.1
    dev: false

+  registry.npmmirror.com/httpx/2.2.7:
+    resolution: {integrity: sha512-Wjh2JOAah0pdczfqL8NC5378G7jMt0Zcpn8U+yyxAiejjlagzSTQgJHuVvka2VNPQlKfoGehYRc79WKq9E4gDw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/httpx/-/httpx-2.2.7.tgz}
+    name: httpx
+    version: 2.2.7
+    dependencies:
+      '@types/node': registry.npmmirror.com/@types/node/14.18.41
+      debug: registry.npmmirror.com/debug/4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
  registry.npmmirror.com/human-signals/3.0.1:
    resolution: {integrity: sha512-rQLskxnM/5OCldHo+wNXbpVgDn5A17CUoKX+7Sokwaknlq7CdSnphy0W39GU8dw59XiCXmFXDg4fRuckQRKewQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/human-signals/-/human-signals-3.0.1.tgz}
    name: human-signals
@@ -7723,6 +7867,14 @@ packages:
      commander: registry.npmmirror.com/commander/8.3.0
    dev: false

+  registry.npmmirror.com/kitx/2.1.0:
+    resolution: {integrity: sha512-C/5v9MtIX7aHGOjwn5BmrrbNkJSf7i0R5mRzmh13GSAdRqQ7bYQo/Su2pTYNylFicqKNTVX3HML9k1u8k51+pQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/kitx/-/kitx-2.1.0.tgz}
+    name: kitx
+    version: 2.1.0
+    dependencies:
+      '@types/node': registry.npmmirror.com/@types/node/12.20.55
+    dev: false
+
  registry.npmmirror.com/kleur/4.1.5:
    resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/kleur/-/kleur-4.1.5.tgz}
    name: kleur
@@ -9808,6 +9960,12 @@ packages:
      is-fullwidth-code-point: registry.npmmirror.com/is-fullwidth-code-point/4.0.0
    dev: true

+  registry.npmmirror.com/sm3/1.0.3:
+    resolution: {integrity: sha512-KyFkIfr8QBlFG3uc3NaljaXdYcsbRy1KrSfc4tsQV8jW68jAktGeOcifu530Vx/5LC+PULHT0Rv8LiI8Gw+c1g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sm3/-/sm3-1.0.3.tgz}
+    name: sm3
+    version: 1.0.3
+    dev: false
+
  registry.npmmirror.com/smart-buffer/4.2.0:
    resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/smart-buffer/-/smart-buffer-4.2.0.tgz}
    name: smart-buffer
@@ -10663,6 +10821,16 @@ packages:
    name: wrappy
    version: 1.0.2

+  registry.npmmirror.com/xml2js/0.4.23:
+    resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xml2js/-/xml2js-0.4.23.tgz}
+    name: xml2js
+    version: 0.4.23
+    engines: {node: '>=4.0.0'}
+    dependencies:
+      sax: registry.npmmirror.com/sax/1.1.6
+      xmlbuilder: registry.npmmirror.com/xmlbuilder/11.0.1
+    dev: false
+
  registry.npmmirror.com/xmlbuilder/10.1.1:
    resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz}
    name: xmlbuilder
@@ -10670,6 +10838,13 @@ packages:
    engines: {node: '>=4.0'}
    dev: false

+  registry.npmmirror.com/xmlbuilder/11.0.1:
+    resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz}
+    name: xmlbuilder
+    version: 11.0.1
+    engines: {node: '>=4.0'}
+    dev: false
+
  registry.npmmirror.com/xtend/4.0.2:
    resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz}
    name: xtend
--- a/src/components/ScrollData/index.tsx
+++ b/src/components/ScrollData/index.tsx
@@ -53,7 +53,7 @@ const ScrollData = ({
  }, [elementRef, nextPage]);

  return (
-    <Box {...props} ref={elementRef} overflow={'auto'} position={'relative'}>
+    <Box {...props} ref={elementRef} overflowY={'auto'} position={'relative'}>
      {children}
      <Box
        mt={2}
--- a/src/constants/common.ts
+++ b/src/constants/common.ts
@@ -44,6 +44,9 @@ export const introPage = `
 `;

 export const chatProblem = `
+**模型问题**
+一般情况下，请直接选择 chatGPT 模型，价格低效果好。
+
 **代理出错**
 服务器代理不稳定，可以过一会儿再尝试。  

--- a/src/pages/data/list.tsx
+++ b/src/pages/data/list.tsx
@@ -122,7 +122,7 @@ const DataList = () => {
              训练数据管理
            </Box>
            <Box fontSize={'xs'} color={'blackAlpha.600'}>
-              允许你将任意文本数据拆分成 QA 的形式。你可以使用这些 QA 去微调你的对话模型。
+              允许你将任意文本数据拆分成 QA 形式，或者进行文本摘要总结。
            </Box>
          </Box>
          <Button variant={'outline'} onClick={onOpenCreateDataModal}>
--- a/src/service/events/generateAbstract.ts
+++ b/src/service/events/generateAbstract.ts
@@ -12,7 +12,7 @@ export async function generateAbstract(next = false): Promise<any> {

  const systemPrompt: ChatCompletionRequestMessage = {
    role: 'system',
-    content: `请从长文本中总结出5至15个摘要，尽量详细，请务必按以下格式返回: "(1):"\n"(2):"\n"(3):"\n`
+    content: `请从长文本中总结出5至15个摘要，尽量详细，并按以下格式返回: "A:"\n"A:"\n"A:"\n`
  };
  let dataItem: DataItemSchema | null = null;

@@ -97,7 +97,7 @@ export async function generateAbstract(next = false): Promise<any> {
          console.log('获取词向量错误: ', item);
          return {
            abstract: splitContents[i].abstract,
-            abstractVector: ''
+            abstractVector: []
          };
        }
        return {
@@ -120,7 +120,7 @@ export async function generateAbstract(next = false): Promise<any> {

    // 计费
    !userApiKey &&
-      vectorResponse.length > 0 &&
+      splitContents.length > 0 &&
      pushSplitDataBill({
        userId: dataItem.userId,
        type: 'abstract',
@@ -134,7 +134,7 @@ export async function generateAbstract(next = false): Promise<any> {
      '生成摘要成功，time:',
      `${(Date.now() - startTime) / 1000}s`,
      '摘要数量：',
-      vectorResponse.length
+      splitContents.length
    );
  } catch (error: any) {
    console.log('error: 生成摘要错误', dataItem?._id);
@@ -157,7 +157,7 @@ export async function generateAbstract(next = false): Promise<any> {
 * 检查文本是否按格式返回
 */
 function splitText(text: string) {
-  const regex = /\(\d+\):(\s*)(.*)(\s*)/g;
+  const regex = /A:(\s*)(.*)(\s*)/g;
  const matches = text.matchAll(regex); // 获取所有匹配到的结果

  const result = []; // 存储最终的结果