From f19c2d2ca1626856444a909227477a5fd891c8a3 Mon Sep 17 00:00:00 2001
From: archer <545436317@qq.com>
Date: Fri, 7 Apr 2023 16:12:43 +0800
Subject: [PATCH] =?UTF-8?q?perf:=20=E5=8E=BB=E6=8E=89raw=20content?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/pages/api/model/data/exportModelData.ts   | 21 +++++--------------
 src/pages/api/model/data/pushModelDataJson.ts |  4 +---
 src/service/events/generateVector.ts          |  2 --
 src/service/events/pushBill.ts                |  3 +--
 4 files changed, 7 insertions(+), 23 deletions(-)

diff --git a/src/pages/api/model/data/exportModelData.ts b/src/pages/api/model/data/exportModelData.ts
index 314796c34..68145e5d9 100644
--- a/src/pages/api/model/data/exportModelData.ts
+++ b/src/pages/api/model/data/exportModelData.ts
@@ -32,7 +32,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       VecModelDataIdx,
       `@modelId:{${modelId}} @userId:{${userId}}`,
       {
-        RETURN: ['q', 'text', 'rawVector'],
+        RETURN: ['q', 'text'],
         LIMIT: {
           from: 0,
           size: 10000
@@ -40,21 +40,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       }
     );
 
-    const data = searchRes.documents
-      .filter((item) => {
-        if (!item?.value?.rawVector) return false;
-        try {
-          JSON.parse(item.value.rawVector as string);
-          return true;
-        } catch (error) {
-          return false;
-        }
-      })
-      .map((item: any) => ({
-        prompt: item.value.q,
-        completion: item.value.text,
-        vector: JSON.parse(item.value.rawVector)
-      }));
+    const data = searchRes.documents.map((item: any) => ({
+      prompt: item.value.q,
+      completion: item.value.text
+    }));
 
     jsonRes(res, {
       data: JSON.stringify(data)
diff --git a/src/pages/api/model/data/pushModelDataJson.ts b/src/pages/api/model/data/pushModelDataJson.ts
index 39ec6ab2f..8231acb5f 100644
--- a/src/pages/api/model/data/pushModelDataJson.ts
+++ b/src/pages/api/model/data/pushModelDataJson.ts
@@ -53,9 +53,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
           userId,
           'modelId',
           String(modelId),
-          ...(vector
-            ? ['vector', vectorToBuffer(formatVector(vector)), 'rawVector', JSON.stringify(vector)]
-            : []),
+          ...(vector ? ['vector', vectorToBuffer(formatVector(vector))] : []),
           'q',
           item.prompt,
           'text',
diff --git a/src/service/events/generateVector.ts b/src/service/events/generateVector.ts
index 0f7d1e524..19c395003 100644
--- a/src/service/events/generateVector.ts
+++ b/src/service/events/generateVector.ts
@@ -68,8 +68,6 @@ export async function generateVector(next = false): Promise<any> {
       dataItem.id,
       'vector',
       vectorToBuffer(vector),
-      'rawVector',
-      JSON.stringify(vector),
       'status',
       ModelDataStatusEnum.ready
     ]);
diff --git a/src/service/events/pushBill.ts b/src/service/events/pushBill.ts
index ec2bf3f01..1f17ee1e1 100644
--- a/src/service/events/pushBill.ts
+++ b/src/service/events/pushBill.ts
@@ -1,7 +1,6 @@
 import { connectToDatabase, Bill, User } from '../mongo';
 import { modelList, ChatModelNameEnum } from '@/constants/model';
 import { encode } from 'gpt-token-utils';
-import { formatPrice } from '@/utils/user';
 import { BillTypeEnum } from '@/constants/user';
 import type { DataType } from '@/types/data';
 
@@ -22,7 +21,7 @@ export const pushChatBill = async ({
 
   try {
     // 计算 token 数量
-    const tokens = encode(text).length;
+    const tokens = Math.floor(encode(text).length * 0.7);
 
     console.log(`chat generate success. text len: ${text.length}. token len: ${tokens}`);