perf: 改用hash索引

README.md

@@ -106,5 +106,6 @@ echo "Restart clash"
 
 ```bash
 # 索引
-FT.CREATE idx:model:data ON JSON PREFIX 1 model:data: SCHEMA $.modelId AS modelId TAG $.dataId AS dataId TAG $.vector AS vector VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT32
+# FT.CREATE idx:model:data ON JSON PREFIX 1 model:data: SCHEMA $.modelId AS modelId TAG $.dataId AS dataId TAG $.vector AS vector VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT32
+FT.CREATE idx:model:data:hash ON HASH PREFIX 1 model:data: SCHEMA modelId TAG dataId TAG vector VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT32
 ```

src/pages/api/chat/vectorGpt.ts

@@ -14,11 +14,6 @@ import { connectRedis } from '@/service/redis';
 import { VecModelDataIndex } from '@/constants/redis';
 import { vectorToBuffer } from '@/utils/tools';
 
-let vectorData = [
-  -0.025028639, -0.010407282, 0.026523087, -0.0107438695, -0.006967359, 0.010043768, -0.012043097,
-  0.008724345, -0.028919589, -0.0117738275, 0.0050690062, 0.02961969
-].concat(new Array(1524).fill(0));
-
 /* 发送提示词 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   let step = 0; // step=1时，表示开始了流响应
@@ -78,12 +73,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       )
       .then((res) => res?.data?.data?.[0]?.embedding || []);
 
-    const binary = vectorToBuffer(promptVector);
-
     // 搜索系统提示词, 按相似度从 redis 中搜出前3条不同 dataId 的数据
     const redisData: any[] = await redis.sendCommand([
       'FT.SEARCH',
-      `idx:${VecModelDataIndex}`,
+      `idx:${VecModelDataIndex}:hash`,
       `@modelId:{${String(
         chat.modelId._id
       )}} @vector:[VECTOR_RANGE 0.15 $blob]=>{$YIELD_DISTANCE_AS: score}`,
@@ -96,7 +89,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       'PARAMS',
       '2',
       'blob',
-      binary,
+      vectorToBuffer(promptVector),
       'LIMIT',
       '0',
       '20',

src/pages/api/timer/clearAuthCode.ts

@@ -5,8 +5,8 @@ import { AuthCode } from '@/service/models/authCode';
 import { connectToDatabase } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  if (req.headers.auth !== 'archer') {
+  if (process.env.NODE_ENV !== 'development') {
-    throw new Error('凭证错误');
+    throw new Error('不是开发环境');
   }
   try {
     await connectToDatabase();

src/pages/api/timer/clearChatWindow.ts

@@ -4,8 +4,8 @@ import { connectToDatabase, Chat } from '@/service/mongo';
 
 /* 定时删除那些不活跃的内容 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  if (req.headers.auth !== 'archer') {
+  if (process.env.NODE_ENV !== 'development') {
-    throw new Error('凭证错误');
+    throw new Error('不是开发环境');
   }
   try {
     await connectToDatabase();

src/pages/api/timer/initBill.ts

@@ -7,8 +7,8 @@ import type { BillSchema } from '@/types/mongoSchema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    if (req.headers.auth !== 'archer') {
+    if (process.env.NODE_ENV !== 'development') {
-      throw new Error('凭证错误');
+      throw new Error('不是开发环境');
     }
     await connectToDatabase();
 

src/pages/api/timer/initDataItemTime.ts

@@ -5,8 +5,8 @@ import { connectToDatabase, DataItem, Data } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    if (req.headers.auth !== 'archer') {
+    if (process.env.NODE_ENV !== 'development') {
-      throw new Error('凭证错误');
+      throw new Error('不是开发环境');
     }
     await connectToDatabase();
 

src/pages/api/timer/testVector.ts

@@ -0,0 +1,68 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Bill } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import type { BillSchema } from '@/types/mongoSchema';
+import { VecModelDataIndex } from '@/constants/redis';
+import { connectRedis } from '@/service/redis';
+import { vectorToBuffer } from '@/utils/tools';
+
+let vectorData = [
+  -0.025028639, -0.010407282, 0.026523087, -0.0107438695, -0.006967359, 0.010043768, -0.012043097,
+  0.008724345, -0.028919589, -0.0117738275, 0.0050690062, 0.02961969
+].concat(new Array(1524).fill(0));
+let vectorData2 = [
+  0.025028639, 0.010407282, 0.026523087, 0.0107438695, -0.006967359, 0.010043768, -0.012043097,
+  0.008724345, 0.028919589, 0.0117738275, 0.0050690062, 0.02961969
+].concat(new Array(1524).fill(0));
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    if (process.env.NODE_ENV !== 'development') {
+      throw new Error('不是开发环境');
+    }
+    await connectToDatabase();
+
+    const redis = await connectRedis();
+
+    await redis.sendCommand([
+      'HMSET',
+      'model:data:333',
+      'vector',
+      vectorToBuffer(vectorData2),
+      'modelId',
+      '1133',
+      'dataId',
+      'safadfa'
+    ]);
+
+    // search
+    const response = await redis.sendCommand([
+      'FT.SEARCH',
+      'idx:model:data:hash',
+      '@modelId:{1133} @vector:[VECTOR_RANGE 0.15 $blob]=>{$YIELD_DISTANCE_AS: score}',
+      'RETURN',
+      '2',
+      'modelId',
+      'dataId',
+      'PARAMS',
+      '2',
+      'blob',
+      vectorToBuffer(vectorData2),
+      'SORTBY',
+      'score',
+      'DIALECT',
+      '2'
+    ]);
+
+    jsonRes(res, {
+      data: response
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/timer/updateTraining.ts

@@ -12,8 +12,8 @@ import { httpsAgent } from '@/service/utils/tools';
 import { ModelPopulate } from '@/types/mongoSchema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  if (req.headers.auth !== 'archer') {
+  if (process.env.NODE_ENV !== 'development') {
-    throw new Error('凭证错误');
+    throw new Error('不是开发环境');
   }
   try {
     await connectToDatabase();

src/service/events/generateVector.ts

@@ -3,6 +3,7 @@ import { httpsAgent } from '@/service/utils/tools';
 import { ModelData } from '../models/modelData';
 import { connectRedis } from '../redis';
 import { VecModelDataIndex } from '@/constants/redis';
+import { vectorToBuffer } from '@/utils/tools';
 
 export async function generateVector(next = false): Promise<any> {
   if (global.generatingVector && !next) return;
@@ -47,14 +48,14 @@ export async function generateVector(next = false): Promise<any> {
           .then((res) => res?.data?.data?.[0]?.embedding || [])
           .then((vector) =>
             redis.sendCommand([
-              'JSON.SET',
+              'HMSET',
               `${VecModelDataIndex}:${item.id}`,
-              '$',
+              'vector',
-              JSON.stringify({
+              vectorToBuffer(vector),
-                dataId,
+              'modelId',
-                modelId: String(dataItem.modelId),
+              String(dataItem.modelId),
-                vector
+              'dataId',
-              })
+              String(dataId)
             ])
           )
       )

src/utils/tools.ts

@@ -126,13 +126,7 @@ export const readDocContent = (file: File) =>
   });
 
 export const vectorToBuffer = (vector: number[]) => {
-  const float32Arr = new Float32Array(vector);
+  let npVector = new Float32Array(vector);
-  const myBuffer = new ArrayBuffer(float32Arr.length * Float32Array.BYTES_PER_ELEMENT);
-  const myView = new DataView(myBuffer);
 
-  for (let i = 0; i < float32Arr.length; i++) {
+  return Buffer.from(npVector.buffer);
-    myView.setFloat32(i * Float32Array.BYTES_PER_ELEMENT, float32Arr[i], true);
-  }
-
-  return Buffer.from(myBuffer);
 };