perf: search prompt, upload step and psw len

client/public/locales/en/common.json

@@ -22,6 +22,9 @@
     "New Chat": "New Chat",
     "You need to a chat app": "You need to a chat app"
   },
+  "commom": {
+    "Password inconsistency": "Password inconsistency"
+  },
   "common": {
     "Add": "Add",
     "Cancel": "Cancel",

client/public/locales/zh/common.json

@@ -22,11 +22,15 @@
     "New Chat": "新对话",
     "You need to a chat app": "你需要创建一个应用"
   },
+  "commom": {
+    "Password inconsistency": "两次密码不一致"
+  },
   "common": {
     "Add": "添加",
     "Cancel": "取消",
     "Collect": "收藏",
     "Copy": "复制",
+    "Course": "",
     "Delete": "删除",
     "Filed is repeat": "",
     "Filed is repeated": "字段重复了",

client/src/components/ChatBox/QuoteModal.tsx

@@ -1,5 +1,5 @@
 import React, { useCallback, useState } from 'react';
-import { ModalBody, ModalCloseButton, ModalHeader, Box, useTheme } from '@chakra-ui/react';
+import { ModalBody, Box, useTheme } from '@chakra-ui/react';
 import { getKbDataItemById } from '@/api/plugins/kb';
 import { useLoading } from '@/hooks/useLoading';
 import { useToast } from '@/hooks/useToast';
@@ -9,13 +9,21 @@ import MyIcon from '@/components/Icon';
 import InputDataModal from '@/pages/kb/detail/components/InputDataModal';
 import MyModal from '../MyModal';
 
+type SearchType = {
+  kb_id?: string;
+  id?: string;
+  q: string;
+  a?: string;
+  source?: string | undefined;
+};
+
 const QuoteModal = ({
   onUpdateQuote,
   rawSearch = [],
   onClose
 }: {
   onUpdateQuote: (quoteId: string, sourceText: string) => Promise<void>;
-  rawSearch: QuoteItemType[];
+  rawSearch: SearchType[];
   onClose: () => void;
 }) => {
   const theme = useTheme();
@@ -32,7 +40,8 @@ const QuoteModal = ({
    * click edit, get new kbDataItem
    */
   const onclickEdit = useCallback(
-    async (item: QuoteItemType) => {
+    async (item: SearchType) => {
+      if (!item.id) return;
       try {
         setIsLoading(true);
         const data = (await getKbDataItemById(item.id)) as QuoteItemType;
@@ -77,9 +86,9 @@ const QuoteModal = ({
         }
       >
         <ModalBody pt={0} whiteSpace={'pre-wrap'} textAlign={'justify'} fontSize={'sm'}>
-          {rawSearch.map((item) => (
+          {rawSearch.map((item, i) => (
             <Box
-              key={item.id}
+              key={i}
               flex={'1 0 0'}
               p={2}
               borderRadius={'lg'}
@@ -91,31 +100,33 @@ const QuoteModal = ({
               {item.source && <Box color={'myGray.600'}>({item.source})</Box>}
               <Box>{item.q}</Box>
               <Box>{item.a}</Box>
-              <Box
+              {item.id && (
-                className="edit"
+                <Box
-                display={'none'}
+                  className="edit"
-                position={'absolute'}
+                  display={'none'}
-                right={0}
+                  position={'absolute'}
-                top={0}
+                  right={0}
-                bottom={0}
+                  top={0}
-                w={'40px'}
+                  bottom={0}
-                bg={'rgba(255,255,255,0.9)'}
+                  w={'40px'}
-                alignItems={'center'}
+                  bg={'rgba(255,255,255,0.9)'}
-                justifyContent={'center'}
+                  alignItems={'center'}
-                boxShadow={'-10px 0 10px rgba(255,255,255,1)'}
+                  justifyContent={'center'}
-              >
+                  boxShadow={'-10px 0 10px rgba(255,255,255,1)'}
-                <MyIcon
+                >
-                  name={'edit'}
+                  <MyIcon
-                  w={'18px'}
+                    name={'edit'}
-                  h={'18px'}
+                    w={'18px'}
-                  cursor={'pointer'}
+                    h={'18px'}
-                  color={'myGray.600'}
+                    cursor={'pointer'}
-                  _hover={{
+                    color={'myGray.600'}
-                    color: 'myBlue.700'
+                    _hover={{
-                  }}
+                      color: 'myBlue.700'
-                  onClick={() => onclickEdit(item)}
+                    }}
-                />
+                    onClick={() => onclickEdit(item)}
-              </Box>
+                  />
+                </Box>
+              )}
             </Box>
           ))}
         </ModalBody>

client/src/pages/account/components/UpdatePswModal.tsx

@@ -6,9 +6,15 @@ import { useForm } from 'react-hook-form';
 import { useRequest } from '@/hooks/useRequest';
 import { updatePasswordByOld } from '@/api/user';
 
+type FormType = {
+  oldPsw: string;
+  newPsw: string;
+  confirmPsw: string;
+};
+
 const UpdatePswModal = ({ onClose }: { onClose: () => void }) => {
   const { t } = useTranslation();
-  const { register, handleSubmit } = useForm({
+  const { register, handleSubmit } = useForm<FormType>({
     defaultValues: {
       oldPsw: '',
       newPsw: '',
@@ -17,7 +23,10 @@ const UpdatePswModal = ({ onClose }: { onClose: () => void }) => {
   });
 
   const { mutate: onSubmit, isLoading } = useRequest({
-    mutationFn: (data) => {
+    mutationFn: (data: FormType) => {
+      if (data.newPsw !== data.confirmPsw) {
+        return Promise.reject(t('commom.Password inconsistency'));
+      }
       return updatePasswordByOld(data);
     },
     onSuccess() {
@@ -36,11 +45,31 @@ const UpdatePswModal = ({ onClose }: { onClose: () => void }) => {
         </Flex>
         <Flex alignItems={'center'} mt={5}>
           <Box flex={'0 0 70px'}>新密码:</Box>
-          <Input flex={1} type={'password'} {...register('newPsw', { required: true })}></Input>
+          <Input
+            flex={1}
+            type={'password'}
+            {...register('newPsw', {
+              required: true,
+              maxLength: {
+                value: 20,
+                message: '密码最少 4 位最多 20 位'
+              }
+            })}
+          ></Input>
         </Flex>
         <Flex alignItems={'center'} mt={5}>
           <Box flex={'0 0 70px'}>确认密码:</Box>
-          <Input flex={1} type={'password'} {...register('confirmPsw', { required: true })}></Input>
+          <Input
+            flex={1}
+            type={'password'}
+            {...register('confirmPsw', {
+              required: true,
+              maxLength: {
+                value: 20,
+                message: '密码最少 4 位最多 20 位'
+              }
+            })}
+          ></Input>
         </Flex>
       </ModalBody>
       <ModalFooter>

client/src/pages/api/openapi/kb/pushData.ts

@@ -90,15 +90,14 @@ export async function pushDataToKb({
   data.forEach((item) => {
     const text = item.q + item.a;
 
-    if (mode === TrainingModeEnum.qa) {
+    // count token
-      // count token
+    const token = modelToolMap.countTokens({
-      const token = modelToolMap.countTokens({
+      model: 'gpt-3.5-turbo',
-        model: 'gpt-3.5-turbo-16k',
+      messages: [{ obj: 'System', value: item.q }]
-        messages: [{ obj: 'System', value: item.q }]
+    });
-      });
+
-      if (token > modeMaxToken[TrainingModeEnum.qa]) {
+    if (token > modeMaxToken[TrainingModeEnum.qa]) {
-        return;
+      return;
-      }
     }
 
     if (!set.has(text)) {

client/src/pages/kb/detail/components/Import/Chunk.tsx

@@ -142,7 +142,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
 
       // subsection import
       let success = 0;
-      const step = 100;
+      const step = 500;
       for (let i = 0; i < chunks.length; i += step) {
         const { insertLen } = await postKbDataFromList({
           kbId,

client/src/pages/kb/detail/components/Import/Csv.tsx

@@ -13,7 +13,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
 import FileSelect from './FileSelect';
 import { useRouter } from 'next/router';
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
-import { fileDownload, readCsvContent } from '@/utils/file';
+import { readCsvContent } from '@/utils/file';
 
 const fileExtension = '.csv';
 
@@ -98,7 +98,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
 
       // subsection import
       let success = 0;
-      const step = 100;
+      const step = 500;
       for (let i = 0; i < chunks.length; i += step) {
         const { insertLen } = await postKbDataFromList({
           kbId,

client/src/pages/kb/detail/components/Import/QA.tsx

@@ -132,7 +132,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
 
       // subsection import
       let success = 0;
-      const step = 100;
+      const step = 500;
       for (let i = 0; i < chunks.length; i += step) {
         const { insertLen } = await postKbDataFromList({
           kbId,

client/src/pages/login/components/ForgetPasswordForm.tsx

@@ -129,11 +129,11 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
               required: '密码不能为空',
               minLength: {
                 value: 4,
-                message: '密码最少4位最多12位'
+                message: '密码最少 4 位最多 20 位'
               },
               maxLength: {
-                value: 12,
+                value: 20,
-                message: '密码最少4位最多12位'
+                message: '密码最少 4 位最多 20 位'
               }
             })}
           ></Input>

client/src/pages/login/components/LoginForm.tsx

@@ -97,8 +97,8 @@ const LoginForm = ({ setPageType, loginSuccess }: Props) => {
             {...register('password', {
               required: '密码不能为空',
               maxLength: {
-                value: 12,
+                value: 20,
-                message: '密码最多12位'
+                message: '密码最多 20 位'
               }
             })}
           ></Input>

client/src/pages/login/components/RegisterForm.tsx

@@ -142,11 +142,11 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
               required: '密码不能为空',
               minLength: {
                 value: 4,
-                message: '密码最少4位最多12位'
+                message: '密码最少 4 位最多 20 位'
               },
               maxLength: {
-                value: 12,
+                value: 20,
-                message: '密码最少4位最多12位'
+                message: '密码最少 4 位最多 20 位'
               }
             })}
           ></Input>

client/src/service/moduleDispatch/agent/classifyQuestion.ts

@@ -30,6 +30,10 @@ const maxTokens = 3000;
 export const dispatchClassifyQuestion = async (props: Record<string, any>): Promise<CQResponse> => {
   const { agents, systemPrompt, history = [], userChatInput, userOpenaiAccount } = props as CQProps;
 
+  if (!userChatInput) {
+    return Promise.reject('Input is empty');
+  }
+
   const messages: ChatItemType[] = [
     ...(systemPrompt
       ? [

client/src/service/moduleDispatch/chat/oneapi.ts

@@ -192,7 +192,7 @@ function filterQuote({
     maxToken: model.quoteMaxToken,
     messages: quoteQA.map((item, i) => ({
       obj: ChatRoleEnum.System,
-      value: `${i + 1}. [${item.q}\n${item.a}]`
+      value: item.a ? `{instruction:${item.q},output:${item.a}}` : `{instruction:${item.q}}`
     }))
   });
 
@@ -202,7 +202,9 @@ function filterQuote({
   const quotePrompt =
     filterQuoteQA.length > 0
       ? `下面是知识库内容:
-${filterQuoteQA.map((item) => `{Q:${item.q},A:${item.a}}`).join('\n')}
+${filterQuoteQA
+  .map((item) => (item.a ? `{instruction:${item.q},output:${item.a}}` : `{instruction:${item.q}}`))
+  .join('\n')}
 `
       : '';
 

client/src/service/utils/chat/index.ts

@@ -21,13 +21,16 @@ export type StreamResponseType = {
 /* slice chat context by tokens */
 export const ChatContextFilter = ({
   model,
-  prompts,
+  prompts = [],
   maxTokens
 }: {
   model: string;
   prompts: ChatItemType[];
   maxTokens: number;
 }) => {
+  if (!Array.isArray(prompts)) {
+    return [];
+  }
   const rawTextLen = prompts.reduce((sum, item) => sum + item.value.length, 0);
 
   // If the text length is less than half of the maximum token, no calculation is required