V4.9.1 feature (#4206)

* fix: remove DefaultTeam (#4037)

* fix ：Get application bound knowledge base information logical rewrite (#4057)

* fix ：Get application bound knowledge base information logical rewrite

* fix ：Get application bound knowledge base information logical rewrite

* fix ：Get application bound knowledge base information logical rewrite

* fix ：Get application bound knowledge base information logical rewrite

* update package

* fix: import dataset step error;perf: ai proxy avatar (#4074)

* perf: pg config params

* perf: ai proxy avatar

* fix: import dataset step error

* feat: data input ux

* perf: app dataset rewite

* fix: 文本提取不支持arrayString,arrayNumber等jsonSchema (#4079)

* update doc ;perf: model test (#4098)

* perf: extract array

* update doc

* perf: model test

* perf: model test

* perf: think tag parse (#4102)

* chat quote reader (#3912)

* init chat quote full text reader

* linked structure

* dataset data linked

* optimize code

* fix ts build

* test finish

* delete log

* fix

* fix ts

* fix ts

* remove nextId

* initial scroll

* fix

* fix

* perf: chunk read   (#4109)

* package

* perf: chunk read

* feat: api dataset support pdf parse;fix: chunk reader auth (#4117)

* feat: api dataset support pdf parse

* fix: chunk reader auth

* feat: invitation link (#3979)

* feat: invitation link schema and apis

* feat: add invitation link

* feat: member status: active, leave, forbidden

* fix: expires show hours and minutes

* feat: invalid invitation link hint

* fix: typo

* chore: fix typo & i18n

* fix

* pref: fe

* feat: add ttl index for 30-day-clean-up

* perf: invite member code (#4118)

* perf: invite member code

* fix: ts

* fix: model test channel id;fix: quote reader (#4123)

* fix: model test channel id

* fix: quote reader

* fix chat quote reader (#4125)

* perf: model test;perf: sidebar trigger (#4127)

* fix: import dataset step error;perf: ai proxy avatar (#4074)

* perf: pg config params

* perf: ai proxy avatar

* fix: import dataset step error

* feat: data input ux

* perf: app dataset rewite

* perf: model test

* perf: sidebar trigger

* lock

* update nanoid version

* fix: select component ux

* fix: ts

* fix: vitest

* remove test

* fix: prompt toolcall ui (#4139)

* load log error adapt

* fix: prompt toolcall ui

* perf: commercial function tip

* update package

* pref: copy link (#4147)

* fix(i18n): namespace (#4143)

* hiden dataset source (#4152)

* hiden dataset source

* perf: reader

* chore: move all tests into a single folder (#4160)

* fix modal close scroll (#4162)

* fix modal close scroll

* update refresh

* feat: rerank modal select and weight (#4164)

* fix loadInitData refresh (#4169)

* fix

* fix

* form input number default & api dataset max token

* feat: mix search weight (#4170)

* feat: mix search weight

* feat: svg render

* fix: avatar error remove (#4173)

* fix: avatar error remove

* fix: index

* fix: guide

* fix: auth

* update package;fix: input data model ui (#4181)

* update package

* fix: ts

* update config

* update jieba package

* add type sign

* fix: input data ui

* fix: page title refresh (#4186)

* fix: ts

* update jieba package

* fix: page title refresh

* fix: remove member length check when opening invite create modal (#4193)

* add env to check internal ip (#4187)

* fix: ts

* update jieba package

* add env to check internal ip

* package

* fix: jieba

* reset package

* update config

* fix: jieba package

* init shell

* init version

* change team reload

* update jieba package (#4200)

* update jieba package

* package

* update package

* remove invalid code

* action

* package (#4201)

* package

* update package

* remove invalid code

* package

* remove i18n tip (#4202)

* doc (#4205)

* fix: i18n (#4208)

* fix: next config (#4207)

* reset package

* i18n

* update config

* i18n

* remove log

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>
Co-authored-by: shilin <39396378+shilin66@users.noreply.github.com>
Co-authored-by: heheer <heheer@sealos.io>

packages/service/common/file/image/controller.ts

@@ -76,7 +76,7 @@ export const refreshSourceAvatar = async (
   const newId = getIdFromPath(path);
   const oldId = getIdFromPath(oldPath);
 
-  if (!newId) return;
+  if (!newId || newId === oldId) return;
 
   await MongoImage.updateOne({ _id: newId }, { $unset: { expiredTime: 1 } }, { session });
 

packages/service/common/file/read/utils.ts

@@ -300,6 +300,9 @@ export const readRawContentByFileBuffer = async ({
     return systemParse();
   };
 
+  const start = Date.now();
+  addLog.debug(`Start parse file`, { extension });
+
   let { rawText, formatText, imageList } = await (async () => {
     if (extension === 'pdf') {
       return await pdfParseFn();
@@ -307,6 +310,8 @@ export const readRawContentByFileBuffer = async ({
     return await systemParse();
   })();
 
+  addLog.debug(`Parse file success, time: ${Date.now() - start}ms. Uploading file image.`);
+
   // markdown data format
   if (imageList) {
     await batchRun(imageList, async (item) => {
@@ -341,5 +346,7 @@ export const readRawContentByFileBuffer = async ({
     }
   }
 
+  addLog.debug(`Upload file image success, time: ${Date.now() - start}ms`);
+
   return { rawText, formatText, imageList };
 };

packages/service/common/string/jieba/index.ts

@@ -1,4 +1,13 @@
-import { cut } from '@node-rs/jieba';
+import { Jieba } from '@node-rs/jieba';
+
+let jieba: Jieba | undefined;
+
+(async () => {
+  const dictData = await import('./dict.json');
+  // @ts-ignore
+  const dictBuffer = Buffer.from(dictData.dict?.replace(/\\n/g, '\n'), 'utf-8');
+  jieba = Jieba.withDict(dictBuffer);
+})();
 
 const stopWords = new Set([
   '--',
@@ -1509,8 +1518,10 @@ const stopWords = new Set([
   ]
 ]);
 
-export function jiebaSplit({ text }: { text: string }) {
-  const tokens = cut(text, true);
+export async function jiebaSplit({ text }: { text: string }) {
+  text = text.replace(/[#*`_~>[\](){}|]/g, '').replace(/\S*https?\S*/gi, '');
+
+  const tokens = (await jieba!.cutAsync(text, true)) as string[];
 
   return (
     tokens

packages/service/common/system/utils.ts

@@ -30,6 +30,8 @@ export const isInternalAddress = (url: string): boolean => {
       return true;
     }
 
+    if (process.env.CHECK_INTERNAL_IP !== 'true') return false;
+
     // For IP addresses, check if they are internal
     const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}$/;
     if (!ipv4Pattern.test(hostname)) {

packages/service/common/vectorStore/pg/class.ts

@@ -38,6 +38,27 @@ export class PgVectorCtrl {
       await PgClient.query(
         `CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${DatasetVectorTableName} USING btree(createtime);`
       );
+      // 10w rows
+      // await PgClient.query(`
+      //   ALTER TABLE modeldata SET (
+      //     autovacuum_vacuum_scale_factor = 0.1,
+      //     autovacuum_analyze_scale_factor = 0.05,
+      //     autovacuum_vacuum_threshold = 50,
+      //     autovacuum_analyze_threshold = 50,
+      //     autovacuum_vacuum_cost_delay = 20,
+      //     autovacuum_vacuum_cost_limit = 200
+      //   );`);
+
+      // 100w rows
+      // await PgClient.query(`
+      //   ALTER TABLE modeldata SET (
+      //   autovacuum_vacuum_scale_factor = 0.01,
+      //   autovacuum_analyze_scale_factor = 0.02,
+      //   autovacuum_vacuum_threshold = 1000,
+      //   autovacuum_analyze_threshold = 1000,
+      //   autovacuum_vacuum_cost_delay = 10,
+      //   autovacuum_vacuum_cost_limit = 2000
+      // );`)
 
       addLog.info('init pg successful');
     } catch (error) {