Change embedding (#1463)

* rebuild embedding queue

* dataset menu

* feat: rebuild data api

* feat: ui change embedding model

* dataset ui

* feat: rebuild index ui

* rename collection
This commit is contained in:
Archer
2024-05-13 14:51:42 +08:00
committed by GitHub
parent 59fd94384d
commit 80a84a5733
37 changed files with 1260 additions and 419 deletions

View File

@@ -19,8 +19,6 @@ import type {
GetTrainingQueueProps,
GetTrainingQueueResponse,
PostPreviewFilesChunksProps,
PostPreviewFilesChunksResponse,
PostPreviewTableChunksResponse,
SearchTestProps,
SearchTestResponse
} from '@/global/core/dataset/api.d';
@@ -29,7 +27,6 @@ import type {
CreateDatasetParams,
InsertOneDatasetDataProps
} from '@/global/core/dataset/api.d';
import type { PushDatasetDataResponse } from '@fastgpt/global/core/dataset/api.d';
import type { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
import {
DatasetCollectionSyncResultEnum,
@@ -38,6 +35,8 @@ import {
import type { DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
import { PagingData } from '@/types';
import type { getDatasetTrainingQueueResponse } from '@/pages/api/core/dataset/training/getDatasetTrainingQueue';
import type { rebuildEmbeddingBody } from '@/pages/api/core/dataset/training/rebuildEmbedding';
/* ======================== dataset ======================= */
export const getDatasets = (data: { parentId?: string; type?: `${DatasetTypeEnum}` }) =>
@@ -124,9 +123,17 @@ export const delOneDatasetDataById = (id: string) =>
DELETE<string>(`/core/dataset/data/delete`, { id });
/* ================ training ==================== */
export const postRebuildEmbedding = (data: rebuildEmbeddingBody) =>
POST(`/core/dataset/training/rebuildEmbedding`, data);
/* get length of system training queue */
export const getTrainingQueueLen = (data: GetTrainingQueueProps) =>
GET<GetTrainingQueueResponse>(`/core/dataset/training/getQueueLen`, data);
export const getDatasetTrainingQueue = (datasetId: string) =>
GET<getDatasetTrainingQueueResponse>(`/core/dataset/training/getDatasetTrainingQueue`, {
datasetId
});
export const getPreviewChunks = (data: PostPreviewFilesChunksProps) =>
POST<{ q: string; a: string }[]>('/core/dataset/file/getPreviewChunks', data);

View File

@@ -0,0 +1,119 @@
import { useQuery } from '@tanstack/react-query';
import { ReactNode, useMemo } from 'react';
import { useTranslation } from 'next-i18next';
import { createContext } from 'use-context-selector';
import { getDatasetTrainingQueue, getTrainingQueueLen } from '../api';
import { useDatasetStore } from '../store/dataset';
type DatasetPageContextType = {
vectorTrainingMap: {
colorSchema: string;
tip: string;
};
agentTrainingMap: {
colorSchema: string;
tip: string;
};
rebuildingCount: number;
trainingCount: number;
refetchDatasetTraining: () => void;
};
type DatasetPageContextValueType = {
datasetId: string;
};
export const DatasetPageContext = createContext<DatasetPageContextType>({
vectorTrainingMap: {
colorSchema: '',
tip: ''
},
agentTrainingMap: {
colorSchema: '',
tip: ''
},
rebuildingCount: 0,
trainingCount: 0,
refetchDatasetTraining: function (): void {
throw new Error('Function not implemented.');
}
});
export const DatasetPageContextProvider = ({
children,
value
}: {
children: ReactNode;
value: DatasetPageContextValueType;
}) => {
const { t } = useTranslation();
const { datasetId } = value;
const { datasetDetail } = useDatasetStore();
// global queue
const { data: { vectorTrainingCount = 0, agentTrainingCount = 0 } = {} } = useQuery(
['getTrainingQueueLen'],
() =>
getTrainingQueueLen({
vectorModel: datasetDetail.vectorModel.model,
agentModel: datasetDetail.agentModel.model
}),
{
refetchInterval: 10000
}
);
const { vectorTrainingMap, agentTrainingMap } = useMemo(() => {
const vectorTrainingMap = (() => {
if (vectorTrainingCount < 1000)
return {
colorSchema: 'green',
tip: t('core.dataset.training.Leisure')
};
if (vectorTrainingCount < 10000)
return {
colorSchema: 'yellow',
tip: t('core.dataset.training.Waiting')
};
return {
colorSchema: 'red',
tip: t('core.dataset.training.Full')
};
})();
const agentTrainingMap = (() => {
if (agentTrainingCount < 100)
return {
colorSchema: 'green',
tip: t('core.dataset.training.Leisure')
};
if (agentTrainingCount < 1000)
return {
colorSchema: 'yellow',
tip: t('core.dataset.training.Waiting')
};
return {
colorSchema: 'red',
tip: t('core.dataset.training.Full')
};
})();
return {
vectorTrainingMap,
agentTrainingMap
};
}, [agentTrainingCount, t, vectorTrainingCount]);
// training and rebuild queue
const { data: { rebuildingCount = 0, trainingCount = 0 } = {}, refetch: refetchDatasetTraining } =
useQuery(['getDatasetTrainingQueue'], () => getDatasetTrainingQueue(datasetId), {
refetchInterval: 10000
});
const contextValue: DatasetPageContextType = {
vectorTrainingMap,
agentTrainingMap,
rebuildingCount,
trainingCount,
refetchDatasetTraining
};
return <DatasetPageContext.Provider value={contextValue}>{children}</DatasetPageContext.Provider>;
};

View File

@@ -0,0 +1,18 @@
import { ReactNode } from 'react';
import { createContext } from 'use-context-selector';
type DatasetContextType = {};
type DatasetContextValueType = {};
export const DatasetContext = createContext<DatasetContextType>({});
export const DatasetContextProvider = ({
children,
value
}: {
children: ReactNode;
value: DatasetContextValueType;
}) => {
return <DatasetContext.Provider value={value}>{children}</DatasetContext.Provider>;
};