mirror of
https://github.com/labring/FastGPT.git
synced 2025-08-03 13:38:00 +00:00
Change embedding (#1463)
* rebuild embedding queue * dataset menu * feat: rebuild data api * feat: ui change embedding model * dataset ui * feat: rebuild index ui * rename collection
This commit is contained in:
@@ -19,8 +19,6 @@ import type {
|
||||
GetTrainingQueueProps,
|
||||
GetTrainingQueueResponse,
|
||||
PostPreviewFilesChunksProps,
|
||||
PostPreviewFilesChunksResponse,
|
||||
PostPreviewTableChunksResponse,
|
||||
SearchTestProps,
|
||||
SearchTestResponse
|
||||
} from '@/global/core/dataset/api.d';
|
||||
@@ -29,7 +27,6 @@ import type {
|
||||
CreateDatasetParams,
|
||||
InsertOneDatasetDataProps
|
||||
} from '@/global/core/dataset/api.d';
|
||||
import type { PushDatasetDataResponse } from '@fastgpt/global/core/dataset/api.d';
|
||||
import type { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
|
||||
import {
|
||||
DatasetCollectionSyncResultEnum,
|
||||
@@ -38,6 +35,8 @@ import {
|
||||
import type { DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
|
||||
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
|
||||
import { PagingData } from '@/types';
|
||||
import type { getDatasetTrainingQueueResponse } from '@/pages/api/core/dataset/training/getDatasetTrainingQueue';
|
||||
import type { rebuildEmbeddingBody } from '@/pages/api/core/dataset/training/rebuildEmbedding';
|
||||
|
||||
/* ======================== dataset ======================= */
|
||||
export const getDatasets = (data: { parentId?: string; type?: `${DatasetTypeEnum}` }) =>
|
||||
@@ -124,9 +123,17 @@ export const delOneDatasetDataById = (id: string) =>
|
||||
DELETE<string>(`/core/dataset/data/delete`, { id });
|
||||
|
||||
/* ================ training ==================== */
|
||||
export const postRebuildEmbedding = (data: rebuildEmbeddingBody) =>
|
||||
POST(`/core/dataset/training/rebuildEmbedding`, data);
|
||||
|
||||
/* get length of system training queue */
|
||||
export const getTrainingQueueLen = (data: GetTrainingQueueProps) =>
|
||||
GET<GetTrainingQueueResponse>(`/core/dataset/training/getQueueLen`, data);
|
||||
export const getDatasetTrainingQueue = (datasetId: string) =>
|
||||
GET<getDatasetTrainingQueueResponse>(`/core/dataset/training/getDatasetTrainingQueue`, {
|
||||
datasetId
|
||||
});
|
||||
|
||||
export const getPreviewChunks = (data: PostPreviewFilesChunksProps) =>
|
||||
POST<{ q: string; a: string }[]>('/core/dataset/file/getPreviewChunks', data);
|
||||
|
||||
|
119
projects/app/src/web/core/dataset/context/datasetPageContext.tsx
Normal file
119
projects/app/src/web/core/dataset/context/datasetPageContext.tsx
Normal file
@@ -0,0 +1,119 @@
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { ReactNode, useMemo } from 'react';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { createContext } from 'use-context-selector';
|
||||
import { getDatasetTrainingQueue, getTrainingQueueLen } from '../api';
|
||||
import { useDatasetStore } from '../store/dataset';
|
||||
|
||||
type DatasetPageContextType = {
|
||||
vectorTrainingMap: {
|
||||
colorSchema: string;
|
||||
tip: string;
|
||||
};
|
||||
agentTrainingMap: {
|
||||
colorSchema: string;
|
||||
tip: string;
|
||||
};
|
||||
rebuildingCount: number;
|
||||
trainingCount: number;
|
||||
refetchDatasetTraining: () => void;
|
||||
};
|
||||
|
||||
type DatasetPageContextValueType = {
|
||||
datasetId: string;
|
||||
};
|
||||
|
||||
export const DatasetPageContext = createContext<DatasetPageContextType>({
|
||||
vectorTrainingMap: {
|
||||
colorSchema: '',
|
||||
tip: ''
|
||||
},
|
||||
agentTrainingMap: {
|
||||
colorSchema: '',
|
||||
tip: ''
|
||||
},
|
||||
rebuildingCount: 0,
|
||||
trainingCount: 0,
|
||||
refetchDatasetTraining: function (): void {
|
||||
throw new Error('Function not implemented.');
|
||||
}
|
||||
});
|
||||
|
||||
export const DatasetPageContextProvider = ({
|
||||
children,
|
||||
value
|
||||
}: {
|
||||
children: ReactNode;
|
||||
value: DatasetPageContextValueType;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const { datasetId } = value;
|
||||
const { datasetDetail } = useDatasetStore();
|
||||
|
||||
// global queue
|
||||
const { data: { vectorTrainingCount = 0, agentTrainingCount = 0 } = {} } = useQuery(
|
||||
['getTrainingQueueLen'],
|
||||
() =>
|
||||
getTrainingQueueLen({
|
||||
vectorModel: datasetDetail.vectorModel.model,
|
||||
agentModel: datasetDetail.agentModel.model
|
||||
}),
|
||||
{
|
||||
refetchInterval: 10000
|
||||
}
|
||||
);
|
||||
const { vectorTrainingMap, agentTrainingMap } = useMemo(() => {
|
||||
const vectorTrainingMap = (() => {
|
||||
if (vectorTrainingCount < 1000)
|
||||
return {
|
||||
colorSchema: 'green',
|
||||
tip: t('core.dataset.training.Leisure')
|
||||
};
|
||||
if (vectorTrainingCount < 10000)
|
||||
return {
|
||||
colorSchema: 'yellow',
|
||||
tip: t('core.dataset.training.Waiting')
|
||||
};
|
||||
return {
|
||||
colorSchema: 'red',
|
||||
tip: t('core.dataset.training.Full')
|
||||
};
|
||||
})();
|
||||
const agentTrainingMap = (() => {
|
||||
if (agentTrainingCount < 100)
|
||||
return {
|
||||
colorSchema: 'green',
|
||||
tip: t('core.dataset.training.Leisure')
|
||||
};
|
||||
if (agentTrainingCount < 1000)
|
||||
return {
|
||||
colorSchema: 'yellow',
|
||||
tip: t('core.dataset.training.Waiting')
|
||||
};
|
||||
return {
|
||||
colorSchema: 'red',
|
||||
tip: t('core.dataset.training.Full')
|
||||
};
|
||||
})();
|
||||
return {
|
||||
vectorTrainingMap,
|
||||
agentTrainingMap
|
||||
};
|
||||
}, [agentTrainingCount, t, vectorTrainingCount]);
|
||||
|
||||
// training and rebuild queue
|
||||
const { data: { rebuildingCount = 0, trainingCount = 0 } = {}, refetch: refetchDatasetTraining } =
|
||||
useQuery(['getDatasetTrainingQueue'], () => getDatasetTrainingQueue(datasetId), {
|
||||
refetchInterval: 10000
|
||||
});
|
||||
|
||||
const contextValue: DatasetPageContextType = {
|
||||
vectorTrainingMap,
|
||||
agentTrainingMap,
|
||||
rebuildingCount,
|
||||
trainingCount,
|
||||
refetchDatasetTraining
|
||||
};
|
||||
|
||||
return <DatasetPageContext.Provider value={contextValue}>{children}</DatasetPageContext.Provider>;
|
||||
};
|
@@ -0,0 +1,18 @@
|
||||
import { ReactNode } from 'react';
|
||||
import { createContext } from 'use-context-selector';
|
||||
|
||||
type DatasetContextType = {};
|
||||
|
||||
type DatasetContextValueType = {};
|
||||
|
||||
export const DatasetContext = createContext<DatasetContextType>({});
|
||||
|
||||
export const DatasetContextProvider = ({
|
||||
children,
|
||||
value
|
||||
}: {
|
||||
children: ReactNode;
|
||||
value: DatasetContextValueType;
|
||||
}) => {
|
||||
return <DatasetContext.Provider value={value}>{children}</DatasetContext.Provider>;
|
||||
};
|
Reference in New Issue
Block a user