refactor: snapshot store to diff (#3155)

* refactor: snapshot store to diff

* change initial state position

* fix old snapshot format

* encapsulate json diff

projects/app/package.json

@@ -42,6 +42,7 @@
     "jest": "^29.5.0",
     "js-yaml": "^4.1.0",
     "json5": "^2.2.3",
+    "jsondiffpatch": "^0.6.0",
     "jsonwebtoken": "^9.0.2",
     "lodash": "^4.17.21",
     "mermaid": "^10.2.3",

projects/app/src/pages/app/detail/components/Plugin/Header.tsx

@@ -13,7 +13,11 @@ import { useTranslation } from 'next-i18next';
 
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useContextSelector } from 'use-context-selector';
-import { WorkflowContext, WorkflowSnapshotsType } from '../WorkflowComponents/context';
+import {
+  WorkflowContext,
+  WorkflowSnapshotsType,
+  WorkflowStateType
+} from '../WorkflowComponents/context';
 import { AppContext, TabEnum } from '../context';
 import RouteTab from '../RouteTab';
 import { useRouter } from 'next/router';
@@ -34,6 +38,7 @@ import {
   WorkflowInitContext
 } from '../WorkflowComponents/context/workflowInitContext';
 import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
+import { applyDiff } from '@/web/core/app/diff';
 
 const Header = () => {
   const { t } = useTranslation();
@@ -76,11 +81,14 @@ const Header = () => {
         [...future].reverse().find((snapshot) => snapshot.isSaved) ||
         past.find((snapshot) => snapshot.isSaved);
 
+      const initialState = past[past.length - 1]?.state;
+      const savedSnapshotState = applyDiff(initialState, savedSnapshot?.diff);
+
       const val = compareSnapshot(
         {
-          nodes: savedSnapshot?.nodes,
-          edges: savedSnapshot?.edges,
-          chatConfig: savedSnapshot?.chatConfig
+          nodes: savedSnapshotState?.nodes,
+          edges: savedSnapshotState?.edges,
+          chatConfig: savedSnapshotState?.chatConfig
         },
         {
           nodes: nodes,

projects/app/src/pages/app/detail/components/SimpleApp/Edit.tsx

@@ -17,6 +17,7 @@ import styles from './styles.module.scss';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import { useTranslation } from 'next-i18next';
 import { onSaveSnapshotFnType, SimpleAppSnapshotType } from './useSnapshots';
+import { applyDiff } from '@/web/core/app/diff';
 
 const Edit = ({
   appForm,
@@ -39,16 +40,19 @@ const Edit = ({
     // show selected dataset
     loadAllDatasets();
 
-    // Get the latest snapshot
-    if (past?.[0]?.appForm) {
-      return setAppForm(past[0].appForm);
-    }
-
     const appForm = appWorkflow2Form({
       nodes: appDetail.modules,
       chatConfig: appDetail.chatConfig
     });
 
+    // Get the latest snapshot
+    if (past?.[0]?.diff) {
+      const pastState = applyDiff(past[past.length - 1].state, past[0].diff);
+
+      return setAppForm(pastState);
+    }
+
+    setAppForm(appForm);
     // Set the first snapshot
     if (past.length === 0) {
       saveSnapshot({
@@ -58,8 +62,6 @@ const Edit = ({
       });
     }
 
-    setAppForm(appForm);
-
     if (appDetail.version !== 'v2') {
       setAppForm(
         appWorkflow2Form({

projects/app/src/pages/app/detail/components/SimpleApp/Header.tsx

@@ -1,4 +1,4 @@
-import React, { useCallback, useState } from 'react';
+import React, { useCallback, useMemo, useState } from 'react';
 import { useContextSelector } from 'use-context-selector';
 import { AppContext } from '../context';
 import FolderPath from '@/components/common/folder/Path';
@@ -29,6 +29,7 @@ import {
 } from './useSnapshots';
 import PublishHistories from '../PublishHistoriesSlider';
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
+import { applyDiff } from '@/web/core/app/diff';
 
 const Header = ({
   forbiddenSaveSnapshot,
@@ -48,9 +49,20 @@ const Header = ({
   const { t } = useTranslation();
   const { isPc } = useSystem();
   const router = useRouter();
-  const { appId, onSaveApp, currentTab } = useContextSelector(AppContext, (v) => v);
+  const { appId, onSaveApp, currentTab, appLatestVersion } = useContextSelector(
+    AppContext,
+    (v) => v
+  );
   const { lastAppListRouteType } = useSystemStore();
   const { allDatasets } = useDatasetStore();
+  const initialAppForm = useMemo(
+    () =>
+      appWorkflow2Form({
+        nodes: appLatestVersion?.nodes || [],
+        chatConfig: appLatestVersion?.chatConfig || {}
+      }),
+    [appLatestVersion]
+  );
 
   const { data: paths = [] } = useRequest2(() => getAppFolderPath(appId), {
     manual: false,
@@ -104,7 +116,8 @@ const Header = ({
 
   const onSwitchTmpVersion = useCallback(
     (data: SimpleAppSnapshotType, customTitle: string) => {
-      setAppForm(data.appForm);
+      const pastState = applyDiff(initialAppForm, data.diff);
+      setAppForm(pastState);
 
       // Remove multiple "copy-"
       const copyText = t('app:version_copy');
@@ -112,11 +125,11 @@ const Header = ({
       const title = customTitle.replace(regex, `$1`);
 
       return saveSnapshot({
-        appForm: data.appForm,
+        appForm: pastState,
         title
       });
     },
-    [saveSnapshot, setAppForm, t]
+    [initialAppForm, saveSnapshot, setAppForm, t]
   );
   const onSwitchCloudVersion = useCallback(
     (appVersion: AppVersionSchemaType) => {
@@ -143,7 +156,8 @@ const Header = ({
   useDebounceEffect(
     () => {
       const savedSnapshot = past.find((snapshot) => snapshot.isSaved);
-      const val = compareSimpleAppSnapshot(savedSnapshot?.appForm, appForm);
+      const pastState = applyDiff(initialAppForm, savedSnapshot?.diff);
+      const val = compareSimpleAppSnapshot(pastState, appForm);
       setIsPublished(val);
     },
     [past, allDatasets],

projects/app/src/pages/app/detail/components/SimpleApp/useSnapshots.tsx

@@ -3,11 +3,13 @@ import { SetStateAction, useEffect, useRef } from 'react';
 import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
 import { AppSimpleEditFormType } from '@fastgpt/global/core/app/type';
 import { isEqual } from 'lodash';
+import { applyDiff, createDiff } from '@/web/core/app/diff';
 
 export type SimpleAppSnapshotType = {
-  appForm: AppSimpleEditFormType;
+  diff?: Record<string, any>;
   title: string;
   isSaved?: boolean;
+  state?: AppSimpleEditFormType;
 };
 export type onSaveSnapshotFnType = (props: {
   appForm: AppSimpleEditFormType;
@@ -66,14 +68,32 @@ export const useSimpleAppSnapshots = (appId: string) => {
       return false;
     }
 
-    const pastState = past[0];
+    if (past.length === 0) {
+      setPast([
+        {
+          title: title || formatTime2YMDHMS(new Date()),
+          isSaved,
+          state: appForm
+        }
+      ]);
+      return true;
+    }
 
-    const isPastEqual = compareSimpleAppSnapshot(pastState?.appForm, appForm);
-    if (isPastEqual) return false;
+    const initialState = past[past.length - 1].state;
+    if (!initialState) return false;
+
+    if (past.length > 0) {
+      const pastState = applyDiff(initialState, past[0].diff);
+
+      const isPastEqual = compareSimpleAppSnapshot(pastState, appForm);
+      if (isPastEqual) return false;
+    }
+
+    const diff = createDiff(initialState, appForm);
 
     setPast((past) => [
       {
-        appForm,
+        diff,
         title: title || formatTime2YMDHMS(new Date()),
         isSaved
       },

projects/app/src/pages/app/detail/components/Workflow/Header.tsx

@@ -13,7 +13,7 @@ import { useTranslation } from 'next-i18next';
 
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useContextSelector } from 'use-context-selector';
-import { WorkflowContext } from '../WorkflowComponents/context';
+import { WorkflowContext, WorkflowStateType } from '../WorkflowComponents/context';
 import { AppContext, TabEnum } from '../context';
 import RouteTab from '../RouteTab';
 import { useRouter } from 'next/router';
@@ -34,6 +34,7 @@ import {
   WorkflowInitContext
 } from '../WorkflowComponents/context/workflowInitContext';
 import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
+import { applyDiff } from '@/web/core/app/diff';
 
 const Header = () => {
   const { t } = useTranslation();
@@ -81,11 +82,14 @@ const Header = () => {
         [...future].reverse().find((snapshot) => snapshot.isSaved) ||
         past.find((snapshot) => snapshot.isSaved);
 
+      const initialState = past[past.length - 1]?.state;
+      const savedSnapshotState = applyDiff(initialState, savedSnapshot?.diff);
+
       const val = compareSnapshot(
         {
-          nodes: savedSnapshot?.nodes,
-          edges: savedSnapshot?.edges,
-          chatConfig: savedSnapshot?.chatConfig
+          nodes: savedSnapshotState?.nodes,
+          edges: savedSnapshotState?.edges,
+          chatConfig: savedSnapshotState?.chatConfig
         },
         {
           nodes: nodes,

projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/components/ContextMenu.tsx

@@ -5,7 +5,6 @@ import { useTranslation } from 'react-i18next';
 import { nodeTemplate2FlowNode } from '@/web/core/workflow/utils';
 import { CommentNode } from '@fastgpt/global/core/workflow/template/system/comment';
 import { useContextSelector } from 'use-context-selector';
-import { WorkflowContext } from '../../context';
 import { useReactFlow } from 'reactflow';
 import { WorkflowNodeEdgeContext } from '../../context/workflowInitContext';
 import { WorkflowEventContext } from '../../context/workflowEventContext';

projects/app/src/pages/app/detail/components/WorkflowComponents/context/index.tsx

@@ -2,6 +2,7 @@ import { postWorkflowDebug } from '@/web/core/workflow/api';
 import {
   checkWorkflowNodeAndConnection,
   compareSnapshot,
+  simplifyNodes,
   storeEdgesRenderEdge,
   storeNode2FlowNode
 } from '@/web/core/workflow/utils';
@@ -41,6 +42,7 @@ import { cloneDeep } from 'lodash';
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
 import WorkflowInitContextProvider, { WorkflowNodeEdgeContext } from './workflowInitContext';
 import WorkflowEventContextProvider from './workflowEventContext';
+import { applyDiff, createDiff } from '@/web/core/app/diff';
 
 /* 
   Context
@@ -67,14 +69,22 @@ export const ReactFlowCustomProvider = ({
   );
 };
 
-type OnChange<ChangesType> = (changes: ChangesType[]) => void;
-
 export type WorkflowSnapshotsType = {
+  diff?: any;
+  title: string;
+  isSaved?: boolean;
+  state?: WorkflowStateType;
+
+  // old format
+  nodes?: Node[];
+  edges?: Edge[];
+  chatConfig?: AppChatConfigType;
+};
+
+export type WorkflowStateType = {
   nodes: Node[];
   edges: Edge[];
-  title: string;
   chatConfig: AppChatConfigType;
-  isSaved?: boolean;
 };
 
 type WorkflowContextType = {
@@ -751,7 +761,7 @@ const WorkflowContextProvider = ({
     defaultValue: []
   }) as [WorkflowSnapshotsType[], (value: SetStateAction<WorkflowSnapshotsType[]>) => void];
 
-  const resetSnapshot = useMemoizedFn((state: Omit<WorkflowSnapshotsType, 'title' | 'isSaved'>) => {
+  const resetSnapshot = useMemoizedFn((state: WorkflowStateType) => {
     setNodes(state.nodes);
     setEdges(state.edges);
     setAppDetail((detail) => ({
@@ -759,20 +769,9 @@ const WorkflowContextProvider = ({
       chatConfig: state.chatConfig
     }));
   });
+
   const pushPastSnapshot = useMemoizedFn(
-    ({
-      pastNodes,
-      pastEdges,
-      customTitle,
-      chatConfig,
-      isSaved
-    }: {
-      pastNodes: Node[];
-      pastEdges: Edge[];
-      customTitle?: string;
-      chatConfig: AppChatConfigType;
-      isSaved?: boolean;
-    }) => {
+    ({ pastNodes, pastEdges, chatConfig, customTitle, isSaved }) => {
       if (!pastNodes || !pastEdges || !chatConfig) return false;
 
       if (forbiddenSaveSnapshot.current) {
@@ -780,7 +779,13 @@ const WorkflowContextProvider = ({
         return false;
       }
 
-      const pastState = past[0];
+      // Get initial state
+      const initialState = past[past.length - 1]?.state;
+      if (!initialState) return false;
+
+      // Apply latest diff to get past state
+      const pastState = applyDiff(initialState, past[0].diff);
+
       const isPastEqual = compareSnapshot(
         {
           nodes: pastNodes,
@@ -796,13 +801,21 @@ const WorkflowContextProvider = ({
 
       if (isPastEqual) return false;
 
+      // Create current state object
+      const newState = {
+        nodes: simplifyNodes(pastNodes),
+        edges: pastEdges,
+        chatConfig
+      };
+
+      // Calculate diff from initial state
+      const diff = createDiff(initialState, newState);
+
       setFuture([]);
       setPast((past) => [
         {
-          nodes: pastNodes,
-          edges: pastEdges,
+          diff,
           title: customTitle || formatTime2YMDHMS(new Date()),
-          chatConfig,
           isSaved
         },
         ...past.slice(0, 199)
@@ -811,18 +824,20 @@ const WorkflowContextProvider = ({
       return true;
     }
   );
+
   const onSwitchTmpVersion = useMemoizedFn((params: WorkflowSnapshotsType, customTitle: string) => {
     // Remove multiple "copy-"
     const copyText = t('app:version_copy');
     const regex = new RegExp(`(${copyText}-)\\1+`, 'g');
     const title = customTitle.replace(regex, `$1`);
+    const pastState = applyDiff(past[past.length - 1].state, params.diff);
 
-    resetSnapshot(params);
+    resetSnapshot(pastState);
 
     return pushPastSnapshot({
-      pastNodes: params.nodes,
-      pastEdges: params.edges,
-      chatConfig: params.chatConfig,
+      pastNodes: pastState.nodes,
+      pastEdges: pastState.edges,
+      chatConfig: pastState.chatConfig,
       customTitle: title
     });
   });
@@ -848,15 +863,19 @@ const WorkflowContextProvider = ({
     if (past[1]) {
       setFuture((future) => [past[0], ...future]);
       setPast((past) => past.slice(1));
-      resetSnapshot(past[1]);
+      const pastState = applyDiff(past[past.length - 1].state, past[1].diff);
+      resetSnapshot(pastState);
     }
   });
   const redo = useMemoizedFn(() => {
-    const futureState = future[0];
+    if (!future[0]) return;
+
+    const futureState = applyDiff(past[past.length - 1].state, future[0].diff);
 
     if (futureState) {
       setPast((past) => [future[0], ...past]);
       setFuture((future) => future.slice(1));
+
       resetSnapshot(futureState);
     }
   });
@@ -873,45 +892,113 @@ const WorkflowContextProvider = ({
     });
   }, [appId]);
 
-  const initData = useCallback(
-    async (e: Parameters<WorkflowContextType['initData']>[0], isInit?: boolean) => {
-      // Refresh web page, load init
-      if (isInit && past.length > 0) {
-        return resetSnapshot(past[0]);
+  // Convert old history format to new format
+  const convertOldFormatHistory = (past: WorkflowSnapshotsType[]) => {
+    const baseState = {
+      nodes: past[past.length - 1].state?.nodes || [],
+      edges: past[past.length - 1].state?.edges || [],
+      chatConfig: past[past.length - 1].state?.chatConfig || {}
+    };
+
+    return past.map((item, index) => {
+      if (index === past.length - 1) {
+        return {
+          title: item.title,
+          isSaved: item.isSaved,
+          state: baseState
+        };
       }
-      // If it is the initial data, save the initial snapshot
+
+      const currentState = {
+        nodes: item.nodes || [],
+        edges: item.edges || [],
+        chatConfig: item.chatConfig || {}
+      };
+
+      const diff = createDiff(baseState, currentState);
+
+      return {
+        title: item.title || formatTime2YMDHMS(new Date()),
+        isSaved: item.isSaved,
+        diff
+      };
+    });
+  };
+
+  const initData = useCallback(
+    async (
+      e: {
+        nodes: StoreNodeItemType[];
+        edges: StoreEdgeItemType[];
+        chatConfig?: AppChatConfigType;
+      },
+      isInit?: boolean
+    ) => {
+      const nodes = e.nodes?.map((item) => storeNode2FlowNode({ item, t })) || [];
+      const edges = e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })) || [];
+
+      const initialState = {
+        nodes: simplifyNodes(nodes),
+        edges,
+        chatConfig: e.chatConfig || appDetail.chatConfig
+      };
+
+      if (isInit && past.length > 0) {
+        // new format
+        if (past[0].diff) {
+          const targetState = applyDiff(
+            past[past.length - 1].state,
+            past[0].diff
+          ) as WorkflowStateType;
+
+          setNodes(targetState.nodes);
+          setEdges(targetState.edges);
+          setAppDetail((state) => ({
+            ...state,
+            chatConfig: targetState.chatConfig
+          }));
+          return;
+        }
+
+        // old format
+        if (past.some((item) => !item.state && (item.nodes || item.edges))) {
+          const newPast = convertOldFormatHistory(past);
+
+          setPast(newPast);
+
+          const latestState = applyDiff(
+            newPast[newPast.length - 1].state,
+            newPast[0].diff
+          ) as WorkflowStateType;
+
+          setNodes(latestState.nodes);
+          setEdges(latestState.edges);
+          setAppDetail((state) => ({
+            ...state,
+            chatConfig: latestState.chatConfig
+          }));
+          return;
+        }
+      }
+
+      setNodes(nodes);
+      setEdges(edges);
+      if (e.chatConfig) {
+        setAppDetail((state) => ({ ...state, chatConfig: e.chatConfig as AppChatConfigType }));
+      }
+
       if (isInit && past.length === 0) {
-        pushPastSnapshot({
-          pastNodes: e.nodes?.map((item) => storeNode2FlowNode({ item, t })) || [],
-          pastEdges: e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })) || [],
-          customTitle: t(`app:app.version_initial`),
-          chatConfig: appDetail.chatConfig,
-          isSaved: true
-        });
+        setPast([
+          {
+            title: t(`app:app.version_initial`),
+            isSaved: true,
+            state: initialState
+          }
+        ]);
         forbiddenSaveSnapshot.current = true;
       }
-
-      setNodes(e.nodes?.map((item) => storeNode2FlowNode({ item, t })) || []);
-      setEdges(e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })) || []);
-
-      const chatConfig = e.chatConfig;
-      if (chatConfig) {
-        setAppDetail((state) => ({
-          ...state,
-          chatConfig
-        }));
-      }
     },
-    [
-      appDetail.chatConfig,
-      past,
-      resetSnapshot,
-      pushPastSnapshot,
-      setAppDetail,
-      setEdges,
-      setNodes,
-      t
-    ]
+    [appDetail.chatConfig, past, setAppDetail, setEdges, setNodes, setPast, t]
   );
 
   const value = useMemo(

projects/app/src/web/core/app/diff.ts

@@ -0,0 +1,20 @@
+import { create } from 'jsondiffpatch';
+
+const createWorkflowDiffPatcher = () =>
+  create({
+    objectHash: (obj: any) => obj.id || obj.nodeId || obj._id,
+    propertyFilter: (name: string) => name !== 'selected'
+  });
+
+const diffPatcher = createWorkflowDiffPatcher();
+
+export const createDiff = <T extends Record<string, unknown>>(initialState?: T, newState?: T) => {
+  return diffPatcher.diff(initialState, newState);
+};
+
+export const applyDiff = <T extends Record<string, unknown>>(
+  initialState?: T,
+  diff?: ReturnType<typeof diffPatcher.diff>
+) => {
+  return diffPatcher.patch(structuredClone(initialState), diff) as T;
+};

projects/app/src/web/core/workflow/utils.ts

@@ -631,3 +631,14 @@ export const compareSnapshot = (
 
   return isEqual(node1, node2);
 };
+
+// remove node size
+export const simplifyNodes = (nodes: Node[]) => {
+  return nodes.map((node) => ({
+    id: node.id,
+    type: node.type,
+    position: node.position,
+    data: node.data,
+    zIndex: node.zIndex
+  }));
+};