Migrate from localStorage to indexed DB. (#886)

* Replace localstorage with WebStorageManager. (#882)

* Use singleton and merge data from localStorage.

* Add toast for displaying error messages.

* Fix build error.

* Fix typescript warning.

Co-authored-by: SimoTw <36868079+SimoTw@users.noreply.github.com>

package.json

@@ -25,6 +25,7 @@
         "exif-js": "^2.3.0",
         "file-type": "^14.6.2",
         "jszip": "^3.5.0",
+        "localforage": "^1.9.0",
         "lodash": "^4.17.20",
         "ol": "^5.3.3",
         "path-to-regexp": "^6.2.0",

src/common/webStorage.ts

@@ -0,0 +1,53 @@
+import * as localforage from "localforage";
+import { toast } from "react-toastify";
+
+class WebStorage {
+    private isStorageReady: boolean = false;
+
+    constructor() {
+        localforage.config({
+            name: "FOTT-webStorage",
+            driver: [localforage.INDEXEDDB, localforage.LOCALSTORAGE],
+        });
+    }
+
+    private async waitForStorageReady() {
+        if (!this.isStorageReady) {
+            try {
+                await localforage.ready();
+                this.isStorageReady = true;
+            } catch (error) {
+                toast.error(`No usable storage driver is found.`);
+            }
+        }
+    }
+
+    public async getItem(key: string) {
+        try {
+            await this.waitForStorageReady();
+            return await localforage.getItem(key);
+        } catch (err) {
+            toast.error(`WebStorage getItem("${key}") error: ${err}`);
+        }
+    }
+
+    public async setItem(key: string, value: any) {
+        try {
+            await this.waitForStorageReady();
+            await localforage.setItem(key, value);
+        } catch (err) {
+            toast.error(`WebStorage setItem("${key}") error: ${err}`);
+        }
+    }
+
+    public async removeItem(key: string) {
+        try {
+            await this.waitForStorageReady();
+            await localforage.removeItem(key);
+        } catch (err) {
+            toast.error(`WebStorage removeItem("${key}") error: ${err}`);
+        }
+    }
+}
+
+export const webStorage = new WebStorage();

src/index.tsx

@@ -16,36 +16,38 @@ import { registerIcons } from "./registerIcons";
 import registerProviders from "./registerProviders";
 import registerMixins from "./registerMixins";
 
-registerIcons();
-registerMixins();
-registerProviders();
-const defaultState: IApplicationState = initialState;
-const store = createReduxStore(defaultState, true);
+(async () => {
+    registerIcons();
+    registerMixins();
+    registerProviders();
+    const defaultState: IApplicationState = initialState;
+    const store = await createReduxStore(defaultState, true);
 
-let noFocusOutline = true;
-document.body.classList.add("no-focus-outline");
+    let noFocusOutline = true;
+    document.body.classList.add("no-focus-outline");
 
-document.body.addEventListener("mousedown", () => {
-    if (!noFocusOutline) {
-        noFocusOutline = true;
-        document.body.classList.add("no-focus-outline");
-    }
-});
+    document.body.addEventListener("mousedown", () => {
+        if (!noFocusOutline) {
+            noFocusOutline = true;
+            document.body.classList.add("no-focus-outline");
+        }
+    });
 
-document.body.addEventListener("keydown", (event) => {
-    if (event.keyCode === 9 && noFocusOutline) {
-        noFocusOutline = false;
-        document.body.classList.remove("no-focus-outline");
-    }
-});
+    document.body.addEventListener("keydown", (event) => {
+        if (event.keyCode === 9 && noFocusOutline) {
+            noFocusOutline = false;
+            document.body.classList.remove("no-focus-outline");
+        }
+    });
 
-ReactDOM.render(
-    <Provider store={store}>
-        <App/>
-    </Provider>
-    , document.getElementById("rootdiv"));
+    ReactDOM.render(
+        <Provider store={store}>
+            <App />
+        </Provider>
+        , document.getElementById("rootdiv"));
 
-// If you want your app to work offline and load faster, you can change
-// unregister() to register() below. Note this comes with some pitfalls.
-// Learn more about service workers: https://bit.ly/CRA-PWA
-serviceWorker.unregister();
+    // If you want your app to work offline and load faster, you can change
+    // unregister() to register() below. Note this comes with some pitfalls.
+    // Learn more about service workers: https://bit.ly/CRA-PWA
+    serviceWorker.unregister();
+})();

src/react/components/pages/appSettings/appSettingsPage.test.tsx

@@ -35,8 +35,8 @@ describe("App Settings Page", () => {
         toast.success = jest.fn(() => 2);
     });
 
-    it("renders correctly", () => {
-        const store = createStore();
+    it("renders correctly", async () => {
+        const store = await createStore();
         const wrapper = createComponent(store);
         expect(wrapper.find(AppSettingsForm).length).toEqual(1);
         expect(wrapper.find("button#toggleDevTools").length).toEqual(1);
@@ -45,7 +45,7 @@ describe("App Settings Page", () => {
 
     it("Saves app settings when click on save button", async () => {
         const appSettings = MockFactory.appSettings();
-        const store = createStore(appSettings);
+        const store = await createStore(appSettings);
         const props = createProps();
         const saveAppSettingsSpy = jest.spyOn(props.actions, "saveAppSettings");
         const goBackSpy = jest.spyOn(props.history, "goBack");
@@ -59,8 +59,8 @@ describe("App Settings Page", () => {
         expect(goBackSpy).toBeCalled();
     });
 
-    it("Navigates the user back to the previous page on cancel", () => {
-        const store = createStore();
+    it("Navigates the user back to the previous page on cancel", async () => {
+        const store = await createStore();
         const props = createProps();
         const goBackSpy = jest.spyOn(props.history, "goBack");
 
@@ -106,7 +106,7 @@ describe("App Settings Page", () => {
         };
     }
 
-    function createStore(appSettings: IAppSettings = null): Store<IApplicationState, AnyAction> {
+    async function createStore(appSettings: IAppSettings = null): Promise<Store<IApplicationState, AnyAction>> {
         const initialState: IApplicationState = {
             currentProject: null,
             appSettings: appSettings || MockFactory.appSettings(),
@@ -114,6 +114,6 @@ describe("App Settings Page", () => {
             recentProjects: [],
         };
 
-        return createReduxStore(initialState);
+        return await createReduxStore(initialState);
     }
 });

src/react/components/pages/connections/connectionsPage.test.tsx

@@ -320,6 +320,6 @@ function createProps(route: string): IConnectionPageProps {
     };
 }
 
-function createStore(state?: IApplicationState): Store<any, AnyAction> {
-    return createReduxStore(state);
+async function createStore(state?: IApplicationState): Promise<Store<any, AnyAction>> {
+    return await createReduxStore(state);
 }

src/react/components/pages/editorPage/editorPage.test.tsx

@@ -529,7 +529,7 @@ describe("Editor Page Component", () => {
     });
 });
 
-function createStore(project: IProject, setCurrentProject: boolean = false): Store<any, AnyAction> {
+async function createStore(project: IProject, setCurrentProject: boolean = false): Promise<Store<any, AnyAction>> {
     const initialState: IApplicationState = {
         currentProject: setCurrentProject ? project : null,
         appSettings: MockFactory.appSettings(),
@@ -537,7 +537,7 @@ function createStore(project: IProject, setCurrentProject: boolean = false): Sto
         recentProjects: [project],
     };
 
-    return createReduxStore(initialState);
+    return await createReduxStore(initialState);
 }
 
 async function waitForSelectedAsset(wrapper: ReactWrapper) {

src/react/components/pages/homepage/homePage.test.tsx

@@ -53,12 +53,12 @@ describe("Homepage Component", () => {
         toast.info = jest.fn(() => 2);
     });
 
-    beforeEach(() => {
+    beforeEach(async () => {
         const projectServiceMock = ProjectService as jest.Mocked<typeof ProjectService>;
         projectServiceMock.prototype.load = jest.fn((project) => Promise.resolve(project));
         projectServiceMock.prototype.delete = jest.fn(() => Promise.resolve());
 
-        store = createStore(recentProjects);
+        store = await createStore(recentProjects);
         props = createProps();
         deleteProjectSpy = jest.spyOn(props.actions, "deleteProject");
         closeProjectSpy = jest.spyOn(props.actions, "closeProject");
@@ -174,7 +174,7 @@ describe("Homepage Component", () => {
         };
     }
 
-    function createStore(recentProjects: IProject[]): Store<IApplicationState, AnyAction> {
+    async function createStore(recentProjects: IProject[]): Promise<Store<IApplicationState, AnyAction>> {
         const initialState: IApplicationState = {
             currentProject: null,
             appSettings: MockFactory.appSettings(),
@@ -183,6 +183,6 @@ describe("Homepage Component", () => {
             appError: null,
         };
 
-        return createReduxStore(initialState);
+        return await createReduxStore(initialState);
     }
 });

src/react/components/pages/projectSettings/projectSettingsPage.test.tsx

@@ -5,6 +5,7 @@ import { mount, ReactWrapper } from "enzyme";
 import React from "react";
 import { Provider } from "react-redux";
 import { BrowserRouter as Router } from "react-router-dom";
+import localforage from "localforage";
 import MockFactory from "../../../../common/mockFactory";
 import createReduxStore from "../../../../redux/store/store";
 import ProjectSettingsPage, { IProjectSettingsPageProps, IProjectSettingsPageState } from "./projectSettingsPage";
@@ -134,7 +135,7 @@ describe("Project settings page", () => {
             securityToken: `${project.name} Token`,
         });
 
-        expect(localStorage.removeItem).toBeCalledWith("projectForm");
+        expect(localforage.removeItem).toBeCalledWith("projectForm");
     });
 
     describe("project does not exists", () => {
@@ -181,7 +182,7 @@ describe("Project settings page", () => {
                 .find(ProjectSettingsPage)
                 .childAt(0) as ReactWrapper<IProjectSettingsPageProps, IProjectSettingsPageState>;
 
-            expect(localStorage.getItem).toBeCalledWith("projectForm");
+            expect(localforage.getItem).toBeCalledWith("projectForm");
             expect(projectSettingsPage.state().project).toEqual(partialProject);
         });
 
@@ -195,7 +196,7 @@ describe("Project settings page", () => {
             const projectForm = wrapper.find(ProjectForm) as ReactWrapper<IProjectFormProps>;
             projectForm.props().onChange(partialProject);
 
-            expect(localStorage.setItem).toBeCalledWith("projectForm", JSON.stringify(partialProject));
+            expect(localforage.setItem).toBeCalledWith("projectForm", JSON.stringify(partialProject));
         });
 
         it("Does NOT store empty project in local storage", () => {
@@ -209,7 +210,7 @@ describe("Project settings page", () => {
             const projectForm = wrapper.find(ProjectForm) as ReactWrapper<IProjectFormProps>;
             projectForm.props().onChange(emptyProject);
 
-            expect(localStorage.setItem).not.toBeCalled();
+            expect(localforage.setItem).not.toBeCalled();
         });
     });
 });

src/react/components/pages/projectSettings/projectSettingsPage.tsx

@@ -161,8 +161,8 @@ export default class ProjectSettingsPage extends React.Component<IProjectSetting
         }
     }
 
-    private newProjectSetting = (): void => {
-        const projectJson = getStorageItem(constants.projectFormTempKey);
+    private newProjectSetting = async (): Promise<void> => {
+        const projectJson = await getStorageItem(constants.projectFormTempKey);
         if (projectJson) {
             this.setState({ project: JSON.parse(projectJson) });
         }

src/react/components/pages/train/trainPage.tsx

@@ -29,6 +29,7 @@ import TrainPanel from "./trainPanel";
 import {ITrainRecordProps} from "./trainRecord";
 import TrainTable from "./trainTable";
 import { getAPIVersion } from "../../../../common/utils";
+import { webStorage } from "../../../../common/webStorage";
 
 export interface ITrainPageProps extends RouteComponentProps, React.Props<TrainPage> {
     connections: IConnection[];
@@ -258,11 +259,11 @@ export default class TrainPage extends React.Component<ITrainPageProps, ITrainPa
         );
     }
 
-    private checkAndUpdateInputsInLocalStorage = (projectId: string) => {
-        const storedTrainInputs = JSON.parse(window.localStorage.getItem("trainPage_inputs"));
+    private checkAndUpdateInputsInLocalStorage = async (projectId: string) => {
+        const storedTrainInputs = JSON.parse(await webStorage.getItem("trainPage_inputs") as string);
 
         if (storedTrainInputs?.id !== projectId) {
-            window.localStorage.removeItem("trainPage_inputs");
+            await webStorage.removeItem("trainPage_inputs");
             UseLocalStorage.setItem("trainPage_inputs", "projectId", projectId);
         }
 
@@ -339,8 +340,8 @@ export default class TrainPage extends React.Component<ITrainPageProps, ITrainPa
                 currTrainRecord: this.getProjectTrainRecord(),
                 modelName: "",
             }));
-            // reset localStorage successful train process
-            localStorage.setItem("trainPage_inputs", "{}");
+            // reset webStorage successful train process
+            await webStorage.setItem("trainPage_inputs", "{}");
         }).catch((err) => {
             this.setState({
                 isTraining: false,

src/react/components/shell/mainContentRouter.test.tsx

@@ -55,6 +55,6 @@ describe("Main Content Router", () => {
     });
 });
 
-function createStore(state?: IApplicationState): Store<any, AnyAction> {
-    return createReduxStore(state);
+async function createStore(state?: IApplicationState): Promise<Store<any, AnyAction>> {
+    return await createReduxStore(state);
 }

src/redux/actions/connectionActions.ts

@@ -6,6 +6,8 @@ import { ActionTypes } from "./actionTypes";
 import { IPayloadAction, createPayloadAction } from "./actionCreators";
 import { Dispatch } from "redux";
 import ConnectionService from "../../services/connectionService";
+import { webStorage } from "../../common/webStorage";
+import { constants } from "../../common/constants";
 
 /**
  * Actions to be performed in relation to connections
@@ -35,6 +37,12 @@ export function saveConnection(connection: IConnection): (dispatch: Dispatch) =>
     return async (dispatch: Dispatch) => {
         const connectionService = new ConnectionService();
         await connectionService.save(connection);
+        const projectJson = await webStorage.getItem(constants.projectFormTempKey);
+        if (projectJson) {
+            const project = JSON.parse(projectJson as string);
+            project.sourceConnection = connection;
+            await webStorage.setItem(constants.projectFormTempKey, JSON.stringify(project));
+        }
         dispatch(saveConnectionAction(connection));
         return Promise.resolve(connection);
     };

src/redux/middleware/localStorage.ts

@@ -3,6 +3,7 @@
 
 import { Middleware, Dispatch, AnyAction, MiddlewareAPI } from "redux";
 import { constants } from "../../common/constants";
+import { webStorage } from "../../common/webStorage";
 
 export interface ILocalStorageMiddlewareOptions {
     paths: string[];
@@ -24,26 +25,44 @@ export function createLocalStorage(config: ILocalStorageMiddlewareOptions): Midd
     };
 }
 
-export function mergeInitialState(state: any, paths: string[]) {
+export async function mergeInitialState(state: any, paths: string[]) {
     const initialState = { ...state };
-    paths.forEach((path) => {
-        const json = getStorageItem(path);
+
+    for (const path of paths) {
+        const isArray = Array.isArray(initialState[path]);
+
+        let legacyItem = isArray ? [] : {};
+        let item = isArray ? [] : {};
+
+        const json = (await getStorageItem(path)) as string;
         if (json) {
-            initialState[path] = JSON.parse(json);
+            item = JSON.parse(json);
         }
-    });
+
+        const legacyJson = localStorage.getItem(`${constants.version}_${path}`);
+        if (legacyJson) {
+            legacyItem = JSON.parse(legacyJson);
+            localStorage.removeItem(`${constants.version}_${path}`);
+        }
+
+        if (isArray) {
+            initialState[path] = [].concat(legacyItem, item);
+        } else {
+            initialState[path] = { ...item, ...legacyItem };
+        }
+    }
 
     return initialState;
 }
 
 export function setStorageItem(key: string, value: string) {
-    localStorage.setItem(`${constants.version}_${key}`, value);
+    webStorage.setItem(`${constants.version}_${key}`, value);
 }
 
-export function getStorageItem(key: string) {
-    return localStorage.getItem(`${constants.version}_${key}`);
+export async function getStorageItem(key: string): Promise<string> {
+    return (await webStorage.getItem(`${constants.version}_${key}`)) as string;
 }
 
 export function removeStorageItem(key: string) {
-    return localStorage.removeItem(`${constants.version}_${key}`);
+    return webStorage.removeItem(`${constants.version}_${key}`);
 }

src/redux/reducers/connectionsReducer.ts

@@ -1,11 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import { constants } from "../../common/constants";
 import { ActionTypes } from "../actions/actionTypes";
 import { IConnection } from "../../models/applicationState";
 import { AnyAction } from "../actions/actionCreators";
-import { getStorageItem, setStorageItem } from "../middleware/localStorage";
 
 /**
  * Reducer for application connections. Actions handled:
@@ -22,12 +20,6 @@ export const reducer = (state: IConnection[] = [], action: AnyAction): IConnecti
 
     switch (action.type) {
         case ActionTypes.SAVE_CONNECTION_SUCCESS:
-            const projectJson = getStorageItem(constants.projectFormTempKey);
-            if (projectJson) {
-                const project = JSON.parse(projectJson);
-                project.sourceConnection = action.payload;
-                setStorageItem(constants.projectFormTempKey, JSON.stringify(project));
-            }
             return [
                 { ...action.payload },
                 ...state.filter((connection) => connection.id !== action.payload.id),

src/redux/store/store.ts

@@ -13,9 +13,9 @@ import { Env } from "../../common/environment";
  * @param initialState - Initial state of application
  * @param useLocalStorage - Whether or not to use localStorage middleware
  */
-export default function createReduxStore(
+export default async function createReduxStore(
     initialState?: IApplicationState,
-    useLocalStorage: boolean = false): Store {
+    useLocalStorage: boolean = false): Promise<Store> {
     const paths: string[] = ["appSettings", "connections", "recentProjects", "prebuiltSettings"];
 
     let middlewares = [thunk];
@@ -39,9 +39,11 @@ export default function createReduxStore(
         ];
     }
 
+    const mergedInitialState = await mergeInitialState(initialState, paths);
+
     return createStore(
         rootReducer,
-        useLocalStorage ? mergeInitialState(initialState, paths) : initialState,
+        useLocalStorage ? mergedInitialState : initialState,
         applyMiddleware(...middlewares),
     );
 }

src/services/useLocalStorage.ts

@@ -1,7 +1,9 @@
+import { webStorage } from "../common/webStorage";
+
 export default class UseLocalStorage {
-    public static setItem(name: string, key: string, value: string) {
+    public static setItem = async (name: string, key: string, value: string) => {
         // Get the existing data
-        const existingData = window.localStorage.getItem(name);
+        const existingData = await webStorage.getItem(name) as string;
 
         // If no existing data, create an {}
         // Otherwise, convert the localStorage string to an {}
@@ -11,6 +13,6 @@ export default class UseLocalStorage {
         newLsData[key] = value;
 
         // Save back to localStorage
-        window.localStorage.setItem(name, JSON.stringify(newLsData));
+        webStorage.setItem(name, JSON.stringify(newLsData));
     }
 }

yarn.lock

@@ -8012,6 +8012,13 @@ levn@^0.3.0, levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
+lie@3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
+  integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=
+  dependencies:
+    immediate "~3.0.5"
+
 lie@~3.3.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
@@ -8086,6 +8093,13 @@ loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4
     emojis-list "^3.0.0"
     json5 "^1.0.1"
 
+localforage@^1.9.0:
+  version "1.9.0"
+  resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.9.0.tgz#f3e4d32a8300b362b4634cc4e066d9d00d2f09d1"
+  integrity sha512-rR1oyNrKulpe+VM9cYmcFn6tsHuokyVHFaCM3+osEmxaHTbEk8oQu6eGDfS6DQLWi/N67XRmB8ECG37OES368g==
+  dependencies:
+    lie "3.1.1"
+
 localized-strings@^0.2.0:
   version "0.2.4"
   resolved "https://registry.yarnpkg.com/localized-strings/-/localized-strings-0.2.4.tgz#9d61c06b60cc7b5edf7c46e6c7f2d1ecb84aeb2c"