floccusaddon
/
floccus
зеркало из https://github.com/floccusaddon/floccus.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Refactor: Add location to tree items 

- Merge Overwrite and Slave into Unidirectional
- Dry up syncProcess#reconcile
This commit is contained in:
Marcel Klehr 2021-01-13 16:35:22 +01:00
Родитель 63b696a015
Коммит f95f02862e
20 изменённых файлов: 1306 добавлений и 1897 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1675
package-lock.json сгенерированный
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

18
package.json
Просмотреть файл

@ -22,8 +22,8 @@
"@babel/core": "^7.12.10",
"@babel/preset-env": "^7.12.11",
"@babel/preset-react": "^7.12.10",
"@typescript-eslint/eslint-plugin": "^4.10.0",
"@typescript-eslint/parser": "^4.10.0",
"@typescript-eslint/eslint-plugin": "^4.13.0",
"@typescript-eslint/parser": "^4.13.0",
"all-contributors-cli": "^6.19.0",
"babel-eslint": "^10.1.0",
"babel-loader": "^8.2.2",
@ -42,19 +42,19 @@
"mocha": "^8.2.1",
"node-fetch": "^2.6.1",
"prettier": "^2.2.1",
"sass": "^1.30.0",
"sass": "^1.32.3",
"sass-loader": "^8.0.2",
"seedrandom": "^3.0.5",
"selenium-webdriver": "^4.0.0-alpha.8",
"ts-loader": "^8.0.12",
"ts-loader": "^8.0.14",
"typescript": "^4.1.3",
"vuetify-loader": "^1.6.0",
"webpack": "^4.44.2",
"webpack": "^4.46.0",
"webpack-cli": "^3.3.12",
"webpack-merge": "^4.2.2"
},
"dependencies": {
"async-lock": "^1.2.6",
"async-lock": "^1.2.8",
"async-parallel": "^1.2.3",
"babel-polyfill": "^6.20.0",
"batching-toposort": "^1.2.0",
@ -68,13 +68,13 @@
"eslint-plugin-promise": "^4.1.1",
"eslint-plugin-standard": "^4.1.0",
"eslint-plugin-vue": "^6.2.2",
"humanize-duration": "^3.25.0",
"humanize-duration": "^3.25.1",
"hyperapp": "^1.2.10",
"install": "^0.13.0",
"js-base64": "^3.6.0",
"lodash": "^4.17.20",
"murmur2js": "^1.0.0",
"npm": "^6.14.10",
"npm": "^6.14.11",
"p-queue": "^5.0.0",
"picostyle": "^1.0.2",
"punycode": "^2.1.1",
@ -86,7 +86,7 @@
"vue-router": "^3.4.9",
"vue-template-compiler": "^2.6.11",
"vuelidate": "^0.7.6",
"vuetify": "^2.3.21",
"vuetify": "^2.4.2",
"vuex": "^3.6.0",
"vuex-router-sync": "^5.0.0"
}

23
src/lib/Account.js
Просмотреть файл

@ -4,16 +4,14 @@ import WebDavAdapter from './adapters/WebDav'
import FakeAdapter from './adapters/Fake'
import LocalTree from './LocalTree'
import DefaultSyncProcess from './strategies/Default'
import SlaveSyncProcess from './strategies/Slave'
import OverwriteSyncProcess from './strategies/Overwrite'
import UnidirectionalSyncProcess from './strategies/Unidirectional'
import Logger from './Logger'
import browser from './browser-api'
import AdapterFactory from './AdapterFactory'
import MergeSyncProcess from './strategies/Merge'
import LocalTabs from './LocalTabs'
import { Folder } from './Tree'
import MergeOverwrite from './strategies/OverwriteMerge'
import MergeSlave from './strategies/SlaveMerge'
import { Folder, ItemLocation } from './Tree'
import UnidirectionalMergeSyncProcess from './strategies/UnidirectionalMerge'
// register Adapters
AdapterFactory.register('nextcloud-folders', NextcloudFoldersAdapter)
@ -174,25 +172,27 @@ export default class Account {
mappings = await this.storage.getMappings()
const cacheTree = localResource instanceof LocalTree ? await this.storage.getCache() : new Folder({title: '', id: 'tabs'})
let strategy
let strategy, direction
switch (this.getData().strategy) {
case 'slave':
if (!cacheTree.children.length) {
Logger.log('Using "merge slave" strategy (no cache available)')
strategy = MergeSlave
strategy = UnidirectionalMergeSyncProcess
} else {
Logger.log('Using slave strategy')
strategy = SlaveSyncProcess
strategy = UnidirectionalSyncProcess
}
direction = ItemLocation.LOCAL
break
case 'overwrite':
if (!cacheTree.children.length) {
Logger.log('Using "merge overwrite" strategy (no cache available)')
strategy = MergeOverwrite
strategy = UnidirectionalMergeSyncProcess
} else {
Logger.log('Using "overwrite" strategy')
strategy = OverwriteSyncProcess
strategy = UnidirectionalSyncProcess
}
direction = ItemLocation.SERVER
break
default:
if (!cacheTree.children.length) {
@ -214,6 +214,9 @@ export default class Account {
this.setData({ ...this.getData(), syncing: progress })
}
)
if (direction) {
this.syncing.setDirection(direction)
}
await this.syncing.sync()
// update cache

134
src/lib/Diff.ts
Просмотреть файл

@ -1,5 +1,5 @@
import { Folder, TItem, ItemType } from './Tree'
import { Mapping } from './Mappings'
import { Folder, TItem, ItemType, TItemLocation, ItemLocation } from './Tree'
import Mappings, { MappingSnapshot } from './Mappings'
import Ordering from './interfaces/Ordering'
import batchingToposort from 'batching-toposort'
import Logger from './Logger'
@ -40,7 +40,7 @@ export interface ReorderAction {
type: 'REORDER',
payload: TItem,
oldItem?: TItem,
order?: Ordering,
order: Ordering,
oldOrder?: Ordering,
}
@ -112,16 +112,6 @@ export default class Diff {
}
}
add(diff: Diff, types:TActionType[] = []):void {
if (types.length === 0) {
diff.getActions().forEach(action => this.commit({...action}))
return
}
types.forEach(type =>
diff.getActions(type).forEach(action => this.commit({...action}))
)
}
getActions(type?: TActionType):Action[] {
if (type) {
return this.actions[type].slice()
@ -170,62 +160,76 @@ export default class Diff {
/**
* on ServerToLocal: don't map removals
* on LocalToServer:
* @param mappings
* @param isLocalToServer
* @param mappingsSnapshot
* @param targetLocation
* @param filter
*/
map(mappings:Mapping, isLocalToServer: boolean, filter: (Action)=>boolean = () => true):void {
map(mappingsSnapshot:MappingSnapshot, targetLocation: TItemLocation, filter: (Action)=>boolean = () => true): Diff {
const newDiff = new Diff
// Map payloads
this.getActions().forEach(action => {
if (action.type === ActionType.REMOVE && !isLocalToServer) {
return
}
this.getActions()
.map(a => a as Action)
.forEach(action => {
let newAction
if (!filter(action)) {
return
}
Logger.log('Mapping action ' + action.type + ' ' + (isLocalToServer ? 'LocalToServer' : 'ServerToLocal'), {...action})
if (action.type === ActionType.REORDER) {
action.oldOrder = action.order
action.order = action.order.slice().map(item => {
return {...item, id: mappings[item.type ][item.id]}
})
}
// needed because we set oldItem in the first section, so we wouldn't know anymore if it was set before
const oldItem = action.oldItem
// We have two sections here, because we want to be able to take IDs from oldItem even for moves
// but not parentIds (which do change during moves, obviously)
if (oldItem && !isLocalToServer) {
const oldId = action.oldItem.id
const newId = action.payload.id
action.oldItem = action.oldItem.clone()
action.payload = action.payload.clone()
action.payload.id = oldId
action.oldItem.id = newId
} else {
const newPayload = action.payload.clone()
newPayload.id = mappings[newPayload.type][newPayload.id]
action.oldItem = action.payload.clone()
action.payload = newPayload
}
if (oldItem && !isLocalToServer && action.type !== ActionType.MOVE) {
const oldParent = action.oldItem.parentId
const newParent = action.payload.parentId
action.payload.parentId = oldParent
action.oldItem.parentId = newParent
} else {
if (typeof action.payload.parentId !== 'undefined' && typeof mappings.folder[action.payload.parentId] === 'undefined') {
throw new Error('Cannot map parentId:' + action.payload.parentId)
if (!filter(action)) {
newDiff.commit(action)
return
}
action.oldItem.parentId = action.payload.parentId
action.payload.parentId = mappings.folder[action.payload.parentId]
}
})
/* if (action.type === ActionType.REMOVE && targetLocation !== ItemLocation.SERVER) {
newDiff.commit({...action})
return
} */
Logger.log('Mapping action ' + action.type + ' to ' + targetLocation, {...action})
// needed because we set oldItem in the first section, so we wouldn't know anymore if it was set before
const oldItem = action.oldItem
// We have two sections here, because we want to be able to take IDs from oldItem even for moves
// but not parentIds (which do change during moves, obviously)
if (oldItem && targetLocation !== ItemLocation.SERVER) {
const oldId = action.oldItem.id
const newId = action.payload.id
newAction = {
...action,
payload: action.payload.clone(false, targetLocation),
oldItem: action.oldItem.clone(false)
}
newAction.payload.id = oldId
newAction.oldItem.id = newId
} else {
newAction = {
...action,
payload: action.payload.clone(false, targetLocation),
oldItem: action.payload.clone(false)
}
newAction.payload.id = Mappings.mapId(mappingsSnapshot, action.payload, targetLocation)
}
if (oldItem && targetLocation !== ItemLocation.SERVER && action.type !== ActionType.MOVE) {
newAction.payload.parentId = action.oldItem.parentId
newAction.oldItem.parentId = action.payload.parentId
} else {
newAction.oldItem.parentId = action.payload.parentId
newAction.payload.parentId = Mappings.mapParentId(mappingsSnapshot, action.payload, targetLocation)
if (typeof newAction.payload.parentId === 'undefined' && typeof action.payload.parentId !== 'undefined') {
throw new Error('Failed to map parentId: ' + action.payload.parentId)
}
}
if (action.type === ActionType.REORDER) {
newAction.oldOrder = action.order
newAction.order = action.order.slice().map(item => {
return {...item, id: mappingsSnapshot[(targetLocation === ItemLocation.LOCAL ? ItemLocation.SERVER : ItemLocation.LOCAL) + 'To' + targetLocation][item.type][item.id]}
})
}
newDiff.commit(newAction)
})
return newDiff
}
}

5
src/lib/LocalTabs.ts
Просмотреть файл

@ -2,7 +2,7 @@ import browser from './browser-api'
import Logger from './Logger'
import { IResource } from './interfaces/Resource'
import PQueue from 'p-queue'
import { Bookmark, Folder } from './Tree'
import { Bookmark, Folder, ItemLocation } from './Tree'
import Ordering from './interfaces/Ordering'
import uniq from 'lodash/uniq'
@ -23,16 +23,19 @@ export default class LocalTabs implements IResource {
return new Folder({
title: '',
id: 'tabs',
location: ItemLocation.LOCAL,
children: uniq(tabs.map(t => t.windowId)).map(windowId => {
return new Folder({
title: '',
id: windowId,
parentId: 'tabs',
location: ItemLocation.LOCAL,
children: tabs.filter(t => t.windowId === windowId).sort(t => t.index).map(t => new Bookmark({
id: t.id,
title: '',
url: t.url,
parentId: windowId,
location: ItemLocation.LOCAL,
}))
})
})

4
src/lib/LocalTree.ts
Просмотреть файл

@ -4,7 +4,7 @@ import * as Tree from './Tree'
import { IResource } from './interfaces/Resource'
import PQueue from 'p-queue'
import Account from './Account'
import { Bookmark, Folder } from './Tree'
import { Bookmark, Folder, ItemLocation } from './Tree'
import Ordering from './interfaces/Ordering'
export default class LocalTree implements IResource {
@ -62,6 +62,7 @@ export default class LocalTree implements IResource {
}
if (node.children) {
const folder = new Tree.Folder({
location: ItemLocation.LOCAL,
id: node.id,
parentId,
title: parentId ? overrideTitle || node.title : undefined,
@ -73,6 +74,7 @@ export default class LocalTree implements IResource {
return folder
} else {
return new Tree.Bookmark({
location: ItemLocation.LOCAL,
id: node.id,
parentId,
title: node.title,

26
src/lib/Mappings.ts
Просмотреть файл

@ -1,4 +1,4 @@
import { TItemType } from './Tree'
import { TItem, TItemLocation, TItemType } from './Tree'
type InternalItemTypeMapping = { LocalToServer: Record<string, string>, ServerToLocal: Record<string, string> }
@ -101,4 +101,28 @@ export default class Mappings {
}
}
}
static mapId(mappingsSnapshot:MappingSnapshot, item: TItem, target: TItemLocation) : string|number {
if (item.location === target) {
return item.id
}
return mappingsSnapshot[item.location + 'To' + target][item.type][item.id]
}
static mapParentId(mappingsSnapshot:MappingSnapshot, item: TItem, target: TItemLocation) : string|number {
if (item.location === target) {
return item.parentId
}
return mappingsSnapshot[item.location + 'To' + target].folder[item.parentId]
}
static mappable(mappingsSnapshot: MappingSnapshot, item1: TItem, item2: TItem) : boolean {
if (Mappings.mapId(mappingsSnapshot, item1, item2.location) === item2.id) {
return true
}
if (Mappings.mapId(mappingsSnapshot, item2, item1.location) === item1.id) {
return true
}
return false
}
}

36
src/lib/Tree.ts
Просмотреть файл

@ -5,6 +5,13 @@ import * as Parallel from 'async-parallel'
const STRANGE_PROTOCOLS = ['data:', 'javascript:', 'about:', 'chrome:']
export const ItemLocation = {
LOCAL: 'Local',
SERVER: 'Server'
} as const
export type TItemLocation = (typeof ItemLocation)[keyof typeof ItemLocation];
export const ItemType = {
FOLDER: 'folder',
BOOKMARK: 'bookmark'
@ -26,13 +33,15 @@ export class Bookmark {
public title: string
public url: string
public tags: string[]
public location: TItemLocation
private hashValue: string
constructor({ id, parentId, url, title, tags }: { id:string|number, parentId:string|number, url:string, title:string, tags?: string[] }) {
constructor({ id, parentId, url, title, tags, location }: { id:string|number, parentId:string|number, url:string, title:string, tags?: string[], location: TItemLocation }) {
this.id = id
this.parentId = parentId
this.title = title
this.tags = tags
this.location = location
// not a regular bookmark
if (STRANGE_PROTOCOLS.some(proto => url.indexOf(proto) === 0)) {
@ -65,8 +74,15 @@ export class Bookmark {
return this.hashValue
}
clone():Bookmark {
return new Bookmark(this)
clone(withHash?: boolean, location?: TItemLocation):Bookmark {
return new Bookmark({...this, location: location ?? this.location})
}
withLocation<T extends TItemLocation>(location: T): Bookmark {
return new Bookmark({
...this,
location,
})
}
createIndex():any {
@ -121,9 +137,10 @@ export class Folder {
public hashValue: Record<string,string>
public isRoot = false
public loaded = true
public location: TItemLocation
private index: IItemIndex
constructor({ id, parentId, title, children, hashValue, loaded }
constructor({ id, parentId, title, children, hashValue, loaded, location }
:{
id:number|string,
parentId?:number|string,
@ -131,7 +148,8 @@ export class Folder {
// eslint-disable-next-line no-use-before-define
children?: TItem[],
hashValue?:Record<'true'|'false',string>,
loaded?: boolean
loaded?: boolean,
location: TItemLocation
}) {
this.id = id
this.parentId = parentId
@ -139,6 +157,7 @@ export class Folder {
this.children = children || []
this.hashValue = {...hashValue} || {}
this.loaded = typeof loaded !== 'undefined' ? loaded : true
this.location = location
}
// eslint-disable-next-line no-use-before-define
@ -247,11 +266,12 @@ export class Folder {
return this.hashValue[String(preserveOrder)]
}
clone(withHash?:boolean):Folder {
clone(withHash?:boolean, location?: TItemLocation):Folder {
return new Folder({
...this,
...(!withHash && { hashValue: {} }),
children: this.children.map(child => child.clone(withHash))
...(location && {location}),
children: this.children.map(child => child.clone(withHash, location ?? this.location))
})
}
@ -320,7 +340,7 @@ export class Folder {
return resource.removeFolder(this)
}
static hydrate(obj: {id: string|number, parentId?: string|number, title?: string, children: any[]}): Folder {
static hydrate(obj: {id: string|number, parentId?: string|number, title?: string, location: TItemLocation, children: any[]}): Folder {
return new Folder({
...obj,
children: obj.children

6
src/lib/adapters/Caching.ts
Просмотреть файл

@ -1,5 +1,5 @@
import * as Tree from '../Tree'
import { Bookmark, Folder } from '../Tree'
import { Bookmark, Folder, ItemLocation } from '../Tree'
import Logger from '../Logger'
import Adapter from '../interfaces/Adapter'
import browser from '../browser-api'
@ -14,7 +14,7 @@ export default class CachingAdapter implements Adapter {
protected server: any
constructor(server: any) {
this.highestId = 0
this.bookmarksCache = new Folder({ id: 0, title: 'root' })
this.bookmarksCache = new Folder({ id: 0, title: 'root', location: ItemLocation.SERVER })
}
getLabel():string {
@ -99,7 +99,7 @@ export default class CachingAdapter implements Adapter {
async createFolder(folder:Folder): Promise<string|number> {
Logger.log('CREATEFOLDER', { folder })
const newFolder = new Tree.Folder({ id: ++this.highestId, parentId: folder.parentId, title: folder.title })
const newFolder = new Tree.Folder({ id: ++this.highestId, parentId: folder.parentId, title: folder.title, location: ItemLocation.SERVER })
const foundParentFolder = this.bookmarksCache.findFolder(newFolder.parentId)
if (!foundParentFolder) {
throw new Error(browser.i18n.getMessage('Error005'))

27
src/lib/adapters/NextcloudFolders.ts
Просмотреть файл

@ -3,7 +3,7 @@
import Adapter from '../interfaces/Adapter'
import HtmlSerializer from '../serializers/Html'
import Logger from '../Logger'
import { Bookmark, Folder, TItem } from '../Tree'
import { Bookmark, Folder, ItemLocation, TItem } from '../Tree'
import { Base64 } from 'js-base64'
import AsyncLock from 'async-lock'
import browser from '../browser-api'
@ -140,7 +140,8 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
id: bm.id as number | string,
url: bm.url as string,
title: bm.title as string,
parentId: null
parentId: null,
location: ItemLocation.SERVER,
}
return bm.folders.map((parentId) => {
@ -211,7 +212,7 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
}
async _findServerRoot():Promise<Folder> {
let tree = new Folder({ id: -1, })
let tree = new Folder({ id: -1, location: ItemLocation.SERVER })
let childFolders
await Parallel.each(
this.server.serverRoot.split('/').slice(1),
@ -237,7 +238,7 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
}
currentChild = { id: json.item.id, children: [], title: json.item.title }
}
tree = new Folder({ id: currentChild.id, title: currentChild.title })
tree = new Folder({ id: currentChild.id, title: currentChild.title, location: ItemLocation.SERVER })
},
1
)
@ -245,7 +246,7 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
}
async getCompleteBookmarksTree():Promise<Folder> {
let tree = new Folder({ id: -1, })
let tree = new Folder({ id: -1, location: ItemLocation.SERVER })
if (this.server.serverRoot) {
tree = await this._findServerRoot()
}
@ -260,7 +261,7 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
this.hasFeatureHashing = true
this.hasFeatureExistenceCheck = true
let tree = new Folder({ id: -1 })
let tree = new Folder({ id: -1, location: ItemLocation.SERVER })
if (this.server.serverRoot) {
tree = await this._findServerRoot()
@ -312,12 +313,14 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
title: item.title,
parentId: folderId,
url: item.url,
location: ItemLocation.SERVER,
})
} else if (item.type === 'folder') {
const childFolder = new Folder({
id: item.id,
parentId: folderId,
title: item.title,
location: ItemLocation.SERVER,
})
childFolder.loaded = Boolean(item.children) // not children.length but whether the whole children field exists
childFolder.children = recurseChildren(item.id, item.children || [])
@ -327,7 +330,7 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
}
return recurseChildren(folderId, children)
} else {
const tree = new Folder({id: folderId})
const tree = new Folder({id: folderId, location: ItemLocation.SERVER})
const [childrenOrder, childFolders, childBookmarks] = await Promise.all([
this._getChildOrder(folderId, layers),
this._getChildFolders(folderId, layers),
@ -352,7 +355,8 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
id: child.id,
title: folder.title,
parentId: tree.id,
loaded: false
loaded: false,
location: ItemLocation.SERVER,
})
tree.children.push(newFolder)
return { newFolder, child, folder}
@ -470,7 +474,7 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
}
parentFolder.children.push(
new Folder({ id: json.item.id, title, parentId })
new Folder({ id: json.item.id, title, parentId, location: ItemLocation.SERVER })
)
this.tree.createIndex()
return json.item.id
@ -517,6 +521,7 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
id,
title,
parentId,
location: ItemLocation.SERVER,
children: children.map((item) => {
if (item.type === 'bookmark') {
return new Bookmark({
@ -524,6 +529,7 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
title: item.title,
url: item.url,
parentId: id,
location: ItemLocation.SERVER,
})
} else if (item.type === 'folder') {
return recurseChildren(item.children, item.id, item.title, id)
@ -633,7 +639,8 @@ export default class NextcloudFoldersAdapter implements Adapter, BulkImportResou
url: bm.url,
title: bm.title,
parentId: parentId,
tags: bm.tags
tags: bm.tags,
location: ItemLocation.SERVER,
})
})
}

10
src/lib/serializers/Xbel.ts
Просмотреть файл

@ -1,5 +1,5 @@
import Serializer from '../interfaces/Serializer'
import { Bookmark, Folder } from '../Tree'
import { Bookmark, Folder, ItemLocation } from '../Tree'
class XbelSerializer implements Serializer {
serialize(folder) {
@ -18,7 +18,7 @@ class XbelSerializer implements Serializer {
)
}
const rootFolder = new Folder({ id: 0, title: 'root' })
const rootFolder = new Folder({ id: 0, title: 'root', location: ItemLocation.SERVER })
this._parseFolder(nodeList[0], rootFolder)
return rootFolder
}
@ -33,13 +33,15 @@ class XbelSerializer implements Serializer {
id: parseInt(node.id),
parentId: folder.id,
url: node.getAttribute('href'),
title: node.firstElementChild.textContent
title: node.firstElementChild.textContent,
location: ItemLocation.SERVER,
})
} else if (node.tagName && node.tagName === 'folder') {
item = new Folder({
id: parseInt(node.getAttribute('id')),
title: node.firstElementChild.textContent,
parentId: folder.id
parentId: folder.id,
location: ItemLocation.SERVER,
})
this._parseFolder(node, item)
} else {

453
src/lib/strategies/Default.ts
Просмотреть файл

@ -1,11 +1,11 @@
import { Bookmark, Folder, TItem, ItemType } from '../Tree'
import { Bookmark, Folder, TItem, ItemType, ItemLocation, TItemLocation } from '../Tree'
import Logger from '../Logger'
import browser from '../browser-api'
import Diff, { Action, ActionType, CreateAction, MoveAction, RemoveAction, ReorderAction, UpdateAction } from '../Diff'
import Scanner from '../Scanner'
import * as Parallel from 'async-parallel'
import { throttle } from 'throttle-debounce'
import Mappings, { Mapping, MappingSnapshot } from '../Mappings'
import Mappings, { MappingSnapshot } from '../Mappings'
import LocalTree from '../LocalTree'
import TResource, { OrderFolderResource } from '../interfaces/Resource'
import { TAdapter } from '../interfaces/Adapter'
@ -83,35 +83,28 @@ export default class SyncProcess {
const {localDiff, serverDiff} = await this.getDiffs()
Logger.log({localDiff, serverDiff})
const {localPlan, serverPlan} = await this.reconcile(localDiff, serverDiff)
const serverPlan = await this.reconcileDiffs(localDiff, serverDiff, ItemLocation.SERVER)
const localPlan = await this.reconcileDiffs(serverDiff, localDiff, ItemLocation.LOCAL)
Logger.log({localPlan, serverPlan})
Logger.log({localTreeRoot: this.localTreeRoot, serverTreeRoot: this.serverTreeRoot, cacheTreeRoot: this.cacheTreeRoot})
this.actionsPlanned = serverPlan.getActions().length + localPlan.getActions().length
// mappings have been updated, reload
mappingsSnapshot = await this.mappings.getSnapshot()
// Weed out modifications to bookmarks root
await this.filterOutRootFolderActions(localPlan)
await this.execute(this.server, serverPlan, mappingsSnapshot.LocalToServer, true)
await this.execute(this.localTree, localPlan, mappingsSnapshot.ServerToLocal, false)
const mappedServerPlan = await this.execute(this.server, serverPlan, ItemLocation.SERVER)
const mappedLocalPlan = await this.execute(this.localTree, localPlan, ItemLocation.LOCAL)
// mappings have been updated, reload
mappingsSnapshot = await this.mappings.getSnapshot()
const localReorder = new Diff()
this.reconcileReorderings(localPlan, serverPlan, mappingsSnapshot.ServerToLocal, false)
localReorder.add(localPlan)
localReorder.map(mappingsSnapshot.ServerToLocal, false, (action) => action.type === ActionType.REORDER)
const localReorder = this.reconcileReorderings(mappedLocalPlan, mappedServerPlan, mappingsSnapshot)
.map(mappingsSnapshot, ItemLocation.LOCAL)
const serverReorder = new Diff()
this.reconcileReorderings(serverPlan, localPlan, mappingsSnapshot.LocalToServer, true)
// localReorder.add(serverPlan)
serverReorder.add(serverPlan)
serverReorder.map(mappingsSnapshot.LocalToServer, true, (action) => action.type === ActionType.REORDER)
const serverReorder = this.reconcileReorderings(mappedServerPlan, mappedLocalPlan, mappingsSnapshot)
.map(mappingsSnapshot, ItemLocation.SERVER)
await this.filterOutRootFolderActions(localReorder)
@ -203,71 +196,69 @@ export default class SyncProcess {
return {localDiff, serverDiff}
}
async reconcile(localDiff:Diff, serverDiff:Diff):Promise<{serverPlan: Diff, localPlan: Diff}> {
async reconcileDiffs(sourceDiff:Diff, targetDiff:Diff, targetLocation: TItemLocation):Promise<Diff> {
let mappingsSnapshot = await this.mappings.getSnapshot()
const serverCreations = serverDiff.getActions(ActionType.CREATE).map(a => a as CreateAction)
const serverRemovals = serverDiff.getActions(ActionType.REMOVE).map(a => a as RemoveAction)
const serverMoves = serverDiff.getActions(ActionType.MOVE).map(a => a as MoveAction)
const targetCreations = targetDiff.getActions(ActionType.CREATE).map(a => a as CreateAction)
const targetRemovals = targetDiff.getActions(ActionType.REMOVE).map(a => a as RemoveAction)
const targetMoves = targetDiff.getActions(ActionType.MOVE).map(a => a as MoveAction)
const targetUpdates = targetDiff.getActions(ActionType.UPDATE).map(a => a as UpdateAction)
const targetReorders = targetDiff.getActions(ActionType.REORDER).map(a => a as ReorderAction)
const localCreations = localDiff.getActions(ActionType.CREATE).map(a => a as CreateAction)
const localRemovals = localDiff.getActions(ActionType.REMOVE).map(a => a as RemoveAction)
const localMoves = localDiff.getActions(ActionType.MOVE).map(a => a as MoveAction)
const localUpdates = localDiff.getActions(ActionType.UPDATE).map(a => a as UpdateAction)
const localReorders = localDiff.getActions(ActionType.REORDER).map(a => a as ReorderAction)
const sourceCreations = sourceDiff.getActions(ActionType.CREATE).map(a => a as CreateAction)
const sourceRemovals = sourceDiff.getActions(ActionType.REMOVE).map(a => a as RemoveAction)
const avoidServerReorders = {}
const avoidLocalReorders = {}
const avoidTargetReorders = {}
// Prepare server plan
const serverPlan = new Diff() // to be mapped
await Parallel.each(localDiff.getActions(), async(action:Action) => {
// Prepare target plan
const targetPlan = new Diff() // to be mapped
await Parallel.each(sourceDiff.getActions(), async(action:Action) => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = serverRemovals.find(a =>
(action.payload.type === a.payload.type && String(action.payload.id) === String(a.payload.id)) ||
a.payload.findItem(ItemType.FOLDER, action.payload.parentId))
const concurrentRemoval = targetRemovals.find(a =>
(action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload)) ||
a.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, action.payload, a.payload.location)))
if (concurrentRemoval) {
// Already deleted on server, do nothing.
// Already deleted on target, do nothing.
return
}
const complexConcurrentRemoval = localRemovals.find(localRemoval => {
const complexConcurrentRemoval = sourceRemovals.find(sourceRemoval => {
let currentAction : RemoveAction|CreateAction|MoveAction =
serverDiff.getActions()
targetDiff.getActions()
.filter(a => a.type === ActionType.CREATE || a.type === ActionType.MOVE)
.map(a => a as CreateAction|MoveAction)
.find(serverAction => serverAction.payload.findItem(ItemType.FOLDER, mappingsSnapshot.LocalToServer.folder[action.payload.parentId]))
while (currentAction && !localRemoval.payload.findItem(ItemType.FOLDER, mappingsSnapshot.ServerToLocal.folder[currentAction.payload.parentId])) {
currentAction = serverDiff.getActions()
.find(targetAction => targetAction.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, action.payload, targetAction.payload.location)))
while (currentAction && !sourceRemoval.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, currentAction.payload, sourceRemoval.payload.location))) {
currentAction = targetDiff.getActions()
.filter(a => a.type === ActionType.CREATE || a.type === ActionType.MOVE)
.map(a => a as CreateAction|MoveAction)
.find(serverAction => serverAction.payload.findItem(ItemType.FOLDER, currentAction.payload.parentId))
.find(targetAction => targetAction.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, currentAction.payload, targetAction.payload.location)))
}
return Boolean(currentAction)
})
if (complexConcurrentRemoval) {
// already deleted by a different REMOVE from this diff (connected via local MOVE|CREATEs)
// already deleted by a different REMOVE from this diff (connected via source MOVE|CREATEs)
return
}
const concurrentMove = serverMoves.find(a =>
String(action.payload.id) === String(mappingsSnapshot.ServerToLocal[a.payload.type][a.payload.id]))
const concurrentMove = targetMoves.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload))
if (concurrentMove) {
// moved on the server, moves take precedence, do nothing (i.e. leave server version intact)
// moved on the target, moves take precedence, do nothing (i.e. leave target version intact)
return
}
}
if (action.type === ActionType.CREATE) {
const concurrentCreation = serverCreations.find(a => (
String(action.payload.parentId) === String(mappingsSnapshot.ServerToLocal.folder[a.payload.parentId]) &&
action.payload.canMergeWith(a.payload)
const concurrentCreation = targetCreations.find(a => (
action.payload.parentId === Mappings.mapParentId(mappingsSnapshot, a.payload, action.payload.location) &&
action.payload.canMergeWith(a.payload)
))
if (concurrentCreation) {
// created on both the server and locally, try to reconcile
// created on both the target and sourcely, try to reconcile
const newMappings = []
const subScanner = new Scanner(
concurrentCreation.payload, // server tree
action.payload, // local tree
concurrentCreation.payload, // target tree
action.payload, // source tree
(oldItem, newItem) => {
if (oldItem.type === newItem.type && oldItem.canMergeWith(newItem)) {
// if two items can be merged, we'll add mappings here directly
@ -282,268 +273,173 @@ export default class SyncProcess {
await subScanner.run()
newMappings.push([concurrentCreation.payload, action.payload.id])
await Parallel.each(newMappings, async([oldItem, newId]) => {
await this.addMapping(this.localTree, oldItem, newId)
await this.addMapping(action.payload.location === ItemLocation.LOCAL ? this.localTree : this.server, oldItem, newId)
},1)
// TODO: subScanner may contain residual CREATE/REMOVE actions that need to be added to mappings
return
}
const concurrentRemoval = serverRemovals.find(a =>
// server removal removed this creation's target
a.payload.findItem(ItemType.FOLDER, action.payload.parentId) ||
// or: server removal removed a move's target that was the target of this creation
localDiff.getActions().filter(a => a.type === ActionType.MOVE || a.type === ActionType.CREATE)
.find(a2 => a.payload.findItem(ItemType.FOLDER, a2.payload.parentId) && a2.payload.findItem(ItemType.FOLDER, action.payload.parentId))
const concurrentRemoval = targetRemovals.find(a =>
// target removal removed this creation's target
a.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, action.payload, a.payload.location)) ||
// or: target removal removed a move's target that was the target of this creation
sourceDiff.getActions().filter(a => a.type === ActionType.MOVE || a.type === ActionType.CREATE)
.find(a2 =>
a.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, a2.payload, a.payload.location)) &&
a2.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, action.payload, a2.payload.location))
)
)
if (concurrentRemoval) {
avoidServerReorders[action.payload.parentId] = true
// Already deleted on server, do nothing.
avoidTargetReorders[action.payload.parentId] = true
// Already deleted on target, do nothing.
return
}
}
if (action.type === ActionType.MOVE) {
const concurrentParentRemoval = serverRemovals.find(a =>
// server-side removal of this move's target
a.payload.findItem(ItemType.FOLDER, action.payload.parentId) ||
// or: server-side removal of a local creation's target which was the target of this move
localDiff.getActions().filter(a => a.type === ActionType.MOVE || a.type === ActionType.CREATE)
.find(a2 => a2.payload.findItem(ItemType.FOLDER, action.payload.parentId) && a.payload.findItem(ItemType.FOLDER, a2.payload.parentId)))
const concurrentParentRemoval = targetRemovals.find(a =>
// target-side removal of this move's target
a.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, action.payload, a.payload.location)) ||
// or: target-side removal of a source creation's target which was the target of this move
sourceDiff.getActions().filter(a => a.type === ActionType.MOVE || a.type === ActionType.CREATE)
.find(a2 =>
a2.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, action.payload, a2.payload.location)) &&
a.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, a2.payload, a.payload.location)))
)
if (concurrentParentRemoval) {
return
}
const concurrentRemoval = serverRemovals.find(a =>
(action.payload.type === a.payload.type && String(action.payload.id) === String(a.payload.id)) ||
a.payload.findItem(ItemType.FOLDER, action.oldItem.parentId))
if (concurrentRemoval && !localRemovals.find(a => a.payload.findItem(action.payload.type, action.payload.id))) {
// moved locally but removed on the server, recreate it on the server
serverPlan.commit({...action, type: ActionType.CREATE, oldItem: null})
const concurrentRemoval = targetRemovals.find(a =>
(action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload)) ||
a.payload.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, action.oldItem, a.payload.location)))
if (concurrentRemoval && !sourceRemovals.find(a => a.payload.findItem(action.payload.type, Mappings.mapId(mappingsSnapshot, action.payload, a.payload.location)))) {
// moved sourcely but removed on the target, recreate it on the target
const originalCreation = sourceCreations.find(creation => creation.payload.findItem(ItemType.FOLDER, action.payload.parentId))
if (originalCreation && originalCreation.payload.type === ItemType.FOLDER) {
// in case the new parent is already a newly created item, merge it into that creation
const folder = originalCreation.payload.findFolder(action.payload.parentId)
folder.children.splice(action.index, 0, action.payload)
} else {
targetPlan.commit({ ...action, type: ActionType.CREATE, oldItem: null })
}
return
}
// Find concurrent moves that form a hierarchy reversal together with this one
const concurrentHierarchyReversals = serverMoves.filter(a => {
const serverFolder = this.serverTreeRoot.findItem(ItemType.FOLDER, a.payload.id)
const localFolder = this.localTreeRoot.findItem(ItemType.FOLDER, action.payload.id)
const localAncestors = Folder.getAncestorsOf(this.localTreeRoot.findItem(ItemType.FOLDER, action.payload.parentId), this.localTreeRoot)
const serverAncestors = Folder.getAncestorsOf(this.serverTreeRoot.findItem(ItemType.FOLDER, a.payload.parentId), this.serverTreeRoot)
if (targetLocation === ItemLocation.LOCAL) {
const concurrentMove = targetMoves.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload))
if (concurrentMove) {
// Moved both on target and sourcely, source has precedence: do nothing sourcely
return
}
}
// Find concurrent moves that form a hierarchy reversal together with this one
const concurrentHierarchyReversals = targetMoves.filter(a => {
let sourceFolder, targetFolder, sourceAncestors, targetAncestors
if (action.payload.location === ItemLocation.LOCAL) {
targetFolder = this.serverTreeRoot.findItem(ItemType.FOLDER, a.payload.id)
sourceFolder = this.localTreeRoot.findItem(ItemType.FOLDER, action.payload.id)
sourceAncestors = Folder.getAncestorsOf(this.localTreeRoot.findItem(ItemType.FOLDER, action.payload.parentId), this.localTreeRoot)
targetAncestors = Folder.getAncestorsOf(this.serverTreeRoot.findItem(ItemType.FOLDER, a.payload.parentId), this.serverTreeRoot)
} else {
sourceFolder = this.serverTreeRoot.findItem(ItemType.FOLDER, action.payload.id)
targetFolder = this.localTreeRoot.findItem(ItemType.FOLDER, a.payload.id)
targetAncestors = Folder.getAncestorsOf(this.localTreeRoot.findItem(ItemType.FOLDER, a.payload.parentId), this.localTreeRoot)
sourceAncestors = Folder.getAncestorsOf(this.serverTreeRoot.findItem(ItemType.FOLDER, action.payload.parentId), this.serverTreeRoot)
}
// If both items are folders, and one of the ancestors of one item is a child of the other item
return action.payload.type === ItemType.FOLDER && a.payload.type === ItemType.FOLDER &&
localAncestors.find(ancestor => serverFolder.findItem(ItemType.FOLDER, mappingsSnapshot.LocalToServer.folder[ancestor.id])) &&
serverAncestors.find(ancestor => localFolder.findItem(ItemType.FOLDER, mappingsSnapshot.ServerToLocal.folder[ancestor.id]))
sourceAncestors.find(ancestor => targetFolder.findItem(ItemType.FOLDER, Mappings.mapId(mappingsSnapshot, ancestor, targetFolder.location))) &&
targetAncestors.find(ancestor => sourceFolder.findItem(ItemType.FOLDER, Mappings.mapId(mappingsSnapshot, ancestor, sourceFolder.location)))
})
if (concurrentHierarchyReversals.length) {
concurrentHierarchyReversals.forEach(a => {
// moved locally but moved in reverse hierarchical order on server
const payload = a.oldItem.clone() // we don't map here as we want this to look like a local action
const oldItem = a.payload.clone()
oldItem.id = mappingsSnapshot.ServerToLocal[oldItem.type][oldItem.id]
oldItem.parentId = mappingsSnapshot.ServerToLocal.folder[oldItem.parentId]
if (targetLocation === ItemLocation.SERVER) {
concurrentHierarchyReversals.forEach(a => {
// moved sourcely but moved in reverse hierarchical order on target
const payload = a.oldItem.clone() // we don't map here as we want this to look like a source action
const oldItem = a.payload.clone()
oldItem.id = Mappings.mapId(mappingsSnapshot, oldItem, action.payload.location)
oldItem.parentId = Mappings.mapParentId(mappingsSnapshot, oldItem, action.payload.location)
if (
serverPlan.getActions(ActionType.MOVE).find(move => move.payload.id === payload.id) ||
localDiff.getActions(ActionType.MOVE).find(move => move.payload.id === payload.id)
) {
if (
targetPlan.getActions(ActionType.MOVE).find(move => move.payload.id === payload.id) ||
sourceDiff.getActions(ActionType.MOVE).find(move => move.payload.id === payload.id)
) {
// Don't create duplicates!
return
}
return
}
// revert server move
serverPlan.commit({...a, payload, oldItem})
avoidServerReorders[payload.parentId] = true
avoidServerReorders[oldItem.parentId] = true
})
serverPlan.commit(action)
// revert target move
targetPlan.commit({ ...a, payload, oldItem })
avoidTargetReorders[payload.parentId] = true
avoidTargetReorders[oldItem.parentId] = true
})
targetPlan.commit(action)
} else {
// Moved sourcely and in reverse hierarchical order on target. source has precedence: do nothing sourcely
avoidTargetReorders[action.payload.parentId] = true
avoidTargetReorders[Mappings.mapParentId(mappingsSnapshot, action.oldItem, ItemLocation.SERVER)] = true
}
return
}
}
if (action.type === ActionType.REORDER) {
if (avoidServerReorders[action.payload.id]) {
if (action.type === ActionType.UPDATE && targetLocation === ItemLocation.LOCAL) {
const concurrentUpdate = targetUpdates.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload))
if (concurrentUpdate) {
// Updated both on target and sourcely, source has precedence: do nothing sourcely
return
}
const concurrentRemoval = serverRemovals.find(a =>
}
if (action.type === ActionType.REORDER) {
if (avoidTargetReorders[action.payload.id]) {
return
}
if (targetLocation === ItemLocation.LOCAL) {
const concurrentReorder = targetReorders.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload))
if (concurrentReorder) {
return
}
}
const concurrentRemoval = targetRemovals.find(a =>
a.payload.findItem('folder', action.payload.id))
if (concurrentRemoval) {
// Already deleted on server, do nothing.
// Already deleted on target, do nothing.
return
}
}
serverPlan.commit(action)
targetPlan.commit(action)
})
// Map payloads
mappingsSnapshot = await this.mappings.getSnapshot() // Necessary because of concurrent creation reconciliation
serverPlan.map(mappingsSnapshot.LocalToServer, true, (action) => action.type !== ActionType.REORDER && action.type !== ActionType.MOVE)
const mappedTargetPlan = targetPlan.map(mappingsSnapshot, targetLocation, (action) => action.type !== ActionType.REORDER && action.type !== ActionType.MOVE)
// Prepare local plan
const localPlan = new Diff()
await Parallel.each(serverDiff.getActions(), async(action:Action) => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = localRemovals.find(a =>
(action.payload.type === a.payload.type && String(action.payload.id) === String(a.payload.id)) ||
a.payload.findItem(ItemType.FOLDER, action.payload.parentId))
if (concurrentRemoval) {
// Already deleted on locally, do nothing.
return
}
const complexConcurrentRemoval = serverRemovals.find(serverRemoval => {
let currentAction : RemoveAction|CreateAction|MoveAction = localDiff.getActions()
.filter(a => a.type === ActionType.CREATE || a.type === ActionType.MOVE)
.map(a => a as CreateAction|MoveAction)
.find(localAction => localAction.payload.findItem(ItemType.FOLDER, action.payload.parentId))
while (currentAction && !serverRemoval.payload.findItem(ItemType.FOLDER, currentAction.payload.parentId)) {
currentAction = localDiff.getActions()
.filter(a => a.type === ActionType.CREATE || a.type === ActionType.MOVE)
.map(a => a as CreateAction|MoveAction)
.find(localAction => localAction.payload.findItem(ItemType.FOLDER, currentAction.payload.parentId))
}
return Boolean(currentAction)
})
if (complexConcurrentRemoval) {
// already deleted by a different REMOVE from this diff (connected via local MOVE|CREATEs)
return
}
const concurrentMove = localMoves.find(a =>
action.payload.type === a.payload.type && String(action.payload.id) === String(a.payload.id))
if (concurrentMove) {
// removed on server, moved locally, do nothing to keep it locally.
return
}
}
if (action.type === ActionType.CREATE) {
const concurrentCreation = localCreations.find(a =>
String(action.payload.parentId) === String(mappingsSnapshot.LocalToServer.folder[a.payload.parentId]) &&
action.payload.canMergeWith(a.payload)
)
if (concurrentCreation) {
// created on both the server and locally, try to reconcile
const newMappings = []
const subScanner = new Scanner(
concurrentCreation.payload,
action.payload,
(oldItem, newItem) => {
if (oldItem.type === newItem.type && oldItem.canMergeWith(newItem)) {
// if two items can be merged, we'll add mappings here directly
newMappings.push([oldItem, newItem.id])
return true
}
return false
},
this.preserveOrder,
false,
)
await subScanner.run()
// also add mappings for the two root folders
newMappings.push([concurrentCreation.payload, action.payload.id])
await Parallel.each(newMappings, async([oldItem, newId]) => {
await this.addMapping(this.server, oldItem, newId)
})
// do nothing locally if the trees differ, serverPlan takes care of adjusting the server tree
return
}
const concurrentRemoval = localRemovals.find(a =>
// local removal removed this creation's target
a.payload.findItem(ItemType.FOLDER, mappingsSnapshot.ServerToLocal.folder[action.payload.parentId]) ||
// or: local removal removed a move or creation's target that was the target of this creation
serverDiff.getActions().filter(a => a.type === ActionType.MOVE || a.type === ActionType.CREATE)
.find(a2 => a.payload.findItem(ItemType.FOLDER, mappingsSnapshot.ServerToLocal.folder[a2.payload.parentId]) && a2.payload.findItem(ItemType.FOLDER, action.payload.parentId))
)
if (concurrentRemoval) {
avoidLocalReorders[action.payload.parentId] = true
// Already deleted locally, do nothing.
return
}
}
if (action.type === ActionType.MOVE) {
const concurrentParentRemoval = localRemovals.find(a =>
// local removal of this move's target
a.payload.findItem(ItemType.FOLDER, mappingsSnapshot.ServerToLocal.folder[action.payload.parentId]) ||
// or: local removal of a server-side creation or move's target which was the target of this move
serverDiff.getActions().filter(a => a.type === ActionType.MOVE || a.type === ActionType.CREATE)
.find(a2 => a2.payload.findItem(ItemType.FOLDER, action.payload.parentId) && a.payload.findItem(ItemType.FOLDER, mappingsSnapshot.ServerToLocal.folder[a2.payload.parentId])))
if (concurrentParentRemoval) {
return
}
const concurrentRemoval = localRemovals.find(a =>
String(action.payload.id) === String(mappingsSnapshot.LocalToServer[a.payload.type][a.payload.id]) ||
a.payload.findItem(ItemType.FOLDER, action.oldItem.parentId))
if (concurrentRemoval) {
localPlan.commit({...action, type: ActionType.CREATE, oldItem: null})
return
}
const concurrentMove = localMoves.find(a =>
String(action.payload.id) === String(mappingsSnapshot.LocalToServer[a.payload.type][a.payload.id]))
if (concurrentMove) {
// Moved both on server and locally, local has precedence: do nothing locally
return
}
const concurrentHierarchyReversals = localMoves.filter(a => {
const serverFolder = this.serverTreeRoot.findItem(ItemType.FOLDER, action.payload.id)
const localFolder = this.localTreeRoot.findItem(ItemType.FOLDER, a.payload.id)
const localAncestors = Folder.getAncestorsOf(this.localTreeRoot.findItem(ItemType.FOLDER, a.payload.parentId), this.localTreeRoot)
const serverAncestors = Folder.getAncestorsOf(this.serverTreeRoot.findItem(ItemType.FOLDER, action.payload.parentId), this.serverTreeRoot)
// If both items are folders, and one of the ancestors of one item is a child of the other item
return action.payload.type === ItemType.FOLDER && a.payload.type === ItemType.FOLDER &&
localAncestors.find(ancestor => serverFolder.findItem(ItemType.FOLDER, mappingsSnapshot.LocalToServer.folder[ancestor.id])) &&
serverAncestors.find(ancestor => localFolder.findItem(ItemType.FOLDER, mappingsSnapshot.ServerToLocal.folder[ancestor.id]))
})
if (concurrentHierarchyReversals.length) {
// Moved locally and in reverse hierarchical order on server. local has precedence: do nothing locally
avoidLocalReorders[action.payload.parentId] = true
avoidLocalReorders[mappingsSnapshot.LocalToServer[action.oldItem.type][action.oldItem.parentId]] = true
return
}
}
if (action.type === ActionType.UPDATE) {
const concurrentUpdate = localUpdates.find(a =>
String(action.payload.id) === String(mappingsSnapshot.LocalToServer[a.payload.type][a.payload.id]))
if (concurrentUpdate) {
// Updated both on server and locally, local has precedence: do nothing locally
return
}
}
if (action.type === ActionType.REORDER) {
if (avoidLocalReorders[action.payload.id]) {
return
}
const concurrentReorder = localReorders.find(a =>
String(action.payload.id) === String(mappingsSnapshot.LocalToServer[a.payload.type][a.payload.id]))
if (concurrentReorder) {
return
}
const concurrentRemoval = serverRemovals.find(a =>
a.payload.findItem('folder', action.payload.id))
if (concurrentRemoval) {
// Already deleted on server, do nothing.
return
}
}
localPlan.commit(action)
})
mappingsSnapshot = await this.mappings.getSnapshot() // Necessary because of concurrent creation reconciliation
localPlan.map(mappingsSnapshot.ServerToLocal, false, (action) => action.type !== ActionType.REORDER && action.type !== ActionType.MOVE)
return { localPlan, serverPlan}
return mappedTargetPlan
}
async execute(resource:TResource, plan:Diff, mappings:Mapping, isLocalToServer:boolean):Promise<void> {
const run = (action) => this.executeAction(resource, action, isLocalToServer)
async execute(resource:TResource, plan:Diff, targetLocation:TItemLocation):Promise<Diff> {
const run = (action) => this.executeAction(resource, action, targetLocation)
await Parallel.each(plan.getActions().filter(action => action.type === ActionType.CREATE || action.type === ActionType.UPDATE), run)
const mappingsSnapshot = await this.mappings.getSnapshot()
plan.map(isLocalToServer ? mappingsSnapshot.LocalToServer : mappingsSnapshot.ServerToLocal, isLocalToServer, (action) => action.type === ActionType.MOVE)
const batches = Diff.sortMoves(plan.getActions(ActionType.MOVE), isLocalToServer ? this.serverTreeRoot : this.localTreeRoot)
const mappedPlan = plan.map(mappingsSnapshot, targetLocation, (action) => action.type === ActionType.MOVE)
const batches = Diff.sortMoves(mappedPlan.getActions(ActionType.MOVE), targetLocation === ItemLocation.SERVER ? this.serverTreeRoot : this.localTreeRoot)
await Parallel.each(batches, batch => Promise.all(batch.map(run)), 1)
await Parallel.each(plan.getActions(ActionType.REMOVE), run)
return mappedPlan
}
async executeAction(resource:TResource, action:Action, isLocalToServer:boolean):Promise<void> {
async executeAction(resource:TResource, action:Action, targetLocation:TItemLocation):Promise<void> {
const item = action.payload
if (this.canceled) {
@ -598,9 +494,9 @@ export default class SyncProcess {
if (action.oldItem && action.oldItem instanceof Folder) {
const subPlan = new Diff
action.oldItem.children.forEach((child) => subPlan.commit({ type: ActionType.CREATE, payload: child }))
const mappingsSnapshot = await this.mappings.getSnapshot()[resource === this.localTree ? 'ServerToLocal' : 'LocalToServer']
subPlan.map(mappingsSnapshot, resource !== this.localTree)
await this.execute(resource, subPlan, mappingsSnapshot, isLocalToServer)
const mappingsSnapshot = await this.mappings.getSnapshot()
const mappedSubPlan = subPlan.map(mappingsSnapshot, targetLocation)
await this.execute(resource, mappedSubPlan, targetLocation)
}
if (item.children.length > 1) {
@ -612,10 +508,10 @@ export default class SyncProcess {
payload: action.oldItem,
order: item.children.map(i => ({ type: i.type, id: i.id }))
})
const mappingsSnapshot = await this.mappings.getSnapshot()[resource === this.localTree ? 'ServerToLocal' : 'LocalToServer']
subOrder.map(mappingsSnapshot, resource !== this.localTree)
const mappingsSnapshot = await this.mappings.getSnapshot()
const mappedOrder = subOrder.map(mappingsSnapshot, targetLocation)
if ('orderFolder' in resource) {
await this.executeReorderings(resource, subOrder)
await this.executeReorderings(resource, mappedOrder)
}
}
}
@ -632,13 +528,23 @@ export default class SyncProcess {
}
}
reconcileReorderings(targetTreePlan:Diff, sourceTreePlan:Diff, sourceToTargetMappings:Mapping, isLocalToServer: boolean) : void {
reconcileReorderings(targetTreePlan: Diff, sourceTreePlan: Diff, mappingSnapshot: MappingSnapshot) : Diff {
const newPlan = new Diff
targetTreePlan
.getActions(ActionType.REORDER)
.map(a => a as ReorderAction)
// MOVEs have oldItem from cacheTree and payload now mapped to their corresponding target tree
// REORDERs have payload in source tree
.forEach(reorderAction => {
.forEach(oldReorderAction => {
// clone action
const reorderAction = {...oldReorderAction, order: oldReorderAction.order.slice()}
const removed = sourceTreePlan.getActions(ActionType.REMOVE)
.filter(removal => removal.payload.findItem(reorderAction.payload.type, removal.payload.id))
if (removed.length) {
return
}
// Find Away-moves
const childAwayMoves = sourceTreePlan.getActions(ActionType.MOVE)
.filter(move =>
@ -693,7 +599,10 @@ export default class SyncProcess {
Logger.log('ReconcileReorders: Inserting moved item into order', {move: a, reorder: reorderAction})
reorderAction.order.splice(a.index, 0, { type: a.payload.type, id: a.payload.id })
})
newPlan.commit(reorderAction)
})
return newPlan
}
async executeReorderings(resource:OrderFolderResource, reorderings:Diff):Promise<void> {

197
src/lib/strategies/Merge.ts
Просмотреть файл

@ -1,9 +1,9 @@
import { Folder, ItemType } from '../Tree'
import { Folder, ItemLocation, ItemType, TItemLocation } from '../Tree'
import Diff, { Action, ActionType } from '../Diff'
import Scanner from '../Scanner'
import * as Parallel from 'async-parallel'
import Default from './Default'
import { Mapping } from '../Mappings'
import Mappings, { MappingSnapshot } from '../Mappings'
import Logger from '../Logger'
export default class MergeSyncProcess extends Default {
@ -43,26 +43,25 @@ export default class MergeSyncProcess extends Default {
return {localDiff, serverDiff}
}
async reconcile(localDiff:Diff, serverDiff:Diff):Promise<{serverPlan: Diff, localPlan: Diff}> {
const mappingsSnapshot = await this.mappings.getSnapshot()
async reconcileDiffs(sourceDiff:Diff, targetDiff:Diff, targetLocation: TItemLocation):Promise<Diff> {
let mappingsSnapshot = await this.mappings.getSnapshot()
const serverCreations = serverDiff.getActions(ActionType.CREATE)
const serverMoves = serverDiff.getActions(ActionType.MOVE)
const targetCreations = targetDiff.getActions(ActionType.CREATE)
const targetMoves = targetDiff.getActions(ActionType.MOVE)
const localCreations = localDiff.getActions(ActionType.CREATE)
const localMoves = localDiff.getActions(ActionType.MOVE)
const localUpdates = localDiff.getActions(ActionType.UPDATE)
const sourceMoves = sourceDiff.getActions(ActionType.MOVE)
const sourceUpdates = sourceDiff.getActions(ActionType.UPDATE)
// Prepare server plan
const serverPlan = new Diff() // to be mapped
await Parallel.each(localDiff.getActions(), async(action:Action) => {
const targetPlan = new Diff() // to be mapped
await Parallel.each(sourceDiff.getActions(), async(action:Action) => {
if (action.type === ActionType.REMOVE) {
// don't execute deletes
return
}
if (action.type === ActionType.CREATE) {
const concurrentCreation = serverCreations.find(a =>
action.payload.parentId === mappingsSnapshot.ServerToLocal.folder[a.payload.parentId] &&
const concurrentCreation = targetCreations.find(a =>
a.payload.parentId === Mappings.mapParentId(mappingsSnapshot, action.payload, a.payload.location) &&
action.payload.canMergeWith(a.payload))
if (concurrentCreation) {
// created on both the server and locally, try to reconcile
@ -84,45 +83,77 @@ export default class MergeSyncProcess extends Default {
await subScanner.run()
newMappings.push([concurrentCreation.payload, action.payload.id])
await Parallel.each(newMappings, async([oldItem, newId]) => {
await this.addMapping(this.localTree, oldItem, newId)
await this.addMapping(action.payload.location === ItemLocation.LOCAL ? this.localTree : this.server, oldItem, newId)
},1)
// TODO: subScanner may contain residual CREATE/REMOVE actions that need to be added to mappings
return
}
}
if (action.type === ActionType.MOVE) {
const concurrentHierarchyReversals = serverMoves.filter(a => {
const serverFolder = this.serverTreeRoot.findItem(ItemType.FOLDER, a.payload.id)
const localFolder = this.localTreeRoot.findItem(ItemType.FOLDER, action.payload.id)
if (targetLocation === ItemLocation.LOCAL) {
const concurrentMove = sourceMoves.find(a =>
(action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload)) ||
(action.payload.type === 'bookmark' && action.payload.canMergeWith(a.payload))
)
if (concurrentMove) {
// Moved both on server and locally, local has precedence: do nothing locally
return
}
}
// Find concurrent moves that form a hierarchy reversal together with this one
const concurrentHierarchyReversals = targetMoves.filter(a => {
let sourceFolder, targetFolder, sourceAncestors, targetAncestors
if (action.payload.location === ItemLocation.LOCAL) {
targetFolder = this.serverTreeRoot.findItem(ItemType.FOLDER, a.payload.id)
sourceFolder = this.localTreeRoot.findItem(ItemType.FOLDER, action.payload.id)
const localAncestors = Folder.getAncestorsOf(this.localTreeRoot.findItem(ItemType.FOLDER, action.payload.parentId), this.localTreeRoot)
const serverAncestors = Folder.getAncestorsOf(this.serverTreeRoot.findItem(ItemType.FOLDER, a.payload.parentId), this.serverTreeRoot)
sourceAncestors = Folder.getAncestorsOf(this.localTreeRoot.findItem(ItemType.FOLDER, action.payload.parentId), this.localTreeRoot)
targetAncestors = Folder.getAncestorsOf(this.serverTreeRoot.findItem(ItemType.FOLDER, a.payload.parentId), this.serverTreeRoot)
} else {
sourceFolder = this.serverTreeRoot.findItem(ItemType.FOLDER, action.payload.id)
targetFolder = this.localTreeRoot.findItem(ItemType.FOLDER, a.payload.id)
targetAncestors = Folder.getAncestorsOf(this.localTreeRoot.findItem(ItemType.FOLDER, a.payload.parentId), this.localTreeRoot)
sourceAncestors = Folder.getAncestorsOf(this.serverTreeRoot.findItem(ItemType.FOLDER, action.payload.parentId), this.serverTreeRoot)
}
// If both items are folders, and one of the ancestors of one item is a child of the other item
return action.payload.type === ItemType.FOLDER && a.payload.type === ItemType.FOLDER &&
localAncestors.find(ancestor => serverFolder.findItem(ItemType.FOLDER, mappingsSnapshot.LocalToServer.folder[ancestor.id])) &&
serverAncestors.find(ancestor => localFolder.findItem(ItemType.FOLDER, mappingsSnapshot.ServerToLocal.folder[ancestor.id]))
sourceAncestors.find(ancestor => targetFolder.findItem(ItemType.FOLDER, Mappings.mapId(mappingsSnapshot, ancestor, targetFolder.location))) &&
targetAncestors.find(ancestor => sourceFolder.findItem(ItemType.FOLDER, Mappings.mapId(mappingsSnapshot, ancestor, sourceFolder.location)))
})
if (concurrentHierarchyReversals.length) {
concurrentHierarchyReversals.forEach(a => {
// moved locally but moved in reverse hierarchical order on server
const payload = a.oldItem.clone() // we don't map here as we want this to look like a local action
const oldItem = a.payload.clone()
oldItem.id = mappingsSnapshot.ServerToLocal[oldItem.type ][oldItem.id]
oldItem.parentId = mappingsSnapshot.ServerToLocal.folder[oldItem.parentId]
if (targetLocation === ItemLocation.SERVER) {
concurrentHierarchyReversals.forEach(a => {
// moved locally but moved in reverse hierarchical order on server
const payload = a.oldItem.clone() // we don't map here as we want this to look like a local action
const oldItem = a.payload.clone()
oldItem.id = Mappings.mapId(mappingsSnapshot, oldItem, action.payload.location)
oldItem.parentId = Mappings.mapParentId(mappingsSnapshot, oldItem, action.payload.location)
if (
serverPlan.getActions(ActionType.MOVE).find(move => move.payload.id === payload.id) ||
localDiff.getActions(ActionType.MOVE).find(move => move.payload.id === payload.id)
) {
// Don't create duplicates!
return
}
if (
targetPlan.getActions(ActionType.MOVE).find(move => move.payload.id === payload.id) ||
sourceDiff.getActions(ActionType.MOVE).find(move => move.payload.id === payload.id)
) {
// Don't create duplicates!
return
}
// revert server move
serverPlan.commit({...a, payload, oldItem})
})
serverPlan.commit(action)
// revert server move
targetPlan.commit({ ...a, payload, oldItem })
})
targetPlan.commit(action)
}
// if target === LOCAL: Moved locally and in reverse hierarchical order on server. local has precedence: do nothing locally
return
}
}
if (action.type === ActionType.UPDATE && targetLocation === ItemLocation.LOCAL) {
const concurrentUpdate = sourceUpdates.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload))
if (concurrentUpdate) {
// Updated both on server and locally, local has precedence: do nothing locally
return
}
}
@ -131,96 +162,18 @@ export default class MergeSyncProcess extends Default {
return
}
serverPlan.commit(action)
targetPlan.commit(action)
})
// Map payloads
serverPlan.map(mappingsSnapshot.LocalToServer, true, (action) => action.type !== ActionType.REORDER && action.type !== ActionType.MOVE)
mappingsSnapshot = await this.mappings.getSnapshot() // Necessary because of concurrent creation reconciliation
const mappedTargetPlan = targetPlan.map(mappingsSnapshot, targetLocation, (action) => action.type !== ActionType.REORDER && action.type !== ActionType.MOVE)
// Prepare local plan
const localPlan = new Diff()
await Parallel.each(serverDiff.getActions(), async(action:Action) => {
if (action.type === ActionType.REMOVE) {
// don't execute deletes
return
}
if (action.type === ActionType.CREATE) {
const concurrentCreation = localCreations.find(a =>
action.payload.parentId === mappingsSnapshot.LocalToServer.folder[a.payload.parentId] &&
action.payload.canMergeWith(a.payload))
if (concurrentCreation) {
// created on both the server and locally, try to reconcile
const newMappings = []
const subScanner = new Scanner(
concurrentCreation.payload,
action.payload,
(oldItem, newItem) => {
if (oldItem.type === newItem.type && oldItem.canMergeWith(newItem)) {
// if two items can be merged, we'll add mappings here directly
newMappings.push([oldItem, newItem.id])
return true
}
return false
},
this.preserveOrder,
false,
)
await subScanner.run()
// also add mappings for the two root folders
newMappings.push([concurrentCreation.payload, action.payload.id])
await Parallel.each(newMappings, async([oldItem, newId]) => {
await this.addMapping(this.server, oldItem, newId)
})
// do nothing locally if the trees differ, serverPlan takes care of adjusting the server tree
return
}
}
if (action.type === ActionType.MOVE) {
const concurrentMove = localMoves.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type][a.payload.id] || (action.payload.type === 'bookmark' && action.payload.canMergeWith(a.payload)))
if (concurrentMove) {
// Moved both on server and locally, local has precedence: do nothing locally
return
}
const concurrentHierarchyReversals = localMoves.filter(a => {
const serverFolder = this.serverTreeRoot.findItem(ItemType.FOLDER, action.payload.id)
const localFolder = this.localTreeRoot.findItem(ItemType.FOLDER, a.payload.id)
const localAncestors = Folder.getAncestorsOf(this.localTreeRoot.findItem(ItemType.FOLDER, a.payload.parentId), this.localTreeRoot)
const serverAncestors = Folder.getAncestorsOf(this.serverTreeRoot.findItem(ItemType.FOLDER, action.payload.parentId), this.serverTreeRoot)
// If both items are folders, and one of the ancestors of one item is a child of the other item
return action.payload.type === ItemType.FOLDER && a.payload.type === ItemType.FOLDER &&
localAncestors.find(ancestor => serverFolder.findItem(ItemType.FOLDER, mappingsSnapshot.LocalToServer.folder[ancestor.id])) &&
serverAncestors.find(ancestor => localFolder.findItem(ItemType.FOLDER, mappingsSnapshot.ServerToLocal.folder[ancestor.id]))
})
if (concurrentHierarchyReversals.length) {
// Moved locally and in reverse hierarchical order on server. local has precedence: do nothing locally
return
}
}
if (action.type === ActionType.UPDATE) {
const concurrentUpdate = localUpdates.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type][a.payload.id])
if (concurrentUpdate) {
// Updated both on server and locally, local has precedence: do nothing locally
return
}
}
if (action.type === ActionType.REORDER) {
// don't reorder in first sync
return
}
localPlan.commit(action)
})
localPlan.map(mappingsSnapshot.ServerToLocal, false, (action) => action.type !== ActionType.REORDER && action.type !== ActionType.MOVE)
return { localPlan, serverPlan}
return mappedTargetPlan
}
reconcileReorderings(targetTreePlan:Diff, sourceTreePlan:Diff, sourceToTargetMappings:Mapping, isLocalToServer: boolean) : void {
super.reconcileReorderings(targetTreePlan, sourceTreePlan, sourceToTargetMappings, true)
reconcileReorderings(targetTreePlan:Diff, sourceTreePlan:Diff, mappingSnapshot:MappingSnapshot) : Diff {
return super.reconcileReorderings(targetTreePlan, sourceTreePlan, mappingSnapshot)
}
async loadChildren():Promise<void> {

86
src/lib/strategies/Overwrite.ts
Просмотреть файл

@ -1,86 +0,0 @@
import DefaultStrategy from './Default'
import Diff, { ActionType } from '../Diff'
import * as Parallel from 'async-parallel'
export default class OverwriteSyncProcess extends DefaultStrategy {
async reconcile(localDiff: Diff, serverDiff: Diff):Promise<{serverPlan: Diff, localPlan: Diff}> {
const mappingsSnapshot = await this.mappings.getSnapshot()
const serverRemovals = serverDiff.getActions().filter(action => action.type === ActionType.REMOVE)
const localRemovals = localDiff.getActions().filter(action => action.type === ActionType.REMOVE)
const localMoves = localDiff.getActions().filter(action => action.type === ActionType.MOVE)
// Prepare server plan
const serverPlan = new Diff()
await Parallel.each(localDiff.getActions(), async action => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = serverRemovals.find(a =>
action.payload.id === mappingsSnapshot.ServerToLocal[a.payload.type ][a.payload.id])
if (concurrentRemoval) {
// Already deleted on server, do nothing.
return
}
}
if (action.type === ActionType.MOVE) {
const concurrentRemoval = serverRemovals.find(a =>
action.payload.id === mappingsSnapshot.ServerToLocal[a.payload.type][a.payload.id])
if (concurrentRemoval) {
// moved locally but removed on the server, recreate it on the server
serverPlan.commit({...action, type: ActionType.CREATE})
return
}
}
serverPlan.commit(action)
})
// Map payloads
serverPlan.map(mappingsSnapshot.LocalToServer, true, (action) => action.type !== ActionType.REORDER && action.type !== ActionType.MOVE)
// Prepare server plan for reversing server changes
await Parallel.each(serverDiff.getActions(), async action => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = localRemovals.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type ][a.payload.id])
if (concurrentRemoval) {
// Already deleted locally, do nothing.
return
}
const concurrentMove = localMoves.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type ][a.payload.id])
if (concurrentMove) {
// removed on the server, moved locally, do nothing to recreate it on the server.
return
}
const payload = action.payload.clone()
payload.id = null
payload.parentId = mappingsSnapshot.LocalToServer.folder[payload.parentId]
// recreate it on the server otherwise
serverPlan.commit({...action, type: ActionType.CREATE, payload, oldItem: action.payload})
return
}
if (action.type === ActionType.CREATE) {
serverPlan.commit({...action, type: ActionType.REMOVE})
return
}
if (action.type === ActionType.MOVE) {
serverPlan.commit({type: ActionType.MOVE, payload: action.oldItem, oldItem: action.payload})
return
}
if (action.type === ActionType.UPDATE) {
const payload = action.oldItem
payload.id = action.payload.id
payload.parentId = action.payload.parentId
const oldItem = action.payload
oldItem.id = action.oldItem.id
oldItem.parentId = action.oldItem.parentId
serverPlan.commit({type: ActionType.UPDATE, payload, oldItem})
}
})
const localPlan = new Diff() // empty, we don't wanna change anything here
return { localPlan, serverPlan}
}
}

101
src/lib/strategies/OverwriteMerge.ts
Просмотреть файл

@ -1,101 +0,0 @@
import Diff, { ActionType } from '../Diff'
import Scanner from '../Scanner'
import OverwriteSyncProcess from './Overwrite'
import * as Parallel from 'async-parallel'
export default class MergeOverwrite extends OverwriteSyncProcess {
async getDiffs():Promise<{localDiff:Diff, serverDiff:Diff}> {
// If there's no cache, diff the two trees directly
const newMappings = []
const localScanner = new Scanner(
this.serverTreeRoot,
this.localTreeRoot,
(serverItem, localItem) => {
if (localItem.type === serverItem.type && serverItem.canMergeWith(localItem)) {
newMappings.push([localItem, serverItem])
return true
}
return false
},
this.preserveOrder,
false
)
const serverScanner = new Scanner(
this.localTreeRoot,
this.serverTreeRoot,
(localItem, serverItem) => {
if (serverItem.type === localItem.type && serverItem.canMergeWith(localItem)) {
newMappings.push([localItem, serverItem])
return true
}
return false
},
this.preserveOrder,
false
)
const [localDiff, serverDiff] = await Promise.all([localScanner.run(), serverScanner.run()])
await Promise.all(newMappings.map(([localItem, serverItem]) => {
this.addMapping(this.server, localItem, serverItem.id)
}))
return {localDiff, serverDiff}
}
async reconcile(localDiff: Diff, serverDiff: Diff):Promise<{serverPlan: Diff, localPlan: Diff}> {
const mappingsSnapshot = await this.mappings.getSnapshot()
const localRemovals = localDiff.getActions().filter(action => action.type === ActionType.REMOVE)
const localMoves = localDiff.getActions().filter(action => action.type === ActionType.MOVE)
// Prepare server plan
const serverPlan = new Diff()
// Prepare server plan for reversing server changes
await Parallel.each(serverDiff.getActions(), async action => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = localRemovals.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type ][a.payload.id])
if (concurrentRemoval) {
// Already deleted locally, do nothing.
return
}
const concurrentMove = localMoves.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type ][a.payload.id])
if (concurrentMove) {
// removed on the server, moved locally, do nothing to recreate it on the server.
return
}
const payload = action.payload.clone()
payload.id = null
payload.parentId = mappingsSnapshot.LocalToServer.folder[payload.parentId]
// recreate it on the server otherwise
serverPlan.commit({...action, type: ActionType.CREATE, payload, oldItem: action.payload})
return
}
if (action.type === ActionType.CREATE) {
serverPlan.commit({...action, type: ActionType.REMOVE})
return
}
if (action.type === ActionType.MOVE) {
serverPlan.commit({type: ActionType.MOVE, payload: action.oldItem, oldItem: action.payload})
return
}
if (action.type === ActionType.UPDATE) {
const payload = action.oldItem
payload.id = action.payload.id
payload.parentId = action.payload.parentId
const oldItem = action.payload
oldItem.id = action.oldItem.id
oldItem.parentId = action.oldItem.parentId
serverPlan.commit({type: ActionType.UPDATE, payload, oldItem})
}
})
const localPlan = new Diff() // empty, we don't wanna change anything here
return { localPlan, serverPlan}
}
async loadChildren() :Promise<void> {
this.serverTreeRoot = await this.server.getBookmarksTree(true)
}
}

103
src/lib/strategies/Slave.ts
Просмотреть файл

@ -1,103 +0,0 @@
import DefaultStrategy from './Default'
import Diff, { ActionType } from '../Diff'
import * as Parallel from 'async-parallel'
import TResource from '../interfaces/Resource'
import { Mapping } from '../Mappings'
export default class SlaveSyncProcess extends DefaultStrategy {
async reconcile(localDiff: Diff, serverDiff: Diff): Promise<{serverPlan: Diff, localPlan: Diff}> {
const mappingsSnapshot = await this.mappings.getSnapshot()
const serverMoves = serverDiff.getActions().filter(action => action.type === ActionType.MOVE)
const serverRemovals = serverDiff.getActions().filter(action => action.type === ActionType.REMOVE)
const localRemovals = localDiff.getActions().filter(action => action.type === ActionType.REMOVE)
// Prepare local plan
const localPlan = new Diff()
await Parallel.each(serverDiff.getActions(), async action => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = localRemovals.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type ][a.payload.id])
if (concurrentRemoval) {
// Already deleted locally, do nothing.
return
}
}
if (action.type === ActionType.MOVE) {
const concurrentRemoval = localRemovals.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type ][a.payload.id])
if (concurrentRemoval) {
// moved on server but removed locally, recreate it on the server
localPlan.commit({...action, type: ActionType.CREATE})
return
}
}
localPlan.commit(action)
})
// Map payloads
localPlan.map(mappingsSnapshot.ServerToLocal, false, (action) => action.type !== ActionType.REORDER && action.type !== ActionType.MOVE)
// Prepare server plan for reversing local changes
await Parallel.each(localDiff.getActions(), async action => {
if (action.type === ActionType.REMOVE) {
let concurrentRemoval = serverRemovals.find(a =>
action.payload.id === mappingsSnapshot.ServerToLocal[a.payload.type ][a.payload.id])
if (concurrentRemoval) {
// Already deleted locally, do nothing.
return
}
concurrentRemoval = serverRemovals.find(a =>
action.payload.id === mappingsSnapshot.ServerToLocal[a.payload.type ][a.payload.id])
if (concurrentRemoval) {
// Already deleted locally, do nothing.
return
}
const concurrentMove = serverMoves.find(a =>
action.payload.id === mappingsSnapshot.ServerToLocal[a.payload.type ][a.payload.id] || (action.payload.type === 'bookmark' && action.payload.canMergeWith(a.payload)))
if (concurrentMove) {
// removed on the server, moved locally, do nothing to recreate it on the server.
return
}
// recreate it on the server otherwise
const oldItem = action.payload.clone()
oldItem.id = mappingsSnapshot.LocalToServer[oldItem.type ][oldItem.id]
oldItem.parentId = mappingsSnapshot.LocalToServer.folder[oldItem.parentId]
localPlan.commit({...action, type: ActionType.CREATE, oldItem})
return
}
if (action.type === ActionType.CREATE) {
localPlan.commit({...action, type: ActionType.REMOVE})
return
}
if (action.type === ActionType.MOVE) {
localPlan.commit({type: ActionType.MOVE, payload: action.oldItem, oldItem: action.payload})
return
}
if (action.type === ActionType.UPDATE) {
const payload = action.oldItem
payload.id = action.payload.id
payload.parentId = action.payload.parentId
const oldItem = action.payload
oldItem.id = action.oldItem.id
oldItem.parentId = action.oldItem.parentId
localPlan.commit({type: ActionType.UPDATE, payload, oldItem})
}
})
const serverPlan = new Diff() // empty, we don't wanna change anything here
return { localPlan, serverPlan}
}
async execute(resource: TResource, plan:Diff, mappings:Mapping, isLocalToServer: boolean): Promise<void> {
const run = (action) => this.executeAction(resource, action, isLocalToServer)
await Parallel.each(plan.getActions().filter(action => action.type === ActionType.CREATE || action.type === ActionType.UPDATE), run)
// Don't map here in slave mode!
await Parallel.each(plan.getActions(ActionType.MOVE), run, 1) // Don't run in parallel for weird hierarchy reversals
await Parallel.each(plan.getActions(ActionType.REMOVE), run)
}
}

83
src/lib/strategies/SlaveMerge.ts
Просмотреть файл

@ -1,83 +0,0 @@
import Diff, { ActionType } from '../Diff'
import Scanner from '../Scanner'
import Slave from './Slave'
import * as Parallel from 'async-parallel'
export default class MergeSlave extends Slave {
async getDiffs():Promise<{localDiff:Diff, serverDiff:Diff}> {
// If there's no cache, diff the two trees directly
const newMappings = []
const localScanner = new Scanner(
this.serverTreeRoot,
this.localTreeRoot,
(serverItem, localItem) => {
if (localItem.type === serverItem.type && serverItem.canMergeWith(localItem)) {
newMappings.push([localItem, serverItem])
return true
}
return false
},
this.preserveOrder,
false
)
const serverScanner = new Scanner(
this.localTreeRoot,
this.serverTreeRoot,
(localItem, serverItem) => {
if (serverItem.type === localItem.type && serverItem.canMergeWith(localItem)) {
newMappings.push([localItem, serverItem])
return true
}
return false
},
this.preserveOrder,
false
)
const [localDiff, serverDiff] = await Promise.all([localScanner.run(), serverScanner.run()])
await Promise.all(newMappings.map(([localItem, serverItem]) => {
this.addMapping(this.server, localItem, serverItem.id)
}))
return {localDiff, serverDiff}
}
async reconcile(localDiff: Diff, serverDiff: Diff): Promise<{serverPlan: Diff, localPlan: Diff}> {
const mappingsSnapshot = await this.mappings.getSnapshot()
const localRemovals = localDiff.getActions().filter(action => action.type === ActionType.REMOVE)
// Prepare local plan
const localPlan = new Diff()
await Parallel.each(serverDiff.getActions(), async action => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = localRemovals.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type ][a.payload.id])
if (concurrentRemoval) {
// Already deleted locally, do nothing.
return
}
}
if (action.type === ActionType.MOVE) {
const concurrentRemoval = localRemovals.find(a =>
action.payload.id === mappingsSnapshot.LocalToServer[a.payload.type][a.payload.id])
if (concurrentRemoval) {
// moved on server but removed locally, recreate it on the server
localPlan.commit({...action, type: ActionType.CREATE})
return
}
}
localPlan.commit(action)
})
// Map payloads
localPlan.map(mappingsSnapshot.ServerToLocal, false, (action) => action.type !== ActionType.REORDER)// && action.type !== ActionType.MOVE)
const serverPlan = new Diff() // empty, we don't wanna change anything here
return { localPlan, serverPlan}
}
async loadChildren() :Promise<void> {
this.serverTreeRoot = await this.server.getBookmarksTree(true)
}
}

119
src/lib/strategies/Unidirectional.ts Normal file
Просмотреть файл

@ -0,0 +1,119 @@
import DefaultStrategy from './Default'
import Diff, { ActionType } from '../Diff'
import * as Parallel from 'async-parallel'
import TResource from '../interfaces/Resource'
import Mappings from '../Mappings'
import { ItemLocation, TItemLocation } from '../Tree'
export default class UnidirectionalSyncProcess extends DefaultStrategy {
protected direction: TItemLocation
setDirection(direction: TItemLocation): void {
this.direction = direction
}
async reconcileDiffs(sourceDiff: Diff, targetDiff: Diff, targetLocation: TItemLocation): Promise<Diff> {
const mappingsSnapshot = await this.mappings.getSnapshot()
const masterMoves = sourceDiff.getActions().filter(action => action.type === ActionType.MOVE)
const masterRemovals = sourceDiff.getActions().filter(action => action.type === ActionType.REMOVE)
const slaveRemovals = targetDiff.getActions().filter(action => action.type === ActionType.REMOVE)
if (targetLocation === this.direction) {
// Prepare slave plan
let slavePlan = new Diff()
await Parallel.each(sourceDiff.getActions(), async action => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = slaveRemovals.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload)
)
if (concurrentRemoval) {
// Already deleted locally, do nothing.
return
}
}
if (action.type === ActionType.MOVE) {
const concurrentRemoval = slaveRemovals.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload)
)
if (concurrentRemoval) {
// moved on server but removed locally, recreate it on the server
slavePlan.commit({ ...action, type: ActionType.CREATE })
return
}
}
slavePlan.commit(action)
})
// Map payloads
slavePlan = slavePlan.map(mappingsSnapshot, targetLocation, (action) => action.type !== ActionType.REORDER && action.type !== ActionType.MOVE)
// Prepare slave plan for reversing slave changes
await Parallel.each(targetDiff.getActions(), async action => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = masterRemovals.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload)
)
if (concurrentRemoval) {
// Already deleted on slave, do nothing.
return
}
const concurrentMove = masterMoves.find(a =>
(action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload)) ||
(action.payload.type === 'bookmark' && action.payload.canMergeWith(a.payload))
)
if (concurrentMove) {
// removed on the master, moved in slave, do nothing to recreate it on the master.
return
}
// recreate it on slave resource otherwise
const payload = action.payload.clone(false, targetLocation)
payload.id = null
payload.parentId = Mappings.mapParentId(mappingsSnapshot, action.payload, targetLocation)
slavePlan.commit({...action, type: ActionType.CREATE, payload, oldItem: action.payload})
return
}
if (action.type === ActionType.CREATE) {
slavePlan.commit({ ...action, type: ActionType.REMOVE })
return
}
if (action.type === ActionType.MOVE) {
slavePlan.commit({ type: ActionType.MOVE, payload: action.oldItem, oldItem: action.payload })
return
}
if (action.type === ActionType.UPDATE) {
const payload = action.oldItem
payload.id = action.payload.id
payload.parentId = action.payload.parentId
const oldItem = action.payload
oldItem.id = action.oldItem.id
oldItem.parentId = action.oldItem.parentId
slavePlan.commit({ type: ActionType.UPDATE, payload, oldItem })
}
})
return slavePlan
} else {
return new Diff() // empty, we don't wanna change anything here
}
}
async execute(resource:TResource, plan:Diff, targetLocation:TItemLocation):Promise<Diff> {
if (this.direction === ItemLocation.LOCAL) {
const run = (action) => this.executeAction(resource, action, targetLocation)
await Parallel.each(plan.getActions().filter(action => action.type === ActionType.CREATE || action.type === ActionType.UPDATE), run)
// Don't map here in slave mode!
const batches = Diff.sortMoves(plan.getActions(ActionType.MOVE), targetLocation === ItemLocation.SERVER ? this.serverTreeRoot : this.localTreeRoot)
await Parallel.each(batches, batch => Promise.all(batch.map(run)), 1)
await Parallel.each(plan.getActions(ActionType.REMOVE), run)
return plan
} else {
return super.execute(resource, plan, targetLocation)
}
}
}

90
src/lib/strategies/UnidirectionalMerge.ts Normal file
Просмотреть файл

@ -0,0 +1,90 @@
import Diff, { ActionType } from '../Diff'
import Scanner from '../Scanner'
import Unidirectional from './Unidirectional'
import * as Parallel from 'async-parallel'
import { TItemLocation } from '../Tree'
import Mappings from '../Mappings'
export default class UnidirectionalMergeSyncProcess extends Unidirectional {
async getDiffs():Promise<{localDiff:Diff, serverDiff:Diff}> {
// If there's no cache, diff the two trees directly
const newMappings = []
const localScanner = new Scanner(
this.serverTreeRoot,
this.localTreeRoot,
(serverItem, localItem) => {
if (localItem.type === serverItem.type && serverItem.canMergeWith(localItem)) {
newMappings.push([localItem, serverItem])
return true
}
return false
},
this.preserveOrder,
false
)
const serverScanner = new Scanner(
this.localTreeRoot,
this.serverTreeRoot,
(localItem, serverItem) => {
if (serverItem.type === localItem.type && serverItem.canMergeWith(localItem)) {
newMappings.push([localItem, serverItem])
return true
}
return false
},
this.preserveOrder,
false
)
const [localDiff, serverDiff] = await Promise.all([localScanner.run(), serverScanner.run()])
await Promise.all(newMappings.map(([localItem, serverItem]) => {
this.addMapping(this.server, localItem, serverItem.id)
}))
return {localDiff, serverDiff}
}
async reconcileDiffs(sourceDiff: Diff, targetDiff: Diff, targetLocation: TItemLocation): Promise<Diff> {
const mappingsSnapshot = await this.mappings.getSnapshot()
if (targetLocation === this.direction) {
const slaveRemovals = targetDiff.getActions().filter(action => action.type === ActionType.REMOVE)
// Prepare local plan
const slavePlan = new Diff()
await Parallel.each(sourceDiff.getActions(), async action => {
if (action.type === ActionType.REMOVE) {
const concurrentRemoval = slaveRemovals.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload)
)
if (concurrentRemoval) {
// Already deleted on slave, do nothing.
return
}
}
if (action.type === ActionType.MOVE) {
const concurrentRemoval = slaveRemovals.find(a =>
action.payload.type === a.payload.type && Mappings.mappable(mappingsSnapshot, action.payload, a.payload)
)
if (concurrentRemoval) {
// moved on master but removed on slave, recreate it on slave
slavePlan.commit({ ...action, type: ActionType.CREATE })
return
}
}
slavePlan.commit(action)
})
// Map payloads
const mappedPlan = slavePlan.map(mappingsSnapshot, targetLocation, (action) => action.type !== ActionType.REORDER)
return mappedPlan
} else {
const serverPlan = new Diff() // empty, we don't wanna change anything here
return serverPlan
}
}
async loadChildren() :Promise<void> {
this.serverTreeRoot = await this.server.getBookmarksTree(true)
}
}

7
src/test/test.js
Просмотреть файл

@ -131,7 +131,8 @@ describe('Floccus', function() {
if (ACCOUNT_DATA.type === 'fake') {
account.server.bookmarksCache = new Folder({
id: '',
title: 'root'
title: 'root',
location: 'Server'
})
}
await account.init()
@ -2421,7 +2422,7 @@ describe('Floccus', function() {
if (ACCOUNT_DATA.type === 'fake') {
// Wrire both accounts to the same fake db
account2.server.bookmarksCache = account1.server.bookmarksCache = new Folder(
{ id: '', title: 'root' }
{ id: '', title: 'root', location: 'Server' }
)
}
})
@ -2989,7 +2990,7 @@ describe('Floccus', function() {
if (ACCOUNT_DATA.type === 'fake') {
// Wrire both accounts to the same fake db
account2.server.bookmarksCache = account1.server.bookmarksCache = new Folder(
{ id: '', title: 'root' }
{ id: '', title: 'root', location: 'Server' }
)
account2.server.__defineSetter__('highestId', (id) => {
account1.server.highestId = id