From f3abf619ddd6be9dee7ed5807b967e06a6d7ef93 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 24 Apr 2014 16:33:29 -0300 Subject: Add splitted POST sync design docs (#5571). --- .../soledad/common/ddocs/syncs/updates/state.js | 99 ++++++++++++++++++++++ .../ddocs/syncs/views/changes_to_return/map.js | 18 ++++ .../common/ddocs/syncs/views/seen_ids/map.js | 9 ++ .../soledad/common/ddocs/syncs/views/state/map.js | 16 ++++ 4 files changed, 142 insertions(+) create mode 100644 common/src/leap/soledad/common/ddocs/syncs/updates/state.js create mode 100644 common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js create mode 100644 common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js create mode 100644 common/src/leap/soledad/common/ddocs/syncs/views/state/map.js diff --git a/common/src/leap/soledad/common/ddocs/syncs/updates/state.js b/common/src/leap/soledad/common/ddocs/syncs/updates/state.js new file mode 100644 index 00000000..cb2b6b7b --- /dev/null +++ b/common/src/leap/soledad/common/ddocs/syncs/updates/state.js @@ -0,0 +1,99 @@ +/** + * This update handler stores information about ongoing synchronization + * attempts from distinct source replicas. + * + * Normally, u1db synchronization occurs during one POST request. In order to + * split that into many serial POST requests, we store the state of each sync + * in the server, using a document with id 'u1db_sync_state'. To identify + * each sync attempt, we use a sync_id sent by the client. If we ever receive + * a new sync_id, we trash current data for that source replica and start + * over. + * + * We expect the following in the document body: + * + * { + * 'source_replica_uid': '', + * 'sync_id': '', + * 'seen_ids': [['', ], ...], // optional + * 'changes_to_return': [ // optional + * 'gen': , + * 'trans_id': '', + * 'changes_to_return': [[', , ''], ...] + * ], + * } + * + * The format of the final document stored on server is: + * + * { + * '_id': '', + * '_rev' '', + * 'ongoing_syncs': { + * '': { + * 'seen_ids': [['', [, ...], + * 'changes_to_return': { + * 'gen': , + * 'trans_id': '', + * 'changes_to_return': [ + * ['', , ''], + * ..., + * ], + * }, + * }, + * ... // info about other source replicas here + * } + * } + */ +function(doc, req) { + + // prevent updates to alien documents + if (doc != null && doc['_id'] != 'u1db_sync_state') + return [null, 'invalid data']; + + // create the document if it doesn't exist + if (!doc) + doc = { + '_id': 'u1db_sync_state', + 'ongoing_syncs': {}, + }; + + // parse and validate incoming data + var body = JSON.parse(req.body); + if (body['source_replica_uid'] == null) + return [null, 'invalid data'] + var source_replica_uid = body['source_replica_uid']; + + // trash outdated sync data for that replica if that exists + if (doc['ongoing_syncs'][source_replica_uid] != null && + doc['ongoing_syncs'][source_replica_uid] == null) + delete doc['ongoing_syncs'][source_replica_uid]; + + // create an entry for that source replica + if (doc['ongoing_syncs'][source_replica_uid] == null) + doc['ongoing_syncs'][source_replica_uid] = { + 'seen_ids': {}, + 'changes_to_return': null, + }; + + // incoming meta-data values should be exclusive, so we count how many + // arrived and deny to accomplish the transaction if the count is high. + var incoming_values = 0; + var info = doc['ongoing_syncs'][source_replica_uid] + + // add incoming seen id + if ('seen_id' in body) { + info['seen_ids'][body['seen_id'][0]] = body['seen_id'][1]; + incoming_values += 1; + } + + // add incoming changes_to_return + if ('changes_to_return' in body) { + info['changes_to_return'] = body['changes_to_return']; + incoming_values += 1; + } + + if (incoming_values != 1) + return [null, 'invalid data']; + + return [doc, 'ok']; +} + diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js new file mode 100644 index 00000000..220345dc --- /dev/null +++ b/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js @@ -0,0 +1,18 @@ +function(doc) { + if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null) + for (var source_replica_uid in doc['ongoing_syncs']) { + var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return']; + if (changes == null) + emit([source_replica_uid, 0], null); + else if (changes.length == 0) + emit([source_replica_uid, 0], []); + for (var i = 0; i < changes['changes_to_return'].length; i++) + emit( + [source_replica_uid, i], + { + 'gen': changes['gen'], + 'trans_id': changes['trans_id'], + 'next_change_to_return': changes['changes_to_return'][i], + }); + } +} diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js new file mode 100644 index 00000000..34c65b3f --- /dev/null +++ b/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js @@ -0,0 +1,9 @@ +function(doc) { + if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null) + for (var source_replica_uid in doc['ongoing_syncs']) + emit( + source_replica_uid, + { + 'seen_ids': doc['ongoing_syncs'][source_replica_uid]['seen_ids'], + }); +} diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js new file mode 100644 index 00000000..1d8f8e84 --- /dev/null +++ b/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js @@ -0,0 +1,16 @@ +function(doc) { + if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null) + for (var source_replica_uid in doc['ongoing_syncs']) { + var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return']; + if (changes == null) + emit(source_replica_uid, null); + else + emit( + source_replica_uid, + { + 'gen': changes['gen'], + 'trans_id': changes['trans_id'], + 'number_of_changes': changes['changes_to_return'].length + }); + } +} -- cgit v1.2.3