summaryrefslogtreecommitdiff
path: root/scripts/migration/0.9.0/migrate_couch_schema/__init__.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/migration/0.9.0/migrate_couch_schema/__init__.py')
-rw-r--r--scripts/migration/0.9.0/migrate_couch_schema/__init__.py192
1 files changed, 192 insertions, 0 deletions
diff --git a/scripts/migration/0.9.0/migrate_couch_schema/__init__.py b/scripts/migration/0.9.0/migrate_couch_schema/__init__.py
new file mode 100644
index 00000000..f0b456e4
--- /dev/null
+++ b/scripts/migration/0.9.0/migrate_couch_schema/__init__.py
@@ -0,0 +1,192 @@
+# __init__.py
+"""
+Support functions for migration script.
+"""
+
+import logging
+
+from couchdb import Server
+from couchdb import ResourceNotFound
+from couchdb import ResourceConflict
+
+from leap.soledad.common.couch import GENERATION_KEY
+from leap.soledad.common.couch import TRANSACTION_ID_KEY
+from leap.soledad.common.couch import REPLICA_UID_KEY
+from leap.soledad.common.couch import DOC_ID_KEY
+from leap.soledad.common.couch import SCHEMA_VERSION_KEY
+from leap.soledad.common.couch import CONFIG_DOC_ID
+from leap.soledad.common.couch import SYNC_DOC_ID_PREFIX
+from leap.soledad.common.couch import SCHEMA_VERSION
+
+
+logger = logging.getLogger(__name__)
+
+
+#
+# support functions
+#
+
+def _get_couch_server(couch_url):
+ return Server(couch_url)
+
+
+def _is_migrateable(db):
+ config_doc = db.get('u1db_config')
+ return bool(config_doc)
+
+
+def _get_transaction_log(db):
+ ddoc_path = ['_design', 'transactions', '_view', 'log']
+ resource = db.resource(*ddoc_path)
+ try:
+ _, _, data = resource.get_json()
+ except ResourceNotFound:
+ logger.warning(
+ '[%s] missing transactions design document, '
+ 'can\'t get transaction log.' % db.name)
+ return []
+ rows = data['rows']
+ transaction_log = []
+ gen = 1
+ for row in rows:
+ transaction_log.append((gen, row['id'], row['value']))
+ gen += 1
+ return transaction_log
+
+
+def _get_user_dbs(server):
+ user_dbs = filter(lambda dbname: dbname.startswith('user-'), server)
+ return user_dbs
+
+
+#
+# migration main functions
+#
+
+def migrate(args, target_version):
+ server = _get_couch_server(args.couch_url)
+ logger.info('starting couch schema migration to %s' % target_version)
+ if not args.do_migrate:
+ logger.warning('dry-run: no changes will be made to databases')
+ user_dbs = _get_user_dbs(server)
+ for dbname in user_dbs:
+ db = server[dbname]
+ if not _is_migrateable(db):
+ logger.warning("[%s] skipping not migrateable user db" % dbname)
+ continue
+ logger.info("[%s] starting migration of user db" % dbname)
+ try:
+ _migrate_user_db(db, args.do_migrate)
+ logger.info("[%s] finished migration of user db" % dbname)
+ except:
+ logger.exception('[%s] error migrating user db' % dbname)
+ logger.error('continuing with next database.')
+ logger.info('finished couch schema migration to %s' % target_version)
+
+
+def _migrate_user_db(db, do_migrate):
+ _migrate_transaction_log(db, do_migrate)
+ _migrate_sync_docs(db, do_migrate)
+ _delete_design_docs(db, do_migrate)
+ _migrate_config_doc(db, do_migrate)
+
+
+def _migrate_transaction_log(db, do_migrate):
+ transaction_log = _get_transaction_log(db)
+ for gen, doc_id, trans_id in transaction_log:
+ gen_doc_id = 'gen-%s' % str(gen).zfill(10)
+ doc = {
+ '_id': gen_doc_id,
+ GENERATION_KEY: gen,
+ DOC_ID_KEY: doc_id,
+ TRANSACTION_ID_KEY: trans_id,
+ }
+ logger.debug('[%s] creating gen doc: %s' % (db.name, gen_doc_id))
+ if do_migrate:
+ try:
+ db.save(doc)
+ except ResourceConflict:
+ # this gen document already exists. if documents are the same,
+ # continue with migration.
+ existing_doc = db.get(gen_doc_id)
+ for key in [GENERATION_KEY, DOC_ID_KEY, TRANSACTION_ID_KEY]:
+ if existing_doc[key] != doc[key]:
+ raise
+
+
+def _migrate_config_doc(db, do_migrate):
+ old_doc = db['u1db_config']
+ new_doc = {
+ '_id': CONFIG_DOC_ID,
+ REPLICA_UID_KEY: old_doc[REPLICA_UID_KEY],
+ SCHEMA_VERSION_KEY: SCHEMA_VERSION,
+ }
+ logger.info("[%s] moving config doc: %s -> %s"
+ % (db.name, old_doc['_id'], new_doc['_id']))
+ if do_migrate:
+ # the config doc must not exist, otherwise we would have skipped this
+ # database.
+ db.save(new_doc)
+ db.delete(old_doc)
+
+
+def _migrate_sync_docs(db, do_migrate):
+ logger.info('[%s] moving sync docs' % db.name)
+ view = db.view(
+ '_all_docs',
+ startkey='u1db_sync',
+ endkey='u1db_synd',
+ include_docs='true')
+ for row in view.rows:
+ old_doc = row['doc']
+ old_id = old_doc['_id']
+
+ # older schemas used different documents with ids starting with
+ # "u1db_sync" to store sync-related data:
+ #
+ # - u1db_sync_log: was used to store the whole sync log.
+ # - u1db_sync_state: was used to store the sync state.
+ #
+ # if any of these documents exist in the current db, they are leftover
+ # from previous migrations, and should just be removed.
+ if old_id in ['u1db_sync_log', 'u1db_sync_state']:
+ logger.info('[%s] removing leftover document: %s'
+ % (db.name, old_id))
+ if do_migrate:
+ db.delete(old_doc)
+ continue
+
+ replica_uid = old_id.replace('u1db_sync_', '')
+ new_id = "%s%s" % (SYNC_DOC_ID_PREFIX, replica_uid)
+ new_doc = {
+ '_id': new_id,
+ GENERATION_KEY: old_doc['generation'],
+ TRANSACTION_ID_KEY: old_doc['transaction_id'],
+ REPLICA_UID_KEY: replica_uid,
+ }
+ logger.debug("[%s] moving sync doc: %s -> %s"
+ % (db.name, old_id, new_id))
+ if do_migrate:
+ try:
+ db.save(new_doc)
+ except ResourceConflict:
+ # this sync document already exists. if documents are the same,
+ # continue with migration.
+ existing_doc = db.get(new_id)
+ for key in [GENERATION_KEY, TRANSACTION_ID_KEY,
+ REPLICA_UID_KEY]:
+ if existing_doc[key] != new_doc[key]:
+ raise
+ db.delete(old_doc)
+
+
+def _delete_design_docs(db, do_migrate):
+ for ddoc in ['docs', 'syncs', 'transactions']:
+ doc_id = '_design/%s' % ddoc
+ doc = db.get(doc_id)
+ if doc:
+ logger.info("[%s] deleting design doc: %s" % (db.name, doc_id))
+ if do_migrate:
+ db.delete(doc)
+ else:
+ logger.warning("[%s] design doc not found: %s" % (db.name, doc_id))