# __init__.py """ Support functions for migration script. """ import logging from couchdb import Server from couchdb import ResourceNotFound from couchdb import ResourceConflict from leap.soledad.common.couch import GENERATION_KEY from leap.soledad.common.couch import TRANSACTION_ID_KEY from leap.soledad.common.couch import REPLICA_UID_KEY from leap.soledad.common.couch import DOC_ID_KEY from leap.soledad.common.couch import SCHEMA_VERSION_KEY from leap.soledad.common.couch import CONFIG_DOC_ID from leap.soledad.common.couch import SYNC_DOC_ID_PREFIX from leap.soledad.common.couch import SCHEMA_VERSION logger = logging.getLogger(__name__) # # support functions # def _get_couch_server(couch_url): return Server(couch_url) def _has_u1db_config_doc(db): config_doc = db.get('u1db_config') return bool(config_doc) def _get_transaction_log(db): ddoc_path = ['_design', 'transactions', '_view', 'log'] resource = db.resource(*ddoc_path) try: _, _, data = resource.get_json() except ResourceNotFound: logger.warning( '[%s] missing transactions design document, ' 'can\'t get transaction log.' % db.name) return [] rows = data['rows'] transaction_log = [] gen = 1 for row in rows: transaction_log.append((gen, row['id'], row['value'])) gen += 1 return transaction_log def _get_user_dbs(server): user_dbs = filter(lambda dbname: dbname.startswith('user-'), server) return user_dbs # # migration main functions # def _report_missing_u1db_config_doc(dbname, db): config_doc = db.get(CONFIG_DOC_ID) if not config_doc: logger.warning( "[%s] no '%s' or '%s' documents found, possibly an empty db? I " "don't know what to do with this db, so I am skipping it." % (dbname, 'u1db_config', CONFIG_DOC_ID)) else: if SCHEMA_VERSION_KEY in config_doc: version = config_doc[SCHEMA_VERSION_KEY] if version == SCHEMA_VERSION: logger.info( "[%s] '%s' document exists, and schema versions match " "(expected %r and found %r). This database reports to be " "using the new schema version, so I am skipping it." % (dbname, CONFIG_DOC_ID, SCHEMA_VERSION, version)) else: logger.error( "[%s] '%s' document exists, but schema versions don't " "match (expected %r, found %r instead). I don't know " "how to migrate such a db, so I am skipping it." % (dbname, CONFIG_DOC_ID, SCHEMA_VERSION, version)) else: logger.error( "[%s] '%s' document exists, but has no schema version " "information in it. I don't know how to migrate such a db, " "so I am skipping it." % (dbname, CONFIG_DOC_ID)) def migrate(args, target_version): server = _get_couch_server(args.couch_url) logger.info('starting couch schema migration to %s' % target_version) if not args.do_migrate: logger.warning('dry-run: no changes will be made to databases') user_dbs = _get_user_dbs(server) for dbname in user_dbs: db = server[dbname] if not _has_u1db_config_doc(db): _report_missing_u1db_config_doc(dbname, db) continue logger.info("[%s] starting migration of user db" % dbname) try: _migrate_user_db(db, args.do_migrate) logger.info("[%s] finished migration of user db" % dbname) except: logger.exception('[%s] error migrating user db' % dbname) logger.error('continuing with next database.') logger.info('finished couch schema migration to %s' % target_version) def _migrate_user_db(db, do_migrate): _migrate_transaction_log(db, do_migrate) _migrate_sync_docs(db, do_migrate) _delete_design_docs(db, do_migrate) _migrate_config_doc(db, do_migrate) def _migrate_transaction_log(db, do_migrate): transaction_log = _get_transaction_log(db) for gen, doc_id, trans_id in transaction_log: gen_doc_id = 'gen-%s' % str(gen).zfill(10) doc = { '_id': gen_doc_id, GENERATION_KEY: gen, DOC_ID_KEY: doc_id, TRANSACTION_ID_KEY: trans_id, } logger.debug('[%s] creating gen doc: %s' % (db.name, gen_doc_id)) if do_migrate: try: db.save(doc) except ResourceConflict: # this gen document already exists. if documents are the same, # continue with migration. existing_doc = db.get(gen_doc_id) for key in [GENERATION_KEY, DOC_ID_KEY, TRANSACTION_ID_KEY]: if existing_doc[key] != doc[key]: raise def _migrate_config_doc(db, do_migrate): old_doc = db['u1db_config'] new_doc = { '_id': CONFIG_DOC_ID, REPLICA_UID_KEY: old_doc[REPLICA_UID_KEY], SCHEMA_VERSION_KEY: SCHEMA_VERSION, } logger.info("[%s] moving config doc: %s -> %s" % (db.name, old_doc['_id'], new_doc['_id'])) if do_migrate: # the config doc must not exist, otherwise we would have skipped this # database. db.save(new_doc) db.delete(old_doc) def _migrate_sync_docs(db, do_migrate): logger.info('[%s] moving sync docs' % db.name) view = db.view( '_all_docs', startkey='u1db_sync', endkey='u1db_synd', include_docs='true') for row in view.rows: old_doc = row['doc'] old_id = old_doc['_id'] # older schemas used different documents with ids starting with # "u1db_sync" to store sync-related data: # # - u1db_sync_log: was used to store the whole sync log. # - u1db_sync_state: was used to store the sync state. # # if any of these documents exist in the current db, they are leftover # from previous migrations, and should just be removed. if old_id in ['u1db_sync_log', 'u1db_sync_state']: logger.info('[%s] removing leftover document: %s' % (db.name, old_id)) if do_migrate: db.delete(old_doc) continue replica_uid = old_id.replace('u1db_sync_', '') new_id = "%s%s" % (SYNC_DOC_ID_PREFIX, replica_uid) new_doc = { '_id': new_id, GENERATION_KEY: old_doc['generation'], TRANSACTION_ID_KEY: old_doc['transaction_id'], REPLICA_UID_KEY: replica_uid, } logger.debug("[%s] moving sync doc: %s -> %s" % (db.name, old_id, new_id)) if do_migrate: try: db.save(new_doc) except ResourceConflict: # this sync document already exists. if documents are the same, # continue with migration. existing_doc = db.get(new_id) for key in [GENERATION_KEY, TRANSACTION_ID_KEY, REPLICA_UID_KEY]: if existing_doc[key] != new_doc[key]: raise db.delete(old_doc) def _delete_design_docs(db, do_migrate): for ddoc in ['docs', 'syncs', 'transactions']: doc_id = '_design/%s' % ddoc doc = db.get(doc_id) if doc: logger.info("[%s] deleting design doc: %s" % (db.name, doc_id)) if do_migrate: db.delete(doc) else: logger.warning("[%s] design doc not found: %s" % (db.name, doc_id))