From 0192f5923932ce738656c5b9ec25167a1b74386a Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 30 Aug 2015 05:37:21 -0300 Subject: [pkg] add Beaker as a server dependency Soledad server will use Beaker as cache provider, starting with sync_state being in memory. --- server/pkg/requirements.pip | 1 + 1 file changed, 1 insertion(+) (limited to 'server/pkg') diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index d75678b2..3e1aa992 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -4,6 +4,7 @@ u1db routes PyOpenSSL twisted +Beaker # XXX -- fix me! # oauth is not strictly needed by us, but we need it until u1db adds it to its -- cgit v1.2.3 From b065492f35006c3d108965b2b50144e080fbe678 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 17 Sep 2015 18:30:07 -0300 Subject: [feat] script for user db creation Added a simple script for user db creation and design docs creation. It uses a netrc from /etc/couchdb/couchdb-admin.netrc and same validator used on couch.py for database names. --- server/pkg/create-user-db | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 server/pkg/create-user-db (limited to 'server/pkg') diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db new file mode 100755 index 00000000..edcb8a82 --- /dev/null +++ b/server/pkg/create-user-db @@ -0,0 +1,42 @@ +#!/usr/bin/env python +import os +import sys +import netrc +import argparse +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.common.couch import is_db_name_valid + + +description = """ +Creates a user database. +This is meant to be used by Soledad Server. +""" +parser = argparse.ArgumentParser(description=description) +parser.add_argument('dbname', metavar='user-d34db33f', type=str, + help='database name on the format user-{uuid4}') +NETRC_PATH = '/etc/couchdb/couchdb-admin.netrc' + + +def url_for_db(dbname): + if not os.path.exists(NETRC_PATH): + print ('netrc not found in %s' % NETRC_PATH) + sys.exit(1) + parsed_netrc = netrc.netrc(NETRC_PATH) + host, (login, _, password) = parsed_netrc.hosts.items()[0] + url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % { + 'login':login, + 'password':password, + 'host':host, + 'dbname':dbname}) + return url + + +if __name__ == '__main__': + args = parser.parse_args() + if not is_db_name_valid(args.dbname): + print ("Invalid name! %s" % args.dbname) + sys.exit(1) + url = url_for_db(args.dbname) + db = CouchDatabase.open_database(url=url, create=True, + replica_uid=None, ensure_ddocs=True) + print ('success! Created %s, replica_uid: %s' % (db._dbname, db.replica_uid)) -- cgit v1.2.3 From de0cf00b4412e253a481ff19803bab66ffc4443e Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 24 Sep 2015 21:57:26 -0300 Subject: [refactor] kaliy's review and pep8 fixes README with information about latest change, missing docs and licenses, variable naming and pep8. --- server/pkg/create-user-db | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'server/pkg') diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index edcb8a82..dd68f792 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -1,4 +1,20 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +# create-user-db +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import os import sys import netrc -- cgit v1.2.3 From 3c7a41574ed1a97ae168bbbc50b127d17694734a Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 28 Sep 2015 16:35:19 -0300 Subject: [style] pep8 --- server/pkg/create-user-db | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'server/pkg') diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index dd68f792..1a7e77a7 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -29,7 +29,7 @@ This is meant to be used by Soledad Server. """ parser = argparse.ArgumentParser(description=description) parser.add_argument('dbname', metavar='user-d34db33f', type=str, - help='database name on the format user-{uuid4}') + help='database name on the format user-{uuid4}') NETRC_PATH = '/etc/couchdb/couchdb-admin.netrc' @@ -40,10 +40,10 @@ def url_for_db(dbname): parsed_netrc = netrc.netrc(NETRC_PATH) host, (login, _, password) = parsed_netrc.hosts.items()[0] url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % { - 'login':login, - 'password':password, - 'host':host, - 'dbname':dbname}) + 'login': login, + 'password': password, + 'host': host, + 'dbname': dbname}) return url @@ -55,4 +55,5 @@ if __name__ == '__main__': url = url_for_db(args.dbname) db = CouchDatabase.open_database(url=url, create=True, replica_uid=None, ensure_ddocs=True) - print ('success! Created %s, replica_uid: %s' % (db._dbname, db.replica_uid)) + print ('success! Created %s, replica_uid: %s' % + (db._dbname, db.replica_uid)) -- cgit v1.2.3 From 7a0dba8b7008aca8652f7b334352d9ca63cb2054 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 30 Sep 2015 15:27:21 -0300 Subject: [feat] read netrc path from configuration file netrc file was hardcoded inside create-user-db. Now it reads the path from /etc/leap/soledad-server.conf as done on server process. The new configuration property is called 'admin_netrc'. --- server/pkg/create-user-db | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'server/pkg') diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index 1a7e77a7..ae2f0f57 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -21,6 +21,7 @@ import netrc import argparse from leap.soledad.common.couch import CouchDatabase from leap.soledad.common.couch import is_db_name_valid +from leap.soledad.server import load_configuration description = """ @@ -30,7 +31,7 @@ This is meant to be used by Soledad Server. parser = argparse.ArgumentParser(description=description) parser.add_argument('dbname', metavar='user-d34db33f', type=str, help='database name on the format user-{uuid4}') -NETRC_PATH = '/etc/couchdb/couchdb-admin.netrc' +NETRC_PATH = load_configuration('/etc/leap/soledad-server.conf')['admin_netrc'] def url_for_db(dbname): -- cgit v1.2.3 From a48b630ef48377b7307b59333f8eb02190786012 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 2 Oct 2015 17:37:52 -0300 Subject: [pkg] pin beaker and couchdb for wheezy Wheezy has python-couchdb 0.8 and python-beaker 1.6.3. Pinning them to avoid false positives on tests. --- server/pkg/requirements.pip | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'server/pkg') diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index 3e1aa992..58834d0e 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -1,10 +1,11 @@ configparser -couchdb u1db routes PyOpenSSL twisted -Beaker +#pinned for wheezy compatibility +Beaker==1.6.3 #wheezy +couchdb==0.8 #wheezy # XXX -- fix me! # oauth is not strictly needed by us, but we need it until u1db adds it to its -- cgit v1.2.3 From 147986547f400f79e8bc0d50af5ae7b5d2a140b4 Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 5 Oct 2015 11:54:56 +0200 Subject: [feat] Move config dir to /etc/soledad - Resolves: #7509 --- server/pkg/create-user-db | 2 +- server/pkg/soledad-server | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'server/pkg') diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index ae2f0f57..7eafc945 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -31,7 +31,7 @@ This is meant to be used by Soledad Server. parser = argparse.ArgumentParser(description=description) parser.add_argument('dbname', metavar='user-d34db33f', type=str, help='database name on the format user-{uuid4}') -NETRC_PATH = load_configuration('/etc/leap/soledad-server.conf')['admin_netrc'] +NETRC_PATH = load_configuration('/etc/soledad/soledad-server.conf')['admin_netrc'] def url_for_db(dbname): diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server index 811ad55b..74ed122e 100644 --- a/server/pkg/soledad-server +++ b/server/pkg/soledad-server @@ -11,12 +11,12 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin PIDFILE=/var/run/soledad.pid -RUNDIR=/var/lib/soledad/ OBJ=leap.soledad.server.application LOGFILE=/var/log/soledad.log HTTPS_PORT=2424 -CERT_PATH=/etc/leap/soledad-server.pem -PRIVKEY_PATH=/etc/leap/soledad-server.key +CONFDIR=/etc/soledad +CERT_PATH="${CONFDIR}/soledad-server.pem" +PRIVKEY_PATH="${CONFDIR}/soledad-server.key" TWISTD_PATH=/usr/bin/twistd HOME=/var/lib/soledad/ SSL_METHOD=SSLv23_METHOD @@ -25,7 +25,7 @@ GROUP=soledad [ -r /etc/default/soledad ] && . /etc/default/soledad -test -r /etc/leap/ || exit 0 +test -r ${CONFDIR} || exit 0 . /lib/lsb/init-functions -- cgit v1.2.3 From f8d38125098829fe50199725545365d6d2a889a6 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 26 Oct 2015 18:50:20 -0300 Subject: [feat] read security doc from configuration LEAP Platform needs to granularly allow access on user database for other services, like mx. This is now possible by editing soledad-server.conf file. A new section 'database-security' was added and it is parsed during 'create-user-db' to be set on security design document, present on every per-user database. --- server/pkg/create-user-db | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'server/pkg') diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index 7eafc945..28d1cbd0 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -31,7 +31,8 @@ This is meant to be used by Soledad Server. parser = argparse.ArgumentParser(description=description) parser.add_argument('dbname', metavar='user-d34db33f', type=str, help='database name on the format user-{uuid4}') -NETRC_PATH = load_configuration('/etc/soledad/soledad-server.conf')['admin_netrc'] +CONF = load_configuration('/etc/soledad/soledad-server.conf') +NETRC_PATH = CONF['soledad-server']['admin_netrc'] def url_for_db(dbname): @@ -54,7 +55,9 @@ if __name__ == '__main__': print ("Invalid name! %s" % args.dbname) sys.exit(1) url = url_for_db(args.dbname) + db_security = CONF['database-security'] db = CouchDatabase.open_database(url=url, create=True, - replica_uid=None, ensure_ddocs=True) + replica_uid=None, ensure_ddocs=True, + database_security=db_security) print ('success! Created %s, replica_uid: %s' % (db._dbname, db.replica_uid)) -- cgit v1.2.3 From bc8e8bea61b01e0b93593e24a816574f5a19ac61 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 27 Oct 2015 13:30:07 -0300 Subject: [feat] adds --migrate-all to create-user-db script This parameter applies the ensure_database to all existing databases, making all of them use the latest design documents. This can be used to migrate security documents, update handlers or any other kind of design document. --- server/pkg/create-user-db | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) (limited to 'server/pkg') diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index 28d1cbd0..ae5d15dc 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -21,6 +21,7 @@ import netrc import argparse from leap.soledad.common.couch import CouchDatabase from leap.soledad.common.couch import is_db_name_valid +from leap.soledad.common.couch import list_users_dbs from leap.soledad.server import load_configuration @@ -30,7 +31,10 @@ This is meant to be used by Soledad Server. """ parser = argparse.ArgumentParser(description=description) parser.add_argument('dbname', metavar='user-d34db33f', type=str, + default='', nargs='?', help='database name on the format user-{uuid4}') +parser.add_argument('--migrate-all', action='store_true', + help="recreate all design docs for all existing account") CONF = load_configuration('/etc/soledad/soledad-server.conf') NETRC_PATH = CONF['soledad-server']['admin_netrc'] @@ -49,15 +53,34 @@ def url_for_db(dbname): return url -if __name__ == '__main__': - args = parser.parse_args() - if not is_db_name_valid(args.dbname): - print ("Invalid name! %s" % args.dbname) +def ensure_database(dbname): + """ + This method will ensure that a database named `dbname` will exist + or created if it doesn't. Calling it twice will ensure that design + documents are present and updated. + The database name has to match this criteria to be considered valid: + user-[a-f0-9]+ + + :param dbname: name of the user database + :type dbname: str + """ + if not is_db_name_valid(dbname): + print ("Invalid name! %s" % dbname) sys.exit(1) - url = url_for_db(args.dbname) + url = url_for_db(dbname) db_security = CONF['database-security'] db = CouchDatabase.open_database(url=url, create=True, replica_uid=None, ensure_ddocs=True, database_security=db_security) - print ('success! Created %s, replica_uid: %s' % + print ('success! Ensured that database %s exists, with replica_uid: %s' % (db._dbname, db.replica_uid)) + + +if __name__ == '__main__': + args = parser.parse_args() + if args.migrate_all: + couch_url = url_for_db('') + for dbname in list_users_dbs(couch_url): + ensure_database(dbname) + else: + ensure_database(args.dbname) -- cgit v1.2.3 From 36f476c53f6f3532652d7428e306dcef1c37f879 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Fri, 30 Oct 2015 16:26:07 +0100 Subject: [feat] bump twisted dependency to 12.3.0 Needed after the SSL monkeypatch removal at 3b869fb7f --- server/pkg/requirements.pip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/pkg') diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index 58834d0e..f9cce08e 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -2,7 +2,7 @@ configparser u1db routes PyOpenSSL -twisted +twisted>=12.3.0 #pinned for wheezy compatibility Beaker==1.6.3 #wheezy couchdb==0.8 #wheezy -- cgit v1.2.3 From b0557f9c1d5e6f153f926ba3cb5876453ef23a10 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 1 Oct 2015 15:07:25 -0300 Subject: [refactor] separate SoledadBackend from CouchDatabase CouchDatabase was renamed to SoledadBackend and a new class CouchDatabase was created to hold all couchdb code. This should make SoledadBackend less tied to database implementation. A few more separations are needed to split into modules. --- server/pkg/create-user-db | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'server/pkg') diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index ae5d15dc..a8ba3833 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -19,7 +19,7 @@ import os import sys import netrc import argparse -from leap.soledad.common.couch import CouchDatabase +from leap.soledad.common.couch import SoledadBackend from leap.soledad.common.couch import is_db_name_valid from leap.soledad.common.couch import list_users_dbs from leap.soledad.server import load_configuration @@ -69,7 +69,7 @@ def ensure_database(dbname): sys.exit(1) url = url_for_db(dbname) db_security = CONF['database-security'] - db = CouchDatabase.open_database(url=url, create=True, + db = SoledadBackend.open_database(url=url, create=True, replica_uid=None, ensure_ddocs=True, database_security=db_security) print ('success! Ensured that database %s exists, with replica_uid: %s' % -- cgit v1.2.3 From f0b96af943dcb6c8cde4f6d4280186d78c78096c Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 13 Oct 2015 21:34:40 -0300 Subject: [refactor] split out backend from couch database First step of splitting classes across files on common. backend.py holds SoledadBackend (generic backend logic) couch/ is now a directory with old code inside __init__.py and CouchServerState on state.py Also removed mock IndexedSoledadBackend, since Soledad does not support indexing due to encryption on server side. Also fixed DesignDocUnknownError to show up what is the message of the original exception. It was being lost. --- server/pkg/create-user-db | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'server/pkg') diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index a8ba3833..54856643 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -19,8 +19,8 @@ import os import sys import netrc import argparse -from leap.soledad.common.couch import SoledadBackend -from leap.soledad.common.couch import is_db_name_valid +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.common.couch.state import is_db_name_valid from leap.soledad.common.couch import list_users_dbs from leap.soledad.server import load_configuration @@ -69,7 +69,7 @@ def ensure_database(dbname): sys.exit(1) url = url_for_db(dbname) db_security = CONF['database-security'] - db = SoledadBackend.open_database(url=url, create=True, + db = CouchDatabase.open_database(url=url, create=True, replica_uid=None, ensure_ddocs=True, database_security=db_security) print ('success! Ensured that database %s exists, with replica_uid: %s' % -- cgit v1.2.3