Merge remote-tracking branch 'refs/remotes/drebs/bug/5739_fix-multipart-put-problem-2' into develop

4 files changed, 224 insertions, 222 deletions

diff --git a/scripts/build_debian_package.sh b/scripts/build_debian_package.sh
index cc62c3ac..1ec9b00a 100755
--- a/scripts/build_debian_package.sh
+++ b/scripts/build_debian_package.sh
@@ -26,7 +26,7 @@ export GIT_DIR=${workdir}/soledad/.git
 export GIT_WORK_TREE=${workdir}/soledad
 git remote add leapcode ${SOLEDAD_MAIN_REPO}
 git fetch leapcode
-git checkout debian
+git checkout -b debian leapcode/debian
 git merge --no-edit ${branch}
 (cd ${workdir}/soledad && debuild -uc -us)
 echo "Packages generated in ${workdir}"
diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py
index 2bf4ab5e..6c456c41 100644
--- a/scripts/db_access/client_side_db.py
+++ b/scripts/db_access/client_side_db.py
@@ -15,10 +15,14 @@ import srp._pysrp as srp
 import binascii
 import logging
 
+
 from leap.common.config import get_path_prefix
 from leap.soledad.client import Soledad
 
 
+from util import ValidateUserHandle
+
+
 # create a logger
 logger = logging.getLogger(__name__)
 LOG_FORMAT = '%(asctime)s %(message)s'
@@ -122,26 +126,23 @@ def get_soledad_instance(username, provider, passphrase, basedir):
 
 if __name__ == '__main__':
 
-    class ValidateUserHandle(argparse.Action):
-        def __call__(self, parser, namespace, values, option_string=None):
-            m = re.compile('^([^@]+)@([^@]+\.[^@]+)$')
-            res = m.match(values)
-            if res == None:
-                parser.error('User handle should have the form user@provider.')
-            setattr(namespace, 'username', res.groups()[0])
-            setattr(namespace, 'provider', res.groups()[1])
-
     # parse command line
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'user@provider', action=ValidateUserHandle, help='the user handle')
     parser.add_argument(
-        '-b', dest='basedir', required=False, default=None, help='the user handle')
+        '-b', dest='basedir', required=False, default=None,
+        help='soledad base directory')
+    parser.add_argument(
+        '-p', dest='passphrase', required=False, default=None,
+        help='the user passphrase')
     args = parser.parse_args()
 
     # get the password
-    passphrase = getpass.getpass(
-        'Password for %s@%s: ' % (args.username, args.provider))
+    passphrase = args.passphrase
+    if passphrase is None:
+        passphrase = getpass.getpass(
+            'Password for %s@%s: ' % (args.username, args.provider))
 
     # get the basedir
     basedir = args.basedir
diff --git a/scripts/profiling/backends_cpu_usage/movingaverage.py b/scripts/profiling/backends_cpu_usage/movingaverage.py
index bac1b3e1..098b0a01 100644..120000
--- a/scripts/profiling/backends_cpu_usage/movingaverage.py
+++ b/scripts/profiling/backends_cpu_usage/movingaverage.py
@@ -1,209 +1 @@
-#!/usr/bin/env python
-#
-#  Sean Reifschneider, tummy.com, ltd.  <jafo@tummy.com>
-#  Released into the Public Domain, 2011-02-06
-
-import itertools
-from itertools import islice
-from collections import deque
-
-
-#########################################################
-def movingaverage(data, subset_size, data_is_list = None,
-		avoid_fp_drift = True):
-	'''Return the moving averages of the data, with a window size of
-	`subset_size`.  `subset_size` must be an integer greater than 0 and
-	less than the length of the input data, or a ValueError will be raised.
-
-	`data_is_list` can be used to tune the algorithm for list or iteratable
-	as an input.  The default value, `None` will auto-detect this.
-	The algorithm used if `data` is a list is almost twice as fast as if
-	it is an iteratable.
-
-	`avoid_fp_drift`, if True (the default) sums every sub-set rather than
-	keeping a "rolling sum" (which may be subject to floating-point drift).
-	While more correct, it is also dramatically slower for subset sizes
-	much larger than 20.
-
-	NOTE: You really should consider setting `avoid_fp_drift = False` unless
-	you are dealing with very small numbers (say, far smaller than 0.00001)
-	or require extreme accuracy at the cost of execution time.  For
-	`subset_size` < 20, the performance difference is very small.
-	'''
-	if subset_size < 1:
-		raise ValueError('subset_size must be 1 or larger')
-
-	if data_is_list is None:
-		data_is_list = hasattr(data, '__getslice__')
-
-	divisor = float(subset_size)
-	if data_is_list:
-		#  This only works if we can re-access old elements, but is much faster.
-		#  In other words, it can't be just an iterable, it needs to be a list.
-
-		if subset_size > len(data):
-			raise ValueError('subset_size must be smaller than data set size')
-
-		if avoid_fp_drift:
-			for x in range(subset_size, len(data) + 1):
-				yield sum(data[x - subset_size:x]) / divisor
-		else:
-			cur = sum(data[0:subset_size])
-			yield cur / divisor
-			for x in range(subset_size, len(data)):
-				cur += data[x] - data[x - subset_size]
-				yield cur / divisor
-	else:
-		#  Based on the recipe at:
-		#     http://docs.python.org/library/collections.html#deque-recipes
-		it = iter(data)
-		d = deque(islice(it, subset_size))
-
-		if subset_size > len(d):
-			raise ValueError('subset_size must be smaller than data set size')
-
-		if avoid_fp_drift:
-			yield sum(d) / divisor
-			for elem in it:
-				d.popleft()
-				d.append(elem)
-				yield sum(d) / divisor
-		else:
-			s = sum(d)
-			yield s / divisor
-			for elem in it:
-				s += elem - d.popleft()
-				d.append(elem)
-				yield s / divisor
-
-
-##########################
-if __name__ == '__main__':
-	import unittest
-
-	class TestMovingAverage(unittest.TestCase):
-		####################
-		def test_List(self):
-			try:
-				list(movingaverage([1,2,3], 0))
-				self.fail('Did not raise ValueError on subset_size=0')
-			except ValueError:
-				pass
-
-			try:
-				list(movingaverage([1,2,3,4,5,6], 7))
-				self.fail('Did not raise ValueError on subset_size > len(data)')
-			except ValueError:
-				pass
-
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 1)), [1,2,3,4,5,6])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 2)),
-					[1.5,2.5,3.5,4.5,5.5])
-			self.assertEqual(list(movingaverage(map(float, [1,2,3,4,5,6]), 2)),
-					[1.5,2.5,3.5,4.5,5.5])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 3)), [2,3,4,5])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 4)), [2.5,3.5,4.5])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 5)), [3,4])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 6)), [3.5])
-
-			self.assertEqual(list(movingaverage([40, 30, 50, 46, 39, 44],
-					3, False)), [40.0,42.0,45.0,43.0])
-			self.assertEqual(list(movingaverage([40, 30, 50, 46, 39, 44],
-					3, True)), [40.0,42.0,45.0,43.0])
-
-
-		######################
-		def test_XRange(self):
-			try:
-				list(movingaverage(xrange(1, 4), 0))
-				self.fail('Did not raise ValueError on subset_size=0')
-			except ValueError:
-				pass
-
-			try:
-				list(movingaverage(xrange(1, 7), 7))
-				self.fail('Did not raise ValueError on subset_size > len(data)')
-			except ValueError:
-				pass
-
-			self.assertEqual(list(movingaverage(xrange(1, 7), 1)), [1,2,3,4,5,6])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 2)),
-					[1.5,2.5,3.5,4.5,5.5])
-			self.assertEqual(list(movingaverage(iter(map(float, xrange(1, 7))),
-					2)), [1.5,2.5,3.5,4.5,5.5])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 3)), [2,3,4,5])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 4)), [2.5,3.5,4.5])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 5)), [3,4])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 6)), [3.5])
-
-
-		###########################
-		def test_ListRolling(self):
-			try:
-				list(movingaverage([1,2,3], 0, avoid_fp_drift = False))
-				self.fail('Did not raise ValueError on subset_size=0')
-			except ValueError:
-				pass
-
-			try:
-				list(movingaverage([1,2,3,4,5,6], 7, avoid_fp_drift = False))
-				self.fail('Did not raise ValueError on subset_size > len(data)')
-			except ValueError:
-				pass
-
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 1,
-					avoid_fp_drift = False)), [1,2,3,4,5,6])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 2,
-					avoid_fp_drift = False)),
-					[1.5,2.5,3.5,4.5,5.5])
-			self.assertEqual(list(movingaverage(map(float, [1,2,3,4,5,6]), 2,
-					avoid_fp_drift = False)), [1.5,2.5,3.5,4.5,5.5])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 3,
-					avoid_fp_drift = False)), [2,3,4,5])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 4,
-					avoid_fp_drift = False)), [2.5,3.5,4.5])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 5,
-					avoid_fp_drift = False)), [3,4])
-			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 6,
-					avoid_fp_drift = False)), [3.5])
-
-			self.assertEqual(list(movingaverage([40, 30, 50, 46, 39, 44],
-					3, False, avoid_fp_drift = False)), [40.0,42.0,45.0,43.0])
-			self.assertEqual(list(movingaverage([40, 30, 50, 46, 39, 44],
-					3, True, avoid_fp_drift = False)), [40.0,42.0,45.0,43.0])
-
-
-		#############################
-		def test_XRangeRolling(self):
-			try:
-				list(movingaverage(xrange(1, 4), 0, avoid_fp_drift = False))
-				self.fail('Did not raise ValueError on subset_size=0')
-			except ValueError:
-				pass
-
-			try:
-				list(movingaverage(xrange(1, 7), 7, avoid_fp_drift = False))
-				self.fail('Did not raise ValueError on subset_size > len(data)')
-			except ValueError:
-				pass
-
-			self.assertEqual(list(movingaverage(xrange(1, 7), 1,
-					avoid_fp_drift = False)), [1,2,3,4,5,6])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 2,
-					avoid_fp_drift = False)), [1.5,2.5,3.5,4.5,5.5])
-			self.assertEqual(list(movingaverage(iter(map(float, xrange(1, 7))),
-					2, avoid_fp_drift = False)), [1.5,2.5,3.5,4.5,5.5])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 3,
-					avoid_fp_drift = False)), [2,3,4,5])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 4,
-					avoid_fp_drift = False)), [2.5,3.5,4.5])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 5,
-					avoid_fp_drift = False)), [3,4])
-			self.assertEqual(list(movingaverage(xrange(1, 7), 6,
-					avoid_fp_drift = False)), [3.5])
-
-
-	######################################################################
-	suite = unittest.TestLoader().loadTestsFromTestCase(TestMovingAverage)
-	unittest.TextTestRunner(verbosity = 2).run(suite)
-
+../movingaverage.py
\ No newline at end of file
diff --git a/scripts/profiling/movingaverage.py b/scripts/profiling/movingaverage.py
new file mode 100644
index 00000000..bac1b3e1
--- /dev/null
+++ b/scripts/profiling/movingaverage.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+#
+#  Sean Reifschneider, tummy.com, ltd.  <jafo@tummy.com>
+#  Released into the Public Domain, 2011-02-06
+
+import itertools
+from itertools import islice
+from collections import deque
+
+
+#########################################################
+def movingaverage(data, subset_size, data_is_list = None,
+		avoid_fp_drift = True):
+	'''Return the moving averages of the data, with a window size of
+	`subset_size`.  `subset_size` must be an integer greater than 0 and
+	less than the length of the input data, or a ValueError will be raised.
+
+	`data_is_list` can be used to tune the algorithm for list or iteratable
+	as an input.  The default value, `None` will auto-detect this.
+	The algorithm used if `data` is a list is almost twice as fast as if
+	it is an iteratable.
+
+	`avoid_fp_drift`, if True (the default) sums every sub-set rather than
+	keeping a "rolling sum" (which may be subject to floating-point drift).
+	While more correct, it is also dramatically slower for subset sizes
+	much larger than 20.
+
+	NOTE: You really should consider setting `avoid_fp_drift = False` unless
+	you are dealing with very small numbers (say, far smaller than 0.00001)
+	or require extreme accuracy at the cost of execution time.  For
+	`subset_size` < 20, the performance difference is very small.
+	'''
+	if subset_size < 1:
+		raise ValueError('subset_size must be 1 or larger')
+
+	if data_is_list is None:
+		data_is_list = hasattr(data, '__getslice__')
+
+	divisor = float(subset_size)
+	if data_is_list:
+		#  This only works if we can re-access old elements, but is much faster.
+		#  In other words, it can't be just an iterable, it needs to be a list.
+
+		if subset_size > len(data):
+			raise ValueError('subset_size must be smaller than data set size')
+
+		if avoid_fp_drift:
+			for x in range(subset_size, len(data) + 1):
+				yield sum(data[x - subset_size:x]) / divisor
+		else:
+			cur = sum(data[0:subset_size])
+			yield cur / divisor
+			for x in range(subset_size, len(data)):
+				cur += data[x] - data[x - subset_size]
+				yield cur / divisor
+	else:
+		#  Based on the recipe at:
+		#     http://docs.python.org/library/collections.html#deque-recipes
+		it = iter(data)
+		d = deque(islice(it, subset_size))
+
+		if subset_size > len(d):
+			raise ValueError('subset_size must be smaller than data set size')
+
+		if avoid_fp_drift:
+			yield sum(d) / divisor
+			for elem in it:
+				d.popleft()
+				d.append(elem)
+				yield sum(d) / divisor
+		else:
+			s = sum(d)
+			yield s / divisor
+			for elem in it:
+				s += elem - d.popleft()
+				d.append(elem)
+				yield s / divisor
+
+
+##########################
+if __name__ == '__main__':
+	import unittest
+
+	class TestMovingAverage(unittest.TestCase):
+		####################
+		def test_List(self):
+			try:
+				list(movingaverage([1,2,3], 0))
+				self.fail('Did not raise ValueError on subset_size=0')
+			except ValueError:
+				pass
+
+			try:
+				list(movingaverage([1,2,3,4,5,6], 7))
+				self.fail('Did not raise ValueError on subset_size > len(data)')
+			except ValueError:
+				pass
+
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 1)), [1,2,3,4,5,6])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 2)),
+					[1.5,2.5,3.5,4.5,5.5])
+			self.assertEqual(list(movingaverage(map(float, [1,2,3,4,5,6]), 2)),
+					[1.5,2.5,3.5,4.5,5.5])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 3)), [2,3,4,5])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 4)), [2.5,3.5,4.5])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 5)), [3,4])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 6)), [3.5])
+
+			self.assertEqual(list(movingaverage([40, 30, 50, 46, 39, 44],
+					3, False)), [40.0,42.0,45.0,43.0])
+			self.assertEqual(list(movingaverage([40, 30, 50, 46, 39, 44],
+					3, True)), [40.0,42.0,45.0,43.0])
+
+
+		######################
+		def test_XRange(self):
+			try:
+				list(movingaverage(xrange(1, 4), 0))
+				self.fail('Did not raise ValueError on subset_size=0')
+			except ValueError:
+				pass
+
+			try:
+				list(movingaverage(xrange(1, 7), 7))
+				self.fail('Did not raise ValueError on subset_size > len(data)')
+			except ValueError:
+				pass
+
+			self.assertEqual(list(movingaverage(xrange(1, 7), 1)), [1,2,3,4,5,6])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 2)),
+					[1.5,2.5,3.5,4.5,5.5])
+			self.assertEqual(list(movingaverage(iter(map(float, xrange(1, 7))),
+					2)), [1.5,2.5,3.5,4.5,5.5])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 3)), [2,3,4,5])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 4)), [2.5,3.5,4.5])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 5)), [3,4])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 6)), [3.5])
+
+
+		###########################
+		def test_ListRolling(self):
+			try:
+				list(movingaverage([1,2,3], 0, avoid_fp_drift = False))
+				self.fail('Did not raise ValueError on subset_size=0')
+			except ValueError:
+				pass
+
+			try:
+				list(movingaverage([1,2,3,4,5,6], 7, avoid_fp_drift = False))
+				self.fail('Did not raise ValueError on subset_size > len(data)')
+			except ValueError:
+				pass
+
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 1,
+					avoid_fp_drift = False)), [1,2,3,4,5,6])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 2,
+					avoid_fp_drift = False)),
+					[1.5,2.5,3.5,4.5,5.5])
+			self.assertEqual(list(movingaverage(map(float, [1,2,3,4,5,6]), 2,
+					avoid_fp_drift = False)), [1.5,2.5,3.5,4.5,5.5])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 3,
+					avoid_fp_drift = False)), [2,3,4,5])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 4,
+					avoid_fp_drift = False)), [2.5,3.5,4.5])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 5,
+					avoid_fp_drift = False)), [3,4])
+			self.assertEqual(list(movingaverage([1,2,3,4,5,6], 6,
+					avoid_fp_drift = False)), [3.5])
+
+			self.assertEqual(list(movingaverage([40, 30, 50, 46, 39, 44],
+					3, False, avoid_fp_drift = False)), [40.0,42.0,45.0,43.0])
+			self.assertEqual(list(movingaverage([40, 30, 50, 46, 39, 44],
+					3, True, avoid_fp_drift = False)), [40.0,42.0,45.0,43.0])
+
+
+		#############################
+		def test_XRangeRolling(self):
+			try:
+				list(movingaverage(xrange(1, 4), 0, avoid_fp_drift = False))
+				self.fail('Did not raise ValueError on subset_size=0')
+			except ValueError:
+				pass
+
+			try:
+				list(movingaverage(xrange(1, 7), 7, avoid_fp_drift = False))
+				self.fail('Did not raise ValueError on subset_size > len(data)')
+			except ValueError:
+				pass
+
+			self.assertEqual(list(movingaverage(xrange(1, 7), 1,
+					avoid_fp_drift = False)), [1,2,3,4,5,6])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 2,
+					avoid_fp_drift = False)), [1.5,2.5,3.5,4.5,5.5])
+			self.assertEqual(list(movingaverage(iter(map(float, xrange(1, 7))),
+					2, avoid_fp_drift = False)), [1.5,2.5,3.5,4.5,5.5])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 3,
+					avoid_fp_drift = False)), [2,3,4,5])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 4,
+					avoid_fp_drift = False)), [2.5,3.5,4.5])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 5,
+					avoid_fp_drift = False)), [3,4])
+			self.assertEqual(list(movingaverage(xrange(1, 7), 6,
+					avoid_fp_drift = False)), [3.5])
+
+
+	######################################################################
+	suite = unittest.TestLoader().loadTestsFromTestCase(TestMovingAverage)
+	unittest.TextTestRunner(verbosity = 2).run(suite)
+