Convert to utf8 all the strings before saving them

3 files changed, 37 insertions, 1 deletions

diff --git a/changes/utf8_all_the_things b/changes/utf8_all_the_things
new file mode 100644
index 00000000..c213b890
--- /dev/null
+++ b/changes/utf8_all_the_things
@@ -0,0 +1 @@
+  o Save only UTF8 strings. Related to #3660.
diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip
index df6ddb94..8733f320 100644
--- a/client/pkg/requirements.pip
+++ b/client/pkg/requirements.pip
@@ -20,3 +20,5 @@ oauth
 
 # pysqlite should not be a dep, see #2945
 pysqlite
+
+cchardet
\ No newline at end of file
diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index 1f54ef8c..13a3b68f 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -32,6 +32,8 @@ import socket
 import ssl
 import urlparse
 
+import cchardet
+
 from hashlib import sha256
 
 from u1db.remote import http_client
@@ -755,6 +757,37 @@ class Soledad(object):
         """
         return self._db.get_all_docs(include_deleted)
 
+    def _convert_to_utf8(self, content):
+        """
+        Converts content to utf8 (or all the strings in content)
+
+        NOTE: Even though this method supports any type, it will
+        currently ignore contents of lists, tuple or any other
+        iterable than dict. We don't need support for these at the
+        moment
+
+        :param content: content to convert
+        :type content: object
+
+        :rtype: object
+        """
+
+        if isinstance(content, unicode):
+            return content
+        elif isinstance(content, str):
+            try:
+                result = cchardet.detect(content)
+                content = content.decode(result["encoding"]).encode("utf-8")\
+                                                            .decode("utf-8")
+            except UnicodeError:
+                pass
+            return content
+        else:
+            if isinstance(content, dict):
+                for key in content.keys():
+                    content[key] = self._convert_to_utf8(content[key])
+        return content
+
     def create_doc(self, content, doc_id=None):
         """
         Create a new document in the local encrypted database.
@@ -767,7 +800,7 @@ class Soledad(object):
         :return: the new document
         :rtype: SoledadDocument
         """
-        return self._db.create_doc(content, doc_id=doc_id)
+        return self._db.create_doc(self._convert_to_utf8(content), doc_id=doc_id)
 
     def create_doc_from_json(self, json, doc_id=None):
         """