From cfb55e5f1a26ae38e842a44df280f48e1e31eb06 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 30 Sep 2013 12:54:58 -0400 Subject: Updates to client docstrings * Add a more precise description of cert_file * change docstring placeholder to be compliant with sphinx rst --- client/src/leap/soledad/client/__init__.py | 255 ++++++++++++------------ client/src/leap/soledad/client/auth.py | 24 +-- client/src/leap/soledad/client/crypto.py | 62 +++--- client/src/leap/soledad/client/shared_db.py | 58 +++--- client/src/leap/soledad/client/sqlcipher.py | 290 ++++++++++++++-------------- client/src/leap/soledad/client/target.py | 132 ++++++------- 6 files changed, 412 insertions(+), 409 deletions(-) diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 4c6a41fc..1f54ef8c 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -226,25 +226,26 @@ class Soledad(object): """ Initialize configuration, cryptographic keys and dbs. - @param uuid: User's uuid. - @type uuid: str - @param passphrase: The passphrase for locking and unlocking encryption - secrets for local and remote storage. - @type passphrase: str - @param secrets_path: Path for storing encrypted key used for - symmetric encryption. - @type secrets_path: str - @param local_db_path: Path for local encrypted storage db. - @type local_db_path: str - @param server_url: URL for Soledad server. This is used either to sync + :param uuid: User's uuid. + :type uuid: str + :param passphrase: The passphrase for locking and unlocking encryption + secrets for local and remote storage. + :type passphrase: str + :param secrets_path: Path for storing encrypted key used for + symmetric encryption. + :type secrets_path: str + :param local_db_path: Path for local encrypted storage db. + :type local_db_path: str + :param server_url: URL for Soledad server. This is used either to sync with the user's remote db and to interact with the shared recovery database. - @type server_url: str - @param cert_file: Path to the SSL certificate to use in the - connection to the server_url. - @type cert_file: str - @param auth_token: Authorization token for accessing remote databases. - @type auth_token: str + :type server_url: str + :param cert_file: Path to the certificate of the ca used + to validate the SSL certificate used by the remote + soledad server. + :type cert_file: str + :param auth_token: Authorization token for accessing remote databases. + :type auth_token: str """ # get config params self._uuid = uuid @@ -339,7 +340,7 @@ class Soledad(object): """ Create work directories. - @raise OSError: in case file exists and is not a dir. + :raise OSError: in case file exists and is not a dir. """ paths = map( lambda x: os.path.dirname(x), @@ -418,8 +419,8 @@ class Soledad(object): Storage secret is encrypted before being stored. This method decrypts and returns the stored secret. - @return: The storage secret. - @rtype: str + :return: The storage secret. + :rtype: str """ # calculate the encryption key key = scrypt.hash( @@ -478,8 +479,8 @@ class Soledad(object): """ Return whether there is a storage secret available for use or not. - @return: Whether there's a storage secret for symmetric encryption. - @rtype: bool + :return: Whether there's a storage secret for symmetric encryption. + :rtype: bool """ if self._secret_id is None or self._secret_id not in self._secrets: try: @@ -515,8 +516,8 @@ class Soledad(object): } } - @return: The id of the generated secret. - @rtype: str + :return: The id of the generated secret. + :rtype: str """ signal(SOLEDAD_CREATING_KEYS, self._uuid) # generate random secret @@ -571,10 +572,10 @@ class Soledad(object): """ Change the passphrase that encrypts the storage secret. - @param new_passphrase: The new passphrase. - @type new_passphrase: str + :param new_passphrase: The new passphrase. + :type new_passphrase: str - @raise NoStorageSecret: Raised if there's no storage secret available. + :raise NoStorageSecret: Raised if there's no storage secret available. """ # maybe we want to add more checks to guarantee passphrase is # reasonable? @@ -616,8 +617,8 @@ class Soledad(object): Calculate a hash for storing/retrieving key material on shared database, based on user's uuid. - @return: the hash - @rtype: str + :return: the hash + :rtype: str """ return sha256( '%s%s' % ( @@ -639,8 +640,8 @@ class Soledad(object): Retrieve the document with encrypted key material from the shared database. - @return: a document with encrypted key material in its contents - @rtype: SoledadDocument + :return: a document with encrypted key material in its contents + :rtype: SoledadDocument """ signal(SOLEDAD_DOWNLOADING_KEYS, self._uuid) db = self._shared_db() @@ -687,11 +688,11 @@ class Soledad(object): """ Update a document in the local encrypted database. - @param doc: the document to update - @type doc: SoledadDocument + :param doc: the document to update + :type doc: SoledadDocument - @return: the new revision identifier for the document - @rtype: str + :return: the new revision identifier for the document + :rtype: str """ return self._db.put_doc(doc) @@ -699,11 +700,11 @@ class Soledad(object): """ Delete a document from the local encrypted database. - @param doc: the document to delete - @type doc: SoledadDocument + :param doc: the document to delete + :type doc: SoledadDocument - @return: the new revision identifier for the document - @rtype: str + :return: the new revision identifier for the document + :rtype: str """ return self._db.delete_doc(doc) @@ -711,15 +712,15 @@ class Soledad(object): """ Retrieve a document from the local encrypted database. - @param doc_id: the unique document identifier - @type doc_id: str - @param include_deleted: if True, deleted documents will be - returned with empty content; otherwise asking for a deleted - document will return None - @type include_deleted: bool + :param doc_id: the unique document identifier + :type doc_id: str + :param include_deleted: if True, deleted documents will be + returned with empty content; otherwise asking + for a deleted document will return None + :type include_deleted: bool - @return: the document object or None - @rtype: SoledadDocument + :return: the document object or None + :rtype: SoledadDocument """ return self._db.get_doc(doc_id, include_deleted=include_deleted) @@ -728,15 +729,15 @@ class Soledad(object): """ Get the content for many documents. - @param doc_ids: a list of document identifiers - @type doc_ids: list - @param check_for_conflicts: if set False, then the conflict check will + :param doc_ids: a list of document identifiers + :type doc_ids: list + :param check_for_conflicts: if set False, then the conflict check will be skipped, and 'None' will be returned instead of True/False - @type check_for_conflicts: bool + :type check_for_conflicts: bool - @return: iterable giving the Document object for each document id + :return: iterable giving the Document object for each document id in matching doc_ids order. - @rtype: generator + :rtype: generator """ return self._db.get_docs(doc_ids, check_for_conflicts=check_for_conflicts, @@ -745,12 +746,12 @@ class Soledad(object): def get_all_docs(self, include_deleted=False): """Get the JSON content for all documents in the database. - @param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted documents will not - be included in the results. - @return: (generation, [Document]) - The current generation of the database, followed by a list of all - the documents in the database. + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted + documents will not be included in the results. + :return: (generation, [Document]) + The current generation of the database, followed by a list of + all the documents in the database. """ return self._db.get_all_docs(include_deleted) @@ -758,13 +759,13 @@ class Soledad(object): """ Create a new document in the local encrypted database. - @param content: the contents of the new document - @type content: dict - @param doc_id: an optional identifier specifying the document id - @type doc_id: str + :param content: the contents of the new document + :type content: dict + :param doc_id: an optional identifier specifying the document id + :type doc_id: str - @return: the new document - @rtype: SoledadDocument + :return: the new document + :rtype: SoledadDocument """ return self._db.create_doc(content, doc_id=doc_id) @@ -778,12 +779,12 @@ class Soledad(object): If the database specifies a maximum document size and the document exceeds it, create will fail and raise a DocumentTooBig exception. - @param json: The JSON document string - @type json: str - @param doc_id: An optional identifier specifying the document id. - @type doc_id: - @return: The new cocument - @rtype: SoledadDocument + :param json: The JSON document string + :type json: str + :param doc_id: An optional identifier specifying the document id. + :type doc_id: + :return: The new cocument + :rtype: SoledadDocument """ return self._db.create_doc_from_json(json, doc_id=doc_id) @@ -796,11 +797,11 @@ class Soledad(object): Creating an index will block until the expressions have been evaluated and the index generated. - @param index_name: A unique name which can be used as a key prefix - @type index_name: str - @param index_expressions: index expressions defining the index - information. - @type index_expressions: dict + :param index_name: A unique name which can be used as a key prefix + :type index_name: str + :param index_expressions: index expressions defining the index + information. + :type index_expressions: dict Examples: @@ -816,8 +817,8 @@ class Soledad(object): """ Remove a named index. - @param index_name: The name of the index we are removing - @type index_name: str + :param index_name: The name of the index we are removing + :type index_name: str """ if self._db: return self._db.delete_index(index_name) @@ -826,8 +827,8 @@ class Soledad(object): """ List the definitions of all known indexes. - @return: A list of [('index-name', ['field', 'field2'])] definitions. - @rtype: list + :return: A list of [('index-name', ['field', 'field2'])] definitions. + :rtype: list """ if self._db: return self._db.list_indexes() @@ -843,14 +844,14 @@ class Soledad(object): It is also possible to append a '*' to the last supplied value (eg 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - @param index_name: The index to query - @type index_name: str - @param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - @type key_values: tuple - @return: List of [Document] - @rtype: list + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: List of [Document] + :rtype: list """ if self._db: return self._db.get_from_index(index_name, *key_values) @@ -869,18 +870,18 @@ class Soledad(object): possible to append a '*' to the last supplied value (eg 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - @param index_name: The index to query - @type index_name: str - @param start_values: tuples of values that define the lower bound of + :param index_name: The index to query + :type index_name: str + :param start_values: tuples of values that define the lower bound of the range. eg, if you have an index with 3 fields then you would have: (val1, val2, val3) - @type start_values: tuple - @param end_values: tuples of values that define the upper bound of the + :type start_values: tuple + :param end_values: tuples of values that define the upper bound of the range. eg, if you have an index with 3 fields then you would have: (val1, val2, val3) - @type end_values: tuple - @return: List of [Document] - @rtype: list + :type end_values: tuple + :return: List of [Document] + :rtype: list """ if self._db: return self._db.get_range_from_index( @@ -890,10 +891,10 @@ class Soledad(object): """ Return all keys under which documents are indexed in this index. - @param index_name: The index to query - @type index_name: str - @return: [] A list of tuples of indexed keys. - @rtype: list + :param index_name: The index to query + :type index_name: str + :return: [] A list of tuples of indexed keys. + :rtype: list """ if self._db: return self._db.get_index_keys(index_name) @@ -902,11 +903,11 @@ class Soledad(object): """ Get the list of conflicts for the given document. - @param doc_id: the document id - @type doc_id: str + :param doc_id: the document id + :type doc_id: str - @return: a list of the document entries that are conflicted - @rtype: list + :return: a list of the document entries that are conflicted + :rtype: list """ if self._db: return self._db.get_doc_conflicts(doc_id) @@ -915,11 +916,11 @@ class Soledad(object): """ Mark a document as no longer conflicted. - @param doc: a document with the new content to be inserted. - @type doc: SoledadDocument - @param conflicted_doc_revs: a list of revisions that the new content - supersedes. - @type conflicted_doc_revs: list + :param doc: a document with the new content to be inserted. + :type doc: SoledadDocument + :param conflicted_doc_revs: a list of revisions that the new content + supersedes. + :type conflicted_doc_revs: list """ if self._db: return self._db.resolve_doc(doc, conflicted_doc_revs) @@ -928,12 +929,12 @@ class Soledad(object): """ Synchronize the local encrypted replica with a remote replica. - @param url: the url of the target replica to sync with - @type url: str + :param url: the url of the target replica to sync with + :type url: str - @return: the local generation before the synchronisation was + :return: the local generation before the synchronisation was performed. - @rtype: str + :rtype: str """ if self._db: local_gen = self._db.sync( @@ -946,11 +947,11 @@ class Soledad(object): """ Return if local db replica differs from remote url's replica. - @param url: The remote replica to compare with local replica. - @type url: str + :param url: The remote replica to compare with local replica. + :type url: str - @return: Whether remote replica and local replica differ. - @rtype: bool + :return: Whether remote replica and local replica differ. + :rtype: bool """ target = SoledadSyncTarget(url, creds=self._creds, crypto=self._crypto) info = target.get_sync_info(self._db._get_replica_uid()) @@ -972,8 +973,8 @@ class Soledad(object): 'token': '' } - @param token: The authentication token. - @type token: str + :param token: The authentication token. + :type token: str """ self._creds = { 'token': { @@ -1004,11 +1005,11 @@ class Soledad(object): self.UUID_KEY: '', # (optional) } - @param include_uuid: Should the uuid be included? - @type include_uuid: bool + :param include_uuid: Should the uuid be included? + :type include_uuid: bool - @return: The recovery document. - @rtype: dict + :return: The recovery document. + :rtype: dict """ data = {self.STORAGE_SECRETS_KEY: self._secrets} if include_uuid: @@ -1027,8 +1028,8 @@ class Soledad(object): self.UUID_KEY: '', # (optional) } - @param data: The recovery document. - @type data: dict + :param data: The recovery document. + :type data: dict """ # include new secrets in our secret pool. for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items(): @@ -1101,7 +1102,9 @@ SOLEDAD_TIMEOUT = 10 class VerifiedHTTPSConnection(httplib.HTTPSConnection): - """HTTPSConnection verifying server side certificates.""" + """ + HTTPSConnection verifying server side certificates. + """ # derived from httplib.py def connect(self): diff --git a/client/src/leap/soledad/client/auth.py b/client/src/leap/soledad/client/auth.py index 3cd6dabe..d85e3ba6 100644 --- a/client/src/leap/soledad/client/auth.py +++ b/client/src/leap/soledad/client/auth.py @@ -37,10 +37,10 @@ class TokenBasedAuth(object): """ Store given credentials so we can sign the request later. - @param uuid: The user's uuid. - @type uuid: str - @param token: The authentication token. - @type token: str + :param uuid: The user's uuid. + :type uuid: str + :param token: The authentication token. + :type token: str """ self._creds = {'token': (uuid, token)} @@ -51,15 +51,15 @@ class TokenBasedAuth(object): [('Authorization', 'Token PRAGMA rekey_kdf_iter. These are deprecated and should not be used. Instead, use sqlcipher_export(). - @param db_handle: A handle to the SQLCipher database. - @type db_handle: pysqlcipher.Connection - @param new_key: The new key. - @type new_key: str - @param raw_key: Whether C{password} is a raw 64-char hex string or a + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param new_key: The new key. + :type new_key: str + :param raw_key: Whether C{password} is a raw 64-char hex string or a passphrase that should be hashed to obtain the encyrption key. - @type raw_key: bool + :type raw_key: bool """ if raw_key: cls._pragma_rekey_raw(db_handle, key) @@ -659,10 +659,10 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): PBKDF2 key derivation. The result is used as the encryption key for the database. - @param db_handle: A handle to the SQLCipher database. - @type db_handle: pysqlcipher.Connection - @param passphrase: The passphrase used to derive the encryption key. - @type passphrase: str + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param passphrase: The passphrase used to derive the encryption key. + :type passphrase: str """ db_handle.cursor().execute("PRAGMA rekey = '%s'" % passphrase) @@ -676,10 +676,10 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): ensure that the data provided is a 64 character hex string, which will be converted directly to 32 bytes (256 bits) of key data. - @param db_handle: A handle to the SQLCipher database. - @type db_handle: pysqlcipher.Connection - @param key: A 64 character hex string. - @type key: str + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param key: A 64 character hex string. + :type key: str """ if not all(c in string.hexdigits for c in key): raise NotAnHexString(key) diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index d0bc3706..65639887 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -98,19 +98,19 @@ def mac_doc(crypto, doc_id, doc_rev, ciphertext, mac_method): * msg: doc_id + doc_rev + ciphertext * digestmod: sha256 - @param crypto: A SoledadCryto instance used to perform the encryption. - @type crypto: leap.soledad.crypto.SoledadCrypto - @param doc_id: The id of the document. - @type doc_id: str - @param doc_rev: The revision of the document. - @type doc_rev: str - @param ciphertext: The content of the document. - @type ciphertext: str - @param mac_method: The MAC method to use. - @type mac_method: str - - @return: The calculated MAC. - @rtype: str + :param crypto: A SoledadCryto instance used to perform the encryption. + :type crypto: leap.soledad.crypto.SoledadCrypto + :param doc_id: The id of the document. + :type doc_id: str + :param doc_rev: The revision of the document. + :type doc_rev: str + :param ciphertext: The content of the document. + :type ciphertext: str + :param mac_method: The MAC method to use. + :type mac_method: str + + :return: The calculated MAC. + :rtype: str """ if mac_method == MacMethods.HMAC: return hmac.new( @@ -137,14 +137,14 @@ def encrypt_doc(crypto, doc): MAC_METHOD_KEY: 'hmac' } - @param crypto: A SoledadCryto instance used to perform the encryption. - @type crypto: leap.soledad.crypto.SoledadCrypto - @param doc: The document with contents to be encrypted. - @type doc: SoledadDocument + :param crypto: A SoledadCryto instance used to perform the encryption. + :type crypto: leap.soledad.crypto.SoledadCrypto + :param doc: The document with contents to be encrypted. + :type doc: SoledadDocument - @return: The JSON serialization of the dict representing the encrypted + :return: The JSON serialization of the dict representing the encrypted content. - @rtype: str + :rtype: str """ soledad_assert(doc.is_tombstone() is False) # encrypt content using AES-256 CTR mode @@ -191,13 +191,13 @@ def decrypt_doc(crypto, doc): EncryptionSchemes.SYMKEY and C{enc_method} is EncryptionMethods.AES_256_CTR. - @param crypto: A SoledadCryto instance to perform the encryption. - @type crypto: leap.soledad.crypto.SoledadCrypto - @param doc: The document to be decrypted. - @type doc: SoledadDocument + :param crypto: A SoledadCryto instance to perform the encryption. + :type crypto: leap.soledad.crypto.SoledadCrypto + :param doc: The document to be decrypted. + :type doc: SoledadDocument - @return: The JSON serialization of the decrypted content. - @rtype: str + :return: The JSON serialization of the decrypted content. + :rtype: str """ soledad_assert(doc.is_tombstone() is False) soledad_assert(ENC_JSON_KEY in doc.content) @@ -258,10 +258,10 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): """ Store given credentials so we can sign the request later. - @param uuid: The user's uuid. - @type uuid: str - @param token: The authentication token. - @type token: str + :param uuid: The user's uuid. + :type uuid: str + :param token: The authentication token. + :type token: str """ TokenBasedAuth.set_token_credentials(self, uuid, token) @@ -269,15 +269,15 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): """ Return an authorization header to be included in the HTTP request. - @param method: The HTTP method. - @type method: str - @param url_query: The URL query string. - @type url_query: str - @param params: A list with encoded query parameters. - @type param: list + :param method: The HTTP method. + :type method: str + :param url_query: The URL query string. + :type url_query: str + :param params: A list with encoded query parameters. + :type param: list - @return: The Authorization header. - @rtype: list of tuple + :return: The Authorization header. + :rtype: list of tuple """ return TokenBasedAuth._sign_request(self, method, url_query, params) @@ -293,14 +293,14 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): """ Initialize the SoledadSyncTarget. - @param url: The url of the target replica to sync with. - @type url: str - @param creds: optional dictionary giving credentials. + :param url: The url of the target replica to sync with. + :type url: str + :param creds: optional dictionary giving credentials. to authorize the operation with the server. - @type creds: dict - @param soledad: An instance of Soledad so we can encrypt/decrypt + :type creds: dict + :param soledad: An instance of Soledad so we can encrypt/decrypt document contents when syncing. - @type soledad: soledad.Soledad + :type soledad: soledad.Soledad """ HTTPSyncTarget.__init__(self, url, creds) self._crypto = crypto @@ -314,19 +314,19 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): EncryptionSchemes.SYMKEY, then this method will decrypt it with Soledad's symmetric key. - @param data: The body of the HTTP response. - @type data: str - @param return_doc_cb: A callback to insert docs from target. - @type return_doc_cb: function - @param ensure_callback: A callback to ensure we have the correct + :param data: The body of the HTTP response. + :type data: str + :param return_doc_cb: A callback to insert docs from target. + :type return_doc_cb: function + :param ensure_callback: A callback to ensure we have the correct target_replica_uid, if it was just created. - @type ensure_callback: function + :type ensure_callback: function - @raise BrokenSyncStream: If C{data} is malformed. + :raise BrokenSyncStream: If C{data} is malformed. - @return: A dictionary representing the first line of the response got + :return: A dictionary representing the first line of the response got from remote replica. - @rtype: list of str + :rtype: list of str """ parts = data.splitlines() # one at a time if not parts or parts[0] != '[': @@ -381,25 +381,25 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): This does the same as the parent's method but encrypts content before syncing. - @param docs_by_generations: A list of (doc_id, generation, trans_id) + :param docs_by_generations: A list of (doc_id, generation, trans_id) of local documents that were changed since the last local generation the remote replica knows about. - @type docs_by_generations: list of tuples - @param source_replica_uid: The uid of the source replica. - @type source_replica_uid: str - @param last_known_generation: Target's last known generation. - @type last_known_generation: int - @param last_known_trans_id: Target's last known transaction id. - @type last_known_trans_id: str - @param return_doc_cb: A callback for inserting received documents from + :type docs_by_generations: list of tuples + :param source_replica_uid: The uid of the source replica. + :type source_replica_uid: str + :param last_known_generation: Target's last known generation. + :type last_known_generation: int + :param last_known_trans_id: Target's last known transaction id. + :type last_known_trans_id: str + :param return_doc_cb: A callback for inserting received documents from target. - @type return_doc_cb: function - @param ensure_callback: A callback that ensures we know the target + :type return_doc_cb: function + :param ensure_callback: A callback that ensures we know the target replica uid if the target replica was just created. - @type ensure_callback: function + :type ensure_callback: function - @return: The new generation and transaction id of the target replica. - @rtype: tuple + :return: The new generation and transaction id of the target replica. + :rtype: tuple """ self._ensure_connection() if self._trace_hook: # for tests -- cgit v1.2.3 From e7c90a407c08ee169ae912db8f62f218531e41a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Wed, 2 Oct 2013 17:05:58 -0300 Subject: Convert to utf8 all the strings before saving them --- changes/utf8_all_the_things | 1 + client/pkg/requirements.pip | 2 ++ client/src/leap/soledad/client/__init__.py | 35 +++++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 changes/utf8_all_the_things diff --git a/changes/utf8_all_the_things b/changes/utf8_all_the_things new file mode 100644 index 00000000..c213b890 --- /dev/null +++ b/changes/utf8_all_the_things @@ -0,0 +1 @@ + o Save only UTF8 strings. Related to #3660. diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index df6ddb94..8733f320 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -20,3 +20,5 @@ oauth # pysqlite should not be a dep, see #2945 pysqlite + +cchardet \ No newline at end of file diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 1f54ef8c..13a3b68f 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -32,6 +32,8 @@ import socket import ssl import urlparse +import cchardet + from hashlib import sha256 from u1db.remote import http_client @@ -755,6 +757,37 @@ class Soledad(object): """ return self._db.get_all_docs(include_deleted) + def _convert_to_utf8(self, content): + """ + Converts content to utf8 (or all the strings in content) + + NOTE: Even though this method supports any type, it will + currently ignore contents of lists, tuple or any other + iterable than dict. We don't need support for these at the + moment + + :param content: content to convert + :type content: object + + :rtype: object + """ + + if isinstance(content, unicode): + return content + elif isinstance(content, str): + try: + result = cchardet.detect(content) + content = content.decode(result["encoding"]).encode("utf-8")\ + .decode("utf-8") + except UnicodeError: + pass + return content + else: + if isinstance(content, dict): + for key in content.keys(): + content[key] = self._convert_to_utf8(content[key]) + return content + def create_doc(self, content, doc_id=None): """ Create a new document in the local encrypted database. @@ -767,7 +800,7 @@ class Soledad(object): :return: the new document :rtype: SoledadDocument """ - return self._db.create_doc(content, doc_id=doc_id) + return self._db.create_doc(self._convert_to_utf8(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): """ -- cgit v1.2.3 From 33f2b3373ae1c2ceb040fc8cc6504462976e8d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Fri, 4 Oct 2013 11:27:08 -0300 Subject: Fold in changes --- CHANGELOG | 4 ++++ changes/utf8_all_the_things | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 changes/utf8_all_the_things diff --git a/CHANGELOG b/CHANGELOG index d400fc52..a7254c40 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +0.4.1 Oct 4: +Client: + o Save only UTF8 strings. Related to #3660. + 0.4.0 Sep 20: Client: o Remove redundant logging when creating data dirs. diff --git a/changes/utf8_all_the_things b/changes/utf8_all_the_things deleted file mode 100644 index c213b890..00000000 --- a/changes/utf8_all_the_things +++ /dev/null @@ -1 +0,0 @@ - o Save only UTF8 strings. Related to #3660. -- cgit v1.2.3