diff options
author | Kali Kaneko <kali@leap.se> | 2016-04-25 22:13:19 -0400 |
---|---|---|
committer | Kali Kaneko <kali@leap.se> | 2016-04-26 21:33:34 -0400 |
commit | 3b3731d873664db00c02603363f61d34c41a3990 (patch) | |
tree | 13293a9aee4253f8e4bcad5c37f97230b8f200eb /src/pixelated/support | |
parent | 005f3f5e5157bddf792f46a983bfadd0d4398a89 (diff) |
embed pixelated
Diffstat (limited to 'src/pixelated/support')
-rw-r--r-- | src/pixelated/support/__init__.py | 81 | ||||
-rw-r--r-- | src/pixelated/support/date.py | 29 | ||||
-rw-r--r-- | src/pixelated/support/encrypted_file_storage.py | 153 | ||||
-rw-r--r-- | src/pixelated/support/error_handler.py | 27 | ||||
-rw-r--r-- | src/pixelated/support/functional.py | 37 | ||||
-rw-r--r-- | src/pixelated/support/mail_generator.py | 157 | ||||
-rw-r--r-- | src/pixelated/support/markov.py | 94 | ||||
-rw-r--r-- | src/pixelated/support/replier.py | 35 | ||||
-rw-r--r-- | src/pixelated/support/tls_adapter.py | 47 |
9 files changed, 660 insertions, 0 deletions
diff --git a/src/pixelated/support/__init__.py b/src/pixelated/support/__init__.py new file mode 100644 index 00000000..56283918 --- /dev/null +++ b/src/pixelated/support/__init__.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import time +import logging +from functools import wraps +from twisted.internet import defer + + +log = logging.getLogger(__name__) + + +def _start_stopwatch(): + return (time.time(), time.clock()) + + +def _stop_stopwatch(start): + start_time, start_clock = start + end_clock = time.clock() + end_time = time.time() + clock_duration = end_clock - start_clock + time_duration = end_time - start_time + if time_duration < 0.00000001: # avoid division by zero + time_duration = 0.00000001 + + estimate_percent_io = ( + (time_duration - clock_duration) / time_duration) * 100.0 + + return time_duration, clock_duration, estimate_percent_io + + +def log_time(f): + + @wraps(f) + def wrapper(*args, **kwds): + start = _start_stopwatch() + + result = f(*args, **kwds) + + time_duration, clock_duration, estimate_percent_io = _stop_stopwatch( + start) + log.info('Needed %fs (%fs cpu time, %.2f%% spent outside process) to execute %s' % ( + time_duration, clock_duration, estimate_percent_io, f)) + + return result + + return wrapper + + +def log_time_deferred(f): + + def log_time(result, start): + time_duration, clock_duration, estimate_percent_io = _stop_stopwatch( + start) + log.info('after callback: Needed %fs (%fs cpu time, %.2f%% spent outside process) to execute %s' % ( + time_duration, clock_duration, estimate_percent_io, f)) + return result + + @wraps(f) + def wrapper(*args, **kwds): + start = _start_stopwatch() + result = f(*args, **kwds) + if isinstance(result, defer.Deferred): + result.addCallback(log_time, start=start) + else: + log.warn('No Deferred returned, perhaps need to re-order annotations?') + return result + + return wrapper diff --git a/src/pixelated/support/date.py b/src/pixelated/support/date.py new file mode 100644 index 00000000..0012aeea --- /dev/null +++ b/src/pixelated/support/date.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import datetime + +import dateutil.parser +from email.utils import formatdate +from dateutil.tz import tzlocal + + +def iso_now(): + return datetime.datetime.now(tzlocal()).isoformat() + + +def mail_date_now(): + date = dateutil.parser.parse(iso_now()) + return formatdate(float(date.strftime('%s'))) diff --git a/src/pixelated/support/encrypted_file_storage.py b/src/pixelated/support/encrypted_file_storage.py new file mode 100644 index 00000000..d4e06e0b --- /dev/null +++ b/src/pixelated/support/encrypted_file_storage.py @@ -0,0 +1,153 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import with_statement + +import hmac +import io +import os +from hashlib import sha256 + +from leap.soledad.client.crypto import decrypt_sym +from leap.soledad.client.crypto import encrypt_sym +from whoosh.filedb.filestore import FileStorage +from whoosh.filedb.structfile import BufferFile, StructFile +from whoosh.util import random_name + + +class DelayedCloseBytesIO(io.BytesIO): + + def __init__(self, name): + super(DelayedCloseBytesIO, self).__init__() + self._name = name + self.shouldClose = False + + def close(self): + + self.shouldClose = True + + def explicit_close(self): + super(DelayedCloseBytesIO, self).close() + + +class DelayedCloseStructFile(StructFile): + + def __init__(self, fileobj, name=None, onclose=None): + super(DelayedCloseStructFile, self).__init__(fileobj, name, onclose) + + def close(self): + """Closes the wrapped file. + """ + + if self.is_closed: + raise Exception("This file is already closed") + if self.onclose: + self.onclose(self) + if hasattr(self.file, "explicit_close"): + self.file.explicit_close() + self.is_closed = True + + +class EncryptedFileStorage(FileStorage): + + def __init__(self, path, masterkey=None): + FileStorage.__init__(self, path, supports_mmap=False) + self.masterkey = masterkey[:32] + self.signkey = masterkey[32:] + self._tmp_storage = self.temp_storage + self.length_cache = {} + self._open_files = {} + + def open_file(self, name, **kwargs): + return self._open_encrypted_file(name) + + def create_file(self, name, excl=False, mode="w+b", **kwargs): + f = DelayedCloseStructFile(DelayedCloseBytesIO( + name), name=name, onclose=self._encrypt_index_on_close(name)) + f.is_real = False + self._open_files[name] = f + return f + + def delete_file(self, name): + super(EncryptedFileStorage, self).delete_file(name) + if name in self._open_files: + del self._open_files[name] + + def temp_storage(self, name=None): + name = name or "%s.tmp" % random_name() + path = os.path.join(self.folder, name) + return EncryptedFileStorage(path, self.masterkey).create() + + def file_length(self, name): + return self.length_cache[name][0] + + def gen_mac(self, iv, ciphertext): + verifiable_payload = ''.join((iv, ciphertext)) + return hmac.new(self.signkey, verifiable_payload, sha256).digest() + + def encrypt(self, content): + iv, ciphertext = encrypt_sym(content, self.masterkey) + mac = self.gen_mac(iv, ciphertext) + return ''.join((mac, iv, ciphertext)) + + def decrypt(self, payload): + payload_mac, iv, ciphertext = payload[ + :32], payload[32:57], payload[57:] + generated_mac = self.gen_mac(iv, ciphertext) + if sha256(payload_mac).digest() != sha256(generated_mac).digest(): + raise Exception( + "EncryptedFileStorage - Error opening file. Wrong MAC") + return decrypt_sym(ciphertext, self.masterkey, iv) + + def _encrypt_index_on_close(self, name): + def wrapper(struct_file): + struct_file.seek(0) + content = struct_file.file.read() + file_hash = sha256(content).digest() + if name in self.length_cache and file_hash == self.length_cache[name][1]: + return + self.length_cache[name] = (len(content), file_hash) + encrypted_content = self.encrypt(content) + with open(self._fpath(name), 'w+b') as f: + f.write(encrypted_content) + + return wrapper + + def _open_encrypted_file(self, name, onclose=lambda x: None): + if not self.file_exists(name): + if name in self._open_files: + f = self._open_files[name] + if not f.is_closed: + state = 'closed' if f.file.shouldClose else 'open' + if state == 'closed': + self._store_file(name, f.file.getvalue()) + f.close() + del self._open_files[name] + else: + raise NameError(name) + file_content = open(self._fpath(name), "rb").read() + decrypted = self.decrypt(file_content) + self.length_cache[name] = (len(decrypted), sha256(decrypted).digest()) + return BufferFile(buffer(decrypted), name=name, onclose=onclose) + + def _store_file(self, name, content): + try: + encrypted_content = self.encrypt(content) + with open(self._fpath(name), 'w+b') as f: + f.write(encrypted_content) + except Exception, e: + print e + raise diff --git a/src/pixelated/support/error_handler.py b/src/pixelated/support/error_handler.py new file mode 100644 index 00000000..1a0e1a11 --- /dev/null +++ b/src/pixelated/support/error_handler.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from requests.exceptions import SSLError + + +def error_handler(excp): + if excp.type is SSLError: + print """ + SSL Error: Please check your certificates or read our wiki for further info: + https://github.com/pixelated-project/pixelated-user-agent/wiki/Configuring-and-using-SSL-Certificates-for-LEAP-provider + Error reference: %s + """ % excp.getErrorMessage() + else: + raise excp diff --git a/src/pixelated/support/functional.py b/src/pixelated/support/functional.py new file mode 100644 index 00000000..2e293625 --- /dev/null +++ b/src/pixelated/support/functional.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from itertools import chain + + +def flatten(_list): + return list(chain.from_iterable(_list)) + + +def unique(_list): + seen = set() + seen_add = seen.add + return [x for x in _list if not (x in seen or seen_add(x))] + + +def compact(_list): + return [a for a in _list if a] + + +def to_unicode(text): + if text and not isinstance(text, unicode): + encoding = 'utf-8' + return unicode(text, encoding=encoding) + return text diff --git a/src/pixelated/support/mail_generator.py b/src/pixelated/support/mail_generator.py new file mode 100644 index 00000000..e01274f3 --- /dev/null +++ b/src/pixelated/support/mail_generator.py @@ -0,0 +1,157 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +from email.mime.text import MIMEText +from email.utils import formatdate +from random import Random +from pixelated.support.markov import MarkovGenerator +import re +from collections import Counter +import time + + +def filter_two_line_on_wrote(lines): + skip_next = False + if len(lines) > 0: + for i in xrange(len(lines) - 1): + if skip_next: + skip_next = False + continue + + if lines[i].startswith('On') and lines[i + 1].endswith('wrote:'): + skip_next = True + else: + yield lines[i].strip() + + yield lines[-1] + + +def filter_lines(text): + pattern = re.compile('\s*[>-].*') + wrote_pattern = re.compile('\s*On.*wrote.*') + + lines = text.splitlines() + + lines = filter(lambda line: not pattern.match(line), lines) + lines = filter(lambda line: not len(line.strip()) == 0, lines) + lines = filter(lambda line: not wrote_pattern.match(line), lines) + lines = filter(lambda line: not line.endswith('writes:'), lines) + lines = filter(lambda line: ' ' in line.strip(), lines) + + lines = filter_two_line_on_wrote(lines) + + return ' '.join(lines) + + +def decode_multipart_mail_text(mail): + for payload in mail.get_payload(): + if payload.get_content_type() == 'text/plain': + return payload.get_payload(decode=True) + return '' + + +def search_for_tags(content): + words = content.split() + + only_alnum = filter(lambda word: word.isalnum(), words) + only_longer = filter(lambda word: len(word) > 5, only_alnum) + lower_case = map(lambda word: word.lower(), only_longer) + + counter = Counter(lower_case) + potential_tags = counter.most_common(10) + + return map(lambda tag: tag[0], potential_tags) + + +def filter_too_short_texts(texts): + return [text for text in texts if text is not None and len(text.split()) >= 3] + + +def load_all_mails(mail_list): + subjects = set() + mail_bodies = [] + + for mail in mail_list: + subjects.add(mail['Subject']) + if mail.is_multipart(): + mail_bodies.append(filter_lines(decode_multipart_mail_text(mail))) + else: + if mail.get_content_type() == 'text/plain': + mail_bodies.append(filter_lines(mail.get_payload(decode=True))) + else: + raise Exception(mail.get_content_type()) + + return filter_too_short_texts(subjects), filter_too_short_texts(mail_bodies) + + +class MailGenerator(object): + + NAMES = ['alice', 'bob', 'eve'] + + def __init__(self, receiver, domain_name, sample_mail_list, random=None): + self._random = random if random else Random() + self._receiver = receiver + self._domain_name = domain_name + self._subjects, self._bodies = load_all_mails(sample_mail_list) + + self._potential_tags = search_for_tags(' '.join(self._bodies)) + self._subject_markov = MarkovGenerator( + self._subjects, random=self._random) + self._body_markov = MarkovGenerator( + self._bodies, random=self._random, add_paragraph_on_empty_chain=True) + + def generate_mail(self): + body = self._body_markov.generate(150) + mail = MIMEText(body) + + mail['Subject'] = self._subject_markov.generate(8) + mail['To'] = '%s@%s' % (self._receiver, self._domain_name) + mail['From'] = self._random_from() + mail['Date'] = self._random_date() + mail['X-Tags'] = self._random_tags() + mail['X-Leap-Encryption'] = self._random_encryption_state() + mail['X-Leap-Signature'] = self._random_signature_state() + + return mail + + def _random_date(self): + now = int(time.time()) + ten_days = 60 * 60 * 24 * 10 + mail_time = self._random.randint(now - ten_days, now) + + return formatdate(mail_time) + + def _random_encryption_state(self): + return self._random.choice(['true', 'decrypted']) + + def _random_signature_state(self): + return self._random.choice(['could not verify', 'valid']) + + def _random_from(self): + name = self._random.choice( + filter(lambda name: name != self._receiver, MailGenerator.NAMES)) + + return '%s@%s' % (name, self._domain_name) + + def _random_tags(self): + barrier = 0.5 + tags = set() + while self._random.random() > barrier: + tags.add(self._random.choice(self._potential_tags)) + barrier += 0.15 + + return ' '.join(tags) diff --git a/src/pixelated/support/markov.py b/src/pixelated/support/markov.py new file mode 100644 index 00000000..8f7c0ef3 --- /dev/null +++ b/src/pixelated/support/markov.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from random import Random + +NEW_PARAGRAPH = '\n\n' + + +class MarkovGenerator(object): + + def __init__(self, texts, random=None, add_paragraph_on_empty_chain=False): + self._markov_chain = {} + self._random = random if random else Random() + self._add_paragraph_on_empty_chain = add_paragraph_on_empty_chain + + for text in filter(lambda _: _ is not None, texts): + self._extend_chain_with(text) + + def add(self, text): + self._extend_chain_with(text) + + @staticmethod + def _triplet_generator(words): + if len(words) < 3: + raise ValueError('Expected input with at least three words') + + for i in xrange(len(words) - 2): + yield ((words[i], words[i + 1]), words[i + 2]) + + def _extend_chain_with(self, input_text): + words = input_text.split() + gen = self._triplet_generator(words) + + for key, value in gen: + if key in self._markov_chain: + self._markov_chain[key].add(value) + else: + self._markov_chain[key] = {value} + + def _generate_chain(self, length): + seed_pair = self._find_good_seed() + word, next_word = seed_pair + new_seed = False + + for i in xrange(length): + yield word + + if new_seed: + word, next_word = self._find_good_seed() + if self._add_paragraph_on_empty_chain: + yield NEW_PARAGRAPH + new_seed = False + else: + prev_word, word = word, next_word + + try: + next_word = self._random_next_word(prev_word, word) + except KeyError: + new_seed = True + + def _random_next_word(self, prev_word, word): + return self._random.choice(list(self._markov_chain[(prev_word, word)])) + + def _find_good_seed(self): + max_tries = len(self._markov_chain.keys()) + try_count = 0 + + seed_pair = self._random.choice(self._markov_chain.keys()) + while not seed_pair[0][0].isupper() and try_count <= max_tries: + seed_pair = self._random.choice(self._markov_chain.keys()) + try_count += 1 + + if try_count > max_tries: + raise ValueError('Not able find start word with captial letter') + + return seed_pair + + def generate(self, length): + if len(self._markov_chain.keys()) == 0: + raise ValueError('Expected at least three words input') + return ' '.join(self._generate_chain(length)) diff --git a/src/pixelated/support/replier.py b/src/pixelated/support/replier.py new file mode 100644 index 00000000..83324eae --- /dev/null +++ b/src/pixelated/support/replier.py @@ -0,0 +1,35 @@ +from email.utils import parseaddr + + +def generate_recipients(sender, to, ccs, current_user): + result = {'single': None, 'all': {'to-field': [], 'cc-field': []}} + + to.append(sender) + to = remove_duplicates(to) + ccs = remove_duplicates(ccs) + + result['single'] = swap_recipient_if_needed( + sender, remove_address(to, current_user), current_user) + result['all'][ + 'to-field'] = remove_address(to, current_user) if len(to) > 1 else to + result['all'][ + 'cc-field'] = remove_address(ccs, current_user) if len(ccs) > 1 else ccs + return result + + +def remove_duplicates(recipients): + return list(set(recipients)) + + +def remove_address(recipients, current_user): + return [recipient for recipient in recipients if not parsed_mail_matches(recipient, current_user)] + + +def parsed_mail_matches(to_parse, expected): + return parseaddr(to_parse)[1] == expected + + +def swap_recipient_if_needed(sender, recipients, current_user): + if len(recipients) == 1 and parsed_mail_matches(sender, current_user): + return recipients[0] + return sender diff --git a/src/pixelated/support/tls_adapter.py b/src/pixelated/support/tls_adapter.py new file mode 100644 index 00000000..301a2123 --- /dev/null +++ b/src/pixelated/support/tls_adapter.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import ssl +from requests.adapters import HTTPAdapter +try: + from urllib3.poolmanager import PoolManager +except: + from requests.packages.urllib3.poolmanager import PoolManager + +VERIFY_HOSTNAME = None + + +def latest_available_ssl_version(): + try: + return ssl.PROTOCOL_TLSv1_2 + except AttributeError: + return ssl.PROTOCOL_TLSv1 + + +class EnforceTLSv1Adapter(HTTPAdapter): + __slots__ = ('_assert_hostname', '_assert_fingerprint') + + def __init__(self, assert_hostname=VERIFY_HOSTNAME, assert_fingerprint=None): + self._assert_hostname = assert_hostname + self._assert_fingerprint = assert_fingerprint + super(EnforceTLSv1Adapter, self).__init__() + + def init_poolmanager(self, connections, maxsize, block=False): + self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, + block=block, + assert_hostname=self._assert_hostname, + assert_fingerprint=self._assert_fingerprint, + cert_reqs=ssl.CERT_REQUIRED) |