summaryrefslogtreecommitdiff
path: root/service/src/pixelated/support
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2017-07-25 11:40:11 -0400
committerKali Kaneko <kali@leap.se>2017-07-25 11:40:29 -0400
commit91e4481c450eb7eb928debc1cb7fa59bdb63dd7b (patch)
tree8fd7e6e77b6df669c33d96b7edad6db3cbe14dfe /service/src/pixelated/support
parente4f755309d4cf5cfb6b0bcc62ed73d6070956ab5 (diff)
[pkg] packaging and path changes
- move all the pixelated python package under src/ - move the pixelated_www package under the leap namespace - allow to set globally the static folder - add hours and minutes to the timestamp in package version, to allow for several releases a day.
Diffstat (limited to 'service/src/pixelated/support')
-rw-r--r--service/src/pixelated/support/__init__.py76
-rw-r--r--service/src/pixelated/support/clock.py32
-rw-r--r--service/src/pixelated/support/date.py29
-rw-r--r--service/src/pixelated/support/encrypted_file_storage.py147
-rw-r--r--service/src/pixelated/support/error_handler.py27
-rw-r--r--service/src/pixelated/support/functional.py37
-rw-r--r--service/src/pixelated/support/language.py24
-rw-r--r--service/src/pixelated/support/loglinegenerator.py23
-rw-r--r--service/src/pixelated/support/mail_generator.py154
-rw-r--r--service/src/pixelated/support/markov.py94
-rw-r--r--service/src/pixelated/support/replier.py48
-rw-r--r--service/src/pixelated/support/tls_adapter.py47
12 files changed, 738 insertions, 0 deletions
diff --git a/service/src/pixelated/support/__init__.py b/service/src/pixelated/support/__init__.py
new file mode 100644
index 00000000..0685f48d
--- /dev/null
+++ b/service/src/pixelated/support/__init__.py
@@ -0,0 +1,76 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import time
+from functools import wraps
+from twisted.internet import defer
+from twisted.logger import Logger
+
+
+log = Logger()
+
+
+def _start_stopwatch():
+ return (time.time(), time.clock())
+
+
+def _stop_stopwatch(start):
+ start_time, start_clock = start
+ end_clock = time.clock()
+ end_time = time.time()
+ clock_duration = end_clock - start_clock
+ time_duration = end_time - start_time
+ if time_duration < 0.00000001: # avoid division by zero
+ time_duration = 0.00000001
+
+ estimate_percent_io = ((time_duration - clock_duration) / time_duration) * 100.0
+
+ return time_duration, clock_duration, estimate_percent_io
+
+
+def log_time(f):
+
+ @wraps(f)
+ def wrapper(*args, **kwds):
+ start = _start_stopwatch()
+
+ result = f(*args, **kwds)
+
+ time_duration, clock_duration, estimate_percent_io = _stop_stopwatch(start)
+ log.info('Needed %fs (%fs cpu time, %.2f%% spent outside process) to execute %s' % (time_duration, clock_duration, estimate_percent_io, f))
+
+ return result
+
+ return wrapper
+
+
+def log_time_deferred(f):
+
+ def log_time(result, start):
+ time_duration, clock_duration, estimate_percent_io = _stop_stopwatch(start)
+ log.info('after callback: Needed %fs (%fs cpu time, %.2f%% spent outside process) to execute %s' % (time_duration, clock_duration, estimate_percent_io, f))
+ return result
+
+ @wraps(f)
+ def wrapper(*args, **kwds):
+ start = _start_stopwatch()
+ result = f(*args, **kwds)
+ if isinstance(result, defer.Deferred):
+ result.addCallback(log_time, start=start)
+ else:
+ log.warn('No Deferred returned, perhaps need to re-order annotations?')
+ return result
+
+ return wrapper
diff --git a/service/src/pixelated/support/clock.py b/service/src/pixelated/support/clock.py
new file mode 100644
index 00000000..9cab8857
--- /dev/null
+++ b/service/src/pixelated/support/clock.py
@@ -0,0 +1,32 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+from datetime import datetime
+from os.path import expanduser
+
+
+class Clock():
+
+ def __init__(self, label, user=None):
+ self.start = datetime.now()
+ self.label = label
+ self.user = user
+
+ def stop(self, fresh=False, user=None):
+ end = datetime.now()
+ with open(expanduser('~/MetricsTime'), 'a') as f:
+ flag = ' fresh-account' if fresh else ''
+ f.write('{} {:.5f} {} {}\n'.format((self.user or user or 'Unknown'), (end - self.start).total_seconds(), self.label, flag))
diff --git a/service/src/pixelated/support/date.py b/service/src/pixelated/support/date.py
new file mode 100644
index 00000000..0012aeea
--- /dev/null
+++ b/service/src/pixelated/support/date.py
@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import datetime
+
+import dateutil.parser
+from email.utils import formatdate
+from dateutil.tz import tzlocal
+
+
+def iso_now():
+ return datetime.datetime.now(tzlocal()).isoformat()
+
+
+def mail_date_now():
+ date = dateutil.parser.parse(iso_now())
+ return formatdate(float(date.strftime('%s')))
diff --git a/service/src/pixelated/support/encrypted_file_storage.py b/service/src/pixelated/support/encrypted_file_storage.py
new file mode 100644
index 00000000..a1dbffa8
--- /dev/null
+++ b/service/src/pixelated/support/encrypted_file_storage.py
@@ -0,0 +1,147 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import with_statement
+
+import hmac
+import io
+import os
+from hashlib import sha256
+
+from leap.soledad.client.crypto import decrypt_sym
+from leap.soledad.client.crypto import encrypt_sym
+from whoosh.filedb.filestore import FileStorage
+from whoosh.filedb.structfile import BufferFile, StructFile
+from whoosh.util import random_name
+
+
+class DelayedCloseBytesIO(io.BytesIO):
+ def __init__(self, name):
+ super(DelayedCloseBytesIO, self).__init__()
+ self._name = name
+ self.shouldClose = False
+
+ def close(self):
+
+ self.shouldClose = True
+
+ def explicit_close(self):
+ super(DelayedCloseBytesIO, self).close()
+
+
+class DelayedCloseStructFile(StructFile):
+ def __init__(self, fileobj, name=None, onclose=None):
+ super(DelayedCloseStructFile, self).__init__(fileobj, name, onclose)
+
+ def close(self):
+ """Closes the wrapped file.
+ """
+
+ if self.is_closed:
+ raise Exception("This file is already closed")
+ if self.onclose:
+ self.onclose(self)
+ if hasattr(self.file, "explicit_close"):
+ self.file.explicit_close()
+ self.is_closed = True
+
+
+class EncryptedFileStorage(FileStorage):
+ def __init__(self, path, masterkey=None):
+ FileStorage.__init__(self, path, supports_mmap=False)
+ self.masterkey = masterkey[:32]
+ self.signkey = masterkey[32:]
+ self._tmp_storage = self.temp_storage
+ self.length_cache = {}
+ self._open_files = {}
+
+ def open_file(self, name, **kwargs):
+ return self._open_encrypted_file(name)
+
+ def create_file(self, name, excl=False, mode="w+b", **kwargs):
+ f = DelayedCloseStructFile(DelayedCloseBytesIO(name), name=name, onclose=self._encrypt_index_on_close(name))
+ f.is_real = False
+ self._open_files[name] = f
+ return f
+
+ def delete_file(self, name):
+ super(EncryptedFileStorage, self).delete_file(name)
+ if name in self._open_files:
+ del self._open_files[name]
+
+ def temp_storage(self, name=None):
+ name = name or "%s.tmp" % random_name()
+ path = os.path.join(self.folder, name)
+ return EncryptedFileStorage(path, self.masterkey).create()
+
+ def file_length(self, name):
+ return self.length_cache[name][0]
+
+ def gen_mac(self, iv, ciphertext):
+ verifiable_payload = ''.join((iv, ciphertext))
+ return hmac.new(self.signkey, verifiable_payload, sha256).digest()
+
+ def encrypt(self, content):
+ iv, ciphertext = encrypt_sym(content, self.masterkey)
+ mac = self.gen_mac(iv, ciphertext)
+ return ''.join((mac, iv, ciphertext))
+
+ def decrypt(self, payload):
+ payload_mac, iv, ciphertext = payload[:32], payload[32:57], payload[57:]
+ generated_mac = self.gen_mac(iv, ciphertext)
+ if sha256(payload_mac).digest() != sha256(generated_mac).digest():
+ raise Exception("EncryptedFileStorage - Error opening file. Wrong MAC")
+ return decrypt_sym(ciphertext, self.masterkey, iv)
+
+ def _encrypt_index_on_close(self, name):
+ def wrapper(struct_file):
+ struct_file.seek(0)
+ content = struct_file.file.read()
+ file_hash = sha256(content).digest()
+ if name in self.length_cache and file_hash == self.length_cache[name][1]:
+ return
+ self.length_cache[name] = (len(content), file_hash)
+ encrypted_content = self.encrypt(content)
+ with open(self._fpath(name), 'w+b') as f:
+ f.write(encrypted_content)
+
+ return wrapper
+
+ def _open_encrypted_file(self, name, onclose=lambda x: None):
+ if not self.file_exists(name):
+ if name in self._open_files:
+ f = self._open_files[name]
+ if not f.is_closed:
+ state = 'closed' if f.file.shouldClose else 'open'
+ if state == 'closed':
+ self._store_file(name, f.file.getvalue())
+ f.close()
+ del self._open_files[name]
+ else:
+ raise NameError(name)
+ file_content = open(self._fpath(name), "rb").read()
+ decrypted = self.decrypt(file_content)
+ self.length_cache[name] = (len(decrypted), sha256(decrypted).digest())
+ return BufferFile(buffer(decrypted), name=name, onclose=onclose)
+
+ def _store_file(self, name, content):
+ try:
+ encrypted_content = self.encrypt(content)
+ with open(self._fpath(name), 'w+b') as f:
+ f.write(encrypted_content)
+ except Exception, e:
+ print e
+ raise
diff --git a/service/src/pixelated/support/error_handler.py b/service/src/pixelated/support/error_handler.py
new file mode 100644
index 00000000..1a0e1a11
--- /dev/null
+++ b/service/src/pixelated/support/error_handler.py
@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from requests.exceptions import SSLError
+
+
+def error_handler(excp):
+ if excp.type is SSLError:
+ print """
+ SSL Error: Please check your certificates or read our wiki for further info:
+ https://github.com/pixelated-project/pixelated-user-agent/wiki/Configuring-and-using-SSL-Certificates-for-LEAP-provider
+ Error reference: %s
+ """ % excp.getErrorMessage()
+ else:
+ raise excp
diff --git a/service/src/pixelated/support/functional.py b/service/src/pixelated/support/functional.py
new file mode 100644
index 00000000..2e293625
--- /dev/null
+++ b/service/src/pixelated/support/functional.py
@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from itertools import chain
+
+
+def flatten(_list):
+ return list(chain.from_iterable(_list))
+
+
+def unique(_list):
+ seen = set()
+ seen_add = seen.add
+ return [x for x in _list if not (x in seen or seen_add(x))]
+
+
+def compact(_list):
+ return [a for a in _list if a]
+
+
+def to_unicode(text):
+ if text and not isinstance(text, unicode):
+ encoding = 'utf-8'
+ return unicode(text, encoding=encoding)
+ return text
diff --git a/service/src/pixelated/support/language.py b/service/src/pixelated/support/language.py
new file mode 100644
index 00000000..cd455f89
--- /dev/null
+++ b/service/src/pixelated/support/language.py
@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2017 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+
+def parse_accept_language(all_headers):
+ accepted_languages = ['pt-BR', 'en-US']
+ languages = all_headers.get('accept-language', '').split(';')[0]
+ for language in accepted_languages:
+ if language in languages:
+ return language
+ return 'en-US'
diff --git a/service/src/pixelated/support/loglinegenerator.py b/service/src/pixelated/support/loglinegenerator.py
new file mode 100644
index 00000000..d8a8fd5b
--- /dev/null
+++ b/service/src/pixelated/support/loglinegenerator.py
@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+from zope.interface import Interface
+
+
+class ILogLineGenerator(Interface):
+ def getLogLine(self):
+ """ Return a string that will be logged, or None. This method will be called every second.
+ """
diff --git a/service/src/pixelated/support/mail_generator.py b/service/src/pixelated/support/mail_generator.py
new file mode 100644
index 00000000..e5232370
--- /dev/null
+++ b/service/src/pixelated/support/mail_generator.py
@@ -0,0 +1,154 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+
+from email.mime.text import MIMEText
+from email.utils import formatdate
+from random import Random
+from pixelated.support.markov import MarkovGenerator
+import re
+from collections import Counter
+import time
+
+
+def filter_two_line_on_wrote(lines):
+ skip_next = False
+ if len(lines) > 0:
+ for i in xrange(len(lines) - 1):
+ if skip_next:
+ skip_next = False
+ continue
+
+ if lines[i].startswith('On') and lines[i + 1].endswith('wrote:'):
+ skip_next = True
+ else:
+ yield lines[i].strip()
+
+ yield lines[-1]
+
+
+def filter_lines(text):
+ pattern = re.compile('\s*[>-].*')
+ wrote_pattern = re.compile('\s*On.*wrote.*')
+
+ lines = text.splitlines()
+
+ lines = filter(lambda line: not pattern.match(line), lines)
+ lines = filter(lambda line: not len(line.strip()) == 0, lines)
+ lines = filter(lambda line: not wrote_pattern.match(line), lines)
+ lines = filter(lambda line: not line.endswith('writes:'), lines)
+ lines = filter(lambda line: ' ' in line.strip(), lines)
+
+ lines = filter_two_line_on_wrote(lines)
+
+ return ' '.join(lines)
+
+
+def decode_multipart_mail_text(mail):
+ for payload in mail.get_payload():
+ if payload.get_content_type() == 'text/plain':
+ return payload.get_payload(decode=True)
+ return ''
+
+
+def search_for_tags(content):
+ words = content.split()
+
+ only_alnum = filter(lambda word: word.isalnum(), words)
+ only_longer = filter(lambda word: len(word) > 5, only_alnum)
+ lower_case = map(lambda word: word.lower(), only_longer)
+
+ counter = Counter(lower_case)
+ potential_tags = counter.most_common(10)
+
+ return map(lambda tag: tag[0], potential_tags)
+
+
+def filter_too_short_texts(texts):
+ return [text for text in texts if text is not None and len(text.split()) >= 3]
+
+
+def load_all_mails(mail_list):
+ subjects = set()
+ mail_bodies = []
+
+ for mail in mail_list:
+ subjects.add(mail['Subject'])
+ if mail.is_multipart():
+ mail_bodies.append(filter_lines(decode_multipart_mail_text(mail)))
+ else:
+ if mail.get_content_type() == 'text/plain':
+ mail_bodies.append(filter_lines(mail.get_payload(decode=True)))
+ else:
+ raise Exception(mail.get_content_type())
+
+ return filter_too_short_texts(subjects), filter_too_short_texts(mail_bodies)
+
+
+class MailGenerator(object):
+
+ NAMES = ['alice', 'bob', 'eve']
+
+ def __init__(self, receiver, domain_name, sample_mail_list, random=None):
+ self._random = random if random else Random()
+ self._receiver = receiver
+ self._domain_name = domain_name
+ self._subjects, self._bodies = load_all_mails(sample_mail_list)
+
+ self._potential_tags = search_for_tags(' '.join(self._bodies))
+ self._subject_markov = MarkovGenerator(self._subjects, random=self._random)
+ self._body_markov = MarkovGenerator(self._bodies, random=self._random, add_paragraph_on_empty_chain=True)
+
+ def generate_mail(self):
+ body = self._body_markov.generate(150)
+ mail = MIMEText(body)
+
+ mail['Subject'] = self._subject_markov.generate(8)
+ mail['To'] = '%s@%s' % (self._receiver, self._domain_name)
+ mail['From'] = self._random_from()
+ mail['Date'] = self._random_date()
+ mail['X-Tags'] = self._random_tags()
+ mail['X-Leap-Encryption'] = self._random_encryption_state()
+ mail['X-Leap-Signature'] = self._random_signature_state()
+
+ return mail
+
+ def _random_date(self):
+ now = int(time.time())
+ ten_days = 60 * 60 * 24 * 10
+ mail_time = self._random.randint(now - ten_days, now)
+
+ return formatdate(mail_time)
+
+ def _random_encryption_state(self):
+ return self._random.choice(['true', 'decrypted'])
+
+ def _random_signature_state(self):
+ return self._random.choice(['could not verify', 'valid'])
+
+ def _random_from(self):
+ name = self._random.choice(filter(lambda name: name != self._receiver, MailGenerator.NAMES))
+
+ return '%s@%s' % (name, self._domain_name)
+
+ def _random_tags(self):
+ barrier = 0.5
+ tags = set()
+ while self._random.random() > barrier:
+ tags.add(self._random.choice(self._potential_tags))
+ barrier += 0.15
+
+ return ' '.join(tags)
diff --git a/service/src/pixelated/support/markov.py b/service/src/pixelated/support/markov.py
new file mode 100644
index 00000000..8f7c0ef3
--- /dev/null
+++ b/service/src/pixelated/support/markov.py
@@ -0,0 +1,94 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+from random import Random
+
+NEW_PARAGRAPH = '\n\n'
+
+
+class MarkovGenerator(object):
+
+ def __init__(self, texts, random=None, add_paragraph_on_empty_chain=False):
+ self._markov_chain = {}
+ self._random = random if random else Random()
+ self._add_paragraph_on_empty_chain = add_paragraph_on_empty_chain
+
+ for text in filter(lambda _: _ is not None, texts):
+ self._extend_chain_with(text)
+
+ def add(self, text):
+ self._extend_chain_with(text)
+
+ @staticmethod
+ def _triplet_generator(words):
+ if len(words) < 3:
+ raise ValueError('Expected input with at least three words')
+
+ for i in xrange(len(words) - 2):
+ yield ((words[i], words[i + 1]), words[i + 2])
+
+ def _extend_chain_with(self, input_text):
+ words = input_text.split()
+ gen = self._triplet_generator(words)
+
+ for key, value in gen:
+ if key in self._markov_chain:
+ self._markov_chain[key].add(value)
+ else:
+ self._markov_chain[key] = {value}
+
+ def _generate_chain(self, length):
+ seed_pair = self._find_good_seed()
+ word, next_word = seed_pair
+ new_seed = False
+
+ for i in xrange(length):
+ yield word
+
+ if new_seed:
+ word, next_word = self._find_good_seed()
+ if self._add_paragraph_on_empty_chain:
+ yield NEW_PARAGRAPH
+ new_seed = False
+ else:
+ prev_word, word = word, next_word
+
+ try:
+ next_word = self._random_next_word(prev_word, word)
+ except KeyError:
+ new_seed = True
+
+ def _random_next_word(self, prev_word, word):
+ return self._random.choice(list(self._markov_chain[(prev_word, word)]))
+
+ def _find_good_seed(self):
+ max_tries = len(self._markov_chain.keys())
+ try_count = 0
+
+ seed_pair = self._random.choice(self._markov_chain.keys())
+ while not seed_pair[0][0].isupper() and try_count <= max_tries:
+ seed_pair = self._random.choice(self._markov_chain.keys())
+ try_count += 1
+
+ if try_count > max_tries:
+ raise ValueError('Not able find start word with captial letter')
+
+ return seed_pair
+
+ def generate(self, length):
+ if len(self._markov_chain.keys()) == 0:
+ raise ValueError('Expected at least three words input')
+ return ' '.join(self._generate_chain(length))
diff --git a/service/src/pixelated/support/replier.py b/service/src/pixelated/support/replier.py
new file mode 100644
index 00000000..bab23179
--- /dev/null
+++ b/service/src/pixelated/support/replier.py
@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+from email.utils import parseaddr
+
+
+def generate_recipients(sender, to, ccs, current_user):
+ result = {'single': None, 'all': {'to-field': [], 'cc-field': []}}
+
+ to.append(sender)
+ to = remove_duplicates(to)
+ ccs = remove_duplicates(ccs)
+
+ result['single'] = swap_recipient_if_needed(sender, remove_address(to, current_user), current_user)
+ result['all']['to-field'] = remove_address(to, current_user) if len(to) > 1 else to
+ result['all']['cc-field'] = remove_address(ccs, current_user) if len(ccs) > 1 else ccs
+ return result
+
+
+def remove_duplicates(recipients):
+ return list(set(recipients))
+
+
+def remove_address(recipients, current_user):
+ return [recipient for recipient in recipients if not parsed_mail_matches(recipient, current_user)]
+
+
+def parsed_mail_matches(to_parse, expected):
+ return parseaddr(to_parse)[1] == expected
+
+
+def swap_recipient_if_needed(sender, recipients, current_user):
+ if len(recipients) == 1 and parsed_mail_matches(sender, current_user):
+ return recipients[0]
+ return sender
diff --git a/service/src/pixelated/support/tls_adapter.py b/service/src/pixelated/support/tls_adapter.py
new file mode 100644
index 00000000..301a2123
--- /dev/null
+++ b/service/src/pixelated/support/tls_adapter.py
@@ -0,0 +1,47 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+import ssl
+from requests.adapters import HTTPAdapter
+try:
+ from urllib3.poolmanager import PoolManager
+except:
+ from requests.packages.urllib3.poolmanager import PoolManager
+
+VERIFY_HOSTNAME = None
+
+
+def latest_available_ssl_version():
+ try:
+ return ssl.PROTOCOL_TLSv1_2
+ except AttributeError:
+ return ssl.PROTOCOL_TLSv1
+
+
+class EnforceTLSv1Adapter(HTTPAdapter):
+ __slots__ = ('_assert_hostname', '_assert_fingerprint')
+
+ def __init__(self, assert_hostname=VERIFY_HOSTNAME, assert_fingerprint=None):
+ self._assert_hostname = assert_hostname
+ self._assert_fingerprint = assert_fingerprint
+ super(EnforceTLSv1Adapter, self).__init__()
+
+ def init_poolmanager(self, connections, maxsize, block=False):
+ self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
+ block=block,
+ assert_hostname=self._assert_hostname,
+ assert_fingerprint=self._assert_fingerprint,
+ cert_reqs=ssl.CERT_REQUIRED)