summaryrefslogtreecommitdiff
path: root/src/leap/soledad/client/http_target/fetch_protocol.py
blob: 851eb3a17403b13c240aa9571ef519d2279ef933 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
# -*- coding: utf-8 -*-
# fetch_protocol.py
# Copyright (C) 2016 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
from functools import partial
from six import StringIO
from twisted.web._newclient import ResponseDone
from leap.soledad.common.l2db import errors
from leap.soledad.common.l2db.remote import utils
from leap.soledad.common.log import getLogger
from .support import ReadBodyProtocol
from .support import readBody

logger = getLogger(__name__)


class DocStreamReceiver(ReadBodyProtocol):
    """
    A protocol implementation that can parse incoming data from server based
    on a line format specified on u1db implementation. Except that we split doc
    attributes from content to ease parsing and increment throughput for larger
    documents.
    [\r\n
    {metadata},\r\n
    {doc_info},\r\n
    {content},\r\n
    ...
    {doc_info},\r\n
    {content},\r\n
    ]
    """

    def __init__(self, response, deferred, doc_reader):
        self.deferred = deferred
        self.status = response.code if response else None
        self.message = response.phrase if response else None
        self.headers = response.headers if response else {}
        self.delimiter = '\r\n'
        self.metadata = ''
        self._doc_reader = doc_reader
        self.reset()

    def reset(self):
        self._line = 0
        self._buffer = StringIO()
        self._properly_finished = False

    def connectionLost(self, reason):
        """
        Deliver the accumulated response bytes to the waiting L{Deferred}, if
        the response body has been completely received without error.
        """
        if self.deferred.called:
            return
        try:
            if reason.check(ResponseDone):
                self.dataBuffer = self.metadata
            else:
                self.dataBuffer = self.finish()
        except errors.BrokenSyncStream as e:
            return self.deferred.errback(e)
        return ReadBodyProtocol.connectionLost(self, reason)

    def consumeBufferLines(self):
        """
        Consumes lines from buffer and rewind it, writing remaining data
        that didn't formed a line back into buffer.
        """
        content = self._buffer.getvalue()[0:self._buffer.tell()]
        self._buffer.seek(0)
        lines = content.split(self.delimiter)
        self._buffer.write(lines.pop(-1))
        return lines

    def dataReceived(self, data):
        """
        Buffer incoming data until a line breaks comes in. We check only
        the incoming data for efficiency.
        """
        self._buffer.write(data)
        if '\n' not in data:
            return
        lines = self.consumeBufferLines()
        while lines:
            line, _ = utils.check_and_strip_comma(lines.pop(0))
            self.lineReceived(line)
            self._line += 1

    def lineReceived(self, line):
        """
        Protocol implementation.
        0:      [\r\n
        1:      {metadata},\r\n
        (even): {doc_info},\r\n
        (odd):  {data},\r\n
        (last): ]
        """
        if self._properly_finished:
            raise errors.BrokenSyncStream("Reading a finished stream")
        if ']' == line:
            self._properly_finished = True
        elif self._line == 0:
            if line is not '[':
                raise errors.BrokenSyncStream("Invalid start")
        elif self._line == 1:
            self.metadata = line
            if 'error' in self.metadata:
                raise errors.BrokenSyncStream("Error from server: %s" % line)
            self.total = json.loads(line).get('number_of_changes', -1)
        elif (self._line % 2) == 0:
            self.current_doc = json.loads(line)
            if 'error' in self.current_doc:
                raise errors.BrokenSyncStream("Error from server: %s" % line)
        else:
            d = self._doc_reader(
                self.current_doc, line.strip() or None, self.total)
            d.addErrback(self.deferred.errback)

    def finish(self):
        """
        Checks that ']' came and stream was properly closed.
        """
        if not self._properly_finished:
            raise errors.BrokenSyncStream('Stream not properly closed')
        content = self._buffer.getvalue()[0:self._buffer.tell()]
        self._buffer.close()
        return content


def build_body_reader(doc_reader):
    """
    Get the documents from a sync stream and call doc_reader on each
    doc received.

    @param doc_reader: Function to be called for processing an incoming doc.
        Will be called with doc metadata (dict parsed from 1st line) and doc
        content (string)
    @type doc_reader: function

    @return: A function that can be called by the http Agent to create and
    configure the proper protocol.
    """
    protocolClass = partial(DocStreamReceiver, doc_reader=doc_reader)
    return partial(readBody, protocolClass=protocolClass)