Avoid replication task crashes by imporoving timeout handling.

Closes COUCHDB-597.

Patch by Randall Leeds.

git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@916868 13f79535-47bb-0310-9956-ffa450edef68

2 files changed, 33 insertions, 5 deletions

diff --git a/src/couchdb/couch_rep_att.erl b/src/couchdb/couch_rep_att.erl
index 6b576a01..ee4f0184 100644
--- a/src/couchdb/couch_rep_att.erl
+++ b/src/couchdb/couch_rep_att.erl
@@ -25,7 +25,11 @@ convert_stub(#att{data=stub, name=Name} = Attachment,
         qs = [{rev, couch_doc:rev_to_str({Pos,RevId})}]
     },
     Ref = make_ref(),
-    RcvFun = fun() -> attachment_receiver(Ref, Request) end,
+    RcvFun = fun() ->
+                 Bin = attachment_receiver(Ref, Request),
+                 cleanup(),
+                 Bin
+             end,
     Attachment#att{data=RcvFun}.
 
 cleanup() ->
@@ -34,6 +38,8 @@ cleanup() ->
         %% TODO maybe log, didn't expect to have data here
         cleanup();
     {ibrowse_async_response_end, _} -> 
+        cleanup();
+    {ibrowse_async_headers, _, _, _} ->
         cleanup()
     after 0 ->
         erase(),
@@ -43,13 +49,27 @@ cleanup() ->
 % internal funs
 
 attachment_receiver(Ref, Request) ->
-    case get(Ref) of
+    try case get(Ref) of
     undefined ->
         {ReqId, ContentEncoding} = start_http_request(Request),
         put(Ref, {ReqId, ContentEncoding}),
         receive_data(Ref, ReqId, ContentEncoding);
     {ReqId, ContentEncoding} ->
         receive_data(Ref, ReqId, ContentEncoding)
+    end
+    catch
+    throw:{attachment_request_failed, timeout} ->
+        case {Request#http_db.retries, Request#http_db.pause} of
+        {0, _} ->
+             ?LOG_INFO("request for ~p failed", [Request#http_db.resource]),
+             throw({attachment_request_failed, max_retries_reached});
+        {N, Pause} when N > 0 ->
+            ?LOG_INFO("request for ~p timed out, retrying in ~p seconds",
+                [Request#http_db.resource, Pause/1000]),
+            timer:sleep(Pause),
+            cleanup(),
+            attachment_receiver(Ref, Request#http_db{retries = N-1})
+        end
     end.
 
 receive_data(Ref, ReqId, ContentEncoding) ->
@@ -71,6 +91,8 @@ receive_data(Ref, ReqId, ContentEncoding) ->
     {ibrowse_async_response_end, ReqId} ->
         ?LOG_ERROR("streaming att. ended but more data requested ~p", [ReqId]),
         throw({attachment_request_failed, premature_end})
+    after 31000 ->
+        throw({attachment_request_failed, timeout})
     end.
 
 start_http_request(Req) ->
@@ -84,6 +106,8 @@ start_http_request(Req) ->
         {ok, ContentEncoding, NewReqId} ->
             {NewReqId, ContentEncoding}
         end
+    after 10000 ->
+        throw({attachment_request_failed, timeout})
     end.
 
 validate_headers(_Req, 200, Headers) ->
diff --git a/src/couchdb/couch_rep_changes_feed.erl b/src/couchdb/couch_rep_changes_feed.erl
index 67cfabe4..7c308800 100644
--- a/src/couchdb/couch_rep_changes_feed.erl
+++ b/src/couchdb/couch_rep_changes_feed.erl
@@ -40,7 +40,9 @@ start_link(Parent, Source, StartSeq, PostProps) ->
     gen_server:start_link(?MODULE, [Parent, Source, StartSeq, PostProps], []).
 
 next(Server) ->
-    gen_server:call(Server, next_changes, infinity).
+    try gen_server:call(Server, next_changes, 31000)
+    catch exit:{timeout, _Reason} -> throw(changes_timeout)
+    end.
 
 stop(Server) ->
     gen_server:call(Server, stop).
@@ -83,8 +85,7 @@ init([_Parent, #http_db{}=Source, Since, PostProps] = Args) ->
         resource = "_changes",
         qs = QS,
         conn = Pid,
-        options = [{stream_to, {self(), once}}, {response_format, binary},
-            {inactivity_timeout, 31000}], % miss 3 heartbeats, assume death
+        options = [{stream_to, {self(), once}}, {response_format, binary}],
         headers = Source#http_db.headers -- [{"Accept-Encoding", "gzip"}]
     },
     {ibrowse_req_id, ReqId} = couch_rep_httpc:request(Req),
@@ -202,6 +203,9 @@ handle_info({'EXIT', From, Reason}, #state{changes_loop=From} = State) ->
     ?LOG_ERROR("changes_loop died with reason ~p", [Reason]),
     {stop, changes_loop_died, State};
 
+handle_info({'EXIT', _From, normal}, State) ->
+    {noreply, State};
+
 handle_info(Msg, State) ->
     ?LOG_DEBUG("unexpected message at changes_feed ~p", [Msg]),
     {noreply, State}.