summaryrefslogtreecommitdiff
path: root/src/couchdb/couch_rep.erl
diff options
context:
space:
mode:
authorAdam Kocoloski <kocolosk@apache.org>2009-05-18 18:11:17 +0000
committerAdam Kocoloski <kocolosk@apache.org>2009-05-18 18:11:17 +0000
commitda50be8a9b46b69e0ed9a9d5182be37cf54ed21c (patch)
tree8e875afd27441737b8af3d0e831cb4283781030e /src/couchdb/couch_rep.erl
parent2a585a7029fae665b8370b20da6fcd07fec58a63 (diff)
first crack at binary backoff for failed replicator requests
git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@776031 13f79535-47bb-0310-9956-ffa450edef68
Diffstat (limited to 'src/couchdb/couch_rep.erl')
-rw-r--r--src/couchdb/couch_rep.erl72
1 files changed, 47 insertions, 25 deletions
diff --git a/src/couchdb/couch_rep.erl b/src/couchdb/couch_rep.erl
index c356e189..6dfe6fee 100644
--- a/src/couchdb/couch_rep.erl
+++ b/src/couchdb/couch_rep.erl
@@ -403,14 +403,14 @@ attachment_stub_converter(DbS, Id, Rev, {Name, {stub, Type, Length}}) ->
{Name, {Type, {RcvFun, Length}}}.
make_attachment_stub_receiver(Url, Headers, Name, Type, Length) ->
- make_attachment_stub_receiver(Url, Headers, Name, Type, Length, 10).
+ make_attachment_stub_receiver(Url, Headers, Name, Type, Length, 10, 1000).
-make_attachment_stub_receiver(Url, _Headers, _Name, _Type, _Length, 0) ->
+make_attachment_stub_receiver(Url, _Headers, _Name, _Type, _Length, 0, _Pause) ->
?LOG_ERROR("streaming attachment request failed after 10 retries: ~s",
[Url]),
exit({attachment_request_failed, ?l2b(["failed to replicate ", Url])});
-make_attachment_stub_receiver(Url, Headers, Name, Type, Length, Retries) ->
+make_attachment_stub_receiver(Url, Headers, Name, Type, Length, Retries, Pause) ->
%% start the process that receives attachment data from ibrowse
#url{host=Host, port=Port} = ibrowse_lib:parse_url(Url),
{ok, Conn} = ibrowse:spawn_link_worker_process(Host, Port),
@@ -420,22 +420,37 @@ make_attachment_stub_receiver(Url, Headers, Name, Type, Length, Retries) ->
Opts = [{stream_to, Pid}, {response_format, binary}],
ReqId =
case ibrowse:send_req_direct(Conn, Url, Headers, get, [], Opts, infinity) of
- {ibrowse_req_id, X} -> X;
- {error, Reason} ->
- exit({attachment_request_failed,
- ?l2b(["ibrowse error on ", Url, " : ", atom_to_list(Reason)])})
+ {ibrowse_req_id, X} ->
+ X;
+ {error, Reason} ->
+ ?LOG_INFO("retrying couch_rep attachment request in ~p " ++
+ "seconds due to {error, ~p}: ~s", [Pause/1000, Reason, Url]),
+ catch ibrowse:stop_worker_process(Conn),
+ timer:sleep(Pause),
+ make_attachment_stub_receiver(Url, Headers, Name, Type, Length,
+ Retries-1, 2*Pause)
end,
%% tell our receiver about the ReqId it needs to look for
Pid ! {self(), {set_req_id, ReqId}},
- receive {Pid, {ok, ReqId}} -> ok end,
+ receive
+ {Pid, {ok, ReqId}} ->
+ ok;
+ {'EXIT', Pid, _Reason} ->
+ catch ibrowse:stop_worker_process(Conn),
+ timer:sleep(Pause),
+ make_attachment_stub_receiver(Url, Headers, Name, Type, Length,
+ Retries-1, 2*Pause)
+ end,
%% wait for headers to ensure that we have a 200 status code
%% this is where we follow redirects etc
Pid ! {self(), gimme_status},
receive
{'EXIT', Pid, attachment_request_failed} ->
- make_attachment_stub_receiver(Url, Headers, Name, Type, Length, Retries-1);
+ catch ibrowse:stop_worker_process(Conn),
+ make_attachment_stub_receiver(Url, Headers, Name, Type, Length,
+ Retries-1, Pause);
{Pid, {status, StreamStatus, StreamHeaders}} ->
?LOG_DEBUG("streaming attachment Status ~p Headers ~p",
[StreamStatus, StreamHeaders]),
@@ -462,8 +477,9 @@ make_attachment_stub_receiver(Url, Headers, Name, Type, Length, Retries) ->
Pid ! {self(), stop_ok},
RedirectUrl = mochiweb_headers:get_value("Location",
mochiweb_headers:make(StreamHeaders)),
- make_attachment_stub_receiver(RedirectUrl, Headers, Name, Type, Length,
- Retries - 1);
+ catch ibrowse:stop_worker_process(Conn),
+ make_attachment_stub_receiver(RedirectUrl, Headers, Name, Type,
+ Length, Retries - 1, Pause);
ResponseCode >= 400, ResponseCode < 500 ->
% an error... log and fail
?LOG_ERROR("streaming attachment failed with code ~p: ~s",
@@ -472,10 +488,13 @@ make_attachment_stub_receiver(Url, Headers, Name, Type, Length, Retries) ->
exit(attachment_request_failed);
ResponseCode == 500 ->
% an error... log and retry
- ?LOG_INFO("retrying streaming attachment request due to 500 error: ~s", [Url]),
+ ?LOG_INFO("retrying couch_rep attachment request in ~p " ++
+ "seconds due to 500 response: ~s", [Pause/1000, Url]),
Pid ! {self(), fail},
+ catch ibrowse:stop_worker_process(Conn),
+ timer:sleep(Pause),
make_attachment_stub_receiver(Url, Headers, Name, Type, Length,
- Retries - 1)
+ Retries - 1, 2*Pause)
end
end.
@@ -588,15 +607,15 @@ do_http_request(Url, Action, Headers) ->
do_http_request(Url, Action, Headers, []).
do_http_request(Url, Action, Headers, JsonBody) ->
- do_http_request(Url, Action, Headers, JsonBody, 10).
+ do_http_request(Url, Action, Headers, JsonBody, 10, 1000).
-do_http_request(Url, Action, Headers, Body, Retries) when is_binary(Url) ->
- do_http_request(?b2l(Url), Action, Headers, Body, Retries);
-do_http_request(Url, Action, _Headers, _JsonBody, 0) ->
+do_http_request(Url, Action, Headers, Body, Retries, Pause) when is_binary(Url) ->
+ do_http_request(?b2l(Url), Action, Headers, Body, Retries, Pause);
+do_http_request(Url, Action, _Headers, _JsonBody, 0, _Pause) ->
?LOG_ERROR("couch_rep HTTP ~p request failed after 10 retries: ~s",
[Action, Url]),
exit({http_request_failed, ?l2b(["failed to replicate ", Url])});
-do_http_request(Url, Action, Headers, JsonBody, Retries) ->
+do_http_request(Url, Action, Headers, JsonBody, Retries, Pause) ->
?LOG_DEBUG("couch_rep HTTP ~p request: ~s", [Action, Url]),
Body =
case JsonBody of
@@ -622,18 +641,21 @@ do_http_request(Url, Action, Headers, JsonBody, Retries) ->
ResponseCode >= 300, ResponseCode < 400 ->
RedirectUrl = mochiweb_headers:get_value("Location",
mochiweb_headers:make(ResponseHeaders)),
- do_http_request(RedirectUrl, Action, Headers, JsonBody, Retries-1);
+ do_http_request(RedirectUrl, Action, Headers, JsonBody, Retries-1,
+ Pause);
ResponseCode >= 400, ResponseCode < 500 ->
?JSON_DECODE(ResponseBody);
ResponseCode == 500 ->
- ?LOG_INFO("retrying couch_rep HTTP ~p request due to 500 error: ~s",
- [Action, Url]),
- do_http_request(Url, Action, Headers, JsonBody, Retries - 1)
+ ?LOG_INFO("retrying couch_rep HTTP ~p request in ~p seconds " ++
+ "due to 500 error: ~s", [Action, Pause/1000, Url]),
+ timer:sleep(Pause),
+ do_http_request(Url, Action, Headers, JsonBody, Retries - 1, 2*Pause)
end;
{error, Reason} ->
- ?LOG_INFO("retrying couch_rep HTTP ~p request due to {error, ~p}: ~s",
- [Action, Reason, Url]),
- do_http_request(Url, Action, Headers, JsonBody, Retries - 1)
+ ?LOG_INFO("retrying couch_rep HTTP ~p request in ~p seconds due to " ++
+ "{error, ~p}: ~s", [Action, Pause/1000, Reason, Url]),
+ timer:sleep(Pause),
+ do_http_request(Url, Action, Headers, JsonBody, Retries - 1, 2*Pause)
end.
ensure_full_commit(#http_db{uri=DbUrl, headers=Headers}) ->