Replicator changes:

1) avoid badmatch exceptions when there's a failure opening a single document from a remote source;
2) add error log messages describing which remote documents couldn't be opened and why;
3) when replications errors can't be encoded as json, convert and send them as json strings



git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@996492 13f79535-47bb-0310-9956-ffa450edef68

4 files changed, 37 insertions, 20 deletions

diff --git a/src/couchdb/couch_httpd_misc_handlers.erl b/src/couchdb/couch_httpd_misc_handlers.erl
index 46aba9eb..70db79f6 100644
--- a/src/couchdb/couch_httpd_misc_handlers.erl
+++ b/src/couchdb/couch_httpd_misc_handlers.erl
@@ -93,7 +93,12 @@ handle_replicate_req(#httpd{method='POST'}=Req) ->
     {error, not_found} ->
         send_json(Req, 404, {[{error, not_found}]});
     {error, Reason} ->
-        send_json(Req, 500, {[{error, Reason}]})
+        try
+            send_json(Req, 500, {[{error, Reason}]})
+        catch
+        exit:{json_encode, _} ->
+            send_json(Req, 500, {[{error, couch_util:to_binary(Reason)}]})
+        end
     catch
     throw:{db_not_found, Msg} ->
         send_json(Req, 404, {[{error, db_not_found}, {reason, Msg}]})
diff --git a/src/couchdb/couch_rep.erl b/src/couchdb/couch_rep.erl
index 57b30088..92142b77 100644
--- a/src/couchdb/couch_rep.erl
+++ b/src/couchdb/couch_rep.erl
@@ -377,16 +377,10 @@ close_db(Db) ->
     couch_db:close(Db).
 
 dbname(#http_db{url = Url}) ->
-    strip_password(Url);
+    couch_util:url_strip_password(Url);
 dbname(#db{name = Name}) ->
     Name.
 
-strip_password(Url) ->
-    re:replace(Url,
-        "http(s)?://([^:]+):[^@]+@(.*)$",
-        "http\\1://\\2:*****@\\3",
-        [{return, list}]).
-
 dbinfo(#http_db{} = Db) ->
     {DbProps} = couch_rep_httpc:request(Db),
     [{list_to_existing_atom(?b2l(K)), V} || {K,V} <- DbProps];
diff --git a/src/couchdb/couch_rep_reader.erl b/src/couchdb/couch_rep_reader.erl
index 3edc1f37..5c824cbc 100644
--- a/src/couchdb/couch_rep_reader.erl
+++ b/src/couchdb/couch_rep_reader.erl
@@ -230,7 +230,7 @@ update_sequence_lists(Seq, State) ->
         opened_seqs = Opened
     }.
 
-open_doc_revs(#http_db{} = DbS, DocId, Revs) ->
+open_doc_revs(#http_db{url = Url} = DbS, DocId, Revs) ->
     %% all this logic just splits up revision lists that are too long for
     %% MochiWeb into multiple requests
     BaseQS = [{revs,true}, {latest,true}, {att_encoding_info,true}],
@@ -246,28 +246,39 @@ open_doc_revs(#http_db{} = DbS, DocId, Revs) ->
     JsonResults = lists:flatten([couch_rep_httpc:request(R) || R <- Requests]),
 
     Transform =
-    fun({[{<<"missing">>, Rev}]}) ->
-        {{not_found, missing}, couch_doc:parse_rev(Rev)};
-    ({[{<<"ok">>, Json}]}) ->
+    fun({[{<<"ok">>, Json}]}, Acc) ->
         #doc{id=Id, revs=Rev, atts=Atts} = Doc = couch_doc:from_json_obj(Json),
-        Doc#doc{atts=[couch_rep_att:convert_stub(A, {DbS,Id,Rev}) || A <- Atts]}
+        Doc1 = Doc#doc{
+            atts=[couch_rep_att:convert_stub(A, {DbS,Id,Rev}) || A <- Atts]
+        },
+        [Doc1 | Acc];
+    ({ErrorProps}, Acc) ->
+        Err = couch_util:get_value(<<"error">>, ErrorProps,
+            ?JSON_ENCODE({ErrorProps})),
+        ?LOG_ERROR("Replicator: error accessing doc ~s at ~s, reason: ~s",
+           [DocId, couch_util:url_strip_password(Url), Err]),
+        Acc
     end,
-    [Transform(Result) || Result <- JsonResults].
+    lists:reverse(lists:foldl(Transform, [], JsonResults)).
 
-open_doc(#http_db{} = DbS, DocId) ->
+open_doc(#http_db{url = Url} = DbS, DocId) ->
     % get latest rev of the doc
     Req = DbS#http_db{
         resource=url_encode(DocId),
         qs=[{att_encoding_info, true}]
     },
-    case couch_rep_httpc:request(Req) of
-    {[{<<"error">>,<<"not_found">>}, {<<"reason">>,<<"missing">>}]} ->
-        [];
-    Json ->
+    {Props} = Json = couch_rep_httpc:request(Req),
+    case couch_util:get_value(<<"_id">>, Props) of
+    Id when is_binary(Id) ->
         #doc{id=Id, revs=Rev, atts=Atts} = Doc = couch_doc:from_json_obj(Json),
         [Doc#doc{
             atts=[couch_rep_att:convert_stub(A, {DbS,Id,Rev}) || A <- Atts]
-        }]
+        }];
+    undefined ->
+        Err = couch_util:get_value(<<"error">>, Props, ?JSON_ENCODE(Json)),
+        ?LOG_ERROR("Replicator: error accessing doc ~s at ~s, reason: ~s",
+            [DocId, couch_util:url_strip_password(Url), Err]),
+        []
     end.
 
 reader_loop(ReaderServer, Source, DocIds) when is_list(DocIds) ->
diff --git a/src/couchdb/couch_util.erl b/src/couchdb/couch_util.erl
index f2bf6297..5c85c050 100644
--- a/src/couchdb/couch_util.erl
+++ b/src/couchdb/couch_util.erl
@@ -27,6 +27,7 @@
 -export([get_value/2, get_value/3]).
 -export([md5/1, md5_init/0, md5_update/2, md5_final/1]).
 -export([reorder_results/2]).
+-export([url_strip_password/1]).
 
 -include("couch_db.hrl").
 -include_lib("kernel/include/file.hrl").
@@ -432,3 +433,9 @@ reorder_results(Keys, SortedResults) when length(Keys) < 100 ->
 reorder_results(Keys, SortedResults) ->
     KeyDict = dict:from_list(SortedResults),
     [dict:fetch(Key, KeyDict) || Key <- Keys].
+
+url_strip_password(Url) ->
+    re:replace(Url,
+        "http(s)?://([^:]+):[^@]+@(.*)$",
+        "http\\1://\\2:*****@\\3",
+        [{return, list}]).