second cut at _changes feed consumer. still not used by replicator

git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@794851 13f79535-47bb-0310-9956-ffa450edef68

2 files changed, 186 insertions, 71 deletions

diff --git a/src/couchdb/couch_rep_changes_feed.erl b/src/couchdb/couch_rep_changes_feed.erl
index bf69e100..35d6de1d 100644
--- a/src/couchdb/couch_rep_changes_feed.erl
+++ b/src/couchdb/couch_rep_changes_feed.erl
@@ -22,13 +22,22 @@
 -include("couch_db.hrl").
 -include("../ibrowse/ibrowse.hrl").
 
--record (state, {
+-record (remote, {
     conn = nil,
     reqid = nil,
+    complete = false,
     count = 0,
     reply_to = nil,
-    rows = queue:new(),
-    complete = false
+    rows = queue:new()
+}).
+
+-record (local, {
+    changes_from = nil,
+    changes_pid = nil,
+    complete = false,
+    count = 0,
+    reply_to = nil,
+    rows = queue:new()
 }).
 
 start(Url, Options) ->
@@ -38,7 +47,6 @@ start_link(Url, Options) ->
     gen_server:start_link(?MODULE, [Url, Options], []).
 
 next(Server) ->
-    ?LOG_DEBUG("~p at ~p received next_change call", [?MODULE, Server]),
     gen_server:call(Server, next_change, infinity).
 
 stop(Server) ->
@@ -49,64 +57,110 @@ init([{remote, Url}, Options]) ->
     Continuous = proplists:get_value(continuous, Options, false),
     {Pid, ReqId} = start_http_request(lists:concat([Url, "/_changes",
         "?style=all_docs", "&since=", Since, "&continuous=", Continuous])),
-    {ok, #state{conn=Pid, reqid=ReqId}};
+    {ok, #remote{conn=Pid, reqid=ReqId}};
 
+init([{local, DbName}, Options]) when is_list(DbName) ->
+    init([{local, ?l2b(DbName)}, Options]);
 init([{local, DbName}, Options]) ->
-    init([{remote, "http://" ++ couch_config:get("httpd", "bind_address") ++ ":"
-        ++ couch_config:get("httpd", "port") ++ "/" ++ DbName}, Options]).
+    ?LOG_DEBUG("initializing local changes feed for ~s with ~p", [DbName, Options]),
+    process_flag(trap_exit, true),
+    Server = self(),
+    Since = proplists:get_value(since, Options, 0),
+    ChangesPid =
+    case proplists:get_value(continuous, Options, false) of
+    false ->
+        spawn_link(fun() -> send_local_changes_once(Server, DbName, Since) end);
+    true ->
+        spawn_link(fun() -> send_local_changes_forever(Server, DbName, Since) end)
+    end,
+    {ok, #local{changes_pid=ChangesPid}}.
+
+handle_call({add, Row}, _From, #local{count=Count, rows=Rows}=State) 
+        when Count < ?MIN_BUFFER_SIZE->
+    case State of
+    #local{reply_to=nil} ->
+        {reply, ok, State#local{count=Count+1, rows = queue:in(Row, Rows)}};
+    #local{count=0, reply_to=Requestor}->
+        gen_server:reply(Requestor, Row),
+        {reply, ok, State#local{reply_to=nil}}
+    end;
+handle_call({add, Row}, From, #local{}=State) ->
+    #local{
+        count = Count,
+        rows = Rows
+    } = State,
+    {noreply, State#local{count=Count+1, changes_from=From, rows=queue:in(Row,Rows)}};
 
-handle_call(next_change, From, State) ->
-    #state{
+handle_call(next_change, From, #local{count=0}=State) ->
+    if State#local.complete ->
+        {stop, normal, complete, State};
+    true ->
+        {noreply, State#local{reply_to=From}}
+    end;
+handle_call(next_change, _From, #local{}=State) ->
+    #local{
+        count = Count,
+        changes_from = ChangesFrom,
+        rows = Rows
+    } = State,
+    {{value, Row}, NewRows} = queue:out(Rows),
+    if Count =:= ?MIN_BUFFER_SIZE, ChangesFrom =/= nil ->
+        gen_server:reply(ChangesFrom, ok),
+        {reply, Row, State#local{count=Count-1, changes_from=nil, rows=NewRows}};
+    true ->
+        {reply, Row, State#local{count=Count-1, rows=NewRows}}
+    end;
+
+handle_call(next_change, From, #remote{count=0}=State) ->
+    if State#remote.complete ->
+        {stop, normal, complete, State};
+    true ->
+        {noreply, State#remote{reply_to=From}}
+    end;
+handle_call(next_change, _From, #remote{}=State) ->
+    #remote{
         reqid = Id,
         complete = Complete,
         count = Count,
         rows = Rows
     } = State,
-    
     ok = maybe_stream_next(Complete, Count, Id),
-    
-    case queue:out(Rows) of
-    {{value, Row}, NewRows} ->
-        {reply, Row, State#state{count=Count-1, rows=NewRows}};
-    {empty, Rows} ->
-        if State#state.complete ->
-            {stop, normal, complete, State};
-        % State#state.waiting_on_headers ->
-        %     {noreply, State#state{reply_to=From}};
-        true ->
-            {noreply, State#state{reply_to=From}}
-        end
-    end;
+    {{value, Row}, NewRows} = queue:out(Rows),
+    {reply, Row, State#remote{count=Count-1, rows=NewRows}};
+
+handle_call(stop, _From, #local{changes_pid=ChangesPid} = State) ->
+    exit(ChangesPid, stop),
+    {stop, normal, ok, State};
 
-handle_call(stop, _From, State) ->
-    catch ibrowse:stop_worker_process(State#state.conn),
+handle_call(stop, _From, #remote{conn=Conn} = State) ->
+    catch ibrowse:stop_worker_process(Conn),
     {stop, normal, ok, State}.
 
 handle_cast(_Msg, State) ->
     {noreply, State}.
 
-handle_info({ibrowse_async_headers, Id, "200", _}, #state{reqid=Id}=State) ->
-    #state{
+handle_info({ibrowse_async_headers, Id, "200", _}, #remote{reqid=Id}=State) ->
+    #remote{
         complete = Complete,
         count = Count
     } = State,
     ?LOG_DEBUG("~p reqid ~p ibrowse_async_headers 200", [?MODULE, Id]),
     ok = maybe_stream_next(Complete, Count, Id),
     {noreply, State};
-handle_info({ibrowse_async_headers, Id, "301", Hdrs}, #state{reqid=Id}=State) ->
+handle_info({ibrowse_async_headers, Id, "301", Hdrs}, #remote{reqid=Id}=State) ->
     ?LOG_DEBUG("~p reqid ~p ibrowse_async_headers 301", [?MODULE, Id]),
-    catch ibrowse:stop_worker_process(State#state.conn),
+    catch ibrowse:stop_worker_process(State#remote.conn),
     Url = mochiweb_headers:get_value("Location", mochiweb_headers:make(Hdrs)),
     {Pid, ReqId} = start_http_request(Url),
-    {noreply, State#state{conn=Pid, reqid=ReqId}};
-handle_info({ibrowse_async_headers, Id, Code, Hdrs}, #state{reqid=Id}=State) ->
+    {noreply, State#remote{conn=Pid, reqid=ReqId}};
+handle_info({ibrowse_async_headers, Id, Code, Hdrs}, #remote{reqid=Id}=State) ->
     ?LOG_ERROR("replicator changes feed failed with code ~s and Headers ~n~p",
         [Code,Hdrs]),
     {stop, {error, list_to_integer(Code)}, State};
 
-handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) ->
+handle_info({ibrowse_async_response, Id, Msg}, #remote{reqid=Id} = State) ->
     ?LOG_DEBUG("~p reqid ~p ibrowse_async_response ~p", [?MODULE, Id, Msg]),
-    #state{
+    #remote{
         complete = Complete,
         count = Count,
         rows = Rows
@@ -114,11 +168,11 @@ handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) ->
     try
         Row = decode_row(Msg),
         case State of
-        #state{reply_to=nil} ->
-            {noreply, State#state{count=Count+1, rows = queue:in(Row, Rows)}};
-        #state{count=0, reply_to=From}->
+        #remote{reply_to=nil} ->
+            {noreply, State#remote{count=Count+1, rows = queue:in(Row, Rows)}};
+        #remote{count=0, reply_to=From}->
             gen_server:reply(From, Row),
-            {noreply, State#state{reply_to=nil}}
+            {noreply, State#remote{reply_to=nil}}
         end
     catch
     throw:{invalid_json, Msg} ->
@@ -127,21 +181,32 @@ handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) ->
         {noreply, State}
     end;
 
-handle_info({ibrowse_async_response_end, Id}, #state{reqid=Id} = State) ->
-    ?LOG_DEBUG("got ibrowse_async_response_end ~p", [State#state.reply_to]),
+handle_info({ibrowse_async_response_end, Id}, #remote{reqid=Id} = State) ->
+    ?LOG_DEBUG("got ibrowse_async_response_end ~p", [State#remote.reply_to]),
     case State of
-    #state{reply_to=nil} ->
-        {noreply, State#state{complete=true}};
-    #state{count=0, reply_to=From}->
+    #remote{reply_to=nil} ->
+        {noreply, State#remote{complete=true}};
+    #remote{count=0, reply_to=From}->
         gen_server:reply(From, complete),
         {stop, normal, State}
     end;
 
+handle_info({'EXIT', From, normal}, #local{changes_pid=From} = State) ->
+    if State#local.reply_to =/= nil ->
+        gen_server:reply(State#local.reply_to, complete),
+        {stop, normal, State};
+    true ->
+        {noreply, State#local{complete=true}}
+    end;
+handle_info({'EXIT', From, Reason}, #local{changes_pid=From} = State) ->
+    ?LOG_ERROR("changes_pid died with reason ~p", [Reason]),
+    {stop, changes_pid_died, State};
+
 handle_info(Msg, State) ->
     ?LOG_INFO("unexpected message ~p", [Msg]),
     {noreply, State}.
 
-terminate(_Reason, #state{conn=Pid}) when is_pid(Pid) ->
+terminate(_Reason, #remote{conn=Pid}) when is_pid(Pid) ->
     catch ibrowse:stop_worker_process(Pid),
     ok;
 terminate(_Reason, _State) ->
@@ -150,6 +215,8 @@ terminate(_Reason, _State) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
+%internal funs
+
 decode_row([$,, $\n | Rest]) ->
     decode_row(Rest);
 decode_row(Row) ->
@@ -162,6 +229,35 @@ maybe_stream_next(_Complete, _Count, Id) ->
     ?LOG_DEBUG("~p reqid ~p not streaming", [?MODULE, Id]),
     ok.
 
+send_local_changes_forever(Server, DbName, Since) ->
+    {ok, NewSeq} = send_local_changes_once(Server, DbName, Since),
+    timer:sleep(5000),
+    send_local_changes_forever(Server, DbName, NewSeq).
+
+send_local_changes_once(Server, DbName, Since) ->
+    {ok, Db} = couch_db:open(DbName, []),
+
+    FilterFun =
+    fun(#doc_info{revs=[#rev_info{rev=Rev}|_]}) ->
+        {[{<<"rev">>, couch_doc:rev_to_str(Rev)}]}
+    end,
+
+    ChangesFun =
+    fun([#doc_info{id=Id, high_seq=Seq}|_]=DocInfos, _) ->
+        Results0 = [FilterFun(DocInfo) || DocInfo <- DocInfos],
+        Results = [Result || Result <- Results0, Result /= null],
+        if Results /= [] ->
+            Change = {[{<<"seq">>,Seq}, {<<"id">>,Id}, {<<"changes">>,Results}]},
+            gen_server:call(Server, {add, Change}, infinity);
+        true ->
+            ?LOG_DEBUG("Results was empty ~p", [Results0]),
+            ok
+        end,
+        {ok, Seq}
+    end,
+
+    couch_db:changes_since(Db, all_docs, Since, ChangesFun, Since).
+
 start_http_request(RawUrl) ->
     Url = ibrowse_lib:parse_url(RawUrl),
     {ok, Pid} = ibrowse:spawn_link_worker_process(Url#url.host, Url#url.port),
diff --git a/test/etap/110-replication-changes-feed.t b/test/etap/110-replication-changes-feed.t
index 467e43dd..9b5dec85 100644
--- a/test/etap/110-replication-changes-feed.t
+++ b/test/etap/110-replication-changes-feed.t
@@ -22,7 +22,7 @@ main(_) ->
     code:add_pathz("src/ibrowse"),
     code:add_pathz("src/mochiweb"),
     
-    etap:plan(6),
+    etap:plan(12),
     case (catch test()) of
         ok ->
             etap:end_tests();
@@ -40,47 +40,59 @@ test() ->
     crypto:start(),
 
     couch_server:delete(<<"etap-test-db">>, []),
-    {ok, Db} = couch_db:create(<<"etap-test-db">>, []),
-
-    test_unchanged_db(),
-    test_simple_change(),
-    test_since_parameter(),
-    test_continuous_parameter(),
-    test_conflicts(),
-    test_deleted_conflicts(),
+    {ok, Db1} = couch_db:create(<<"etap-test-db">>, []),
+    test_all(local),
+    couch_db:close(Db1),
+    couch_server:delete(<<"etap-test-db">>, []),
 
-    couch_db:close(Db),
     couch_server:delete(<<"etap-test-db">>, []),
+    {ok, Db2} = couch_db:create(<<"etap-test-db">>, []),
+    test_all(remote),
+    couch_db:close(Db2),
+    couch_server:delete(<<"etap-test-db">>, []),
+
     ok.
 
-test_unchanged_db() ->
-    {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, []),
+test_all(Type) ->
+    test_unchanged_db(Type),
+    test_simple_change(Type),
+    test_since_parameter(Type),
+    test_continuous_parameter(Type),
+    test_conflicts(Type),
+    test_deleted_conflicts(Type).
+
+test_unchanged_db(Type) ->
+    {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, []),
     etap:is(
         couch_rep_changes_feed:next(Pid),
         complete,
-        "changes feed for unchanged DB is automatically complete"
+        io_lib:format(
+            "(~p) changes feed for unchanged DB is automatically complete",
+            [Type])
     ).
 
-test_simple_change() ->
+test_simple_change(Type) ->
     Expect = generate_change(),
-    {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, []),
+    {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, []),
     etap:is(
         {couch_rep_changes_feed:next(Pid), couch_rep_changes_feed:next(Pid)},
         {Expect, complete},
-        "change one document, get one row"
+        io_lib:format("(~p) change one document, get one row", [Type])
     ).
 
-test_since_parameter() ->
-    {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, 
+test_since_parameter(Type) ->
+    {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, 
         [{since, get_update_seq()}]),
     etap:is(
         couch_rep_changes_feed:next(Pid),
         complete,
-        "since query-string parameter allows us to skip changes"
+        io_lib:format(
+            "(~p) since query-string parameter allows us to skip changes",
+            [Type])
     ).
 
-test_continuous_parameter() ->
-    {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"},
+test_continuous_parameter(Type) ->
+    {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)},
         [{since, get_update_seq()}, {continuous, true}]),
 
     % make the changes_feed request before the next update
@@ -94,23 +106,25 @@ test_continuous_parameter() ->
     etap:is(
         receive {actual, Actual} -> Actual end,
         Expect,
-        "continuous query-string parameter picks up new changes"
+        io_lib:format(
+            "(~p) continuous query-string parameter picks up new changes",
+            [Type])
     ),
 
     ok = couch_rep_changes_feed:stop(Pid).
 
-test_conflicts() ->
+test_conflicts(Type) ->
     Since = get_update_seq(),
     Expect = generate_conflict(),
-    {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, 
+    {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, 
         [{since, Since}]),
     etap:is(
         {couch_rep_changes_feed:next(Pid), couch_rep_changes_feed:next(Pid)},
         {Expect, complete},
-        "conflict revisions show up in feed"
+        io_lib:format("(~p) conflict revisions show up in feed", [Type])
     ).
 
-test_deleted_conflicts() ->
+test_deleted_conflicts(Type) ->
     Since = get_update_seq(),
     {ExpectProps} = generate_conflict(),
 
@@ -132,12 +146,12 @@ test_deleted_conflicts() ->
         {<<"changes">>, [Win, {[{<<"rev">>, couch_doc:rev_to_str(Rev)}]}]}
     ]},
     
-    {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, 
+    {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, 
         [{since, Since}]),
     etap:is(
         {couch_rep_changes_feed:next(Pid), couch_rep_changes_feed:next(Pid)},
         {Expect, complete},
-        "deleted conflict revisions show up in feed"
+        io_lib:format("(~p) deleted conflict revisions show up in feed", [Type])
     ).
 
 generate_change() ->
@@ -178,6 +192,11 @@ get_db() ->
     {ok, Db} = couch_db:open(<<"etap-test-db">>, []),
     Db.
 
+get_dbname(local) ->
+    "etap-test-db";
+get_dbname(remote) ->
+    "http://127.0.0.1:5984/etap-test-db".
+
 get_update_seq() ->
     Db = get_db(),
     Seq = couch_db:get_update_seq(Db),