diff options
| -rw-r--r-- | src/couchdb/couch_rep_changes_feed.erl | 182 | ||||
| -rw-r--r-- | test/etap/110-replication-changes-feed.t | 75 |
2 files changed, 186 insertions, 71 deletions
diff --git a/src/couchdb/couch_rep_changes_feed.erl b/src/couchdb/couch_rep_changes_feed.erl index bf69e100..35d6de1d 100644 --- a/src/couchdb/couch_rep_changes_feed.erl +++ b/src/couchdb/couch_rep_changes_feed.erl @@ -22,13 +22,22 @@ -include("couch_db.hrl"). -include("../ibrowse/ibrowse.hrl"). --record (state, { +-record (remote, { conn = nil, reqid = nil, + complete = false, count = 0, reply_to = nil, - rows = queue:new(), - complete = false + rows = queue:new() +}). + +-record (local, { + changes_from = nil, + changes_pid = nil, + complete = false, + count = 0, + reply_to = nil, + rows = queue:new() }). start(Url, Options) -> @@ -38,7 +47,6 @@ start_link(Url, Options) -> gen_server:start_link(?MODULE, [Url, Options], []). next(Server) -> - ?LOG_DEBUG("~p at ~p received next_change call", [?MODULE, Server]), gen_server:call(Server, next_change, infinity). stop(Server) -> @@ -49,64 +57,110 @@ init([{remote, Url}, Options]) -> Continuous = proplists:get_value(continuous, Options, false), {Pid, ReqId} = start_http_request(lists:concat([Url, "/_changes", "?style=all_docs", "&since=", Since, "&continuous=", Continuous])), - {ok, #state{conn=Pid, reqid=ReqId}}; + {ok, #remote{conn=Pid, reqid=ReqId}}; +init([{local, DbName}, Options]) when is_list(DbName) -> + init([{local, ?l2b(DbName)}, Options]); init([{local, DbName}, Options]) -> - init([{remote, "http://" ++ couch_config:get("httpd", "bind_address") ++ ":" - ++ couch_config:get("httpd", "port") ++ "/" ++ DbName}, Options]). + ?LOG_DEBUG("initializing local changes feed for ~s with ~p", [DbName, Options]), + process_flag(trap_exit, true), + Server = self(), + Since = proplists:get_value(since, Options, 0), + ChangesPid = + case proplists:get_value(continuous, Options, false) of + false -> + spawn_link(fun() -> send_local_changes_once(Server, DbName, Since) end); + true -> + spawn_link(fun() -> send_local_changes_forever(Server, DbName, Since) end) + end, + {ok, #local{changes_pid=ChangesPid}}. + +handle_call({add, Row}, _From, #local{count=Count, rows=Rows}=State) + when Count < ?MIN_BUFFER_SIZE-> + case State of + #local{reply_to=nil} -> + {reply, ok, State#local{count=Count+1, rows = queue:in(Row, Rows)}}; + #local{count=0, reply_to=Requestor}-> + gen_server:reply(Requestor, Row), + {reply, ok, State#local{reply_to=nil}} + end; +handle_call({add, Row}, From, #local{}=State) -> + #local{ + count = Count, + rows = Rows + } = State, + {noreply, State#local{count=Count+1, changes_from=From, rows=queue:in(Row,Rows)}}; -handle_call(next_change, From, State) -> - #state{ +handle_call(next_change, From, #local{count=0}=State) -> + if State#local.complete -> + {stop, normal, complete, State}; + true -> + {noreply, State#local{reply_to=From}} + end; +handle_call(next_change, _From, #local{}=State) -> + #local{ + count = Count, + changes_from = ChangesFrom, + rows = Rows + } = State, + {{value, Row}, NewRows} = queue:out(Rows), + if Count =:= ?MIN_BUFFER_SIZE, ChangesFrom =/= nil -> + gen_server:reply(ChangesFrom, ok), + {reply, Row, State#local{count=Count-1, changes_from=nil, rows=NewRows}}; + true -> + {reply, Row, State#local{count=Count-1, rows=NewRows}} + end; + +handle_call(next_change, From, #remote{count=0}=State) -> + if State#remote.complete -> + {stop, normal, complete, State}; + true -> + {noreply, State#remote{reply_to=From}} + end; +handle_call(next_change, _From, #remote{}=State) -> + #remote{ reqid = Id, complete = Complete, count = Count, rows = Rows } = State, - ok = maybe_stream_next(Complete, Count, Id), - - case queue:out(Rows) of - {{value, Row}, NewRows} -> - {reply, Row, State#state{count=Count-1, rows=NewRows}}; - {empty, Rows} -> - if State#state.complete -> - {stop, normal, complete, State}; - % State#state.waiting_on_headers -> - % {noreply, State#state{reply_to=From}}; - true -> - {noreply, State#state{reply_to=From}} - end - end; + {{value, Row}, NewRows} = queue:out(Rows), + {reply, Row, State#remote{count=Count-1, rows=NewRows}}; + +handle_call(stop, _From, #local{changes_pid=ChangesPid} = State) -> + exit(ChangesPid, stop), + {stop, normal, ok, State}; -handle_call(stop, _From, State) -> - catch ibrowse:stop_worker_process(State#state.conn), +handle_call(stop, _From, #remote{conn=Conn} = State) -> + catch ibrowse:stop_worker_process(Conn), {stop, normal, ok, State}. handle_cast(_Msg, State) -> {noreply, State}. -handle_info({ibrowse_async_headers, Id, "200", _}, #state{reqid=Id}=State) -> - #state{ +handle_info({ibrowse_async_headers, Id, "200", _}, #remote{reqid=Id}=State) -> + #remote{ complete = Complete, count = Count } = State, ?LOG_DEBUG("~p reqid ~p ibrowse_async_headers 200", [?MODULE, Id]), ok = maybe_stream_next(Complete, Count, Id), {noreply, State}; -handle_info({ibrowse_async_headers, Id, "301", Hdrs}, #state{reqid=Id}=State) -> +handle_info({ibrowse_async_headers, Id, "301", Hdrs}, #remote{reqid=Id}=State) -> ?LOG_DEBUG("~p reqid ~p ibrowse_async_headers 301", [?MODULE, Id]), - catch ibrowse:stop_worker_process(State#state.conn), + catch ibrowse:stop_worker_process(State#remote.conn), Url = mochiweb_headers:get_value("Location", mochiweb_headers:make(Hdrs)), {Pid, ReqId} = start_http_request(Url), - {noreply, State#state{conn=Pid, reqid=ReqId}}; -handle_info({ibrowse_async_headers, Id, Code, Hdrs}, #state{reqid=Id}=State) -> + {noreply, State#remote{conn=Pid, reqid=ReqId}}; +handle_info({ibrowse_async_headers, Id, Code, Hdrs}, #remote{reqid=Id}=State) -> ?LOG_ERROR("replicator changes feed failed with code ~s and Headers ~n~p", [Code,Hdrs]), {stop, {error, list_to_integer(Code)}, State}; -handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) -> +handle_info({ibrowse_async_response, Id, Msg}, #remote{reqid=Id} = State) -> ?LOG_DEBUG("~p reqid ~p ibrowse_async_response ~p", [?MODULE, Id, Msg]), - #state{ + #remote{ complete = Complete, count = Count, rows = Rows @@ -114,11 +168,11 @@ handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) -> try Row = decode_row(Msg), case State of - #state{reply_to=nil} -> - {noreply, State#state{count=Count+1, rows = queue:in(Row, Rows)}}; - #state{count=0, reply_to=From}-> + #remote{reply_to=nil} -> + {noreply, State#remote{count=Count+1, rows = queue:in(Row, Rows)}}; + #remote{count=0, reply_to=From}-> gen_server:reply(From, Row), - {noreply, State#state{reply_to=nil}} + {noreply, State#remote{reply_to=nil}} end catch throw:{invalid_json, Msg} -> @@ -127,21 +181,32 @@ handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) -> {noreply, State} end; -handle_info({ibrowse_async_response_end, Id}, #state{reqid=Id} = State) -> - ?LOG_DEBUG("got ibrowse_async_response_end ~p", [State#state.reply_to]), +handle_info({ibrowse_async_response_end, Id}, #remote{reqid=Id} = State) -> + ?LOG_DEBUG("got ibrowse_async_response_end ~p", [State#remote.reply_to]), case State of - #state{reply_to=nil} -> - {noreply, State#state{complete=true}}; - #state{count=0, reply_to=From}-> + #remote{reply_to=nil} -> + {noreply, State#remote{complete=true}}; + #remote{count=0, reply_to=From}-> gen_server:reply(From, complete), {stop, normal, State} end; +handle_info({'EXIT', From, normal}, #local{changes_pid=From} = State) -> + if State#local.reply_to =/= nil -> + gen_server:reply(State#local.reply_to, complete), + {stop, normal, State}; + true -> + {noreply, State#local{complete=true}} + end; +handle_info({'EXIT', From, Reason}, #local{changes_pid=From} = State) -> + ?LOG_ERROR("changes_pid died with reason ~p", [Reason]), + {stop, changes_pid_died, State}; + handle_info(Msg, State) -> ?LOG_INFO("unexpected message ~p", [Msg]), {noreply, State}. -terminate(_Reason, #state{conn=Pid}) when is_pid(Pid) -> +terminate(_Reason, #remote{conn=Pid}) when is_pid(Pid) -> catch ibrowse:stop_worker_process(Pid), ok; terminate(_Reason, _State) -> @@ -150,6 +215,8 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +%internal funs + decode_row([$,, $\n | Rest]) -> decode_row(Rest); decode_row(Row) -> @@ -162,6 +229,35 @@ maybe_stream_next(_Complete, _Count, Id) -> ?LOG_DEBUG("~p reqid ~p not streaming", [?MODULE, Id]), ok. +send_local_changes_forever(Server, DbName, Since) -> + {ok, NewSeq} = send_local_changes_once(Server, DbName, Since), + timer:sleep(5000), + send_local_changes_forever(Server, DbName, NewSeq). + +send_local_changes_once(Server, DbName, Since) -> + {ok, Db} = couch_db:open(DbName, []), + + FilterFun = + fun(#doc_info{revs=[#rev_info{rev=Rev}|_]}) -> + {[{<<"rev">>, couch_doc:rev_to_str(Rev)}]} + end, + + ChangesFun = + fun([#doc_info{id=Id, high_seq=Seq}|_]=DocInfos, _) -> + Results0 = [FilterFun(DocInfo) || DocInfo <- DocInfos], + Results = [Result || Result <- Results0, Result /= null], + if Results /= [] -> + Change = {[{<<"seq">>,Seq}, {<<"id">>,Id}, {<<"changes">>,Results}]}, + gen_server:call(Server, {add, Change}, infinity); + true -> + ?LOG_DEBUG("Results was empty ~p", [Results0]), + ok + end, + {ok, Seq} + end, + + couch_db:changes_since(Db, all_docs, Since, ChangesFun, Since). + start_http_request(RawUrl) -> Url = ibrowse_lib:parse_url(RawUrl), {ok, Pid} = ibrowse:spawn_link_worker_process(Url#url.host, Url#url.port), diff --git a/test/etap/110-replication-changes-feed.t b/test/etap/110-replication-changes-feed.t index 467e43dd..9b5dec85 100644 --- a/test/etap/110-replication-changes-feed.t +++ b/test/etap/110-replication-changes-feed.t @@ -22,7 +22,7 @@ main(_) -> code:add_pathz("src/ibrowse"), code:add_pathz("src/mochiweb"), - etap:plan(6), + etap:plan(12), case (catch test()) of ok -> etap:end_tests(); @@ -40,47 +40,59 @@ test() -> crypto:start(), couch_server:delete(<<"etap-test-db">>, []), - {ok, Db} = couch_db:create(<<"etap-test-db">>, []), - - test_unchanged_db(), - test_simple_change(), - test_since_parameter(), - test_continuous_parameter(), - test_conflicts(), - test_deleted_conflicts(), + {ok, Db1} = couch_db:create(<<"etap-test-db">>, []), + test_all(local), + couch_db:close(Db1), + couch_server:delete(<<"etap-test-db">>, []), - couch_db:close(Db), couch_server:delete(<<"etap-test-db">>, []), + {ok, Db2} = couch_db:create(<<"etap-test-db">>, []), + test_all(remote), + couch_db:close(Db2), + couch_server:delete(<<"etap-test-db">>, []), + ok. -test_unchanged_db() -> - {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, []), +test_all(Type) -> + test_unchanged_db(Type), + test_simple_change(Type), + test_since_parameter(Type), + test_continuous_parameter(Type), + test_conflicts(Type), + test_deleted_conflicts(Type). + +test_unchanged_db(Type) -> + {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, []), etap:is( couch_rep_changes_feed:next(Pid), complete, - "changes feed for unchanged DB is automatically complete" + io_lib:format( + "(~p) changes feed for unchanged DB is automatically complete", + [Type]) ). -test_simple_change() -> +test_simple_change(Type) -> Expect = generate_change(), - {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, []), + {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, []), etap:is( {couch_rep_changes_feed:next(Pid), couch_rep_changes_feed:next(Pid)}, {Expect, complete}, - "change one document, get one row" + io_lib:format("(~p) change one document, get one row", [Type]) ). -test_since_parameter() -> - {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, +test_since_parameter(Type) -> + {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, [{since, get_update_seq()}]), etap:is( couch_rep_changes_feed:next(Pid), complete, - "since query-string parameter allows us to skip changes" + io_lib:format( + "(~p) since query-string parameter allows us to skip changes", + [Type]) ). -test_continuous_parameter() -> - {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, +test_continuous_parameter(Type) -> + {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, [{since, get_update_seq()}, {continuous, true}]), % make the changes_feed request before the next update @@ -94,23 +106,25 @@ test_continuous_parameter() -> etap:is( receive {actual, Actual} -> Actual end, Expect, - "continuous query-string parameter picks up new changes" + io_lib:format( + "(~p) continuous query-string parameter picks up new changes", + [Type]) ), ok = couch_rep_changes_feed:stop(Pid). -test_conflicts() -> +test_conflicts(Type) -> Since = get_update_seq(), Expect = generate_conflict(), - {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, + {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, [{since, Since}]), etap:is( {couch_rep_changes_feed:next(Pid), couch_rep_changes_feed:next(Pid)}, {Expect, complete}, - "conflict revisions show up in feed" + io_lib:format("(~p) conflict revisions show up in feed", [Type]) ). -test_deleted_conflicts() -> +test_deleted_conflicts(Type) -> Since = get_update_seq(), {ExpectProps} = generate_conflict(), @@ -132,12 +146,12 @@ test_deleted_conflicts() -> {<<"changes">>, [Win, {[{<<"rev">>, couch_doc:rev_to_str(Rev)}]}]} ]}, - {ok, Pid} = couch_rep_changes_feed:start({local, "etap-test-db"}, + {ok, Pid} = couch_rep_changes_feed:start({Type, get_dbname(Type)}, [{since, Since}]), etap:is( {couch_rep_changes_feed:next(Pid), couch_rep_changes_feed:next(Pid)}, {Expect, complete}, - "deleted conflict revisions show up in feed" + io_lib:format("(~p) deleted conflict revisions show up in feed", [Type]) ). generate_change() -> @@ -178,6 +192,11 @@ get_db() -> {ok, Db} = couch_db:open(<<"etap-test-db">>, []), Db. +get_dbname(local) -> + "etap-test-db"; +get_dbname(remote) -> + "http://127.0.0.1:5984/etap-test-db". + get_update_seq() -> Db = get_db(), Seq = couch_db:get_update_seq(Db), |