summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorAdam Kocoloski <kocolosk@apache.org>2009-07-16 20:54:45 +0000
committerAdam Kocoloski <kocolosk@apache.org>2009-07-16 20:54:45 +0000
commit9e5260008b6b67ecd0d876c3956e26a3f2a7d14a (patch)
tree8b6792353625bad55f4905c1692c79f92fbba0e9 /src
parent8ea7e1aa67d2be6c22c93273a47923718a2d44d2 (diff)
second cut at _changes feed consumer. still not used by replicator
git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@794851 13f79535-47bb-0310-9956-ffa450edef68
Diffstat (limited to 'src')
-rw-r--r--src/couchdb/couch_rep_changes_feed.erl182
1 files changed, 139 insertions, 43 deletions
diff --git a/src/couchdb/couch_rep_changes_feed.erl b/src/couchdb/couch_rep_changes_feed.erl
index bf69e100..35d6de1d 100644
--- a/src/couchdb/couch_rep_changes_feed.erl
+++ b/src/couchdb/couch_rep_changes_feed.erl
@@ -22,13 +22,22 @@
-include("couch_db.hrl").
-include("../ibrowse/ibrowse.hrl").
--record (state, {
+-record (remote, {
conn = nil,
reqid = nil,
+ complete = false,
count = 0,
reply_to = nil,
- rows = queue:new(),
- complete = false
+ rows = queue:new()
+}).
+
+-record (local, {
+ changes_from = nil,
+ changes_pid = nil,
+ complete = false,
+ count = 0,
+ reply_to = nil,
+ rows = queue:new()
}).
start(Url, Options) ->
@@ -38,7 +47,6 @@ start_link(Url, Options) ->
gen_server:start_link(?MODULE, [Url, Options], []).
next(Server) ->
- ?LOG_DEBUG("~p at ~p received next_change call", [?MODULE, Server]),
gen_server:call(Server, next_change, infinity).
stop(Server) ->
@@ -49,64 +57,110 @@ init([{remote, Url}, Options]) ->
Continuous = proplists:get_value(continuous, Options, false),
{Pid, ReqId} = start_http_request(lists:concat([Url, "/_changes",
"?style=all_docs", "&since=", Since, "&continuous=", Continuous])),
- {ok, #state{conn=Pid, reqid=ReqId}};
+ {ok, #remote{conn=Pid, reqid=ReqId}};
+init([{local, DbName}, Options]) when is_list(DbName) ->
+ init([{local, ?l2b(DbName)}, Options]);
init([{local, DbName}, Options]) ->
- init([{remote, "http://" ++ couch_config:get("httpd", "bind_address") ++ ":"
- ++ couch_config:get("httpd", "port") ++ "/" ++ DbName}, Options]).
+ ?LOG_DEBUG("initializing local changes feed for ~s with ~p", [DbName, Options]),
+ process_flag(trap_exit, true),
+ Server = self(),
+ Since = proplists:get_value(since, Options, 0),
+ ChangesPid =
+ case proplists:get_value(continuous, Options, false) of
+ false ->
+ spawn_link(fun() -> send_local_changes_once(Server, DbName, Since) end);
+ true ->
+ spawn_link(fun() -> send_local_changes_forever(Server, DbName, Since) end)
+ end,
+ {ok, #local{changes_pid=ChangesPid}}.
+
+handle_call({add, Row}, _From, #local{count=Count, rows=Rows}=State)
+ when Count < ?MIN_BUFFER_SIZE->
+ case State of
+ #local{reply_to=nil} ->
+ {reply, ok, State#local{count=Count+1, rows = queue:in(Row, Rows)}};
+ #local{count=0, reply_to=Requestor}->
+ gen_server:reply(Requestor, Row),
+ {reply, ok, State#local{reply_to=nil}}
+ end;
+handle_call({add, Row}, From, #local{}=State) ->
+ #local{
+ count = Count,
+ rows = Rows
+ } = State,
+ {noreply, State#local{count=Count+1, changes_from=From, rows=queue:in(Row,Rows)}};
-handle_call(next_change, From, State) ->
- #state{
+handle_call(next_change, From, #local{count=0}=State) ->
+ if State#local.complete ->
+ {stop, normal, complete, State};
+ true ->
+ {noreply, State#local{reply_to=From}}
+ end;
+handle_call(next_change, _From, #local{}=State) ->
+ #local{
+ count = Count,
+ changes_from = ChangesFrom,
+ rows = Rows
+ } = State,
+ {{value, Row}, NewRows} = queue:out(Rows),
+ if Count =:= ?MIN_BUFFER_SIZE, ChangesFrom =/= nil ->
+ gen_server:reply(ChangesFrom, ok),
+ {reply, Row, State#local{count=Count-1, changes_from=nil, rows=NewRows}};
+ true ->
+ {reply, Row, State#local{count=Count-1, rows=NewRows}}
+ end;
+
+handle_call(next_change, From, #remote{count=0}=State) ->
+ if State#remote.complete ->
+ {stop, normal, complete, State};
+ true ->
+ {noreply, State#remote{reply_to=From}}
+ end;
+handle_call(next_change, _From, #remote{}=State) ->
+ #remote{
reqid = Id,
complete = Complete,
count = Count,
rows = Rows
} = State,
-
ok = maybe_stream_next(Complete, Count, Id),
-
- case queue:out(Rows) of
- {{value, Row}, NewRows} ->
- {reply, Row, State#state{count=Count-1, rows=NewRows}};
- {empty, Rows} ->
- if State#state.complete ->
- {stop, normal, complete, State};
- % State#state.waiting_on_headers ->
- % {noreply, State#state{reply_to=From}};
- true ->
- {noreply, State#state{reply_to=From}}
- end
- end;
+ {{value, Row}, NewRows} = queue:out(Rows),
+ {reply, Row, State#remote{count=Count-1, rows=NewRows}};
+
+handle_call(stop, _From, #local{changes_pid=ChangesPid} = State) ->
+ exit(ChangesPid, stop),
+ {stop, normal, ok, State};
-handle_call(stop, _From, State) ->
- catch ibrowse:stop_worker_process(State#state.conn),
+handle_call(stop, _From, #remote{conn=Conn} = State) ->
+ catch ibrowse:stop_worker_process(Conn),
{stop, normal, ok, State}.
handle_cast(_Msg, State) ->
{noreply, State}.
-handle_info({ibrowse_async_headers, Id, "200", _}, #state{reqid=Id}=State) ->
- #state{
+handle_info({ibrowse_async_headers, Id, "200", _}, #remote{reqid=Id}=State) ->
+ #remote{
complete = Complete,
count = Count
} = State,
?LOG_DEBUG("~p reqid ~p ibrowse_async_headers 200", [?MODULE, Id]),
ok = maybe_stream_next(Complete, Count, Id),
{noreply, State};
-handle_info({ibrowse_async_headers, Id, "301", Hdrs}, #state{reqid=Id}=State) ->
+handle_info({ibrowse_async_headers, Id, "301", Hdrs}, #remote{reqid=Id}=State) ->
?LOG_DEBUG("~p reqid ~p ibrowse_async_headers 301", [?MODULE, Id]),
- catch ibrowse:stop_worker_process(State#state.conn),
+ catch ibrowse:stop_worker_process(State#remote.conn),
Url = mochiweb_headers:get_value("Location", mochiweb_headers:make(Hdrs)),
{Pid, ReqId} = start_http_request(Url),
- {noreply, State#state{conn=Pid, reqid=ReqId}};
-handle_info({ibrowse_async_headers, Id, Code, Hdrs}, #state{reqid=Id}=State) ->
+ {noreply, State#remote{conn=Pid, reqid=ReqId}};
+handle_info({ibrowse_async_headers, Id, Code, Hdrs}, #remote{reqid=Id}=State) ->
?LOG_ERROR("replicator changes feed failed with code ~s and Headers ~n~p",
[Code,Hdrs]),
{stop, {error, list_to_integer(Code)}, State};
-handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) ->
+handle_info({ibrowse_async_response, Id, Msg}, #remote{reqid=Id} = State) ->
?LOG_DEBUG("~p reqid ~p ibrowse_async_response ~p", [?MODULE, Id, Msg]),
- #state{
+ #remote{
complete = Complete,
count = Count,
rows = Rows
@@ -114,11 +168,11 @@ handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) ->
try
Row = decode_row(Msg),
case State of
- #state{reply_to=nil} ->
- {noreply, State#state{count=Count+1, rows = queue:in(Row, Rows)}};
- #state{count=0, reply_to=From}->
+ #remote{reply_to=nil} ->
+ {noreply, State#remote{count=Count+1, rows = queue:in(Row, Rows)}};
+ #remote{count=0, reply_to=From}->
gen_server:reply(From, Row),
- {noreply, State#state{reply_to=nil}}
+ {noreply, State#remote{reply_to=nil}}
end
catch
throw:{invalid_json, Msg} ->
@@ -127,21 +181,32 @@ handle_info({ibrowse_async_response, Id, Msg}, #state{reqid=Id} = State) ->
{noreply, State}
end;
-handle_info({ibrowse_async_response_end, Id}, #state{reqid=Id} = State) ->
- ?LOG_DEBUG("got ibrowse_async_response_end ~p", [State#state.reply_to]),
+handle_info({ibrowse_async_response_end, Id}, #remote{reqid=Id} = State) ->
+ ?LOG_DEBUG("got ibrowse_async_response_end ~p", [State#remote.reply_to]),
case State of
- #state{reply_to=nil} ->
- {noreply, State#state{complete=true}};
- #state{count=0, reply_to=From}->
+ #remote{reply_to=nil} ->
+ {noreply, State#remote{complete=true}};
+ #remote{count=0, reply_to=From}->
gen_server:reply(From, complete),
{stop, normal, State}
end;
+handle_info({'EXIT', From, normal}, #local{changes_pid=From} = State) ->
+ if State#local.reply_to =/= nil ->
+ gen_server:reply(State#local.reply_to, complete),
+ {stop, normal, State};
+ true ->
+ {noreply, State#local{complete=true}}
+ end;
+handle_info({'EXIT', From, Reason}, #local{changes_pid=From} = State) ->
+ ?LOG_ERROR("changes_pid died with reason ~p", [Reason]),
+ {stop, changes_pid_died, State};
+
handle_info(Msg, State) ->
?LOG_INFO("unexpected message ~p", [Msg]),
{noreply, State}.
-terminate(_Reason, #state{conn=Pid}) when is_pid(Pid) ->
+terminate(_Reason, #remote{conn=Pid}) when is_pid(Pid) ->
catch ibrowse:stop_worker_process(Pid),
ok;
terminate(_Reason, _State) ->
@@ -150,6 +215,8 @@ terminate(_Reason, _State) ->
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
+%internal funs
+
decode_row([$,, $\n | Rest]) ->
decode_row(Rest);
decode_row(Row) ->
@@ -162,6 +229,35 @@ maybe_stream_next(_Complete, _Count, Id) ->
?LOG_DEBUG("~p reqid ~p not streaming", [?MODULE, Id]),
ok.
+send_local_changes_forever(Server, DbName, Since) ->
+ {ok, NewSeq} = send_local_changes_once(Server, DbName, Since),
+ timer:sleep(5000),
+ send_local_changes_forever(Server, DbName, NewSeq).
+
+send_local_changes_once(Server, DbName, Since) ->
+ {ok, Db} = couch_db:open(DbName, []),
+
+ FilterFun =
+ fun(#doc_info{revs=[#rev_info{rev=Rev}|_]}) ->
+ {[{<<"rev">>, couch_doc:rev_to_str(Rev)}]}
+ end,
+
+ ChangesFun =
+ fun([#doc_info{id=Id, high_seq=Seq}|_]=DocInfos, _) ->
+ Results0 = [FilterFun(DocInfo) || DocInfo <- DocInfos],
+ Results = [Result || Result <- Results0, Result /= null],
+ if Results /= [] ->
+ Change = {[{<<"seq">>,Seq}, {<<"id">>,Id}, {<<"changes">>,Results}]},
+ gen_server:call(Server, {add, Change}, infinity);
+ true ->
+ ?LOG_DEBUG("Results was empty ~p", [Results0]),
+ ok
+ end,
+ {ok, Seq}
+ end,
+
+ couch_db:changes_since(Db, all_docs, Since, ChangesFun, Since).
+
start_http_request(RawUrl) ->
Url = ibrowse_lib:parse_url(RawUrl),
{ok, Pid} = ibrowse:spawn_link_worker_process(Url#url.host, Url#url.port),