From 0fb2f9696d8005eb46d5efeac1ae217fe0fb6a04 Mon Sep 17 00:00:00 2001 From: John Christopher Anderson Date: Wed, 25 Feb 2009 06:34:03 +0000 Subject: Support for reduce views in _list. closes COUCHDB-260. Thanks Jason Davies. git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@747679 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_db.hrl | 5 ++ src/couchdb/couch_httpd_show.erl | 115 ++++++++++++++++++++++++------ src/couchdb/couch_httpd_view.erl | 137 ++++++++++++++++++++++++++---------- src/couchdb/couch_query_servers.erl | 12 +++- 4 files changed, 208 insertions(+), 61 deletions(-) (limited to 'src/couchdb') diff --git a/src/couchdb/couch_db.hrl b/src/couchdb/couch_db.hrl index 443ac73f..89d8965e 100644 --- a/src/couchdb/couch_db.hrl +++ b/src/couchdb/couch_db.hrl @@ -161,6 +161,11 @@ send_row }). +-record(reduce_fold_helper_funs, { + start_response, + send_row +}). + -record(extern_resp_args, { code = 200, stop = false, diff --git a/src/couchdb/couch_httpd_show.erl b/src/couchdb/couch_httpd_show.erl index cc7d4d08..a6c795c6 100644 --- a/src/couchdb/couch_httpd_show.erl +++ b/src/couchdb/couch_httpd_show.erl @@ -19,7 +19,7 @@ -import(couch_httpd, [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2, - start_json_response/2,send_chunk/2,end_json_response/1, + start_json_response/2,send_chunk/2, start_chunked_response/3, send_error/4]). handle_doc_show_req(#httpd{ @@ -87,13 +87,13 @@ send_view_list_response(Lang, ListSrc, ViewName, DesignId, Req, Db) -> MapView = couch_view:extract_map_view(ReduceView), output_map_list(Req, Lang, ListSrc, MapView, Group, Db, QueryArgs); _ -> - throw({not_implemented, reduce_view_lists}) + output_reduce_list(Req, Lang, ListSrc, ReduceView, Group, Db, QueryArgs) end; {not_found, Reason} -> throw({not_found, Reason}) end end. - + output_map_list(#httpd{mochi_req=MReq}=Req, Lang, ListSrc, View, Group, Db, QueryArgs) -> #view_query_args{ limit = Limit, @@ -137,13 +137,18 @@ output_map_list(#httpd{mochi_req=MReq}=Req, Lang, ListSrc, View, Group, Db, Quer stop = StopIter, data = RowBody } = couch_httpd_external:parse_external_response(JsonResp), - RowFront2 = case RowFront of - nil -> []; - _ -> RowFront - end, case StopIter of true -> stop; - _ -> send_chunk(Resp, RowFront2 ++ binary_to_list(RowBody)) + _ -> + RowFront2 = case RowFront of + nil -> []; + _ -> RowFront + end, + Chunk = RowFront2 ++ binary_to_list(RowBody), + case Chunk of + [] -> {ok, Resp}; + _ -> send_chunk(Resp, Chunk) + end end end, @@ -155,26 +160,94 @@ output_map_list(#httpd{mochi_req=MReq}=Req, Lang, ListSrc, View, Group, Db, Quer }), FoldAccInit = {Limit, SkipCount, undefined, []}, FoldResult = couch_view:fold(View, Start, Dir, FoldlFun, FoldAccInit), - finish_view_list(Req, Db, QueryServer, RowCount, FoldResult, StartListRespFun) + finish_list(Req, Db, QueryServer, CurrentEtag, FoldResult, StartListRespFun, RowCount) end). -finish_view_list(Req, Db, QueryServer, TotalRows, - FoldResult, StartListRespFun) -> - case FoldResult of - {ok, {_, _, undefined, _}} -> - {ok, Resp, BeginBody} = StartListRespFun(Req, 200, TotalRows, null), - JsonTail = couch_query_servers:render_list_tail(QueryServer, Req, Db), +output_reduce_list(#httpd{mochi_req=MReq}=Req, Lang, ListSrc, View, Group, Db, QueryArgs) -> + #view_query_args{ + limit = Limit, + direction = Dir, + skip = SkipCount, + start_key = StartKey, + start_docid = StartDocId, + end_key = EndKey, + end_docid = EndDocId, + group_level = GroupLevel + } = QueryArgs, + % get the os process here + % pass it into the view fold with closures + {ok, QueryServer} = couch_query_servers:start_view_list(Lang, ListSrc), + Headers = MReq:get(headers), + Hlist = mochiweb_headers:to_list(Headers), + Accept = proplists:get_value('Accept', Hlist), + CurrentEtag = couch_httpd_view:view_group_etag(Group, {Lang, ListSrc, Accept}), + + StartListRespFun = fun(Req2, _Etag, _, _) -> + JsonResp = couch_query_servers:render_reduce_head(QueryServer, + Req2, Db), #extern_resp_args{ - data = Tail - } = couch_httpd_external:parse_external_response(JsonTail), - send_chunk(Resp, BeginBody ++ Tail), - send_chunk(Resp, []); - {ok, {_, _, Resp, _AccRevRows}} -> + code = Code, + data = BeginBody, + ctype = CType, + headers = ExtHeaders + } = couch_httpd_external:parse_external_response(JsonResp), + JsonHeaders = couch_httpd_external:default_or_content_type(CType, ExtHeaders), + {ok, Resp} = start_chunked_response(Req, Code, JsonHeaders), + {ok, Resp, binary_to_list(BeginBody)} + end, + + SendListRowFun = fun(Resp, {Key, Value}, RowFront) -> + JsonResp = couch_query_servers:render_reduce_row(QueryServer, + Req, Db, {Key, Value}), + #extern_resp_args{ + stop = StopIter, + data = RowBody + } = couch_httpd_external:parse_external_response(JsonResp), + RowFront2 = case RowFront of + nil -> []; + _ -> RowFront + end, + case StopIter of + true -> stop; + _ -> + Chunk = RowFront2 ++ binary_to_list(RowBody), + case Chunk of + [] -> {ok, Resp}; + _ -> send_chunk(Resp, Chunk) + end + end + end, + + {ok, GroupRowsFun, RespFun} = couch_httpd_view:make_reduce_fold_funs(Req, GroupLevel, QueryArgs, CurrentEtag, + #reduce_fold_helper_funs{ + start_response = StartListRespFun, + send_row = SendListRowFun + }), + FoldAccInit = {Limit, SkipCount, undefined, []}, + FoldResult = couch_view:fold_reduce(View, Dir, {StartKey, StartDocId}, + {EndKey, EndDocId}, GroupRowsFun, RespFun, + FoldAccInit), + finish_list(Req, Db, QueryServer, CurrentEtag, FoldResult, StartListRespFun, null). + +finish_list(Req, Db, QueryServer, Etag, FoldResult, StartListRespFun, TotalRows) -> + case FoldResult of + {ok, Acc} -> JsonTail = couch_query_servers:render_list_tail(QueryServer, Req, Db), #extern_resp_args{ data = Tail } = couch_httpd_external:parse_external_response(JsonTail), - send_chunk(Resp, Tail), + {Resp, BeginBody} = case Acc of + {_, _, undefined, _} -> + {ok, Resp2, BeginBody2} = StartListRespFun(Req, Etag, TotalRows, null), + {Resp2, BeginBody2}; + {_, _, Resp2, _} -> + {Resp2, ""} + end, + Chunk = BeginBody ++ binary_to_list(Tail), + case Chunk of + [] -> ok; + _ -> send_chunk(Resp, Chunk) + end, send_chunk(Resp, []); Error -> throw(Error) diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl index c4e2174e..6b9befe1 100644 --- a/src/couchdb/couch_httpd_view.erl +++ b/src/couchdb/couch_httpd_view.erl @@ -16,7 +16,7 @@ -export([handle_view_req/2,handle_temp_view_req/2]). -export([parse_view_query/1,parse_view_query/2,parse_view_query/4,make_view_fold_fun/6, - finish_view_fold/3, view_row_obj/3, view_group_etag/1, view_group_etag/2]). + finish_view_fold/3, view_row_obj/3, view_group_etag/1, view_group_etag/2, make_reduce_fold_funs/5]). -import(couch_httpd, [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2,send_chunk/2, @@ -145,13 +145,11 @@ output_reduce_view(Req, View, Group, QueryArgs, nil) -> } = QueryArgs, CurrentEtag = view_group_etag(Group), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> - {ok, Resp} = start_json_response(Req, 200, [{"Etag",CurrentEtag}]), - {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Resp, GroupLevel), - send_chunk(Resp, "{\"rows\":["), - {ok, _} = couch_view:fold_reduce(View, Dir, {StartKey, StartDocId}, - {EndKey, EndDocId}, GroupRowsFun, RespFun, {"", Skip, Limit}), - send_chunk(Resp, "]}"), - end_json_response(Resp) + {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel, QueryArgs, CurrentEtag, #reduce_fold_helper_funs{}), + FoldAccInit = {Limit, Skip, undefined, []}, + {ok, {_, _, Resp, _}} = couch_view:fold_reduce(View, Dir, {StartKey, StartDocId}, + {EndKey, EndDocId}, GroupRowsFun, RespFun, FoldAccInit), + finish_reduce_fold(Req, Resp) end); output_reduce_view(Req, View, Group, QueryArgs, Keys) -> @@ -165,22 +163,25 @@ output_reduce_view(Req, View, Group, QueryArgs, Keys) -> } = QueryArgs, CurrentEtag = view_group_etag(Group), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> - {ok, Resp} = start_json_response(Req, 200, [{"Etag",CurrentEtag}]), - {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Resp, GroupLevel), - send_chunk(Resp, "{\"rows\":["), - lists:foldl( - fun(Key, AccSeparator) -> - {ok, {NewAcc, _, _}} = couch_view:fold_reduce(View, Dir, {Key, StartDocId}, - {Key, EndDocId}, GroupRowsFun, RespFun, - {AccSeparator, Skip, Limit}), - NewAcc % Switch to comma + {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel, QueryArgs, CurrentEtag, #reduce_fold_helper_funs{}), + {Resp, _} = lists:foldl( + fun(Key, {Resp, AccSeparator}) -> + FoldAccInit = {Limit, Skip, Resp, AccSeparator}, + {_, {_, _, Resp2, NewAcc}} = couch_view:fold_reduce(View, Dir, {Key, StartDocId}, + {Key, EndDocId}, GroupRowsFun, RespFun, FoldAccInit), + % Switch to comma + {Resp2, NewAcc} end, - "", Keys), % Start with no comma - send_chunk(Resp, "]}"), - end_json_response(Resp) + {undefined, []}, Keys), % Start with no comma + finish_reduce_fold(Req, Resp) end). -make_reduce_fold_funs(Resp, GroupLevel) -> +make_reduce_fold_funs(Req, GroupLevel, _QueryArgs, Etag, HelperFuns) -> + #reduce_fold_helper_funs{ + start_response = StartRespFun, + send_row = SendRowFun + } = apply_default_helper_funs(HelperFuns), + GroupRowsFun = fun({_Key1,_}, {_Key2,_}) when GroupLevel == 0 -> true; @@ -190,31 +191,47 @@ make_reduce_fold_funs(Resp, GroupLevel) -> ({Key1,_}, {Key2,_}) -> Key1 == Key2 end, - RespFun = fun(_Key, _Red, {AccSeparator,AccSkip,AccLimit}) when AccSkip > 0 -> - {ok, {AccSeparator,AccSkip-1,AccLimit}}; - (_Key, _Red, {AccSeparator,0,AccLimit}) when AccLimit == 0 -> - {stop, {AccSeparator,0,AccLimit}}; - (_Key, Red, {AccSeparator,0,AccLimit}) when GroupLevel == 0 -> - Json = ?JSON_ENCODE({[{key, null}, {value, Red}]}), - send_chunk(Resp, AccSeparator ++ Json), - {ok, {",",0,AccLimit-1}}; - (Key, Red, {AccSeparator,0,AccLimit}) + RespFun = fun(_Key, _Red, {AccLimit, AccSkip, Resp, AccSeparator}) when AccSkip > 0 -> + {ok, {AccLimit, AccSkip - 1, Resp, AccSeparator}}; + (_Key, _Red, {0, 0, Resp, AccSeparator}) -> + {stop, {0, 0, Resp, AccSeparator}}; + (_Key, Red, {AccLimit, 0, Resp, AccSeparator}) when GroupLevel == 0 -> + {ok, Resp2, RowSep} = case Resp of + undefined -> StartRespFun(Req, Etag, null, null); + _ -> {ok, Resp, nil} + end, + RowResult = case SendRowFun(Resp2, {null, Red}, RowSep) of + stop -> stop; + _ -> ok + end, + {RowResult, {AccLimit - 1, 0, Resp2, AccSeparator}}; + (Key, Red, {AccLimit, 0, Resp, AccSeparator}) when is_integer(GroupLevel) andalso is_list(Key) -> - Json = ?JSON_ENCODE( - {[{key, lists:sublist(Key, GroupLevel)},{value, Red}]}), - send_chunk(Resp, AccSeparator ++ Json), - {ok, {",",0,AccLimit-1}}; - (Key, Red, {AccSeparator,0,AccLimit}) -> - Json = ?JSON_ENCODE({[{key, Key}, {value, Red}]}), - send_chunk(Resp, AccSeparator ++ Json), - {ok, {",",0,AccLimit-1}} + {ok, Resp2, RowSep} = case Resp of + undefined -> StartRespFun(Req, Etag, null, null); + _ -> {ok, Resp, nil} + end, + RowResult = case SendRowFun(Resp2, {lists:sublist(Key, GroupLevel), Red}, RowSep) of + stop -> stop; + _ -> ok + end, + {RowResult, {AccLimit - 1, 0, Resp2, AccSeparator}}; + (Key, Red, {AccLimit, 0, Resp, AccSeparator}) -> + {ok, Resp2, RowSep} = case Resp of + undefined -> StartRespFun(Req, Etag, null, null); + _ -> {ok, Resp, nil} + end, + RowResult = case SendRowFun(Resp2, {Key, Red}, RowSep) of + stop -> stop; + _ -> ok + end, + {RowResult, {AccLimit - 1, 0, Resp2, AccSeparator}} end, {ok, GroupRowsFun, RespFun}. - reverse_key_default(nil) -> {}; reverse_key_default({}) -> nil; reverse_key_default(Key) -> Key. @@ -470,6 +487,25 @@ apply_default_helper_funs(#view_fold_helper_funs{ send_row = SendRow2 }. +apply_default_helper_funs(#reduce_fold_helper_funs{ + start_response = StartResp, + send_row = SendRow +}=Helpers) -> + StartResp2 = case StartResp of + undefined -> fun json_reduce_start_resp/4; + _ -> StartResp + end, + + SendRow2 = case SendRow of + undefined -> fun send_json_reduce_row/3; + _ -> SendRow + end, + + Helpers#reduce_fold_helper_funs{ + start_response = StartResp2, + send_row = SendRow2 + }. + make_passed_end_fun(Dir, EndKey, EndDocId) -> case Dir of fwd -> @@ -496,6 +532,18 @@ send_json_view_row(Resp, Db, {{Key, DocId}, Value}, RowFront, IncludeDocs) -> end, send_chunk(Resp, RowFront2 ++ ?JSON_ENCODE(JsonObj)). +json_reduce_start_resp(Req, Etag, _, _) -> + {ok, Resp} = start_json_response(Req, 200, [{"Etag", Etag}]), + BeginBody = "{\"rows\":[\r\n", + {ok, Resp, BeginBody}. + +send_json_reduce_row(Resp, {Key, Value}, RowFront) -> + RowFront2 = case RowFront of + nil -> ",\r\n"; + _ -> RowFront + end, + send_chunk(Resp, RowFront2 ++ ?JSON_ENCODE({[{key, Key}, {value, Value}]})). + view_group_etag(Group) -> view_group_etag(Group, nil). @@ -557,3 +605,14 @@ finish_view_fold(Req, TotalRows, FoldResult) -> Error -> throw(Error) end. + +finish_reduce_fold(Req, Resp) -> + case Resp of + undefined -> + send_json(Req, 200, {[ + {rows, []} + ]}); + Resp -> + send_chunk(Resp, "\r\n]}"), + end_json_response(Resp) + end. diff --git a/src/couchdb/couch_query_servers.erl b/src/couchdb/couch_query_servers.erl index 5eca63c2..a33ffada 100644 --- a/src/couchdb/couch_query_servers.erl +++ b/src/couchdb/couch_query_servers.erl @@ -18,7 +18,7 @@ -export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2,code_change/3,stop/0]). -export([start_doc_map/2, map_docs/2, stop_doc_map/1]). -export([reduce/3, rereduce/3,validate_doc_update/5]). --export([render_doc_show/5,start_view_list/2,render_list_head/5, render_list_row/4, render_list_tail/3]). +-export([render_doc_show/5,start_view_list/2,render_list_head/5, render_list_row/4, render_list_tail/3, render_reduce_head/3, render_reduce_row/4]). % -export([test/0]). -include("couch_db.hrl"). @@ -160,6 +160,16 @@ render_list_tail({Lang, Pid}, Req, Db) -> JsonResp. +render_reduce_head({_Lang, Pid}, Req, Db) -> + Head = {[]}, + JsonReq = couch_httpd_external:json_req_obj(Req, Db), + couch_os_process:prompt(Pid, [<<"list_begin">>, Head, JsonReq]). + +render_reduce_row({_Lang, Pid}, Req, Db, {Key, Value}) -> + JsonRow = {[{key, Key}, {value, Value}]}, + JsonReq = couch_httpd_external:json_req_obj(Req, Db), + couch_os_process:prompt(Pid, [<<"list_row">>, JsonRow, JsonReq]). + init([]) -> -- cgit v1.2.3