From 0a46c330072a3811d98a5c989d4c6486cff83df2 Mon Sep 17 00:00:00 2001 From: John Christopher Anderson Date: Fri, 23 Jan 2009 00:53:05 +0000 Subject: View list functions can stream views in any format. See list_views test for details. git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@736876 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_db.hrl | 15 +++- src/couchdb/couch_httpd_db.erl | 13 ++- src/couchdb/couch_httpd_external.erl | 13 ++- src/couchdb/couch_httpd_show.erl | 168 +++++++++++++++++++++++++++-------- src/couchdb/couch_httpd_view.erl | 100 +++++++++++++++------ src/couchdb/couch_query_servers.erl | 26 +++++- 6 files changed, 258 insertions(+), 77 deletions(-) (limited to 'src') diff --git a/src/couchdb/couch_db.hrl b/src/couchdb/couch_db.hrl index 083468dc..eb449ab7 100644 --- a/src/couchdb/couch_db.hrl +++ b/src/couchdb/couch_db.hrl @@ -143,7 +143,6 @@ end_key = {}, limit = 10000000000, % a huge huge default number. Picked so we don't have % to do different logic for when there is no limit - % limit stale = false, direction = fwd, start_docid = nil, @@ -154,6 +153,20 @@ include_docs = false }). +-record(view_fold_helper_funs, { + reduce_count, + passed_end, + start_response, + send_row +}). + +-record(extern_resp_args, { + code = 200, + data = <<>>, + ctype = "application/json", + headers = [] +}). + -record(group, {type=view, % can also be slow_view sig=nil, diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl index c7c9ec15..9ef7387c 100644 --- a/src/couchdb/couch_httpd_db.erl +++ b/src/couchdb/couch_httpd_db.erl @@ -192,7 +192,9 @@ db_req(#httpd{method='GET',path_parts=[_,<<"_all_docs_by_seq">>]}=Req, Db) -> TotalRowCount = proplists:get_value(doc_count, Info), FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, Db, - TotalRowCount, fun couch_db:enum_docs_since_reduce_to_count/1), + TotalRowCount, #view_fold_helper_funs{ + reduce_count = fun couch_db:enum_docs_since_reduce_to_count/1 + }), StartKey2 = case StartKey of nil -> 0; <<>> -> 100000000000; @@ -306,7 +308,10 @@ all_docs_view(Req, Db, Keys) -> case Keys of nil -> FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, Db, - TotalRowCount, fun couch_db:enum_docs_reduce_to_count/1, PassedEndFun), + TotalRowCount, #view_fold_helper_funs{ + reduce_count = fun couch_db:enum_docs_reduce_to_count/1, + passed_end = PassedEndFun + }), AdapterFun = fun(#full_doc_info{id=Id}=FullDocInfo, Offset, Acc) -> case couch_doc:to_doc_info(FullDocInfo) of #doc_info{deleted=false, rev=Rev} -> @@ -320,7 +325,9 @@ all_docs_view(Req, Db, Keys) -> couch_httpd_view:finish_view_fold(Req, TotalRowCount, {ok, FoldResult}); _ -> FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, Db, - TotalRowCount, fun(Offset) -> Offset end), + TotalRowCount, #view_fold_helper_funs{ + reduce_count = fun(Offset) -> Offset end + }), KeyFoldFun = case Dir of fwd -> fun lists:foldl/3; diff --git a/src/couchdb/couch_httpd_external.erl b/src/couchdb/couch_httpd_external.erl index 7a682ddd..692044ba 100644 --- a/src/couchdb/couch_httpd_external.erl +++ b/src/couchdb/couch_httpd_external.erl @@ -14,18 +14,12 @@ -export([handle_external_req/2, handle_external_req/3]). -export([send_external_response/2, json_req_obj/2]). +-export([default_or_content_type/2, parse_external_response/1]). -import(couch_httpd,[send_error/4]). -include("couch_db.hrl"). --record(extern_resp_args, { - code = 200, - data = <<>>, - ctype = "application/json", - headers = [] -}). - % handle_external_req/2 % for the old type of config usage: % _external = {couch_httpd_external, handle_external_req} @@ -121,7 +115,10 @@ parse_external_response({Response}) -> {<<"body">>, Value} -> Args#extern_resp_args{data=Value, ctype="text/html"}; {<<"base64">>, Value} -> - Args#extern_resp_args{data=couch_util:decodeBase64(Value), ctype="application/binary"}; + Args#extern_resp_args{ + data=couch_util:decodeBase64(Value), + ctype="application/binary" + }; {<<"headers">>, {Headers}} -> NewHeaders = lists:map(fun({Header, HVal}) -> {binary_to_list(Header), binary_to_list(HVal)} diff --git a/src/couchdb/couch_httpd_show.erl b/src/couchdb/couch_httpd_show.erl index d4b4997b..47d3fb46 100644 --- a/src/couchdb/couch_httpd_show.erl +++ b/src/couchdb/couch_httpd_show.erl @@ -12,7 +12,7 @@ -module(couch_httpd_show). --export([handle_doc_show_req/2]). +-export([handle_doc_show_req/2, handle_view_list_req/2]). -include("couch_db.hrl"). @@ -22,50 +22,142 @@ start_json_response/2,send_chunk/2,end_json_response/1, start_chunked_response/3, send_error/4]). -handle_doc_show_req(#httpd{method='GET',path_parts=[_, _, DesignName, ShowName, Docid]}=Req, Db) -> +handle_doc_show_req(#httpd{ + method='GET', + path_parts=[_, _, DesignName, ShowName, Docid] + }=Req, Db) -> DesignId = <<"_design/", DesignName/binary>>, - % Anyway we can dry up this error handling? - case (catch couch_httpd_db:couch_doc_open(Db, DesignId, [], [])) of - {not_found, missing} -> - throw({not_found, missing_design_doc}); - {not_found, deleted} -> - throw({not_found, deleted_design_doc}); - DesignDoc -> - #doc{body={Props}} = DesignDoc, - Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), - case proplists:get_value(<<"show">>, Props, nil) of - {DocAndViews} -> - case proplists:get_value(<<"docs">>, DocAndViews, nil) of - nil -> - throw({not_found, missing_show_docs}); - {DocShows} -> - case proplists:get_value(ShowName, DocShows, nil) of - nil -> - throw({not_found, missing_show_doc_function}); - ShowSrc -> - case (catch couch_httpd_db:couch_doc_open( - Db, Docid, [], [])) of - {not_found, missing} -> - throw({not_found, missing}); - {not_found, deleted} -> - throw({not_found, deleted}); - Doc -> - % ok we have everythign we need. let's make it happen. - send_doc_show_response(Lang, ShowSrc, Doc, Req, Db) - end - end - end; - nil -> - throw({not_found, missing_show}) - end - end; + #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, [], []), + Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), + ShowSrc = get_nested_json_value({Props}, [<<"shows">>, ShowName]), + Doc = couch_httpd_db:couch_doc_open(Db, Docid, [], []), + send_doc_show_response(Lang, ShowSrc, Doc, Req, Db); handle_doc_show_req(#httpd{method='GET'}=Req, _Db) -> - send_error(Req, 404, <<"form_error">>, <<"Invalid path.">>); + send_error(Req, 404, <<"show_error">>, <<"Invalid path.">>); handle_doc_show_req(Req, _Db) -> send_method_not_allowed(Req, "GET,HEAD"). +handle_view_list_req(#httpd{method='GET',path_parts=[_, _, DesignName, ListName, ViewName]}=Req, Db) -> + DesignId = <<"_design/", DesignName/binary>>, + #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, [], []), + Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), + ListSrc = get_nested_json_value({Props}, [<<"lists">>, ListName]), + send_view_list_response(Lang, ListSrc, ViewName, DesignId, Req, Db); + +handle_view_list_req(Req, _Db) -> + send_method_not_allowed(Req, "GET,HEAD"). + + +get_nested_json_value({Props}, [Key|Keys]) -> + case proplists:get_value(Key, Props, nil) of + nil -> throw({not_found, <<"missing json key: ", Key/binary>>}); + Value -> get_nested_json_value(Value, Keys) + end; +get_nested_json_value(Value, []) -> + Value; +get_nested_json_value(_NotJSONObj, _) -> + throw({not_found, json_mismatch}). + +send_view_list_response(Lang, ListSrc, ViewName, DesignId, Req, Db) -> + % TODO add etags when we get view etags + #view_query_args{ + stale = Stale, + reduce = Reduce + } = QueryArgs = couch_httpd_view:parse_view_query(Req), + case couch_view:get_map_view(Db, DesignId, ViewName, Stale) of + {ok, View} -> + output_map_list(Req, Lang, ListSrc, View, Db, QueryArgs); + {not_found, _Reason} -> + case couch_view:get_reduce_view(Db, DesignId, ViewName, Stale) of + {ok, ReduceView} -> + case Reduce of + false -> + MapView = couch_view:extract_map_view(ReduceView), + output_map_list(Req, Lang, ListSrc, MapView, Db, QueryArgs); + _ -> + throw({not_implemented, reduce_view_lists}) + end; + {not_found, Reason} -> + throw({not_found, Reason}) + end + end. + +output_map_list(Req, Lang, ListSrc, View, Db, QueryArgs) -> + #view_query_args{ + limit = Limit, + direction = Dir, + skip = SkipCount, + start_key = StartKey, + start_docid = StartDocId + } = QueryArgs, + {ok, RowCount} = couch_view:get_row_count(View), + Start = {StartKey, StartDocId}, + % get the os process here + % pass it into the view fold with closures + {ok, QueryServer} = couch_query_servers:start_view_list(Lang, ListSrc), + + StartListRespFun = fun(Req2, Code, TotalViewCount, Offset) -> + JsonResp = couch_query_servers:render_list_head(QueryServer, + Req2, Db, TotalViewCount, Offset), + #extern_resp_args{ + code = Code, + data = BeginBody, + ctype = CType, + headers = Headers + } = couch_httpd_external:parse_external_response(JsonResp), + JsonHeaders = couch_httpd_external:default_or_content_type(CType, Headers), + {ok, Resp} = start_chunked_response(Req, Code, JsonHeaders), + {ok, Resp, binary_to_list(BeginBody)} + end, + + SendListRowFun = fun(Resp, Db2, {{Key, DocId}, Value}, + RowFront, _IncludeDocs) -> + JsonResp = couch_query_servers:render_list_row(QueryServer, + Req, Db2, {{Key, DocId}, Value}), + #extern_resp_args{ + data = RowBody + } = couch_httpd_external:parse_external_response(JsonResp), + RowFront2 = case RowFront of + nil -> []; + _ -> RowFront + end, + send_chunk(Resp, RowFront2 ++ binary_to_list(RowBody)) + end, + + FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, Db, RowCount, + #view_fold_helper_funs{ + reduce_count = fun couch_view:reduce_to_count/1, + start_response = StartListRespFun, + send_row = SendListRowFun + }), + FoldAccInit = {Limit, SkipCount, undefined, []}, + FoldResult = couch_view:fold(View, Start, Dir, FoldlFun, FoldAccInit), + finish_view_list(Req, Db, QueryServer, RowCount, FoldResult, StartListRespFun). + +finish_view_list(Req, Db, QueryServer, TotalRows, + FoldResult, StartListRespFun) -> + case FoldResult of + {ok, {_, _, undefined, _}} -> + {ok, Resp, BeginBody} = StartListRespFun(Req, 200, TotalRows, null), + JsonTail = couch_query_servers:render_list_tail(QueryServer, Req, Db), + #extern_resp_args{ + data = Tail + } = couch_httpd_external:parse_external_response(JsonTail), + send_chunk(Resp, BeginBody ++ Tail), + send_chunk(Resp, []); + {ok, {_, _, Resp, _AccRevRows}} -> + JsonTail = couch_query_servers:render_list_tail(QueryServer, Req, Db), + #extern_resp_args{ + data = Tail + } = couch_httpd_external:parse_external_response(JsonTail), + send_chunk(Resp, Tail), + send_chunk(Resp, []); + Error -> + throw(Error) + end. + send_doc_show_response(Lang, ShowSrc, #doc{revs=[DocRev|_]}=Doc, #httpd{mochi_req=MReq}=Req, Db) -> % make a term with etag-effecting Req components, but not always changing ones. diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl index dca1e6e3..06d89a59 100644 --- a/src/couchdb/couch_httpd_view.erl +++ b/src/couchdb/couch_httpd_view.erl @@ -15,7 +15,8 @@ -export([handle_view_req/2,handle_slow_view_req/2]). --export([parse_view_query/1,parse_view_query/2,make_view_fold_fun/5, make_view_fold_fun/6,finish_view_fold/3]). +-export([parse_view_query/1,parse_view_query/2,make_view_fold_fun/5, + finish_view_fold/3, view_row_obj/3]). -import(couch_httpd, [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2, @@ -92,8 +93,7 @@ output_map_view(Req, View, Db, QueryArgs, nil) -> } = QueryArgs, {ok, RowCount} = couch_view:get_row_count(View), Start = {StartKey, StartDocId}, - FoldlFun = make_view_fold_fun(Req, QueryArgs, Db, RowCount, - fun couch_view:reduce_to_count/1), + FoldlFun = make_view_fold_fun(Req, QueryArgs, Db, RowCount, #view_fold_helper_funs{reduce_count=fun couch_view:reduce_to_count/1}), FoldAccInit = {Limit, SkipCount, undefined, []}, FoldResult = couch_view:fold(View, Start, Dir, FoldlFun, FoldAccInit), finish_view_fold(Req, RowCount, FoldResult); @@ -114,7 +114,10 @@ output_map_view(Req, View, Db, QueryArgs, Keys) -> QueryArgs#view_query_args{ start_key = Key, end_key = Key - }, Db, RowCount, fun couch_view:reduce_to_count/1), + }, Db, RowCount, + #view_fold_helper_funs{ + reduce_count = fun couch_view:reduce_to_count/1 + }), couch_view:fold(View, Start, Dir, FoldlFun, FoldAcc) end, {ok, FoldAccInit}, Keys), finish_view_fold(Req, RowCount, FoldResult). @@ -362,28 +365,21 @@ parse_view_query(Req, Keys, IsReduce) -> end end. - make_view_fold_fun(Req, QueryArgs, Db, - TotalViewCount, ReduceCountFun) -> + TotalViewCount, HelperFuns) -> #view_query_args{ end_key = EndKey, end_docid = EndDocId, direction = Dir } = QueryArgs, - PassedEndFun = - case Dir of - fwd -> - fun(ViewKey, ViewId) -> - couch_view:less_json([EndKey, EndDocId], [ViewKey, ViewId]) - end; - rev-> - fun(ViewKey, ViewId) -> - couch_view:less_json([ViewKey, ViewId], [EndKey, EndDocId]) - end - end, - make_view_fold_fun(Req, QueryArgs, Db, TotalViewCount, ReduceCountFun, PassedEndFun). + + #view_fold_helper_funs{ + passed_end = PassedEndFun, + start_response = StartRespFun, + send_row = SendRowFun, + reduce_count = ReduceCountFun + } = apply_default_helper_funs(HelperFuns, {Dir, EndKey, EndDocId}), -make_view_fold_fun(Req, QueryArgs, Db, TotalViewCount, ReduceCountFun, PassedEndFun) -> #view_query_args{ include_docs = IncludeDocs } = QueryArgs, @@ -401,20 +397,72 @@ make_view_fold_fun(Req, QueryArgs, Db, TotalViewCount, ReduceCountFun, PassedEnd {_, _, AccSkip, _} when AccSkip > 0 -> {ok, {AccLimit, AccSkip - 1, Resp, AccRevRows}}; {_, _, _, undefined} -> - {ok, Resp2} = start_json_response(Req, 200), Offset = ReduceCountFun(OffsetReds), - JsonBegin = io_lib:format("{\"total_rows\":~w,\"offset\":~w,\"rows\":[\r\n", - [TotalViewCount, Offset]), - JsonObj = view_row_obj(Db, {{Key, DocId}, Value}, IncludeDocs), - send_chunk(Resp2, JsonBegin ++ ?JSON_ENCODE(JsonObj)), + {ok, Resp2, BeginBody} = StartRespFun(Req, 200, + TotalViewCount, Offset), + SendRowFun(Resp2, Db, + {{Key, DocId}, Value}, BeginBody, IncludeDocs), {ok, {AccLimit - 1, 0, Resp2, AccRevRows}}; {_, AccLimit, _, Resp} when (AccLimit > 0) -> - JsonObj = view_row_obj(Db, {{Key, DocId}, Value}, IncludeDocs), - send_chunk(Resp, ",\r\n" ++ ?JSON_ENCODE(JsonObj)), + SendRowFun(Resp, Db, + {{Key, DocId}, Value}, nil, IncludeDocs), {ok, {AccLimit - 1, 0, Resp, AccRevRows}} end end. +apply_default_helper_funs(#view_fold_helper_funs{ + passed_end = PassedEnd, + start_response = StartResp, + send_row = SendRow +}=Helpers, {Dir, EndKey, EndDocId}) -> + PassedEnd2 = case PassedEnd of + undefined -> make_passed_end_fun(Dir, EndKey, EndDocId); + _ -> PassedEnd + end, + + StartResp2 = case StartResp of + undefined -> fun json_view_start_resp/4; + _ -> StartResp + end, + + SendRow2 = case SendRow of + undefined -> fun send_json_view_row/5; + _ -> SendRow + end, + + Helpers#view_fold_helper_funs{ + passed_end = PassedEnd2, + start_response = StartResp2, + send_row = SendRow2 + }. + +make_passed_end_fun(Dir, EndKey, EndDocId) -> + case Dir of + fwd -> + fun(ViewKey, ViewId) -> + couch_view:less_json([EndKey, EndDocId], [ViewKey, ViewId]) + end; + rev-> + fun(ViewKey, ViewId) -> + couch_view:less_json([ViewKey, ViewId], [EndKey, EndDocId]) + end + end. + +json_view_start_resp(Req, Code, TotalViewCount, Offset) -> + {ok, Resp} = couch_httpd:start_json_response(Req, Code), + BeginBody = io_lib:format("{\"total_rows\":~w,\"offset\":~w,\"rows\":[\r\n", + [TotalViewCount, Offset]), + {ok, Resp, BeginBody}. + +send_json_view_row(Resp, Db, {{Key, DocId}, Value}, RowFront, IncludeDocs) -> + JsonObj = view_row_obj(Db, {{Key, DocId}, Value}, IncludeDocs), + RowFront2 = case RowFront of + nil -> ",\r\n"; + _ -> RowFront + end, + send_chunk(Resp, RowFront2 ++ ?JSON_ENCODE(JsonObj)). + + view_row_obj(Db, {{Key, DocId}, Value}, IncludeDocs) -> case DocId of error -> diff --git a/src/couchdb/couch_query_servers.erl b/src/couchdb/couch_query_servers.erl index 8d473381..a6bb6e06 100644 --- a/src/couchdb/couch_query_servers.erl +++ b/src/couchdb/couch_query_servers.erl @@ -17,7 +17,8 @@ -export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2,code_change/3,stop/0]). -export([start_doc_map/2, map_docs/2, stop_doc_map/1]). --export([reduce/3, rereduce/3,validate_doc_update/5,render_doc_show/5]). +-export([reduce/3, rereduce/3,validate_doc_update/5]). +-export([render_doc_show/5,start_view_list/2,render_list_head/5, render_list_row/4, render_list_tail/3]). % -export([test/0]). -include("couch_db.hrl"). @@ -134,6 +135,29 @@ render_doc_show(Lang, ShowSrc, Doc, Req, Db) -> ok = ret_os_process(Lang, Pid) end. +start_view_list(Lang, ListSrc) -> + Pid = get_os_process(Lang), + true = couch_os_process:prompt(Pid, [<<"add_fun">>, ListSrc]), + {ok, {Lang, Pid}}. + +render_list_head({_Lang, Pid}, Req, Db, TotalRows, Offset) -> + Head = {[{<<"total_rows">>, TotalRows}, {<<"offset">>, Offset}]}, + JsonReq = couch_httpd_external:json_req_obj(Req, Db), + couch_os_process:prompt(Pid, [<<"list_begin">>, Head, JsonReq]). + +render_list_row({_Lang, Pid}, Req, Db, {{Key, DocId}, Value}) -> + JsonRow = couch_httpd_view:view_row_obj(Db, {{Key, DocId}, Value}, false), + JsonReq = couch_httpd_external:json_req_obj(Req, Db), + couch_os_process:prompt(Pid, [<<"list_row">>, JsonRow, JsonReq]). + +render_list_tail({Lang, Pid}, Req, Db) -> + JsonReq = couch_httpd_external:json_req_obj(Req, Db), + JsonResp = couch_os_process:prompt(Pid, [<<"list_tail">>, JsonReq]), + ok = ret_os_process(Lang, Pid), + JsonResp. + + + init([]) -> % read config and register for configuration changes -- cgit v1.2.3