View list functions can stream views in any format. See list_views test for details.

git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@736876 13f79535-47bb-0310-9956-ffa450edef68

6 files changed, 258 insertions, 77 deletions

diff --git a/src/couchdb/couch_db.hrl b/src/couchdb/couch_db.hrl
index 083468dc..eb449ab7 100644
--- a/src/couchdb/couch_db.hrl
+++ b/src/couchdb/couch_db.hrl
@@ -143,7 +143,6 @@
     end_key = {},
     limit = 10000000000, % a huge huge default number. Picked so we don't have
                          % to do different logic for when there is no limit
-                         % limit
     stale = false,
     direction = fwd,
     start_docid = nil,
@@ -154,6 +153,20 @@
     include_docs = false
 }).
 
+-record(view_fold_helper_funs, {
+    reduce_count,
+    passed_end,
+    start_response,
+    send_row
+}).
+
+-record(extern_resp_args, {
+    code = 200,
+    data = <<>>,
+    ctype = "application/json",
+    headers = []
+}).
+
 -record(group,
     {type=view, % can also be slow_view
     sig=nil,
diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl
index c7c9ec15..9ef7387c 100644
--- a/src/couchdb/couch_httpd_db.erl
+++ b/src/couchdb/couch_httpd_db.erl
@@ -192,7 +192,9 @@ db_req(#httpd{method='GET',path_parts=[_,<<"_all_docs_by_seq">>]}=Req, Db) ->
     TotalRowCount = proplists:get_value(doc_count, Info),
 
     FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, Db,
-        TotalRowCount, fun couch_db:enum_docs_since_reduce_to_count/1),
+        TotalRowCount, #view_fold_helper_funs{
+            reduce_count = fun couch_db:enum_docs_since_reduce_to_count/1
+        }),
     StartKey2 = case StartKey of
         nil -> 0;
         <<>> -> 100000000000;
@@ -306,7 +308,10 @@ all_docs_view(Req, Db, Keys) ->
     case Keys of
     nil ->
         FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, Db,
-            TotalRowCount, fun couch_db:enum_docs_reduce_to_count/1, PassedEndFun),
+            TotalRowCount, #view_fold_helper_funs{
+                reduce_count = fun couch_db:enum_docs_reduce_to_count/1,
+                passed_end = PassedEndFun
+            }),
         AdapterFun = fun(#full_doc_info{id=Id}=FullDocInfo, Offset, Acc) ->
             case couch_doc:to_doc_info(FullDocInfo) of
             #doc_info{deleted=false, rev=Rev} ->
@@ -320,7 +325,9 @@ all_docs_view(Req, Db, Keys) ->
         couch_httpd_view:finish_view_fold(Req, TotalRowCount, {ok, FoldResult});
     _ ->
         FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, Db,
-            TotalRowCount, fun(Offset) -> Offset end),
+            TotalRowCount, #view_fold_helper_funs{
+                reduce_count = fun(Offset) -> Offset end
+            }),
         KeyFoldFun = case Dir of
         fwd ->
             fun lists:foldl/3;
diff --git a/src/couchdb/couch_httpd_external.erl b/src/couchdb/couch_httpd_external.erl
index 7a682ddd..692044ba 100644
--- a/src/couchdb/couch_httpd_external.erl
+++ b/src/couchdb/couch_httpd_external.erl
@@ -14,18 +14,12 @@
 
 -export([handle_external_req/2, handle_external_req/3]).
 -export([send_external_response/2, json_req_obj/2]).
+-export([default_or_content_type/2, parse_external_response/1]).
 
 -import(couch_httpd,[send_error/4]).
 
 -include("couch_db.hrl").
 
--record(extern_resp_args, {
-    code = 200,
-    data = <<>>,
-    ctype = "application/json",
-    headers = []
-}).
-
 % handle_external_req/2
 % for the old type of config usage:
 % _external = {couch_httpd_external, handle_external_req}
@@ -121,7 +115,10 @@ parse_external_response({Response}) ->
             {<<"body">>, Value} ->
                 Args#extern_resp_args{data=Value, ctype="text/html"};
             {<<"base64">>, Value} ->
-                Args#extern_resp_args{data=couch_util:decodeBase64(Value), ctype="application/binary"};
+                Args#extern_resp_args{
+                    data=couch_util:decodeBase64(Value),        
+                    ctype="application/binary"
+                };
             {<<"headers">>, {Headers}} ->
                 NewHeaders = lists:map(fun({Header, HVal}) ->
                     {binary_to_list(Header), binary_to_list(HVal)}
diff --git a/src/couchdb/couch_httpd_show.erl b/src/couchdb/couch_httpd_show.erl
index d4b4997b..47d3fb46 100644
--- a/src/couchdb/couch_httpd_show.erl
+++ b/src/couchdb/couch_httpd_show.erl
@@ -12,7 +12,7 @@
 
 -module(couch_httpd_show).
     
--export([handle_doc_show_req/2]).
+-export([handle_doc_show_req/2, handle_view_list_req/2]).
 
 
 -include("couch_db.hrl").
@@ -22,50 +22,142 @@
     start_json_response/2,send_chunk/2,end_json_response/1,
     start_chunked_response/3, send_error/4]).
     
-handle_doc_show_req(#httpd{method='GET',path_parts=[_, _, DesignName, ShowName, Docid]}=Req, Db) ->
+handle_doc_show_req(#httpd{
+        method='GET',
+        path_parts=[_, _, DesignName, ShowName, Docid]
+    }=Req, Db) ->
     DesignId = <<"_design/", DesignName/binary>>,
-    % Anyway we can dry up this error handling?
-    case (catch couch_httpd_db:couch_doc_open(Db, DesignId, [], [])) of
-    {not_found, missing} ->
-        throw({not_found, missing_design_doc});
-    {not_found, deleted} ->
-        throw({not_found, deleted_design_doc});
-    DesignDoc ->
-        #doc{body={Props}} = DesignDoc,
-        Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>),
-        case proplists:get_value(<<"show">>, Props, nil) of
-        {DocAndViews} ->
-            case proplists:get_value(<<"docs">>, DocAndViews, nil) of 
-            nil ->
-                throw({not_found, missing_show_docs});
-            {DocShows} ->
-                case proplists:get_value(ShowName, DocShows, nil) of
-                nil ->
-                    throw({not_found, missing_show_doc_function});
-                ShowSrc ->
-                    case (catch couch_httpd_db:couch_doc_open(
-                        Db, Docid, [], [])) of
-                    {not_found, missing} ->
-                        throw({not_found, missing});
-                    {not_found, deleted} ->
-                        throw({not_found, deleted});
-                    Doc ->
-                        % ok we have everythign we need. let's make it happen.
-                        send_doc_show_response(Lang, ShowSrc, Doc, Req, Db)
-                    end
-                end
-            end;
-        nil ->
-            throw({not_found, missing_show})
-        end
-    end;
+    #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, [], []),
+    Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>),
+    ShowSrc = get_nested_json_value({Props}, [<<"shows">>, ShowName]),
+    Doc = couch_httpd_db:couch_doc_open(Db, Docid, [], []),
+    send_doc_show_response(Lang, ShowSrc, Doc, Req, Db);
 
 handle_doc_show_req(#httpd{method='GET'}=Req, _Db) ->
-    send_error(Req, 404, <<"form_error">>, <<"Invalid path.">>);
+    send_error(Req, 404, <<"show_error">>, <<"Invalid path.">>);
 
 handle_doc_show_req(Req, _Db) ->
     send_method_not_allowed(Req, "GET,HEAD").
 
+handle_view_list_req(#httpd{method='GET',path_parts=[_, _, DesignName, ListName, ViewName]}=Req, Db) ->
+    DesignId = <<"_design/", DesignName/binary>>,
+    #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, [], []),
+    Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>),
+    ListSrc = get_nested_json_value({Props}, [<<"lists">>, ListName]),
+    send_view_list_response(Lang, ListSrc, ViewName, DesignId, Req, Db);
+
+handle_view_list_req(Req, _Db) ->
+    send_method_not_allowed(Req, "GET,HEAD").
+
+
+get_nested_json_value({Props}, [Key|Keys]) ->
+    case proplists:get_value(Key, Props, nil) of
+    nil -> throw({not_found, <<"missing json key: ", Key/binary>>});
+    Value -> get_nested_json_value(Value, Keys)
+    end;
+get_nested_json_value(Value, []) ->
+    Value;
+get_nested_json_value(_NotJSONObj, _) ->
+    throw({not_found, json_mismatch}).
+
+send_view_list_response(Lang, ListSrc, ViewName, DesignId, Req, Db) ->
+    % TODO add etags when we get view etags
+    #view_query_args{
+        stale = Stale,
+        reduce = Reduce
+    } = QueryArgs = couch_httpd_view:parse_view_query(Req),
+    case couch_view:get_map_view(Db, DesignId, ViewName, Stale) of
+    {ok, View} ->    
+        output_map_list(Req, Lang, ListSrc, View, Db, QueryArgs);
+    {not_found, _Reason} ->
+        case couch_view:get_reduce_view(Db, DesignId, ViewName, Stale) of
+        {ok, ReduceView} ->
+            case Reduce of
+            false ->
+                MapView = couch_view:extract_map_view(ReduceView),
+                output_map_list(Req, Lang, ListSrc, MapView, Db, QueryArgs);
+            _ ->
+                throw({not_implemented, reduce_view_lists})
+            end;
+        {not_found, Reason} ->
+            throw({not_found, Reason})
+        end
+    end.
+    
+output_map_list(Req, Lang, ListSrc, View, Db, QueryArgs) ->
+    #view_query_args{
+        limit = Limit,
+        direction = Dir,
+        skip = SkipCount,
+        start_key = StartKey,
+        start_docid = StartDocId
+    } = QueryArgs,
+    {ok, RowCount} = couch_view:get_row_count(View),
+    Start = {StartKey, StartDocId},
+    % get the os process here
+    % pass it into the view fold with closures
+    {ok, QueryServer} = couch_query_servers:start_view_list(Lang, ListSrc),
+
+    StartListRespFun = fun(Req2, Code, TotalViewCount, Offset) ->
+        JsonResp = couch_query_servers:render_list_head(QueryServer, 
+            Req2, Db, TotalViewCount, Offset),
+        #extern_resp_args{
+            code = Code,
+            data = BeginBody,
+            ctype = CType,
+            headers = Headers
+        } = couch_httpd_external:parse_external_response(JsonResp),
+        JsonHeaders = couch_httpd_external:default_or_content_type(CType, Headers),
+        {ok, Resp} = start_chunked_response(Req, Code, JsonHeaders),
+        {ok, Resp, binary_to_list(BeginBody)}
+    end,
+    
+    SendListRowFun = fun(Resp, Db2, {{Key, DocId}, Value}, 
+        RowFront, _IncludeDocs) ->
+        JsonResp = couch_query_servers:render_list_row(QueryServer, 
+            Req, Db2, {{Key, DocId}, Value}),
+        #extern_resp_args{
+            data = RowBody
+        } = couch_httpd_external:parse_external_response(JsonResp),
+        RowFront2 = case RowFront of
+        nil -> [];
+        _ -> RowFront
+        end,
+        send_chunk(Resp, RowFront2 ++ binary_to_list(RowBody))
+    end,
+    
+    FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, Db, RowCount,
+        #view_fold_helper_funs{
+            reduce_count = fun couch_view:reduce_to_count/1,
+            start_response = StartListRespFun,
+            send_row = SendListRowFun
+        }),
+    FoldAccInit = {Limit, SkipCount, undefined, []},
+    FoldResult = couch_view:fold(View, Start, Dir, FoldlFun, FoldAccInit),
+    finish_view_list(Req, Db, QueryServer, RowCount, FoldResult, StartListRespFun).
+
+finish_view_list(Req, Db, QueryServer, TotalRows, 
+    FoldResult, StartListRespFun) ->
+    case FoldResult of
+    {ok, {_, _, undefined, _}} ->
+        {ok, Resp, BeginBody} = StartListRespFun(Req, 200, TotalRows, null),
+        JsonTail = couch_query_servers:render_list_tail(QueryServer, Req, Db),
+        #extern_resp_args{
+            data = Tail
+        } = couch_httpd_external:parse_external_response(JsonTail),
+        send_chunk(Resp, BeginBody ++ Tail),
+        send_chunk(Resp, []);
+    {ok, {_, _, Resp, _AccRevRows}} ->
+        JsonTail = couch_query_servers:render_list_tail(QueryServer, Req, Db),
+        #extern_resp_args{
+            data = Tail
+        } = couch_httpd_external:parse_external_response(JsonTail),
+        send_chunk(Resp, Tail),
+        send_chunk(Resp, []);
+    Error ->
+        throw(Error)
+    end.
+
 
 send_doc_show_response(Lang, ShowSrc, #doc{revs=[DocRev|_]}=Doc, #httpd{mochi_req=MReq}=Req, Db) ->
     % make a term with etag-effecting Req components, but not always changing ones.
diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl
index dca1e6e3..06d89a59 100644
--- a/src/couchdb/couch_httpd_view.erl
+++ b/src/couchdb/couch_httpd_view.erl
@@ -15,7 +15,8 @@
 
 -export([handle_view_req/2,handle_slow_view_req/2]).
 
--export([parse_view_query/1,parse_view_query/2,make_view_fold_fun/5, make_view_fold_fun/6,finish_view_fold/3]).
+-export([parse_view_query/1,parse_view_query/2,make_view_fold_fun/5,
+    finish_view_fold/3, view_row_obj/3]).
 
 -import(couch_httpd,
     [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2,
@@ -92,8 +93,7 @@ output_map_view(Req, View, Db, QueryArgs, nil) ->
     } = QueryArgs,
     {ok, RowCount} = couch_view:get_row_count(View),
     Start = {StartKey, StartDocId},
-    FoldlFun = make_view_fold_fun(Req, QueryArgs, Db, RowCount,
-            fun couch_view:reduce_to_count/1),
+    FoldlFun = make_view_fold_fun(Req, QueryArgs, Db, RowCount, #view_fold_helper_funs{reduce_count=fun couch_view:reduce_to_count/1}),
     FoldAccInit = {Limit, SkipCount, undefined, []},
     FoldResult = couch_view:fold(View, Start, Dir, FoldlFun, FoldAccInit),
     finish_view_fold(Req, RowCount, FoldResult);
@@ -114,7 +114,10 @@ output_map_view(Req, View, Db, QueryArgs, Keys) ->
                 QueryArgs#view_query_args{
                     start_key = Key,
                     end_key = Key
-                }, Db, RowCount, fun couch_view:reduce_to_count/1),
+                }, Db, RowCount, 
+                #view_fold_helper_funs{
+                    reduce_count = fun couch_view:reduce_to_count/1
+                }),
             couch_view:fold(View, Start, Dir, FoldlFun, FoldAcc)
         end, {ok, FoldAccInit}, Keys),
     finish_view_fold(Req, RowCount, FoldResult).
@@ -362,28 +365,21 @@ parse_view_query(Req, Keys, IsReduce) ->
         end
     end.
 
-
 make_view_fold_fun(Req, QueryArgs, Db,
-    TotalViewCount, ReduceCountFun) ->
+    TotalViewCount, HelperFuns) ->
     #view_query_args{
         end_key = EndKey,
         end_docid = EndDocId,
         direction = Dir
     } = QueryArgs,
-    PassedEndFun =
-    case Dir of
-    fwd ->
-        fun(ViewKey, ViewId) ->
-            couch_view:less_json([EndKey, EndDocId], [ViewKey, ViewId])
-        end;
-    rev->
-        fun(ViewKey, ViewId) ->
-            couch_view:less_json([ViewKey, ViewId], [EndKey, EndDocId])
-        end
-    end,
-    make_view_fold_fun(Req, QueryArgs, Db, TotalViewCount, ReduceCountFun, PassedEndFun).
+    
+    #view_fold_helper_funs{
+        passed_end = PassedEndFun,
+        start_response = StartRespFun,
+        send_row = SendRowFun,
+        reduce_count = ReduceCountFun
+    } = apply_default_helper_funs(HelperFuns, {Dir, EndKey, EndDocId}),
 
-make_view_fold_fun(Req, QueryArgs, Db, TotalViewCount, ReduceCountFun, PassedEndFun) ->
     #view_query_args{
         include_docs = IncludeDocs
     } = QueryArgs,
@@ -401,20 +397,72 @@ make_view_fold_fun(Req, QueryArgs, Db, TotalViewCount, ReduceCountFun, PassedEnd
         {_, _, AccSkip, _} when AccSkip > 0 ->
             {ok, {AccLimit, AccSkip - 1, Resp, AccRevRows}};
         {_, _, _, undefined} ->
-            {ok, Resp2} = start_json_response(Req, 200),
             Offset = ReduceCountFun(OffsetReds),
-            JsonBegin = io_lib:format("{\"total_rows\":~w,\"offset\":~w,\"rows\":[\r\n",
-                    [TotalViewCount, Offset]),
-            JsonObj = view_row_obj(Db, {{Key, DocId}, Value}, IncludeDocs),
-            send_chunk(Resp2, JsonBegin ++ ?JSON_ENCODE(JsonObj)),
+            {ok, Resp2, BeginBody} = StartRespFun(Req, 200, 
+                TotalViewCount, Offset),
+            SendRowFun(Resp2, Db, 
+                {{Key, DocId}, Value}, BeginBody, IncludeDocs),
             {ok, {AccLimit - 1, 0, Resp2, AccRevRows}};
         {_, AccLimit, _, Resp} when (AccLimit > 0) ->
-            JsonObj = view_row_obj(Db, {{Key, DocId}, Value}, IncludeDocs),
-            send_chunk(Resp, ",\r\n" ++  ?JSON_ENCODE(JsonObj)),
+            SendRowFun(Resp, Db, 
+                {{Key, DocId}, Value}, nil, IncludeDocs),
             {ok, {AccLimit - 1, 0, Resp, AccRevRows}}
         end
     end.
 
+apply_default_helper_funs(#view_fold_helper_funs{
+    passed_end = PassedEnd,
+    start_response = StartResp,
+    send_row = SendRow
+}=Helpers, {Dir, EndKey, EndDocId}) ->
+    PassedEnd2 = case PassedEnd of
+    undefined -> make_passed_end_fun(Dir, EndKey, EndDocId);
+    _ -> PassedEnd
+    end,
+
+    StartResp2 = case StartResp of
+    undefined -> fun json_view_start_resp/4;
+    _ -> StartResp
+    end,
+
+    SendRow2 = case SendRow of
+    undefined -> fun send_json_view_row/5;
+    _ -> SendRow
+    end,
+
+    Helpers#view_fold_helper_funs{
+        passed_end = PassedEnd2,
+        start_response = StartResp2,
+        send_row = SendRow2
+    }.
+
+make_passed_end_fun(Dir, EndKey, EndDocId) ->
+    case Dir of
+    fwd ->
+        fun(ViewKey, ViewId) ->
+            couch_view:less_json([EndKey, EndDocId], [ViewKey, ViewId])
+        end;
+    rev->
+        fun(ViewKey, ViewId) ->
+            couch_view:less_json([ViewKey, ViewId], [EndKey, EndDocId])
+        end
+    end.
+
+json_view_start_resp(Req, Code, TotalViewCount, Offset) ->
+    {ok, Resp} = couch_httpd:start_json_response(Req, Code),
+    BeginBody = io_lib:format("{\"total_rows\":~w,\"offset\":~w,\"rows\":[\r\n",
+            [TotalViewCount, Offset]),
+    {ok, Resp, BeginBody}.
+
+send_json_view_row(Resp, Db, {{Key, DocId}, Value}, RowFront, IncludeDocs) ->
+    JsonObj = view_row_obj(Db, {{Key, DocId}, Value}, IncludeDocs),
+    RowFront2 = case RowFront of
+    nil -> ",\r\n";
+    _ -> RowFront
+    end,
+    send_chunk(Resp, RowFront2 ++  ?JSON_ENCODE(JsonObj)).
+    
+
 view_row_obj(Db, {{Key, DocId}, Value}, IncludeDocs) ->
     case DocId of
     error ->
diff --git a/src/couchdb/couch_query_servers.erl b/src/couchdb/couch_query_servers.erl
index 8d473381..a6bb6e06 100644
--- a/src/couchdb/couch_query_servers.erl
+++ b/src/couchdb/couch_query_servers.erl
@@ -17,7 +17,8 @@
 
 -export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2,code_change/3,stop/0]).
 -export([start_doc_map/2, map_docs/2, stop_doc_map/1]).
--export([reduce/3, rereduce/3,validate_doc_update/5,render_doc_show/5]).
+-export([reduce/3, rereduce/3,validate_doc_update/5]).
+-export([render_doc_show/5,start_view_list/2,render_list_head/5, render_list_row/4, render_list_tail/3]).
 % -export([test/0]).
 
 -include("couch_db.hrl").
@@ -134,6 +135,29 @@ render_doc_show(Lang, ShowSrc, Doc, Req, Db) ->
         ok = ret_os_process(Lang, Pid)
     end.
 
+start_view_list(Lang, ListSrc) ->
+    Pid = get_os_process(Lang),
+    true = couch_os_process:prompt(Pid, [<<"add_fun">>, ListSrc]),
+    {ok, {Lang, Pid}}.
+
+render_list_head({_Lang, Pid}, Req, Db, TotalRows, Offset) ->
+    Head = {[{<<"total_rows">>, TotalRows}, {<<"offset">>, Offset}]},
+    JsonReq = couch_httpd_external:json_req_obj(Req, Db),
+    couch_os_process:prompt(Pid, [<<"list_begin">>, Head, JsonReq]).
+
+render_list_row({_Lang, Pid}, Req, Db, {{Key, DocId}, Value}) ->
+    JsonRow = couch_httpd_view:view_row_obj(Db, {{Key, DocId}, Value}, false),
+    JsonReq = couch_httpd_external:json_req_obj(Req, Db),
+    couch_os_process:prompt(Pid, [<<"list_row">>, JsonRow, JsonReq]).
+
+render_list_tail({Lang, Pid}, Req, Db) ->
+    JsonReq = couch_httpd_external:json_req_obj(Req, Db),
+    JsonResp = couch_os_process:prompt(Pid, [<<"list_tail">>, JsonReq]),
+    ok = ret_os_process(Lang, Pid),
+    JsonResp.
+    
+    
+
 init([]) ->
     
     % read config and register for configuration changes