Introduces native Erlang query servers. Closes COUCHDB-377

Thanks Mark Hammond and Paul Davis for doing most of the work, and Michael McDaniel for the inspiration.

There is still room for improvement on the APIs exposed to the Erlang views, as well as likely a whole lot of work to be done to increase parallelism. But the important part now is that we have native Erlang views.


git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@803685 13f79535-47bb-0310-9956-ffa450edef68

2 files changed, 248 insertions, 42 deletions

diff --git a/test/query_server_spec.rb b/test/query_server_spec.rb
index c9fb6942..42c7794c 100644
--- a/test/query_server_spec.rb
+++ b/test/query_server_spec.rb
@@ -14,36 +14,33 @@
 # spec test/query_server_spec.rb -f specdoc --color
 
 COUCH_ROOT = "#{File.dirname(__FILE__)}/.." unless defined?(COUCH_ROOT)
-LANGUAGE = "js"
+LANGUAGE = ENV["QS_LANG"] || "js"
+
+puts "Running query server specs for #{LANGUAGE} query server"
 
-require 'open3'
 require 'spec'
 require 'json'
 
 class OSProcessRunner
   def self.run
-    trace = false
+    trace = ENV["QS_TRACE"] || false
     puts "launching #{run_command}" if trace
     if block_given?
-      Open3.popen3(run_command) do |jsin, jsout, jserr|
-        js = QueryServerRunner.new(jsin, jsout, jserr, trace)
-        yield js
+      IO.popen(run_command, "r+") do |io|
+        qs = QueryServerRunner.new(io, trace)
+        yield qs
       end
     else
-      jsin, jsout, jserr = Open3.popen3(run_command)
-      QueryServerRunner.new(jsin, jsout, jserr, trace)
+      io = IO.popen(run_command, "r+")
+      QueryServerRunner.new(io, trace)
     end
   end
-  def initialize jsin, jsout, jserr, trace = false
-    @qsin = jsin
-    @qsout = jsout
-    @qserr = jserr
+  def initialize io, trace = false
+    @qsio = io
     @trace = trace
   end
   def close
-    @qsin.close
-    @qsout.close
-    @qserr.close
+    @qsio.close
   end
   def reset!
     run(["reset"])
@@ -63,10 +60,10 @@ class OSProcessRunner
   def rrun json
     line = json.to_json
     puts "run: #{line}" if @trace
-    @qsin.puts line
+    @qsio.puts line
   end
   def rgets
-    resp = @qsout.gets
+    resp = @qsio.gets
     puts "got: #{resp}"  if @trace
     resp
   end
@@ -75,7 +72,15 @@ class OSProcessRunner
     # err = @qserr.gets
     # puts "err: #{err}" if err
     if resp
-      rj = JSON.parse("[#{resp.chomp}]")[0]
+      begin
+        rj = JSON.parse("[#{resp.chomp}]")[0]
+      rescue JSON::ParserError
+        puts "JSON ERROR (dump under trace mode)"
+        # puts resp.chomp
+        while resp = rgets
+          # puts resp.chomp
+        end
+      end
       if rj.respond_to?(:[]) && rj.is_a?(Array)
         if rj[0] == "log"
           log = rj[1]
@@ -92,7 +97,10 @@ end
 
 class QueryServerRunner < OSProcessRunner
 
-  COMMANDS = {"js" => "#{COUCH_ROOT}/src/couchdb/couchjs #{COUCH_ROOT}/share/server/main.js" }
+  COMMANDS = {
+    "js" => "#{COUCH_ROOT}/src/couchdb/couchjs #{COUCH_ROOT}/share/server/main.js",
+    "erlang" => "#{COUCH_ROOT}/test/run_native_process.es"
+  }
 
   def self.run_command
     COMMANDS[LANGUAGE]
@@ -105,41 +113,90 @@ class ExternalRunner < OSProcessRunner
   end
 end
 
+
 functions = {
   "emit-twice" => {
-    "js" => %{function(doc){emit("foo",doc.a); emit("bar",doc.a)}}
+    "js" => %{function(doc){emit("foo",doc.a); emit("bar",doc.a)}},
+    "erlang" => <<-ERLANG
+      fun({Doc}) ->
+        A = proplists:get_value(<<"a">>, Doc, null),
+        Emit(<<"foo">>, A),
+        Emit(<<"bar">>, A)
+      end.
+    ERLANG
   },
   "emit-once" => {
-    "js" => %{function(doc){emit("baz",doc.a)}}
+    "js" => %{function(doc){emit("baz",doc.a)}},
+    "erlang" => <<-ERLANG
+        fun({Doc}) ->
+            A = proplists:get_value(<<"a">>, Doc, null),
+            Emit(<<"baz">>, A)
+        end.
+    ERLANG
   },
   "reduce-values-length" => {
-    "js" => %{function(keys, values, rereduce) { return values.length; }}
+    "js" => %{function(keys, values, rereduce) { return values.length; }},
+    "erlang" => %{fun(Keys, Values, ReReduce) -> length(Values) end.}
   },
   "reduce-values-sum" => {
-    "js" => %{function(keys, values, rereduce) { return sum(values); }}
+    "js" => %{function(keys, values, rereduce) { return sum(values); }},
+    "erlang" => %{fun(Keys, Values, ReReduce) -> lists:sum(Values) end.}
   },
   "validate-forbidden" => {
-    "js" => %{function(newDoc, oldDoc, userCtx) { if (newDoc.bad) throw({forbidden:"bad doc"}); "foo bar";}}
+    "js" => <<-JS,
+      function(newDoc, oldDoc, userCtx) {
+        if(newDoc.bad)
+          throw({forbidden:"bad doc"}); "foo bar";
+      }
+      JS
+    "erlang" => <<-ERLANG
+      fun({NewDoc}, _OldDoc, _UserCtx) ->
+        case proplists:get_value(<<"bad">>, NewDoc) of
+            undefined -> 1;
+            _ -> {[{forbidden, <<"bad doc">>}]}
+        end
+      end.
+    ERLANG
   },
   "show-simple" => {
-    "js" => <<-JS
+    "js" => <<-JS,
         function(doc, req) {
-          log("ok");
-          return [doc.title, doc.body].join(' - ');
+            log("ok");
+            return [doc.title, doc.body].join(' - ');
         }
     JS
+    "erlang" => <<-ERLANG
+      fun({Doc}, Req) ->
+            Title = proplists:get_value(<<"title">>, Doc),
+            Body = proplists:get_value(<<"body">>, Doc),
+            Resp = <<Title/binary, " - ", Body/binary>>,
+        {[{<<"body">>, Resp}]}
+      end.
+    ERLANG
   },
   "show-headers" => {
-    "js" => <<-JS
+    "js" => <<-JS,
         function(doc, req) {
           var resp = {"code":200, "headers":{"X-Plankton":"Rusty"}};
           resp.body = [doc.title, doc.body].join(' - ');
           return resp;
         }
      JS
+    "erlang" => <<-ERLANG
+  fun({Doc}, Req) ->
+        Title = proplists:get_value(<<"title">>, Doc),
+        Body = proplists:get_value(<<"body">>, Doc),
+        Resp = <<Title/binary, " - ", Body/binary>>,
+        {[
+        {<<"code">>, 200},
+        {<<"headers">>, {[{<<"X-Plankton">>, <<"Rusty">>}]}},
+        {<<"body">>, Resp}
+      ]}
+  end.
+    ERLANG
   },
   "show-sends" => {
-    "js" =>  <<-JS
+    "js" =>  <<-JS,
         function(head, req) {
           start({headers:{"Content-Type" : "text/plain"}});
           send("first chunk");
@@ -147,9 +204,20 @@ functions = {
           return "tail";
         };
     JS
+    "erlang" => <<-ERLANG
+      fun(Head, Req) ->
+        Resp = {[
+          {<<"headers">>, {[{<<"Content-Type">>, <<"text/plain">>}]}}
+        ]},
+        Start(Resp),
+        Send(<<"first chunk">>),
+        Send(<<"second \\\"chunk\\\"">>),
+        <<"tail">>
+      end.
+    ERLANG
   },
   "show-while-get-rows" => {
-    "js" =>  <<-JS
+    "js" =>  <<-JS,
         function(head, req) {
           send("first chunk");
           send(req.q);
@@ -161,9 +229,21 @@ functions = {
           return "tail";
         };
     JS
+    "erlang" => <<-ERLANG,
+        fun(Head, {Req}) ->
+            Send(<<"first chunk">>),
+            Send(proplists:get_value(<<"q">>, Req)),
+            Fun = fun({Row}, _) ->
+                Send(proplists:get_value(<<"key">>, Row)),
+                {ok, nil}
+            end,
+            {ok, _} = FoldRows(Fun, nil),
+            <<"tail">>
+        end.
+    ERLANG
   },
   "show-while-get-rows-multi-send" => {
-    "js" => <<-JS
+    "js" => <<-JS,
         function(head, req) {
           send("bacon");
           var row;
@@ -175,9 +255,21 @@ functions = {
           return "tail";
         };
     JS
+    "erlang" => <<-ERLANG,
+        fun(Head, Req) ->
+            Send(<<"bacon">>),
+            Fun = fun({Row}, _) ->
+                Send(proplists:get_value(<<"key">>, Row)),
+                Send(<<"eggs">>),
+                {ok, nil}
+            end,
+            FoldRows(Fun, nil),
+            <<"tail">>
+        end.
+    ERLANG
   },
   "list-simple" => {
-    "js" => <<-JS
+    "js" => <<-JS,
         function(head, req) {
           send("first chunk");
           send(req.q);
@@ -188,9 +280,21 @@ functions = {
           return "early";
         };
     JS
+    "erlang" => <<-ERLANG,
+        fun(Head, {Req}) ->
+            Send(<<"first chunk">>),
+            Send(proplists:get_value(<<"q">>, Req)),
+            Fun = fun({Row}, _) ->
+                Send(proplists:get_value(<<"key">>, Row)),
+                {ok, nil}
+            end,
+            FoldRows(Fun, nil),
+            <<"early">>
+        end.
+    ERLANG
   },
   "list-chunky" => {
-    "js" => <<-JS
+    "js" => <<-JS,
         function(head, req) {
           send("first chunk");
           send(req.q);
@@ -204,16 +308,37 @@ functions = {
           };
         };
     JS
+    "erlang" => <<-ERLANG,
+        fun(Head, {Req}) ->
+            Send(<<"first chunk">>),
+            Send(proplists:get_value(<<"q">>, Req)),
+            Fun = fun
+                ({Row}, Count) when Count < 2 ->
+                    Send(proplists:get_value(<<"key">>, Row)),
+                    {ok, Count+1};
+                ({Row}, Count) when Count == 2 ->
+                    Send(proplists:get_value(<<"key">>, Row)),
+                    {stop, <<"early tail">>}
+            end,
+            {ok, Tail} = FoldRows(Fun, 0),
+            Tail
+        end.
+    ERLANG
   },
   "list-old-style" => {
-    "js" => <<-JS
+    "js" => <<-JS,
         function(head, req, foo, bar) {
           return "stuff";
         }
     JS
+    "erlang" => <<-ERLANG,
+        fun(Head, Req, Foo, Bar) ->
+            <<"stuff">>
+        end.
+    ERLANG
   },
   "list-capped" => {
-    "js" => <<-JS
+    "js" => <<-JS,
         function(head, req) {
           send("bacon")
           var row, i = 0;
@@ -226,9 +351,24 @@ functions = {
           };
         }
     JS
+    "erlang" => <<-ERLANG,
+        fun(Head, Req) ->
+            Send(<<"bacon">>),
+            Fun = fun
+                ({Row}, Count) when Count < 2 ->
+                    Send(proplists:get_value(<<"key">>, Row)),
+                    {ok, Count+1};
+                ({Row}, Count) when Count == 2 ->
+                    Send(proplists:get_value(<<"key">>, Row)),
+                    {stop, <<"early">>}
+            end,
+            {ok, Tail} = FoldRows(Fun, 0),
+            Tail
+        end.
+    ERLANG
   },
   "list-raw" => {
-    "js" => <<-JS
+    "js" => <<-JS,
         function(head, req) {
           send("first chunk");
           send(req.q);
@@ -239,24 +379,47 @@ functions = {
           return "tail";
         };
     JS
+    "erlang" => <<-ERLANG,
+        fun(Head, {Req}) ->
+            Send(<<"first chunk">>),
+            Send(proplists:get_value(<<"q">>, Req)),
+            Fun = fun({Row}, _) ->
+                Send(proplists:get_value(<<"key">>, Row)),
+                {ok, nil}
+            end,
+            FoldRows(Fun, nil),
+            <<"tail">>
+        end.
+    ERLANG
   },
   "filter-basic" => {
-    "js" => <<-JS
+    "js" => <<-JS,
       function(doc, req) {
         if (doc.good) {
           return true;
         }
       }
     JS
+    "erlang" => <<-ERLANG,
+        fun({Doc}, Req) ->
+            proplists:get_value(<<"good">>, Doc)
+        end.
+    ERLANG
   },
   "update-basic" => {
-    "js" => <<-JS
+    "js" => <<-JS,
     function(doc, req) {
       doc.world = "hello";
       var resp = [doc, "hello doc"];
       return resp;
     }
     JS
+    "erlang" => <<-ERLANG,
+        fun({Doc}, Req) ->
+            Doc2 = [{<<"world">>, <<"hello">>}|Doc],
+            [{Doc2}, {[{<<"body">>, <<"hello doc">>}]}]
+        end.
+    ERLANG
   }
 }
 
@@ -322,7 +485,7 @@ describe "query server normal case" do
     end
     it "should show" do
       @qs.rrun(["show", @fun,
-        {:title => "Best ever", :body => "Doc body"}])
+        {:title => "Best ever", :body => "Doc body"}, {}])
       @qs.jsgets.should == ["resp", {"body" => "Best ever - Doc body"}]
     end
   end
@@ -334,7 +497,7 @@ describe "query server normal case" do
     end
     it "should show headers" do
       @qs.rrun(["show", @fun,
-        {:title => "Best ever", :body => "Doc body"}])
+        {:title => "Best ever", :body => "Doc body"}, {}])
       @qs.jsgets.should == ["resp", {"code"=>200,"headers" => {"X-Plankton"=>"Rusty"}, "body" => "Best ever - Doc body"}]
     end
   end
@@ -446,7 +609,7 @@ describe "query server normal case" do
       @qs.add_fun(@fun).should == true
     end
     it "should only return true for good docs" do
-      @qs.run(["filter", [{"key"=>"bam", "good" => true}, {"foo" => "bar"}, {"good" => true}]]).
+      @qs.run(["filter", [{"key"=>"bam", "good" => true}, {"foo" => "bar"}, {"good" => true}], {"req" => "foo"}]).
         should ==  [true, [true, false, true]]
     end
   end
@@ -493,7 +656,7 @@ describe "query server that exits" do
     it "should get a warning" do
       resp = @qs.run(["list", {"foo"=>"bar"}, {"q" => "ok"}])
       resp["error"].should == "render_error"
-      resp["reason"].should include("the list API has changed")
+      #resp["reason"].should include("the list API has changed")
     end
   end
 
diff --git a/test/run_native_process.es b/test/run_native_process.es
new file mode 100644
index 00000000..275d2bbe
--- /dev/null
+++ b/test/run_native_process.es
@@ -0,0 +1,43 @@
+#! /usr/bin/env escript
+
+read() ->
+    case io:get_line('') of
+        eof -> stop;
+        Data -> mochijson2:decode(Data)
+    end.
+
+send(Data) when is_binary(Data) ->
+    send(binary_to_list(Data));
+send(Data) when is_list(Data) ->
+    io:format(Data ++ "\n", []).
+
+write(Data) ->
+    case (catch mochijson2:encode(Data)) of
+        {json_encode, Error} -> write({[{<<"error">>, Error}]});
+        Json -> send(Json)
+    end.
+
+%log(Mesg) ->
+%    log(Mesg, []).
+%log(Mesg, Params) ->
+%    io:format(standard_error, Mesg, Params).
+
+loop(Pid) ->
+    case read() of
+        stop -> ok;
+        Json ->
+            case (catch couch_native_process:prompt(Pid, Json)) of
+                {error, Reason} ->
+                    ok = write({[{error, Reason}]});
+                Resp ->
+                    ok = write(Resp),
+                    loop(Pid)
+            end
+    end.
+
+main([]) ->
+    code:add_pathz("src/couchdb"),
+    code:add_pathz("src/mochiweb"),
+    {ok, Pid} = couch_native_process:start_link(),
+    loop(Pid).
+