From 2d0503e3bb0c50c99670c01272ee8c1fe7710f8d Mon Sep 17 00:00:00 2001 From: John Christopher Anderson Date: Mon, 4 May 2009 22:06:12 +0000 Subject: reduce_limit error is thrown when the reduce function output is not small enough compared to the input. Errors can be switched off using the config API. git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@771466 13f79535-47bb-0310-9956-ffa450edef68 --- etc/couchdb/default.ini.tpl.in | 3 +++ share/server/loop.js | 6 ++++-- share/server/state.js | 4 +++- share/server/views.js | 12 +++++++++++- share/www/script/test/design_docs.js | 7 +++++++ share/www/script/test/view_errors.js | 26 +++++++++++++++++++++++--- src/couchdb/couch_query_servers.erl | 26 ++++++++++++++++++-------- src/couchdb/couch_view_group.erl | 10 +++++++--- 8 files changed, 76 insertions(+), 18 deletions(-) diff --git a/etc/couchdb/default.ini.tpl.in b/etc/couchdb/default.ini.tpl.in index a9caadcf..8096ad6b 100644 --- a/etc/couchdb/default.ini.tpl.in +++ b/etc/couchdb/default.ini.tpl.in @@ -26,6 +26,9 @@ level = info [query_servers] javascript = %bindir%/%couchjs_command_name% %localdatadir%/server/main.js +[query_server_config] +reduce_limit = true + ; enable external as an httpd handler, then link it with commands here. ; note, this api is still under consideration. ; [external] diff --git a/share/server/loop.js b/share/server/loop.js index 170a8dc8..db6a9702 100644 --- a/share/server/loop.js +++ b/share/server/loop.js @@ -28,7 +28,7 @@ try { // // Responses are json values followed by a new line ("\n") -var cmd, cmdkey; +var line, cmd, cmdkey; var dispatch = { "reset" : State.reset, @@ -43,7 +43,9 @@ var dispatch = { "list_tail" : Render.listTail }; -while (cmd = eval(readline())) { +while (line = eval(readline())) { + cmd = eval(line) + line_length = line.length try { cmdkey = cmd.shift(); if (dispatch[cmdkey]) { diff --git a/share/server/state.js b/share/server/state.js index 05c55a1b..6c7d63b9 100644 --- a/share/server/state.js +++ b/share/server/state.js @@ -13,12 +13,14 @@ // globals used by other modules and functions var funs = []; // holds functions used for computation var funsrc = []; // holds function source for debug info +var query_config = {}; var State = (function() { return { - reset : function() { + reset : function(config) { // clear the globals and run gc funs = []; funsrc = []; + query_config = config; gc(); print("true"); // indicates success }, diff --git a/share/server/views.js b/share/server/views.js index c6d71579..de728ac2 100644 --- a/share/server/views.js +++ b/share/server/views.js @@ -46,7 +46,17 @@ var Views = (function() { reductions[i] = null; } } - print("[true," + toJSON(reductions) + "]"); + var reduce_line = toJSON(reductions); + var reduce_length = reduce_line.length; + if (query_config && query_config.reduce_limit && + reduce_length > 200 && ((reduce_length * 2) > line.length)) { + throw { + error:"reduce_overflow_error", + reason: "Reduce output must shrink more rapidly. Current output: '"+reduce_line+"'" + }; + } else { + print("[true," + reduce_line + "]"); + } }; return { diff --git a/share/www/script/test/design_docs.js b/share/www/script/test/design_docs.js index ebf0e74d..8f003efa 100644 --- a/share/www/script/test/design_docs.js +++ b/share/www/script/test/design_docs.js @@ -16,6 +16,12 @@ couchTests.design_docs = function(debug) { db.createDb(); if (debug) debugger; + run_on_modified_server( + [{section: "query_server_config", + key: "reduce_limit", + value: "false"}], +function() { + var numDocs = 500; function makebigstring(power) { @@ -104,4 +110,5 @@ couchTests.design_docs = function(debug) { restartServer(); T(db.open(designDoc._id) == null); T(db.view("test/no_docs") == null); +}); }; diff --git a/share/www/script/test/view_errors.js b/share/www/script/test/view_errors.js index 12225e67..1a613c04 100644 --- a/share/www/script/test/view_errors.js +++ b/share/www/script/test/view_errors.js @@ -75,13 +75,26 @@ couchTests.view_errors = function(debug) { _id:"_design/test", language: "javascript", views: { - no_reduce: {map:"function(doc) {emit(doc._id, null);}"}, - with_reduce: {map:"function (doc) {emit(doc.integer, doc.integer)};", - reduce:"function (keys, values) { return sum(values); };"}, + "no_reduce": {map:"function(doc) {emit(doc._id, null);}"}, + "with_reduce": { + map:"function (doc) {emit(doc.integer, doc.integer)};", + reduce:"function (keys, values) { return sum(values); };"} } }; T(db.save(designDoc).ok); + var designDoc2 = { + _id:"_design/testbig", + language: "javascript", + views: { + "reduce_too_big" : { + map:"function (doc) {emit(doc.integer, doc.integer)};", + reduce:"function (keys, values) { var chars = []; for (var i=0; i < 1000; i++) {chars.push('wazzap');};return chars; };"} + } + }; + T(db.save(designDoc2).ok); + + try { db.view("test/no_reduce", {group: true}); T(0 == 1); @@ -117,4 +130,11 @@ couchTests.view_errors = function(debug) { result = JSON.parse(xhr.responseText); T(result.error == "bad_request"); T(result.reason == "`keys` member must be a array."); + + // if the reduce grows to fast, throw an overflow error + var path = "/test_suite_db/_design/testbig/_view/reduce_too_big"; + xhr = CouchDB.request("GET", path); + T(xhr.status == 500); + result = JSON.parse(xhr.responseText); + T(result.error == "reduce_overflow_error"); }; diff --git a/src/couchdb/couch_query_servers.erl b/src/couchdb/couch_query_servers.erl index 4ff69a2d..a27943a1 100644 --- a/src/couchdb/couch_query_servers.erl +++ b/src/couchdb/couch_query_servers.erl @@ -85,23 +85,27 @@ rereduce(_Lang, [], _ReducedValues) -> rereduce(Lang, RedSrcs, ReducedValues) -> Pid = get_os_process(Lang), Grouped = group_reductions_results(ReducedValues), - Results = lists:zipwith( + Results = try lists:zipwith( fun(FunSrc, Values) -> [true, [Result]] = couch_os_process:prompt(Pid, [<<"rereduce">>, [FunSrc], Values]), Result - end, RedSrcs, Grouped), - - ok = ret_os_process(Lang, Pid), + end, RedSrcs, Grouped) + after + ok = ret_os_process(Lang, Pid) + end, {ok, Results}. reduce(_Lang, [], _KVs) -> {ok, []}; reduce(Lang, RedSrcs, KVs) -> Pid = get_os_process(Lang), - [true, Results] = couch_os_process:prompt(Pid, - [<<"reduce">>, RedSrcs, KVs]), - ok = ret_os_process(Lang, Pid), + Results = try couch_os_process:prompt(Pid, + [<<"reduce">>, RedSrcs, KVs]) of + [true, Reductions] -> Reductions + after + ok = ret_os_process(Lang, Pid) + end, {ok, Results}. validate_doc_update(Lang, FunSrc, EditDoc, DiskDoc, Ctx) -> @@ -209,7 +213,8 @@ handle_call({get_proc, Lang}, _From, {Langs, PidLangs, Pids, InUse}=Server) -> add_value(PidLangs, Pid, Lang), rem_from_list(Pids, Lang, Pid), add_to_list(InUse, Lang, Pid), - true = couch_os_process:prompt(Pid, [<<"reset">>]), + QueryConfig = get_query_server_config(), + true = couch_os_process:prompt(Pid, [<<"reset">>, QueryConfig]), {reply, Pid, Server}; _ -> {ok, Pid} = new_process(Langs, Lang), @@ -249,6 +254,11 @@ code_change(_OldVsn, State, _Extra) -> % Private API +get_query_server_config() -> + ReduceLimit = list_to_atom( + couch_config:get("query_server_config","reduce_limit","true")), + {[{<<"reduce_limit">>, ReduceLimit}]}. + new_process(Langs, Lang) -> Proc = case ets:lookup(Langs, Lang) of diff --git a/src/couchdb/couch_view_group.erl b/src/couchdb/couch_view_group.erl index 57ee97da..af4ea814 100644 --- a/src/couchdb/couch_view_group.erl +++ b/src/couchdb/couch_view_group.erl @@ -42,9 +42,9 @@ request_group(Pid, Seq) -> {ok, Group, RefCounter} -> couch_ref_counter:add(RefCounter), {ok, Group}; - Else -> - ?LOG_DEBUG("get_updated_group replied with _Else ~p", [Else]), - Else + Error -> + ?LOG_DEBUG("request_group Error ~p", [Error]), + throw(Error) end. @@ -261,6 +261,10 @@ handle_info({'EXIT', FromPid, reset}, handle_info({'EXIT', _FromPid, normal}, State) -> {noreply, State}; +handle_info({'EXIT', FromPid, {{nocatch, Reason}, Trace}}, State) -> + ?LOG_DEBUG("Uncaught throw() in linked pid: ~p", [{FromPid, Reason}]), + {stop, Reason, State}; + handle_info({'EXIT', FromPid, Reason}, State) -> ?LOG_DEBUG("Exit from linked pid: ~p", [{FromPid, Reason}]), {stop, Reason, State}; -- cgit v1.2.3