diff options
24 files changed, 1479 insertions, 1244 deletions
diff --git a/etc/couchdb/default.ini.tpl.in b/etc/couchdb/default.ini.tpl.in index 422292ff..71656d26 100644 --- a/etc/couchdb/default.ini.tpl.in +++ b/etc/couchdb/default.ini.tpl.in @@ -76,7 +76,6 @@ _user = {couch_httpd_auth, handle_user_req} _view_cleanup = {couch_httpd_db, handle_view_cleanup_req} _compact = {couch_httpd_db, handle_compact_req} _design = {couch_httpd_db, handle_design_req} -_view = {couch_httpd_view, handle_db_view_req} _temp_view = {couch_httpd_view, handle_temp_view_req} _changes = {couch_httpd_db, handle_changes_req} diff --git a/share/server/filter.js b/share/server/filter.js index a683146a..8ba77e64 100644 --- a/share/server/filter.js +++ b/share/server/filter.js @@ -11,17 +11,14 @@ // the License. var Filter = { - filter : function(funSrc, docs, req, userCtx) { - var filterFun = compileFunction(funSrc); - + filter : function(fun, ddoc, args) { var results = []; - try { - for (var i=0; i < docs.length; i++) { - results.push((filterFun(docs[i], req, userCtx) && true) || false); - }; - respond([true, results]); - } catch (error) { - respond(error); - } + var docs = args[0]; + var req = args[1]; + var userCtx = args[2]; + for (var i=0; i < docs.length; i++) { + results.push((fun.apply(ddoc, [docs[i], req, userCtx]) && true) || false); + }; + respond([true, results]); } }; diff --git a/share/server/loop.js b/share/server/loop.js index 33e87c98..84e35dc5 100644 --- a/share/server/loop.js +++ b/share/server/loop.js @@ -12,21 +12,21 @@ var sandbox = null; -var init_sandbox = function() { +function init_sandbox() { try { // if possible, use evalcx (not always available) sandbox = evalcx(''); - sandbox.emit = emit; - sandbox.sum = sum; + sandbox.emit = Views.emit; + sandbox.sum = Views.sum; sandbox.log = log; - sandbox.toJSON = toJSON; - sandbox.provides = provides; - sandbox.registerType = registerType; - sandbox.start = start; - sandbox.send = send; - sandbox.getRow = getRow; + sandbox.toJSON = Couch.toJSON; + sandbox.provides = Mime.provides; + sandbox.registerType = Mime.registerType; + sandbox.start = Render.start; + sandbox.send = Render.send; + sandbox.getRow = Render.getRow; } catch (e) { - log(toJSON(e)); + log(e.toSource()); } }; init_sandbox(); @@ -36,37 +36,104 @@ init_sandbox(); // // Responses are json values followed by a new line ("\n") -var line, cmd, cmdkey; +var DDoc = (function() { + var ddoc_dispatch = { + "lists" : Render.list, + "shows" : Render.show, + "filters" : Filter.filter, + "updates" : Render.update, + "validate_doc_update" : Validate.validate + }; + var ddocs = {}; + return { + ddoc : function() { + var args = []; + for (var i=0; i < arguments.length; i++) { + args.push(arguments[i]); + }; + var ddocId = args.shift(); + if (ddocId == "new") { + // get the real ddocId. + ddocId = args.shift(); + // store the ddoc, functions are lazily compiled. + ddocs[ddocId] = args.shift(); + print("true"); + } else { + // Couch makes sure we know this ddoc already. + var ddoc = ddocs[ddocId]; + if (!ddoc) throw(["fatal", "query_protocol_error", "uncached design doc: "+ddocId]); + var funPath = args.shift(); + var cmd = funPath[0]; + // the first member of the fun path determines the type of operation + var funArgs = args.shift(); + if (ddoc_dispatch[cmd]) { + // get the function, call the command with it + var point = ddoc; + for (var i=0; i < funPath.length; i++) { + if (i+1 == funPath.length) { + fun = point[funPath[i]] + if (typeof fun != "function") { + fun = Couch.compileFunction(fun); + // cache the compiled fun on the ddoc + point[funPath[i]] = fun + }; + } else { + point = point[funPath[i]] + } + }; -var dispatch = { - "reset" : State.reset, - "add_fun" : State.addFun, - "map_doc" : Views.mapDoc, - "reduce" : Views.reduce, - "rereduce" : Views.rereduce, - "validate" : Validate.validate, - "show" : Render.show, - "update" : Render.update, - "list" : Render.list, - "filter" : Filter.filter -}; + // run the correct responder with the cmd body + ddoc_dispatch[cmd].apply(null, [fun, ddoc, funArgs]); + } else { + // unknown command, quit and hope the restarted version is better + throw(["fatal", "unknown_command", "unknown ddoc command '" + cmd + "'"]); + } + } + } + }; +})(); -while (line = eval(readline())) { - cmd = eval(line); - line_length = line.length; - try { - cmdkey = cmd.shift(); - if (dispatch[cmdkey]) { - // run the correct responder with the cmd body - dispatch[cmdkey].apply(this, cmd); +var Loop = function() { + var line, cmd, cmdkey, dispatch = { + "ddoc" : DDoc.ddoc, + // "view" : Views.handler, + "reset" : State.reset, + "add_fun" : State.addFun, + "map_doc" : Views.mapDoc, + "reduce" : Views.reduce, + "rereduce" : Views.rereduce + }; + function handleError(e) { + var type = e[0]; + if (type == "fatal") { + e[0] = "error"; // we tell the client it was a fatal error by dying + respond(e); + quit(-1); + } else if (type == "error") { + respond(e); + } else if (e.error && e.reason) { + // compatibility with old error format + respond(["error", e.error, e.reason]); } else { - // unknown command, quit and hope the restarted version is better - respond({ - error: "query_server_error", - reason: "unknown command '" + cmdkey + "'"}); - quit(); + respond(["error","unnamed_error",e.toSource()]); } - } catch(e) { - respond(e); - } + }; + while (line = readline()) { + cmd = eval('('+line+')'); + State.line_length = line.length; + try { + cmdkey = cmd.shift(); + if (dispatch[cmdkey]) { + // run the correct responder with the cmd body + dispatch[cmdkey].apply(null, cmd); + } else { + // unknown command, quit and hope the restarted version is better + throw(["fatal", "unknown_command", "unknown command '" + cmdkey + "'"]); + } + } catch(e) { + handleError(e); + } + }; }; + +Loop(); diff --git a/share/server/render.js b/share/server/render.js index f147af89..e19f31c4 100644 --- a/share/server/render.js +++ b/share/server/render.js @@ -11,152 +11,115 @@ // the License. -// registerType(name, mime-type, mime-type, ...) -// -// Available in query server sandbox. TODO: The list is cleared on reset. -// This registers a particular name with the set of mimetypes it can handle. -// Whoever registers last wins. -// -// Example: -// registerType("html", "text/html; charset=utf-8"); - -mimesByKey = {}; -keysByMime = {}; -registerType = function() { - var mimes = [], key = arguments[0]; - for (var i=1; i < arguments.length; i++) { - mimes.push(arguments[i]); - }; - mimesByKey[key] = mimes; - for (var i=0; i < mimes.length; i++) { - keysByMime[mimes[i]] = key; - }; -}; - -// Some default types -// Ported from Ruby on Rails -// Build list of Mime types for HTTP responses -// http://www.iana.org/assignments/media-types/ -// http://dev.rubyonrails.org/svn/rails/trunk/actionpack/lib/action_controller/mime_types.rb - -registerType("all", "*/*"); -registerType("text", "text/plain; charset=utf-8", "txt"); -registerType("html", "text/html; charset=utf-8"); -registerType("xhtml", "application/xhtml+xml", "xhtml"); -registerType("xml", "application/xml", "text/xml", "application/x-xml"); -registerType("js", "text/javascript", "application/javascript", "application/x-javascript"); -registerType("css", "text/css"); -registerType("ics", "text/calendar"); -registerType("csv", "text/csv"); -registerType("rss", "application/rss+xml"); -registerType("atom", "application/atom+xml"); -registerType("yaml", "application/x-yaml", "text/yaml"); -// just like Rails -registerType("multipart_form", "multipart/form-data"); -registerType("url_encoded_form", "application/x-www-form-urlencoded"); -// http://www.ietf.org/rfc/rfc4627.txt -registerType("json", "application/json", "text/x-json"); - -// Start chunks -var startResp = {}; -function start(resp) { - startResp = resp || {}; -}; - -function sendStart() { - startResp = applyContentType((startResp || {}), responseContentType); - respond(["start", chunks, startResp]); - chunks = []; - startResp = {}; -} - -function applyContentType(resp, responseContentType) { - resp["headers"] = resp["headers"] || {}; - if (responseContentType) { - resp["headers"]["Content-Type"] = resp["headers"]["Content-Type"] || responseContentType; - } - return resp; -} - -// Send chunk -var chunks = []; -function send(chunk) { - chunks.push(chunk.toString()); -}; - -function blowChunks(label) { - respond([label||"chunks", chunks]); - chunks = []; -}; - -var gotRow = false, lastRow = false; -function getRow() { - if (lastRow) return null; - if (!gotRow) { - gotRow = true; - sendStart(); - } else { - blowChunks(); - } - var line = readline(); - var json = eval(line); - if (json[0] == "list_end") { - lastRow = true; - return null; - } - if (json[0] != "list_row") { - respond({ - error: "query_server_error", - reason: "not a row '" + json[0] + "'"}); - quit(); - } - return json[1]; -}; - -var mimeFuns = [], providesUsed, responseContentType; -function provides(type, fun) { - providesUsed = true; - mimeFuns.push([type, fun]); -}; - -function runProvides(req) { - var supportedMimes = [], bestFun, bestKey = null, accept = req.headers["Accept"]; - if (req.query && req.query.format) { - bestKey = req.query.format; - responseContentType = mimesByKey[bestKey][0]; - } else if (accept) { - // log("using accept header: "+accept); - mimeFuns.reverse().forEach(function(mimeFun) { - var mimeKey = mimeFun[0]; - if (mimesByKey[mimeKey]) { - supportedMimes = supportedMimes.concat(mimesByKey[mimeKey]); - } - }); - responseContentType = Mimeparse.bestMatch(supportedMimes, accept); - bestKey = keysByMime[responseContentType]; - } else { - // just do the first one - bestKey = mimeFuns[0][0]; - responseContentType = mimesByKey[bestKey][0]; +var Mime = (function() { + // registerType(name, mime-type, mime-type, ...) + // + // Available in query server sandbox. TODO: The list is cleared on reset. + // This registers a particular name with the set of mimetypes it can handle. + // Whoever registers last wins. + // + // Example: + // registerType("html", "text/html; charset=utf-8"); + + var mimesByKey = {}; + var keysByMime = {}; + function registerType() { + var mimes = [], key = arguments[0]; + for (var i=1; i < arguments.length; i++) { + mimes.push(arguments[i]); + }; + mimesByKey[key] = mimes; + for (var i=0; i < mimes.length; i++) { + keysByMime[mimes[i]] = key; + }; } + + // Some default types + // Ported from Ruby on Rails + // Build list of Mime types for HTTP responses + // http://www.iana.org/assignments/media-types/ + // http://dev.rubyonrails.org/svn/rails/trunk/actionpack/lib/action_controller/mime_types.rb + + registerType("all", "*/*"); + registerType("text", "text/plain; charset=utf-8", "txt"); + registerType("html", "text/html; charset=utf-8"); + registerType("xhtml", "application/xhtml+xml", "xhtml"); + registerType("xml", "application/xml", "text/xml", "application/x-xml"); + registerType("js", "text/javascript", "application/javascript", "application/x-javascript"); + registerType("css", "text/css"); + registerType("ics", "text/calendar"); + registerType("csv", "text/csv"); + registerType("rss", "application/rss+xml"); + registerType("atom", "application/atom+xml"); + registerType("yaml", "application/x-yaml", "text/yaml"); + // just like Rails + registerType("multipart_form", "multipart/form-data"); + registerType("url_encoded_form", "application/x-www-form-urlencoded"); + // http://www.ietf.org/rfc/rfc4627.txt + registerType("json", "application/json", "text/x-json"); - if (bestKey) { - for (var i=0; i < mimeFuns.length; i++) { - if (mimeFuns[i][0] == bestKey) { - bestFun = mimeFuns[i][1]; - break; - } + + var mimeFuns = []; + function provides(type, fun) { + Mime.providesUsed = true; + mimeFuns.push([type, fun]); + }; + + function resetProvides() { + // set globals + Mime.providesUsed = false; + mimeFuns = []; + Mime.responseContentType = null; + }; + + function runProvides(req) { + var supportedMimes = [], bestFun, bestKey = null, accept = req.headers["Accept"]; + if (req.query && req.query.format) { + bestKey = req.query.format; + Mime.responseContentType = mimesByKey[bestKey][0]; + } else if (accept) { + // log("using accept header: "+accept); + mimeFuns.reverse().forEach(function(mimeFun) { + var mimeKey = mimeFun[0]; + if (mimesByKey[mimeKey]) { + supportedMimes = supportedMimes.concat(mimesByKey[mimeKey]); + } + }); + Mime.responseContentType = Mimeparse.bestMatch(supportedMimes, accept); + bestKey = keysByMime[Mime.responseContentType]; + } else { + // just do the first one + bestKey = mimeFuns[0][0]; + Mime.responseContentType = mimesByKey[bestKey][0]; + } + + if (bestKey) { + for (var i=0; i < mimeFuns.length; i++) { + if (mimeFuns[i][0] == bestKey) { + bestFun = mimeFuns[i][1]; + break; + } + }; }; + + if (bestFun) { + return bestFun(); + } else { + var supportedTypes = mimeFuns.map(function(mf) {return mimesByKey[mf[0]].join(', ') || mf[0]}); + throw(["error","not_acceptable", + "Content-Type "+(accept||bestKey)+" not supported, try one of: "+supportedTypes.join(', ')]); + } }; + - if (bestFun) { - // log("responding with: "+bestKey); - return bestFun(); - } else { - var supportedTypes = mimeFuns.map(function(mf) {return mimesByKey[mf[0]].join(', ') || mf[0]}); - throw({error:"not_acceptable", reason:"Content-Type "+(accept||bestKey)+" not supported, try one of: "+supportedTypes.join(', ')}); - } -}; + return { + registerType : registerType, + provides : provides, + resetProvides : resetProvides, + runProvides : runProvides + } +})(); + @@ -167,151 +130,202 @@ function runProvides(req) { //// //// -var Render = { - show : function(funSrc, doc, req) { - var showFun = compileFunction(funSrc); - runShow(showFun, doc, req, funSrc); - }, - update : function(funSrc, doc, req) { - var upFun = compileFunction(funSrc); - runUpdate(upFun, doc, req, funSrc); - }, - list : function(head, req) { - runList(funs[0], head, req, funsrc[0]); +var Render = (function() { + var chunks = []; + + + // Start chunks + var startResp = {}; + function start(resp) { + startResp = resp || {}; + }; + + function sendStart() { + startResp = applyContentType((startResp || {}), Mime.responseContentType); + respond(["start", chunks, startResp]); + chunks = []; + startResp = {}; } -}; -function maybeWrapResponse(resp) { - var type = typeof resp; - if ((type == "string") || (type == "xml")) { - return {body:resp}; - } else { + function applyContentType(resp, responseContentType) { + resp["headers"] = resp["headers"] || {}; + if (responseContentType) { + resp["headers"]["Content-Type"] = resp["headers"]["Content-Type"] || responseContentType; + } return resp; } -}; -function resetProvides() { - // set globals - providesUsed = false; - mimeFuns = []; - responseContentType = null; -}; + function send(chunk) { + chunks.push(chunk.toString()); + }; + + function blowChunks(label) { + respond([label||"chunks", chunks]); + chunks = []; + }; + + var gotRow = false, lastRow = false; + function getRow() { + if (lastRow) return null; + if (!gotRow) { + gotRow = true; + sendStart(); + } else { + blowChunks(); + } + var line = readline(); + var json = eval('('+line+')'); + if (json[0] == "list_end") { + lastRow = true; + return null; + } + if (json[0] != "list_row") { + throw(["fatal", "list_error", "not a row '" + json[0] + "'"]); + } + return json[1]; + }; + + + function maybeWrapResponse(resp) { + var type = typeof resp; + if ((type == "string") || (type == "xml")) { + return {body:resp}; + } else { + return resp; + } + }; -// from http://javascript.crockford.com/remedial.html -function typeOf(value) { + // from http://javascript.crockford.com/remedial.html + function typeOf(value) { var s = typeof value; if (s === 'object') { - if (value) { - if (value instanceof Array) { - s = 'array'; - } - } else { - s = 'null'; + if (value) { + if (value instanceof Array) { + s = 'array'; } + } else { + s = 'null'; + } } return s; -}; - -function runShow(showFun, doc, req, funSrc) { - try { - resetProvides(); - var resp = showFun.apply(null, [doc, req]); - - if (providesUsed) { - resp = runProvides(req); - resp = applyContentType(maybeWrapResponse(resp), responseContentType); - } + }; - var type = typeOf(resp); - if (type == 'object' || type == 'string') { - respond(["resp", maybeWrapResponse(resp)]); - } else { - renderError("undefined response from show function"); - } - } catch(e) { - respondError(e, funSrc, true); - } -}; - -function runUpdate(renderFun, doc, req, funSrc) { - try { - var result = renderFun.apply(null, [doc, req]); - var doc = result[0]; - var resp = result[1]; - if (resp) { - respond(["up", doc, maybeWrapResponse(resp)]); - } else { - renderError("undefined response from update function"); + function runShow(fun, ddoc, args) { + try { + Mime.resetProvides(); + var resp = fun.apply(ddoc, args); + + if (Mime.providesUsed) { + resp = Mime.runProvides(args[1]); + resp = applyContentType(maybeWrapResponse(resp), Mime.responseContentType); + } + + var type = typeOf(resp); + if (type == 'object' || type == 'string') { + respond(["resp", maybeWrapResponse(resp)]); + } else { + throw(["error", "render_error", "undefined response from show function"]); + } + } catch(e) { + renderError(e, fun.toSource()); } - } catch(e) { - respondError(e, funSrc, true); - } -}; - -function resetList() { - gotRow = false; - lastRow = false; - chunks = []; - startResp = {}; -}; - -function runList(listFun, head, req, funSrc) { - try { - if (listFun.arity > 2) { - throw("the list API has changed for CouchDB 0.10, please upgrade your code"); + }; + + function runUpdate(fun, ddoc, args) { + try { + var verb = args[1].verb; + // for analytics logging applications you might want to remove the next line + if (verb == "GET") throw(["error","method_not_allowed","Update functions do not allow GET"]); + var result = fun.apply(ddoc, args); + var doc = result[0]; + var resp = result[1]; + var type = typeOf(resp); + if (type == 'object' || type == 'string') { + respond(["up", doc, maybeWrapResponse(resp)]); + } else { + throw(["error", "render_error", "undefined response from update function"]); + } + } catch(e) { + renderError(e, fun.toSource()); } - - resetProvides(); - resetList(); - - var tail = listFun.apply(null, [head, req]); - - if (providesUsed) { - tail = runProvides(req); + }; + + function resetList() { + gotRow = false; + lastRow = false; + chunks = []; + startResp = {}; + }; + + function runList(listFun, ddoc, args) { + try { + Mime.resetProvides(); + resetList(); + head = args[0] + req = args[1] + var tail = listFun.apply(ddoc, args); + + if (Mime.providesUsed) { + tail = Mime.runProvides(req); + } + if (!gotRow) getRow(); + if (typeof tail != "undefined") { + chunks.push(tail); + } + blowChunks("end"); + } catch(e) { + renderError(e, listFun.toSource()); } - - if (!gotRow) { - getRow(); + }; + + function renderError(e, funSrc) { + if (e.error && e.reason || e[0] == "error" || e[0] == "fatal") { + throw(e); + } else { + var logMessage = "function raised error: "+e.toSource()+" \nstacktrace: "+e.stack; + log(logMessage); + throw(["error", "render_error", logMessage]); } - if (typeof tail != "undefined") { - chunks.push(tail); + }; + + function escapeHTML(string) { + return string && string.replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">"); + }; + + + return { + start : start, + send : send, + getRow : getRow, + show : function(fun, ddoc, args) { + // var showFun = Couch.compileFunction(funSrc); + runShow(fun, ddoc, args); + }, + update : function(fun, ddoc, args) { + // var upFun = Couch.compileFunction(funSrc); + runUpdate(fun, ddoc, args); + }, + list : function(fun, ddoc, args) { + runList(fun, ddoc, args); } - blowChunks("end"); - } catch(e) { - respondError(e, funSrc, false); - } -}; - -function renderError(m) { - respond({error : "render_error", reason : m}); -} - -function respondError(e, funSrc, htmlErrors) { - if (e.error && e.reason) { - respond(e); - } else { - var logMessage = "function raised error: "+e.toString(); - log(logMessage); - log("stacktrace: "+e.stack); - var errorMessage = htmlErrors ? htmlRenderError(e, funSrc) : logMessage; - renderError(errorMessage); - } -} - -function escapeHTML(string) { - return string.replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">"); -} - -function htmlRenderError(e, funSrc) { - var msg = ["<html><body><h1>Render Error</h1>", - "<p>JavaScript function raised error: ", - e.toString(), - "</p><h2>Stacktrace:</h2><code><pre>", - escapeHTML(e.stack), - "</pre></code><h2>Function source:</h2><code><pre>", - escapeHTML(funSrc), - "</pre></code></body></html>"].join(''); - return {body:msg}; -}; + }; +})(); + +// send = Render.send; +// getRow = Render.getRow; +// start = Render.start; + +// unused. this will be handled in the Erlang side of things. +// function htmlRenderError(e, funSrc) { +// var msg = ["<html><body><h1>Render Error</h1>", +// "<p>JavaScript function raised error: ", +// e.toString(), +// "</p><h2>Stacktrace:</h2><code><pre>", +// escapeHTML(e.stack), +// "</pre></code><h2>Function source:</h2><code><pre>", +// escapeHTML(funSrc), +// "</pre></code></body></html>"].join(''); +// return {body:msg}; +// }; diff --git a/share/server/state.js b/share/server/state.js index b9bd87aa..9af9e475 100644 --- a/share/server/state.js +++ b/share/server/state.js @@ -10,26 +10,18 @@ // License for the specific language governing permissions and limitations under // the License. -// globals used by other modules and functions -var funs = []; // holds functions used for computation -var funsrc = []; // holds function source for debug info -var query_config = {}; -var State = (function() { - return { - reset : function(config) { - // clear the globals and run gc - funs = []; - funsrc = []; - query_config = config; - init_sandbox(); - gc(); - print("true"); // indicates success - }, - addFun : function(newFun) { - // Compile to a function and add it to funs array - funsrc.push(newFun); - funs.push(compileFunction(newFun)); - print("true"); - } +var State = { + reset : function(config) { + // clear the globals and run gc + State.funs = []; + State.query_config = config || {}; + init_sandbox(); + gc(); + print("true"); // indicates success + }, + addFun : function(newFun) { + // Compile to a function and add it to funs array + State.funs.push(Couch.compileFunction(newFun)); + print("true"); } -})(); +} diff --git a/share/server/util.js b/share/server/util.js index 1f69bf16..bd4abc1d 100644 --- a/share/server/util.js +++ b/share/server/util.js @@ -10,13 +10,50 @@ // License for the specific language governing permissions and limitations under // the License. -toJSON.subs = {'\b': '\\b', '\t': '\\t', '\n': '\\n', '\f': '\\f', +var Couch = { + // moving this away from global so we can move to json2.js later + toJSON : function (val) { + if (typeof(val) == "undefined") { + throw "Cannot encode 'undefined' value as JSON"; + } + if (typeof(val) == "xml") { // E4X support + val = val.toXMLString(); + } + if (val === null) { return "null"; } + return (Couch.toJSON.dispatcher[val.constructor.name])(val); + }, + compileFunction : function(source) { + if (!source) throw(["error","not_found","missing function"]); + try { + var functionObject = sandbox ? evalcx(source, sandbox) : eval(source); + } catch (err) { + throw(["error", "compilation_error", err.toSource() + " (" + source + ")"]); + }; + if (typeof(functionObject) == "function") { + return functionObject; + } else { + throw(["error","compilation_error", + "Expression does not eval to a function. (" + source.toSource() + ")"]); + }; + }, + recursivelySeal : function(obj) { + // seal() is broken in current Spidermonkey + seal(obj); + for (var propname in obj) { + if (typeof doc[propname] == "object") { + recursivelySeal(doc[propname]); + } + } + } +} + +Couch.toJSON.subs = {'\b': '\\b', '\t': '\\t', '\n': '\\n', '\f': '\\f', '\r': '\\r', '"' : '\\"', '\\': '\\\\'}; -toJSON.dispatcher = { +Couch.toJSON.dispatcher = { "Array": function(v) { var buf = []; for (var i = 0; i < v.length; i++) { - buf.push(toJSON(v[i])); + buf.push(Couch.toJSON(v[i])); } return "[" + buf.join(",") + "]"; }, @@ -42,14 +79,14 @@ toJSON.dispatcher = { if (!v.hasOwnProperty(k) || typeof(k) !== "string" || v[k] === undefined) { continue; } - buf.push(toJSON(k) + ": " + toJSON(v[k])); + buf.push(Couch.toJSON(k) + ": " + Couch.toJSON(v[k])); } return "{" + buf.join(",") + "}"; }, "String": function(v) { if (/["\\\x00-\x1f]/.test(v)) { v = v.replace(/([\x00-\x1f\\"])/g, function(a, b) { - var c = toJSON.subs[b]; + var c = Couch.toJSON.subs[b]; if (c) return c; c = b.charCodeAt(); return '\\u00' + Math.floor(c / 16).toString(16) + (c % 16).toString(16); @@ -59,56 +96,22 @@ toJSON.dispatcher = { } }; -function toJSON(val) { - if (typeof(val) == "undefined") { - throw "Cannot encode 'undefined' value as JSON"; - } - if (typeof(val) == "xml") { // E4X support - val = val.toXMLString(); - } - if (val === null) { return "null"; } - return (toJSON.dispatcher[val.constructor.name])(val); -} - -function compileFunction(source) { - try { - var functionObject = sandbox ? evalcx(source, sandbox) : eval(source); - } catch (err) { - throw {error: "compilation_error", - reason: err.toString() + " (" + source + ")"}; - } - if (typeof(functionObject) == "function") { - return functionObject; - } else { - throw {error: "compilation_error", - reason: "expression does not eval to a function. (" + source + ")"}; - } -} - -function recursivelySeal(obj) { - seal(obj); - for (var propname in obj) { - if (typeof doc[propname] == "object") { - recursivelySeal(doc[propname]); - } - } -} - // prints the object as JSON, and rescues and logs any toJSON() related errors function respond(obj) { try { - print(toJSON(obj)); + print(Couch.toJSON(obj)); } catch(e) { log("Error converting object to JSON: " + e.toString()); + log("error on obj: "+ obj.toSource()); } }; -log = function(message) { - // return; +function log(message) { + // return; // idea: query_server_config option for log level if (typeof message == "undefined") { message = "Error: attempting to log message of 'undefined'."; } else if (typeof message != "string") { - message = toJSON(message); + message = Couch.toJSON(message); } respond(["log", message]); }; diff --git a/share/server/validate.js b/share/server/validate.js index 5e5e5f9f..76a14129 100644 --- a/share/server/validate.js +++ b/share/server/validate.js @@ -11,10 +11,9 @@ // the License. var Validate = { - validate : function(funSrc, newDoc, oldDoc, userCtx) { - var validateFun = compileFunction(funSrc); + validate : function(fun, ddoc, args) { try { - validateFun(newDoc, oldDoc, userCtx); + fun.apply(ddoc, args); print("1"); } catch (error) { respond(error); diff --git a/share/server/views.js b/share/server/views.js index 1f12ad2b..ffe63377 100644 --- a/share/server/views.js +++ b/share/server/views.js @@ -10,58 +10,76 @@ // License for the specific language governing permissions and limitations under // the License. -// globals used by views -var map_results = []; // holds temporary emitted values during doc map -// view helper functions -emit = function(key, value) { - map_results.push([key, value]); -} - -sum = function(values) { - var rv = 0; - for (var i in values) { - rv += values[i]; - } - return rv; -} var Views = (function() { + var map_results = []; // holds temporary emitted values during doc map + function runReduce(reduceFuns, keys, values, rereduce) { for (var i in reduceFuns) { - reduceFuns[i] = compileFunction(reduceFuns[i]); - } + reduceFuns[i] = Couch.compileFunction(reduceFuns[i]); + }; var reductions = new Array(reduceFuns.length); for(var i = 0; i < reduceFuns.length; i++) { try { reductions[i] = reduceFuns[i](keys, values, rereduce); } catch (err) { - if (err == "fatal_error") { - throw { - error: "reduce_runtime_error", - reason: "function raised fatal exception"}; - } - log("function raised exception (" + err + ")"); + handleViewError(err); + // if the error is not fatal, ignore the results and continue reductions[i] = null; } - } - var reduce_line = toJSON(reductions); + }; + var reduce_line = Couch.toJSON(reductions); var reduce_length = reduce_line.length; - if (query_config && query_config.reduce_limit && - reduce_length > 200 && ((reduce_length * 2) > line.length)) { - var reduce_preview = "Current output: '"+(reduce_line.substring(0,100) + "'... (first 100 of "+reduce_length+' bytes)'); - - throw { - error:"reduce_overflow_error", - reason: "Reduce output must shrink more rapidly: "+reduce_preview+"" - }; + // TODO make reduce_limit config into a number + if (State.query_config && State.query_config.reduce_limit && + reduce_length > 200 && ((reduce_length * 2) > State.line_length)) { + var reduce_preview = "Current output: '"+(reduce_line.substring(0,100) + "'... (first 100 of "+reduce_length+" bytes)"); + throw(["error", + "reduce_overflow_error", + "Reduce output must shrink more rapidly: "+reduce_preview]); } else { print("[true," + reduce_line + "]"); } }; + function handleViewError(err, doc) { + if (err == "fatal_error") { + // Only if it's a "fatal_error" do we exit. What's a fatal error? + // That's for the query to decide. + // + // This will make it possible for queries to completely error out, + // by catching their own local exception and rethrowing a + // fatal_error. But by default if they don't do error handling we + // just eat the exception and carry on. + // + // In this case we abort map processing but don't destroy the + // JavaScript process. If you need to destroy the JavaScript + // process, throw the error form matched by the block below. + throw(["error", "map_runtime_error", "function raised 'fatal_error'"]); + } else if (err[0] == "fatal") { + // Throwing errors of the form ["fatal","error_key","reason"] + // will kill the OS process. This is not normally what you want. + throw(err); + } + var message = "function raised exception " + err.toSource(); + if (doc) message += " with doc._id " + doc._id; + log(message); + }; + return { + // view helper functions + emit : function(key, value) { + map_results.push([key, value]); + }, + sum : function(values) { + var rv = 0; + for (var i in values) { + rv += values[i]; + } + return rv; + }, reduce : function(reduceFuns, kvs) { var keys = new Array(kvs.length); var values = new Array(kvs.length); @@ -101,25 +119,15 @@ var Views = (function() { recursivelySeal(doc); // seal to prevent map functions from changing doc */ var buf = []; - for (var i = 0; i < funs.length; i++) { + for (var i = 0; i < State.funs.length; i++) { map_results = []; try { - funs[i](doc); - buf.push(toJSON(map_results)); + State.funs[i](doc); + buf.push(Couch.toJSON(map_results)); } catch (err) { - if (err == "fatal_error") { - // Only if it's a "fatal_error" do we exit. What's a fatal error? - // That's for the query to decide. - // - // This will make it possible for queries to completely error out, - // by catching their own local exception and rethrowing a - // fatal_error. But by default if they don't do error handling we - // just eat the exception and carry on. - throw { - error: "map_runtime_error", - reason: "function raised fatal exception"}; - } - log("function raised exception (" + err + ") with doc._id " + doc._id); + handleViewError(err, doc); + // If the error is not fatal, we treat the doc as if it + // did not emit anything, by buffering an empty array. buf.push("[]"); } } diff --git a/share/www/script/test/changes.js b/share/www/script/test/changes.js index 96ddf1a4..0cbf3bd6 100644 --- a/share/www/script/test/changes.js +++ b/share/www/script/test/changes.js @@ -213,12 +213,12 @@ couchTests.changes = function(debug) { xhr = CouchDB.newXhr(); xhr.open("GET", "/test_suite_db/_changes?feed=longpoll&since=7&filter=changes_filter/bop", true); xhr.send(""); - db.save({"bop" : ""}); // empty string is falsy - var id = db.save({"bop" : "bingo"}).id; + db.save({"_id":"falsy", "bop" : ""}); // empty string is falsy + db.save({"_id":"bingo","bop" : "bingo"}); sleep(100); var resp = JSON.parse(xhr.responseText); T(resp.last_seq == 9); - T(resp.results && resp.results.length > 0 && resp.results[0]["id"] == id, "filter the correct update"); + T(resp.results && resp.results.length > 0 && resp.results[0]["id"] == "bingo", "filter the correct update"); // filter with continuous xhr = CouchDB.newXhr(); @@ -226,30 +226,29 @@ couchTests.changes = function(debug) { xhr.send(""); db.save({"_id":"rusty", "bop" : "plankton"}); T(db.ensureFullCommit().ok); - sleep(200); + sleep(300); var lines = xhr.responseText.split("\n"); - T(JSON.parse(lines[1]).id == id); - T(JSON.parse(lines[2]).id == "rusty"); - T(JSON.parse(lines[3]).last_seq == 10); + T(JSON.parse(lines[1]).id == "bingo", lines[1]); + T(JSON.parse(lines[2]).id == "rusty", lines[2]); + T(JSON.parse(lines[3]).last_seq == 10, lines[3]); } - // error conditions // non-existing design doc var req = CouchDB.request("GET", "/test_suite_db/_changes?filter=nothingtosee/bop"); - TEquals(400, req.status, "should return 400 for non existant design doc"); + TEquals(404, req.status, "should return 404 for non existant design doc"); // non-existing filter var req = CouchDB.request("GET", "/test_suite_db/_changes?filter=changes_filter/movealong"); - TEquals(400, req.status, "should return 400 for non existant filter fun"); + TEquals(404, req.status, "should return 404 for non existant filter fun"); // both var req = CouchDB.request("GET", "/test_suite_db/_changes?filter=nothingtosee/movealong"); - TEquals(400, req.status, - "should return 400 for non existant design doc and filter fun"); + TEquals(404, req.status, + "should return 404 for non existant design doc and filter fun"); // changes get all_docs style with deleted docs var doc = {a:1}; diff --git a/share/www/script/test/design_docs.js b/share/www/script/test/design_docs.js index 82c186f8..9318d2bc 100644 --- a/share/www/script/test/design_docs.js +++ b/share/www/script/test/design_docs.js @@ -12,8 +12,11 @@ couchTests.design_docs = function(debug) { var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"false"}); + var db2 = new CouchDB("test_suite_db_a", {"X-Couch-Full-Commit":"false"}); db.deleteDb(); db.createDb(); + db2.deleteDb(); + db2.createDb(); if (debug) debugger; run_on_modified_server( @@ -45,10 +48,32 @@ function() { reduce:"function (keys, values) { return sum(values); };"}, huge_src_and_results: {map: "function(doc) { if (doc._id == \"1\") { emit(\"" + makebigstring(16) + "\", null) }}", reduce:"function (keys, values) { return \"" + makebigstring(16) + "\"; };"} + }, + shows: { + simple: "function() {return 'ok'};" } } + var xhr = CouchDB.request("PUT", "/test_suite_db_a/_design/test", {body: JSON.stringify(designDoc)}); + var resp = JSON.parse(xhr.responseText); + + TEquals(resp.rev, db.save(designDoc).rev); + + // test that editing a show fun on the ddoc results in a change in output + var xhr = CouchDB.request("GET", "/test_suite_db/_design/test/_show/simple"); + T(xhr.status == 200); + TEquals(xhr.responseText, "ok"); + + designDoc.shows.simple = "function() {return 'ko'};" T(db.save(designDoc).ok); + var xhr = CouchDB.request("GET", "/test_suite_db/_design/test/_show/simple"); + T(xhr.status == 200); + TEquals(xhr.responseText, "ko"); + + var xhr = CouchDB.request("GET", "/test_suite_db_a/_design/test/_show/simple?cache=buster"); + T(xhr.status == 200); + TEquals("ok", xhr.responseText, 'query server used wrong ddoc'); + // test that we get design doc info back var dinfo = db.designInfo("_design/test"); TEquals("test", dinfo.name); diff --git a/share/www/script/test/list_views.js b/share/www/script/test/list_views.js index d0400ff9..68dfe71c 100644 --- a/share/www/script/test/list_views.js +++ b/share/www/script/test/list_views.js @@ -62,12 +62,7 @@ couchTests.list_views = function(debug) { }), simpleForm: stringFun(function(head, req) { log("simpleForm"); - send('<h1>Total Rows: ' - // + head.total_rows - // + ' Offset: ' + head.offset - + '</h1><ul>'); - - // rows + send('<ul>'); var row, row_number = 0, prevKey, firstKey = null; while (row = getRow()) { row_number += 1; @@ -77,8 +72,6 @@ couchTests.list_views = function(debug) { +' Value: '+row.value +' LineNo: '+row_number+'</li>'); } - - // tail return '</ul><p>FirstKey: '+ firstKey + ' LastKey: '+ prevKey+'</p>'; }), acceptSwitch: stringFun(function(head, req) { @@ -208,22 +201,12 @@ couchTests.list_views = function(debug) { T(xhr.status == 200, "standard get should be 200"); T(/head0123456789tail/.test(xhr.responseText)); - var xhr = CouchDB.request("GET", "/test_suite_db/_view/lists/basicView?list=basicBasic"); - T(xhr.status == 200, "standard get should be 200"); - T(/head0123456789tail/.test(xhr.responseText)); - // test that etags are available var etag = xhr.getResponseHeader("etag"); xhr = CouchDB.request("GET", "/test_suite_db/_design/lists/_list/basicBasic/basicView", { headers: {"if-none-match": etag} }); T(xhr.status == 304); - - var etag = xhr.getResponseHeader("etag"); - xhr = CouchDB.request("GET", "/test_suite_db/_view/lists/basicView?list=basicBasic", { - headers: {"if-none-match": etag} - }); - T(xhr.status == 304); // confirm ETag changes with different POST bodies xhr = CouchDB.request("POST", "/test_suite_db/_design/lists/_list/basicBasic/basicView", @@ -262,14 +245,6 @@ couchTests.list_views = function(debug) { // get with query params xhr = CouchDB.request("GET", "/test_suite_db/_design/lists/_list/simpleForm/basicView?startkey=3&endkey=8"); T(xhr.status == 200, "with query params"); - T(/Total Rows/.test(xhr.responseText)); - T(!(/Key: 1/.test(xhr.responseText))); - T(/FirstKey: 3/.test(xhr.responseText)); - T(/LastKey: 8/.test(xhr.responseText)); - - var xhr = CouchDB.request("GET", "/test_suite_db/_view/lists/basicView?list=simpleForm&startkey=3&endkey=8"); - T(xhr.status == 200, "with query params"); - T(/Total Rows/.test(xhr.responseText)); T(!(/Key: 1/.test(xhr.responseText))); T(/FirstKey: 3/.test(xhr.responseText)); T(/LastKey: 8/.test(xhr.responseText)); @@ -277,11 +252,7 @@ couchTests.list_views = function(debug) { // with 0 rows var xhr = CouchDB.request("GET", "/test_suite_db/_design/lists/_list/simpleForm/basicView?startkey=30"); T(xhr.status == 200, "0 rows"); - T(/Total Rows/.test(xhr.responseText)); - - var xhr = CouchDB.request("GET", "/test_suite_db/_view/lists/basicView?list=simpleForm&startkey=30"); - T(xhr.status == 200, "0 rows"); - T(/Total Rows/.test(xhr.responseText)); + T(/<\/ul>/.test(xhr.responseText)); //too many Get Rows var xhr = CouchDB.request("GET", "/test_suite_db/_design/lists/_list/tooManyGetRows/basicView"); @@ -292,19 +263,11 @@ couchTests.list_views = function(debug) { // reduce with 0 rows var xhr = CouchDB.request("GET", "/test_suite_db/_design/lists/_list/simpleForm/withReduce?startkey=30"); T(xhr.status == 200, "reduce 0 rows"); - T(/Total Rows/.test(xhr.responseText)); - T(/LastKey: undefined/.test(xhr.responseText)); - - // reduce with 0 rows - var xhr = CouchDB.request("GET", "/test_suite_db/_view/lists/withReduce?list=simpleForm&startkey=30"); - T(xhr.status == 200, "reduce 0 rows"); - T(/Total Rows/.test(xhr.responseText)); T(/LastKey: undefined/.test(xhr.responseText)); // when there is a reduce present, but not used var xhr = CouchDB.request("GET", "/test_suite_db/_design/lists/_list/simpleForm/withReduce?reduce=false"); T(xhr.status == 200, "reduce false"); - T(/Total Rows/.test(xhr.responseText)); T(/Key: 1/.test(xhr.responseText)); @@ -352,7 +315,6 @@ couchTests.list_views = function(debug) { body: '{"keys":[2,4,5,7]}' }); T(xhr.status == 200, "multi key"); - T(/Total Rows/.test(xhr.responseText)); T(!(/Key: 1 /.test(xhr.responseText))); T(/Key: 2/.test(xhr.responseText)); T(/FirstKey: 2/.test(xhr.responseText)); @@ -416,11 +378,22 @@ couchTests.list_views = function(debug) { "?startkey=-3"; xhr = CouchDB.request("GET", url); T(xhr.status == 200, "multiple design docs."); - T(/Total Rows/.test(xhr.responseText)); T(!(/Key: -4/.test(xhr.responseText))); T(/FirstKey: -3/.test(xhr.responseText)); T(/LastKey: 0/.test(xhr.responseText)); + // Test we do multi-key requests on lists and views in separate docs. + var url = "/test_suite_db/_design/lists/_list/simpleForm/views/basicView" + xhr = CouchDB.request("POST", url, { + body: '{"keys":[-2,-4,-5,-7]}' + }); + + T(xhr.status == 200, "multi key separate docs"); + T(!(/Key: -3/.test(xhr.responseText))); + T(/Key: -7/.test(xhr.responseText)); + T(/FirstKey: -2/.test(xhr.responseText)); + T(/LastKey: -7/.test(xhr.responseText)); + var erlViewTest = function() { T(db.save(erlListDoc).ok); var url = "/test_suite_db/_design/erlang/_list/simple/views/basicView" + diff --git a/share/www/script/test/show_documents.js b/share/www/script/test/show_documents.js index ae4368a8..53ffbc42 100644 --- a/share/www/script/test/show_documents.js +++ b/share/www/script/test/show_documents.js @@ -21,14 +21,11 @@ couchTests.show_documents = function(debug) { language: "javascript", shows: { "hello" : stringFun(function(doc, req) { + log("hello fun"); if (doc) { return "Hello World"; } else { - if(req.docId) { - return "New World"; - } else { - return "Empty World"; - } + return "Empty World"; } }), "just-name" : stringFun(function(doc, req) { @@ -140,7 +137,7 @@ couchTests.show_documents = function(debug) { // hello template world xhr = CouchDB.request("GET", "/test_suite_db/_design/template/_show/hello/"+docid); - T(xhr.responseText == "Hello World"); + T(xhr.responseText == "Hello World", "hello"); T(/charset=utf-8/.test(xhr.getResponseHeader("Content-Type"))) // Fix for COUCHDB-379 @@ -168,8 +165,10 @@ couchTests.show_documents = function(debug) { // // hello template world (non-existing docid) xhr = CouchDB.request("GET", "/test_suite_db/_design/template/_show/hello/nonExistingDoc"); - T(xhr.responseText == "New World"); - + T(xhr.status == 404); + var resp = JSON.parse(xhr.responseText); + T(resp.error == "not_found"); + // show with doc xhr = CouchDB.request("GET", "/test_suite_db/_design/template/_show/just-name/"+docid); T(xhr.responseText == "Just Rusty"); @@ -179,9 +178,9 @@ couchTests.show_documents = function(debug) { // show with missing doc xhr = CouchDB.request("GET", "/test_suite_db/_design/template/_show/just-name/missingdoc"); - - T(xhr.status == 404, 'Doc should be missing'); - T(xhr.responseText == "No such doc"); + T(xhr.status == 404); + var resp = JSON.parse(xhr.responseText); + T(resp.error == "not_found"); // show with missing func xhr = CouchDB.request("GET", "/test_suite_db/_design/template/_show/missing/"+docid); @@ -268,8 +267,8 @@ couchTests.show_documents = function(debug) { xhr = CouchDB.request("GET", "/test_suite_db/_design/template/_show/just-name/"+docid, { headers: {"if-none-match": etag} }); - // should be 304 - T(xhr.status == 304); + // should not be 304 if we change the doc + T(xhr.status != 304, "changed ddoc"); // update design doc function designDoc.shows["just-name"] = (function(doc, req) { diff --git a/share/www/script/test/update_documents.js b/share/www/script/test/update_documents.js index 142e0a88..87fc7352 100644 --- a/share/www/script/test/update_documents.js +++ b/share/www/script/test/update_documents.js @@ -22,17 +22,26 @@ couchTests.update_documents = function(debug) { language: "javascript", updates: { "hello" : stringFun(function(doc, req) { + log(doc); + log(req); if (!doc) { - if (req.docId) { - return [{ - _id : req.docId - }, "New World"] - } - return [null, "Empty World"]; - } + if (req.id) { + return [ + // Creates a new document with the PUT docid, + { _id : req.id, + reqs : [req] }, + // and returns an HTML response to the client. + "<p>New World</p>"]; + }; + // + return [null, "<p>Empty World</p>"]; + }; + // we can update the document inline doc.world = "hello"; + // we can record aspects of the request or use them in application logic. + doc.reqs && doc.reqs.push(req); doc.edited_by = req.userCtx; - return [doc, "hello doc"]; + return [doc, "<p>hello doc</p>"]; }), "in-place" : stringFun(function(doc, req) { var field = req.query.field; @@ -81,7 +90,7 @@ couchTests.update_documents = function(debug) { // hello update world xhr = CouchDB.request("PUT", "/test_suite_db/_design/update/_update/hello/"+docid); T(xhr.status == 201); - T(xhr.responseText == "hello doc"); + T(xhr.responseText == "<p>hello doc</p>"); T(/charset=utf-8/.test(xhr.getResponseHeader("Content-Type"))) doc = db.open(docid); @@ -93,17 +102,17 @@ couchTests.update_documents = function(debug) { // hello update world (no docid) xhr = CouchDB.request("POST", "/test_suite_db/_design/update/_update/hello"); T(xhr.status == 200); - T(xhr.responseText == "Empty World"); + T(xhr.responseText == "<p>Empty World</p>"); // no GET allowed xhr = CouchDB.request("GET", "/test_suite_db/_design/update/_update/hello"); - T(xhr.status == 405); + // T(xhr.status == 405); // TODO allow qs to throw error code as well as error message T(JSON.parse(xhr.responseText).error == "method_not_allowed"); // // hello update world (non-existing docid) xhr = CouchDB.request("PUT", "/test_suite_db/_design/update/_update/hello/nonExistingDoc"); T(xhr.status == 201); - T(xhr.responseText == "New World"); + T(xhr.responseText == "<p>New World</p>"); // in place update xhr = CouchDB.request("PUT", "/test_suite_db/_design/update/_update/in-place/"+docid+'?field=title&value=test'); diff --git a/src/couchdb/couch_doc.erl b/src/couchdb/couch_doc.erl index bdefb95c..ba5c7450 100644 --- a/src/couchdb/couch_doc.erl +++ b/src/couchdb/couch_doc.erl @@ -292,15 +292,13 @@ att_to_iolist(#att{data=DataFun, len=Len}) when is_function(DataFun)-> lists:reverse(fold_streamed_data(DataFun, Len, fun(Data, Acc) -> [Data | Acc] end, [])). -get_validate_doc_fun(#doc{body={Props}}) -> - Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), +get_validate_doc_fun(#doc{body={Props}}=DDoc) -> case proplists:get_value(<<"validate_doc_update">>, Props) of undefined -> nil; - FunSrc -> + _Else -> fun(EditDoc, DiskDoc, Ctx) -> - couch_query_servers:validate_doc_update( - Lang, FunSrc, EditDoc, DiskDoc, Ctx) + couch_query_servers:validate_doc_update(DDoc, EditDoc, DiskDoc, Ctx) end end. diff --git a/src/couchdb/couch_httpd.erl b/src/couchdb/couch_httpd.erl index a61b29fb..baa22d8f 100644 --- a/src/couchdb/couch_httpd.erl +++ b/src/couchdb/couch_httpd.erl @@ -51,7 +51,7 @@ start_link() -> DesignUrlHandlersList = lists:map( fun({UrlKey, SpecStr}) -> - {?l2b(UrlKey), make_arity_2_fun(SpecStr)} + {?l2b(UrlKey), make_arity_3_fun(SpecStr)} end, couch_config:get("httpd_design_handlers")), UrlHandlers = dict:from_list(UrlHandlersList), @@ -110,6 +110,14 @@ make_arity_2_fun(SpecStr) -> fun(Arg1, Arg2) -> Mod:Fun(Arg1, Arg2) end end. +make_arity_3_fun(SpecStr) -> + case couch_util:parse_term(SpecStr) of + {ok, {Mod, Fun, SpecArg}} -> + fun(Arg1, Arg2, Arg3) -> Mod:Fun(Arg1, Arg2, Arg3, SpecArg) end; + {ok, {Mod, Fun}} -> + fun(Arg1, Arg2, Arg3) -> Mod:Fun(Arg1, Arg2, Arg3) end + end. + % SpecStr is "{my_module, my_fun}, {my_module2, my_fun2}" make_arity_1_fun_list(SpecStr) -> [make_arity_1_fun(FunSpecStr) || FunSpecStr <- re:split(SpecStr, "(?<=})\\s*,\\s*(?={)", [{return, list}])]. diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl index 8b955c88..bf24b712 100644 --- a/src/couchdb/couch_httpd_db.erl +++ b/src/couchdb/couch_httpd_db.erl @@ -16,7 +16,7 @@ -export([handle_request/1, handle_compact_req/2, handle_design_req/2, db_req/2, couch_doc_open/4,handle_changes_req/2, update_doc_result_to_json/1, update_doc_result_to_json/2, - handle_design_info_req/2, handle_view_cleanup_req/2]). + handle_design_info_req/3, handle_view_cleanup_req/2]). -import(couch_httpd, [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2, @@ -232,26 +232,18 @@ make_filter_fun(Req, Db) -> end; [DName, FName] -> DesignId = <<"_design/", DName/binary>>, - case couch_db:open_doc(Db, DesignId) of - {ok, #doc{body={Props}}} -> - FilterSrc = try couch_util:get_nested_json_value({Props}, - [<<"filters">>, FName]) - catch - throw:{not_found, _} -> - throw({bad_request, "invalid filter function"}) - end, - Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), - fun(DocInfos) -> - Docs = [Doc || {ok, Doc} <- [ - {ok, Doc} = couch_db:open_doc(Db, DInfo, [deleted]) - || DInfo <- DocInfos]], - {ok, Passes} = couch_query_servers:filter_docs(Lang, FilterSrc, Docs, Req, Db), - [{[{rev, couch_doc:rev_to_str(Rev)}]} - || #doc_info{revs=[#rev_info{rev=Rev}|_]} <- DocInfos, - Pass <- Passes, Pass == true] - end; - _Error -> - throw({bad_request, "invalid design doc"}) + DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), + % validate that the ddoc has the filter fun + #doc{body={Props}} = DDoc, + couch_util:get_nested_json_value({Props}, [<<"filters">>, FName]), + fun(DocInfos) -> + Docs = [Doc || {ok, Doc} <- [ + {ok, Doc} = couch_db:open_doc(Db, DInfo, [deleted]) + || DInfo <- DocInfos]], + {ok, Passes} = couch_query_servers:filter_docs(Req, Db, DDoc, FName, Docs), + [{[{rev, couch_doc:rev_to_str(Rev)}]} + || #doc_info{revs=[#rev_info{rev=Rev}|_]} <- DocInfos, + Pass <- Passes, Pass == true] end; _Else -> throw({bad_request, @@ -279,11 +271,14 @@ handle_view_cleanup_req(Req, _Db) -> handle_design_req(#httpd{ - path_parts=[_DbName,_Design,_DesName, <<"_",_/binary>> = Action | _Rest], + path_parts=[_DbName, _Design, DesignName, <<"_",_/binary>> = Action | _Rest], design_url_handlers = DesignUrlHandlers }=Req, Db) -> + % load ddoc + DesignId = <<"_design/", DesignName/binary>>, + DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), Handler = couch_util:dict_find(Action, DesignUrlHandlers, fun db_req/2), - Handler(Req, Db); + Handler(Req, Db, DDoc); handle_design_req(Req, Db) -> db_req(Req, Db). @@ -291,7 +286,7 @@ handle_design_req(Req, Db) -> handle_design_info_req(#httpd{ method='GET', path_parts=[_DbName, _Design, DesignName, _] - }=Req, Db) -> + }=Req, Db, _DDoc) -> DesignId = <<"_design/", DesignName/binary>>, {ok, GroupInfoList} = couch_view:get_group_info(Db, DesignId), send_json(Req, 200, {[ @@ -299,7 +294,7 @@ handle_design_info_req(#httpd{ {view_index, {GroupInfoList}} ]}); -handle_design_info_req(Req, _Db) -> +handle_design_info_req(Req, _Db, _DDoc) -> send_method_not_allowed(Req, "GET"). create_db_req(#httpd{user_ctx=UserCtx}=Req, DbName) -> @@ -725,7 +720,12 @@ db_doc_req(#httpd{method='GET'}=Req, Db, DocId) -> end; _ -> {DesignName, ShowName} = Format, - couch_httpd_show:handle_doc_show(Req, DesignName, ShowName, DocId, Db) + % load ddoc + DesignId = <<"_design/", DesignName/binary>>, + DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), + % open doc + Doc = couch_doc_open(Db, DocId, Rev, Options), + couch_httpd_show:handle_doc_show(Req, Db, DDoc, ShowName, Doc) end; db_doc_req(#httpd{method='POST'}=Req, Db, DocId) -> diff --git a/src/couchdb/couch_httpd_external.erl b/src/couchdb/couch_httpd_external.erl index 86b2bfc6..13aff847 100644 --- a/src/couchdb/couch_httpd_external.erl +++ b/src/couchdb/couch_httpd_external.erl @@ -13,7 +13,7 @@ -module(couch_httpd_external). -export([handle_external_req/2, handle_external_req/3]). --export([send_external_response/2, json_req_obj/2]). +-export([send_external_response/2, json_req_obj/2, json_req_obj/3]). -export([default_or_content_type/2, parse_external_response/1]). -import(couch_httpd,[send_error/4]). @@ -53,12 +53,12 @@ process_external_req(HttpReq, Db, Name) -> _ -> send_external_response(HttpReq, Response) end. - +json_req_obj(Req, Db) -> json_req_obj(Req, Db, null). json_req_obj(#httpd{mochi_req=Req, method=Verb, path_parts=Path, req_body=ReqBody - }, Db) -> + }, Db, DocId) -> Body = case ReqBody of undefined -> Req:recv_body(); Else -> Else @@ -74,6 +74,7 @@ json_req_obj(#httpd{mochi_req=Req, {ok, Info} = couch_db:get_db_info(Db), % add headers... {[{<<"info">>, {Info}}, + {<<"id">>, DocId}, {<<"verb">>, Verb}, {<<"path">>, Path}, {<<"query">>, to_json_terms(Req:parse_qs())}, diff --git a/src/couchdb/couch_httpd_show.erl b/src/couchdb/couch_httpd_show.erl index 5c95070a..467c0a42 100644 --- a/src/couchdb/couch_httpd_show.erl +++ b/src/couchdb/couch_httpd_show.erl @@ -12,8 +12,8 @@ -module(couch_httpd_show). --export([handle_doc_show_req/2, handle_doc_update_req/2, handle_view_list_req/2, - handle_doc_show/5, handle_view_list/7]). +-export([handle_doc_show_req/3, handle_doc_update_req/3, handle_view_list_req/3, + handle_doc_show/5, handle_view_list/6, get_fun_key/3]). -include("couch_db.hrl"). @@ -22,217 +22,245 @@ start_json_response/2,send_chunk/2,last_chunk/1,send_chunked_error/2, start_chunked_response/3, send_error/4]). +% /db/_design/foo/show/bar/docid +% show converts a json doc to a response of any content-type. +% it looks up the doc an then passes it to the query server. +% then it sends the response from the query server to the http client. handle_doc_show_req(#httpd{ - method='GET', - path_parts=[_DbName, _Design, DesignName, _Show, ShowName, DocId] - }=Req, Db) -> - handle_doc_show(Req, DesignName, ShowName, DocId, Db); + path_parts=[_, _, _, _, ShowName, DocId] + }=Req, Db, DDoc) -> + % open the doc + Doc = couch_httpd_db:couch_doc_open(Db, DocId, nil, [conflicts]), + % we don't handle revs here b/c they are an internal api + % returns 404 if there is no doc with DocId + handle_doc_show(Req, Db, DDoc, ShowName, Doc); handle_doc_show_req(#httpd{ - path_parts=[_DbName, _Design, DesignName, _Show, ShowName] - }=Req, Db) -> - handle_doc_show(Req, DesignName, ShowName, nil, Db); + path_parts=[_, _, _, _, ShowName] + }=Req, Db, DDoc) -> + % with no docid the doc is nil + handle_doc_show(Req, Db, DDoc, ShowName, nil); -handle_doc_show_req(#httpd{method='GET'}=Req, _Db) -> - send_error(Req, 404, <<"show_error">>, <<"Invalid path.">>); +handle_doc_show_req(Req, _Db, _DDoc) -> + send_error(Req, 404, <<"show_error">>, <<"Invalid path.">>). + +handle_doc_show(Req, Db, DDoc, ShowName, Doc) -> + % get responder for ddoc/showname + CurrentEtag = show_etag(Req, Doc, DDoc, []), + couch_httpd:etag_respond(Req, CurrentEtag, fun() -> + JsonReq = couch_httpd_external:json_req_obj(Req, Db), + JsonDoc = couch_query_servers:json_doc(Doc), + [<<"resp">>, ExternalResp] = + couch_query_servers:ddoc_prompt(DDoc, [<<"shows">>, ShowName], [JsonDoc, JsonReq]), + JsonResp = apply_etag(ExternalResp, CurrentEtag), + couch_httpd_external:send_external_response(Req, JsonResp) + end). -handle_doc_show_req(Req, _Db) -> - send_method_not_allowed(Req, "GET,POST,HEAD"). -handle_doc_update_req(#httpd{method = 'GET'}=Req, _Db) -> - send_method_not_allowed(Req, "POST,PUT,DELETE,ETC"); +show_etag(#httpd{user_ctx=UserCtx}=Req, Doc, DDoc, More) -> + Accept = couch_httpd:header_value(Req, "Accept"), + DocPart = case Doc of + nil -> nil; + Doc -> couch_httpd:doc_etag(Doc) + end, + couch_httpd:make_etag({couch_httpd:doc_etag(DDoc), DocPart, Accept, UserCtx#user_ctx.roles, More}). + +get_fun_key(DDoc, Type, Name) -> + #doc{body={Props}} = DDoc, + Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), + Src = couch_util:get_nested_json_value({Props}, [Type, Name]), + {Lang, Src}. + +% /db/_design/foo/update/bar/docid +% updates a doc based on a request +% handle_doc_update_req(#httpd{method = 'GET'}=Req, _Db, _DDoc) -> +% % anything but GET +% send_method_not_allowed(Req, "POST,PUT,DELETE,ETC"); handle_doc_update_req(#httpd{ - path_parts=[_DbName, _Design, DesignName, _Update, UpdateName, DocId] - }=Req, Db) -> - DesignId = <<"_design/", DesignName/binary>>, - #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), - Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), - UpdateSrc = couch_util:get_nested_json_value({Props}, [<<"updates">>, UpdateName]), + path_parts=[_, _, _, _, UpdateName, DocId] + }=Req, Db, DDoc) -> Doc = try couch_httpd_db:couch_doc_open(Db, DocId, nil, [conflicts]) - catch - _ -> nil - end, - send_doc_update_response(Lang, UpdateSrc, DocId, Doc, Req, Db); + catch + _ -> nil + end, + send_doc_update_response(Req, Db, DDoc, UpdateName, Doc, DocId); handle_doc_update_req(#httpd{ - path_parts=[_DbName, _Design, DesignName, _Update, UpdateName] - }=Req, Db) -> - DesignId = <<"_design/", DesignName/binary>>, - #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), - Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), - UpdateSrc = couch_util:get_nested_json_value({Props}, [<<"updates">>, UpdateName]), - send_doc_update_response(Lang, UpdateSrc, nil, nil, Req, Db); + path_parts=[_, _, _, _, UpdateName] + }=Req, Db, DDoc) -> + send_doc_update_response(Req, Db, DDoc, UpdateName, nil, null); -handle_doc_update_req(Req, _Db) -> +handle_doc_update_req(Req, _Db, _DDoc) -> send_error(Req, 404, <<"update_error">>, <<"Invalid path.">>). - - -handle_doc_show(Req, DesignName, ShowName, DocId, Db) -> - DesignId = <<"_design/", DesignName/binary>>, - #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), - Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), - ShowSrc = couch_util:get_nested_json_value({Props}, [<<"shows">>, ShowName]), - Doc = case DocId of - nil -> nil; - _ -> - try couch_httpd_db:couch_doc_open(Db, DocId, nil, [conflicts]) - catch - _ -> nil - end +send_doc_update_response(Req, Db, DDoc, UpdateName, Doc, DocId) -> + JsonReq = couch_httpd_external:json_req_obj(Req, Db, DocId), + JsonDoc = couch_query_servers:json_doc(Doc), + case couch_query_servers:ddoc_prompt(DDoc, [<<"updates">>, UpdateName], [JsonDoc, JsonReq]) of + [<<"up">>, {NewJsonDoc}, JsonResp] -> + Options = case couch_httpd:header_value(Req, "X-Couch-Full-Commit", "false") of + "true" -> + [full_commit]; + _ -> + [] + end, + NewDoc = couch_doc:from_json_obj({NewJsonDoc}), + Code = 201, + {ok, _NewRev} = couch_db:update_doc(Db, NewDoc, Options); + [<<"up">>, _Other, JsonResp] -> + Code = 200, + ok end, - send_doc_show_response(Lang, ShowSrc, DocId, Doc, Req, Db). + JsonResp2 = json_apply_field({<<"code">>, Code}, JsonResp), + % todo set location field + couch_httpd_external:send_external_response(Req, JsonResp2). + % view-list request with view and list from same design doc. handle_view_list_req(#httpd{method='GET', - path_parts=[_DbName, _Design, DesignName, _List, ListName, ViewName]}=Req, Db) -> - handle_view_list(Req, DesignName, ListName, DesignName, ViewName, Db, nil); + path_parts=[_, _, DesignName, _, ListName, ViewName]}=Req, Db, DDoc) -> + handle_view_list(Req, Db, DDoc, ListName, {DesignName, ViewName}, nil); % view-list request with view and list from different design docs. handle_view_list_req(#httpd{method='GET', - path_parts=[_DbName, _Design, DesignName, _List, ListName, ViewDesignName, ViewName]}=Req, Db) -> - handle_view_list(Req, DesignName, ListName, ViewDesignName, ViewName, Db, nil); + path_parts=[_, _, _, _, ListName, ViewDesignName, ViewName]}=Req, Db, DDoc) -> + handle_view_list(Req, Db, DDoc, ListName, {ViewDesignName, ViewName}, nil); -handle_view_list_req(#httpd{method='GET'}=Req, _Db) -> +handle_view_list_req(#httpd{method='GET'}=Req, _Db, _DDoc) -> send_error(Req, 404, <<"list_error">>, <<"Invalid path.">>); handle_view_list_req(#httpd{method='POST', - path_parts=[_DbName, _Design, DesignName, _List, ListName, ViewName]}=Req, Db) -> + path_parts=[_, _, DesignName, _, ListName, ViewName]}=Req, Db, DDoc) -> + % {Props2} = couch_httpd:json_body(Req), ReqBody = couch_httpd:body(Req), {Props2} = ?JSON_DECODE(ReqBody), Keys = proplists:get_value(<<"keys">>, Props2, nil), - handle_view_list(Req#httpd{req_body=ReqBody}, DesignName, ListName, DesignName, ViewName, Db, Keys); - -handle_view_list_req(Req, _Db) -> - send_method_not_allowed(Req, "GET,POST,HEAD"). + handle_view_list(Req#httpd{req_body=ReqBody}, Db, DDoc, ListName, {DesignName, ViewName}, Keys); -handle_view_list(Req, ListDesignName, ListName, ViewDesignName, ViewName, Db, Keys) -> - ListDesignId = <<"_design/", ListDesignName/binary>>, - #doc{body={ListProps}} = couch_httpd_db:couch_doc_open(Db, ListDesignId, nil, []), - if - ViewDesignName == ListDesignName -> - ViewDesignId = ListDesignId; - true -> - ViewDesignId = <<"_design/", ViewDesignName/binary>> - end, +handle_view_list_req(#httpd{method='POST', + path_parts=[_, _, _, _, ListName, ViewDesignName, ViewName]}=Req, Db, DDoc) -> + % {Props2} = couch_httpd:json_body(Req), + ReqBody = couch_httpd:body(Req), + {Props2} = ?JSON_DECODE(ReqBody), + Keys = proplists:get_value(<<"keys">>, Props2, nil), + handle_view_list(Req#httpd{req_body=ReqBody}, Db, DDoc, ListName, {ViewDesignName, ViewName}, Keys); - ListLang = proplists:get_value(<<"language">>, ListProps, <<"javascript">>), - ListSrc = couch_util:get_nested_json_value({ListProps}, [<<"lists">>, ListName]), - send_view_list_response(ListLang, ListSrc, ViewName, ViewDesignId, Req, Db, Keys). - - -send_view_list_response(Lang, ListSrc, ViewName, DesignId, Req, Db, Keys) -> - Stale = couch_httpd_view:get_stale_type(Req), - Reduce = couch_httpd_view:get_reduce_type(Req), - case couch_view:get_map_view(Db, DesignId, ViewName, Stale) of - {ok, View, Group} -> - QueryArgs = couch_httpd_view:parse_view_params(Req, Keys, map), - output_map_list(Req, Lang, ListSrc, View, Group, Db, QueryArgs, Keys); - {not_found, _Reason} -> - case couch_view:get_reduce_view(Db, DesignId, ViewName, Stale) of - {ok, ReduceView, Group} -> - case Reduce of - false -> - QueryArgs = couch_httpd_view:parse_view_params( - Req, Keys, map_red - ), - MapView = couch_view:extract_map_view(ReduceView), - output_map_list(Req, Lang, ListSrc, MapView, Group, Db, QueryArgs, Keys); - _ -> - QueryArgs = couch_httpd_view:parse_view_params( - Req, Keys, reduce - ), - output_reduce_list(Req, Lang, ListSrc, ReduceView, Group, Db, QueryArgs, Keys) - end; - {not_found, Reason} -> - throw({not_found, Reason}) - end - end. +handle_view_list_req(#httpd{method='POST'}=Req, _Db, _DDoc) -> + send_error(Req, 404, <<"list_error">>, <<"Invalid path.">>); +handle_view_list_req(Req, _Db, _DDoc) -> + send_method_not_allowed(Req, "GET,POST,HEAD"). -output_map_list(#httpd{mochi_req=MReq, user_ctx=UserCtx}=Req, Lang, ListSrc, View, Group, Db, QueryArgs, nil) -> +handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) -> + ViewDesignId = <<"_design/", ViewDesignName/binary>>, + {ViewType, View, Group, QueryArgs} = couch_httpd_view:load_view(Req, Db, {ViewDesignId, ViewName}, Keys), + Etag = list_etag(Req, Db, Group, {couch_httpd:doc_etag(DDoc), Keys}), + couch_httpd:etag_respond(Req, Etag, fun() -> + output_list(ViewType, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys) + end). + +list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, More) -> + Accept = couch_httpd:header_value(Req, "Accept"), + couch_httpd_view:view_group_etag(Group, Db, {More, Accept, UserCtx#user_ctx.roles}). + +output_list(map, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys) -> + output_map_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys); +output_list(reduce, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys) -> + output_reduce_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys). + +% next step: +% use with_ddoc_proc/2 to make this simpler +output_map_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys) -> #view_query_args{ limit = Limit, skip = SkipCount } = QueryArgs, + + FoldAccInit = {Limit, SkipCount, undefined, []}, {ok, RowCount} = couch_view:get_row_count(View), - Headers = MReq:get(headers), - Hlist = mochiweb_headers:to_list(Headers), - Accept = proplists:get_value('Accept', Hlist), - CurrentEtag = couch_httpd_view:view_group_etag(Group, Db, {Lang, ListSrc, Accept, UserCtx}), - couch_httpd:etag_respond(Req, CurrentEtag, fun() -> - % get the os process here - % pass it into the view fold with closures - {ok, QueryServer} = couch_query_servers:start_view_list(Lang, ListSrc), + - StartListRespFun = make_map_start_resp_fun(QueryServer, Db), - SendListRowFun = make_map_send_row_fun(QueryServer), + couch_query_servers:with_ddoc_proc(DDoc, fun(QServer) -> + + ListFoldHelpers = #view_fold_helper_funs{ + reduce_count = fun couch_view:reduce_to_count/1, + start_response = StartListRespFun = make_map_start_resp_fun(QServer, Db, LName), + send_row = make_map_send_row_fun(QServer) + }, + + {ok, _, FoldResult} = case Keys of + nil -> + FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, Etag, Db, RowCount, ListFoldHelpers), + couch_view:fold(View, FoldlFun, FoldAccInit, + couch_httpd_view:make_key_options(QueryArgs)); + Keys -> + lists:foldl( + fun(Key, {ok, _, FoldAcc}) -> + QueryArgs2 = QueryArgs#view_query_args{ + start_key = Key, + end_key = Key + }, + FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs2, Etag, Db, RowCount, ListFoldHelpers), + couch_view:fold(View, FoldlFun, FoldAcc, + couch_httpd_view:make_key_options(QueryArgs2)) + end, {ok, nil, FoldAccInit}, Keys) + end, + finish_list(Req, QServer, Etag, FoldResult, StartListRespFun, RowCount) + end). - FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, CurrentEtag, Db, RowCount, - #view_fold_helper_funs{ - reduce_count = fun couch_view:reduce_to_count/1, - start_response = StartListRespFun, - send_row = SendListRowFun - }), - FoldAccInit = {Limit, SkipCount, undefined, []}, - {ok, _, FoldResult} = couch_view:fold(View, FoldlFun, FoldAccInit, - couch_httpd_view:make_key_options(QueryArgs)), - finish_list(Req, QueryServer, CurrentEtag, FoldResult, StartListRespFun, RowCount) - end); -output_map_list(#httpd{mochi_req=MReq, user_ctx=UserCtx}=Req, Lang, ListSrc, View, Group, Db, QueryArgs, Keys) -> +output_reduce_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys) -> #view_query_args{ limit = Limit, - skip = SkipCount + skip = SkipCount, + group_level = GroupLevel } = QueryArgs, - {ok, RowCount} = couch_view:get_row_count(View), - Headers = MReq:get(headers), - Hlist = mochiweb_headers:to_list(Headers), - Accept = proplists:get_value('Accept', Hlist), - CurrentEtag = couch_httpd_view:view_group_etag(Group, Db, {Lang, ListSrc, Accept, UserCtx, Keys}), - couch_httpd:etag_respond(Req, CurrentEtag, fun() -> - % get the os process here - % pass it into the view fold with closures - {ok, QueryServer} = couch_query_servers:start_view_list(Lang, ListSrc), - - StartListRespFun = make_map_start_resp_fun(QueryServer, Db), - SendListRowFun = make_map_send_row_fun(QueryServer), + couch_query_servers:with_ddoc_proc(DDoc, fun(QServer) -> + StartListRespFun = make_reduce_start_resp_fun(QServer, Db, LName), + SendListRowFun = make_reduce_send_row_fun(QServer, Db), + {ok, GroupRowsFun, RespFun} = couch_httpd_view:make_reduce_fold_funs(Req, + GroupLevel, QueryArgs, Etag, + #reduce_fold_helper_funs{ + start_response = StartListRespFun, + send_row = SendListRowFun + }), FoldAccInit = {Limit, SkipCount, undefined, []}, - {ok, _, FoldResult} = lists:foldl( - fun(Key, {ok, _, FoldAcc}) -> - QueryArgs2 = QueryArgs#view_query_args{ - start_key = Key, - end_key = Key - }, - FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs2, CurrentEtag, Db, RowCount, - #view_fold_helper_funs{ - reduce_count = fun couch_view:reduce_to_count/1, - start_response = StartListRespFun, - send_row = SendListRowFun - }), - couch_view:fold(View, FoldlFun, FoldAcc, - couch_httpd_view:make_key_options(QueryArgs2)) - end, {ok, nil, FoldAccInit}, Keys), - finish_list(Req, QueryServer, CurrentEtag, FoldResult, StartListRespFun, RowCount) + {ok, FoldResult} = case Keys of + nil -> + couch_view:fold_reduce(View, RespFun, FoldAccInit, [{key_group_fun, GroupRowsFun} | + couch_httpd_view:make_key_options(QueryArgs)]); + Keys -> + lists:foldl( + fun(Key, {ok, FoldAcc}) -> + couch_view:fold_reduce(View, RespFun, FoldAcc, + [{key_group_fun, GroupRowsFun} | + couch_httpd_view:make_key_options( + QueryArgs#view_query_args{start_key=Key, end_key=Key})] + ) + end, {ok, FoldAccInit}, Keys) + end, + finish_list(Req, QServer, Etag, FoldResult, StartListRespFun, null) end). -make_map_start_resp_fun(QueryServer, Db) -> + +make_map_start_resp_fun(QueryServer, Db, LName) -> fun(Req, Etag, TotalRows, Offset, _Acc) -> Head = {[{<<"total_rows">>, TotalRows}, {<<"offset">>, Offset}]}, - start_list_resp(QueryServer, Req, Db, Head, Etag) + start_list_resp(QueryServer, LName, Req, Db, Head, Etag) end. -make_reduce_start_resp_fun(QueryServer, _Req, Db, _CurrentEtag) -> +make_reduce_start_resp_fun(QueryServer, Db, LName) -> fun(Req2, Etag, _Acc) -> - start_list_resp(QueryServer, Req2, Db, {[]}, Etag) + start_list_resp(QueryServer, LName, Req2, Db, {[]}, Etag) end. -start_list_resp(QueryServer, Req, Db, Head, Etag) -> - [<<"start">>,Chunks,JsonResp] = couch_query_servers:render_list_head(QueryServer, - Req, Db, Head), +start_list_resp(QServer, LName, Req, Db, Head, Etag) -> + JsonReq = couch_httpd_external:json_req_obj(Req, Db), + [<<"start">>,Chunks,JsonResp] = couch_query_servers:ddoc_proc_prompt(QServer, + [<<"lists">>, LName], [Head, JsonReq]), JsonResp2 = apply_etag(JsonResp, Etag), #extern_resp_args{ code = Code, @@ -255,7 +283,7 @@ make_reduce_send_row_fun(QueryServer, Db) -> send_list_row(Resp, QueryServer, Db, Row, RowFront, IncludeDoc) -> try - [Go,Chunks] = couch_query_servers:render_list_row(QueryServer, Db, Row, IncludeDoc), + [Go,Chunks] = prompt_list_row(QueryServer, Db, Row, IncludeDoc), Chunk = RowFront ++ ?b2l(?l2b(Chunks)), send_non_empty_chunk(Resp, Chunk), case Go of @@ -270,78 +298,22 @@ send_list_row(Resp, QueryServer, Db, Row, RowFront, IncludeDoc) -> throw({already_sent, Resp, Error}) end. + +prompt_list_row({Proc, _DDocId}, Db, {{Key, DocId}, Value}, IncludeDoc) -> + JsonRow = couch_httpd_view:view_row_obj(Db, {{Key, DocId}, Value}, IncludeDoc), + couch_query_servers:proc_prompt(Proc, [<<"list_row">>, JsonRow]); + +prompt_list_row({Proc, _DDocId}, _, {Key, Value}, _IncludeDoc) -> + JsonRow = {[{key, Key}, {value, Value}]}, + couch_query_servers:proc_prompt(Proc, [<<"list_row">>, JsonRow]). + send_non_empty_chunk(Resp, Chunk) -> case Chunk of [] -> ok; _ -> send_chunk(Resp, Chunk) end. -output_reduce_list(#httpd{mochi_req=MReq, user_ctx=UserCtx}=Req, Lang, ListSrc, View, Group, Db, QueryArgs, nil) -> - #view_query_args{ - limit = Limit, - skip = SkipCount, - group_level = GroupLevel - } = QueryArgs, - Headers = MReq:get(headers), - Hlist = mochiweb_headers:to_list(Headers), - Accept = proplists:get_value('Accept', Hlist), - CurrentEtag = couch_httpd_view:view_group_etag(Group, Db, {Lang, ListSrc, Accept, UserCtx}), - couch_httpd:etag_respond(Req, CurrentEtag, fun() -> - % get the os process here - % pass it into the view fold with closures - {ok, QueryServer} = couch_query_servers:start_view_list(Lang, ListSrc), - StartListRespFun = make_reduce_start_resp_fun(QueryServer, Req, Db, CurrentEtag), - SendListRowFun = make_reduce_send_row_fun(QueryServer, Db), - - {ok, GroupRowsFun, RespFun} = couch_httpd_view:make_reduce_fold_funs(Req, - GroupLevel, QueryArgs, CurrentEtag, - #reduce_fold_helper_funs{ - start_response = StartListRespFun, - send_row = SendListRowFun - }), - FoldAccInit = {Limit, SkipCount, undefined, []}, - {ok, FoldResult} = couch_view:fold_reduce(View, RespFun, FoldAccInit, - [{key_group_fun, GroupRowsFun} | - couch_httpd_view:make_key_options(QueryArgs)]), - finish_list(Req, QueryServer, CurrentEtag, FoldResult, StartListRespFun, null) - end); - -output_reduce_list(#httpd{mochi_req=MReq, user_ctx=UserCtx}=Req, Lang, ListSrc, View, Group, Db, QueryArgs, Keys) -> - #view_query_args{ - limit = Limit, - skip = SkipCount, - group_level = GroupLevel - } = QueryArgs, - Headers = MReq:get(headers), - Hlist = mochiweb_headers:to_list(Headers), - Accept = proplists:get_value('Accept', Hlist), - CurrentEtag = couch_httpd_view:view_group_etag(Group, Db, {Lang, ListSrc, Accept, UserCtx, Keys}), - couch_httpd:etag_respond(Req, CurrentEtag, fun() -> - % get the os process here - % pass it into the view fold with closures - {ok, QueryServer} = couch_query_servers:start_view_list(Lang, ListSrc), - StartListRespFun = make_reduce_start_resp_fun(QueryServer, Req, Db, CurrentEtag), - SendListRowFun = make_reduce_send_row_fun(QueryServer, Db), - - {ok, GroupRowsFun, RespFun} = couch_httpd_view:make_reduce_fold_funs(Req, - GroupLevel, QueryArgs, CurrentEtag, - #reduce_fold_helper_funs{ - start_response = StartListRespFun, - send_row = SendListRowFun - }), - FoldAccInit = {Limit, SkipCount, undefined, []}, - {ok, FoldResult} = lists:foldl( - fun(Key, {ok, FoldAcc}) -> - couch_view:fold_reduce(View, RespFun, FoldAcc, - [{key_group_fun, GroupRowsFun} | - couch_httpd_view:make_key_options( - QueryArgs#view_query_args{start_key=Key, end_key=Key})] - ) - end, {ok, FoldAccInit}, Keys), - finish_list(Req, QueryServer, CurrentEtag, FoldResult, StartListRespFun, null) - end). - -finish_list(Req, QueryServer, Etag, FoldResult, StartFun, TotalRows) -> +finish_list(Req, {Proc, _DDocId}, Etag, FoldResult, StartFun, TotalRows) -> FoldResult2 = case FoldResult of {Limit, SkipCount, Response, RowAcc} -> {Limit, SkipCount, Response, RowAcc, nil}; @@ -352,16 +324,15 @@ finish_list(Req, QueryServer, Etag, FoldResult, StartFun, TotalRows) -> {_, _, undefined, _, _} -> {ok, Resp, BeginBody} = render_head_for_empty_list(StartFun, Req, Etag, TotalRows), - [<<"end">>, Chunks] = couch_query_servers:render_list_tail(QueryServer), + [<<"end">>, Chunks] = couch_query_servers:proc_prompt(Proc, [<<"list_end">>]), Chunk = BeginBody ++ ?b2l(?l2b(Chunks)), send_non_empty_chunk(Resp, Chunk); {_, _, Resp, stop, _} -> ok; {_, _, Resp, _, _} -> - [<<"end">>, Chunks] = couch_query_servers:render_list_tail(QueryServer), + [<<"end">>, Chunks] = couch_query_servers:proc_prompt(Proc, [<<"list_end">>]), send_non_empty_chunk(Resp, ?b2l(?l2b(Chunks))) end, - couch_query_servers:stop_doc_map(QueryServer), last_chunk(Resp). @@ -370,53 +341,6 @@ render_head_for_empty_list(StartListRespFun, Req, Etag, null) -> render_head_for_empty_list(StartListRespFun, Req, Etag, TotalRows) -> StartListRespFun(Req, Etag, TotalRows, null, []). -send_doc_show_response(Lang, ShowSrc, DocId, nil, #httpd{mochi_req=MReq, user_ctx=UserCtx}=Req, Db) -> - % compute etag with no doc - Headers = MReq:get(headers), - Hlist = mochiweb_headers:to_list(Headers), - Accept = proplists:get_value('Accept', Hlist), - CurrentEtag = couch_httpd:make_etag({Lang, ShowSrc, nil, Accept, UserCtx}), - couch_httpd:etag_respond(Req, CurrentEtag, fun() -> - [<<"resp">>, ExternalResp] = couch_query_servers:render_doc_show(Lang, ShowSrc, - DocId, nil, Req, Db), - JsonResp = apply_etag(ExternalResp, CurrentEtag), - couch_httpd_external:send_external_response(Req, JsonResp) - end); - -send_doc_show_response(Lang, ShowSrc, DocId, #doc{revs=Revs}=Doc, #httpd{mochi_req=MReq, user_ctx=UserCtx}=Req, Db) -> - % calculate the etag - Headers = MReq:get(headers), - Hlist = mochiweb_headers:to_list(Headers), - Accept = proplists:get_value('Accept', Hlist), - CurrentEtag = couch_httpd:make_etag({Lang, ShowSrc, Revs, Accept, UserCtx}), - % We know our etag now - couch_httpd:etag_respond(Req, CurrentEtag, fun() -> - [<<"resp">>, ExternalResp] = couch_query_servers:render_doc_show(Lang, ShowSrc, - DocId, Doc, Req, Db), - JsonResp = apply_etag(ExternalResp, CurrentEtag), - couch_httpd_external:send_external_response(Req, JsonResp) - end). - -send_doc_update_response(Lang, UpdateSrc, DocId, Doc, Req, Db) -> - case couch_query_servers:render_doc_update(Lang, UpdateSrc, - DocId, Doc, Req, Db) of - [<<"up">>, {NewJsonDoc}, JsonResp] -> - Options = case couch_httpd:header_value(Req, "X-Couch-Full-Commit", "false") of - "true" -> - [full_commit]; - _ -> - [] - end, - NewDoc = couch_doc:from_json_obj({NewJsonDoc}), - Code = 201, - % todo set location field - {ok, _NewRev} = couch_db:update_doc(Db, NewDoc, Options); - [<<"up">>, _Other, JsonResp] -> - Code = 200, - ok - end, - JsonResp2 = json_apply_field({<<"code">>, Code}, JsonResp), - couch_httpd_external:send_external_response(Req, JsonResp2). % Maybe this is in the proplists API % todo move to couch_util diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl index af31ac9c..6419ca55 100644 --- a/src/couchdb/couch_httpd_view.erl +++ b/src/couchdb/couch_httpd_view.erl @@ -13,21 +13,21 @@ -module(couch_httpd_view). -include("couch_db.hrl"). --export([handle_view_req/2,handle_temp_view_req/2,handle_db_view_req/2]). +-export([handle_view_req/3,handle_temp_view_req/2]). -export([get_stale_type/1, get_reduce_type/1, parse_view_params/3]). -export([make_view_fold_fun/6, finish_view_fold/4, view_row_obj/3]). -export([view_group_etag/2, view_group_etag/3, make_reduce_fold_funs/5]). -export([design_doc_view/5, parse_bool_param/1, doc_member/2]). --export([make_key_options/1]). +-export([make_key_options/1, load_view/4]). -import(couch_httpd, [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2,send_chunk/2, start_json_response/2, start_json_response/3, end_json_response/1, send_chunked_error/2]). -design_doc_view(Req, Db, Id, ViewName, Keys) -> - DesignId = <<"_design/", Id/binary>>, +design_doc_view(Req, Db, DName, ViewName, Keys) -> + DesignId = <<"_design/", DName/binary>>, Stale = get_stale_type(Req), Reduce = get_reduce_type(Req), Result = case couch_view:get_map_view(Db, DesignId, ViewName, Stale) of @@ -54,11 +54,11 @@ design_doc_view(Req, Db, Id, ViewName, Keys) -> Result. handle_view_req(#httpd{method='GET', - path_parts=[_Db, _Design, DName, _View, ViewName]}=Req, Db) -> + path_parts=[_, _, DName, _, ViewName]}=Req, Db, _DDoc) -> design_doc_view(Req, Db, DName, ViewName, nil); handle_view_req(#httpd{method='POST', - path_parts=[_Db, _Design, DName, _View, ViewName]}=Req, Db) -> + path_parts=[_, _, DName, _, ViewName]}=Req, Db, _DDoc) -> {Fields} = couch_httpd:json_body_obj(Req), case proplists:get_value(<<"keys">>, Fields, nil) of nil -> @@ -71,50 +71,7 @@ handle_view_req(#httpd{method='POST', throw({bad_request, "`keys` member must be a array."}) end; -handle_view_req(Req, _Db) -> - send_method_not_allowed(Req, "GET,POST,HEAD"). - -handle_db_view_req(#httpd{method='GET', - path_parts=[_Db, _View, DName, ViewName]}=Req, Db) -> - QueryArgs = couch_httpd_view:parse_view_params(Req, nil, nil), - #view_query_args{ - list = ListName - } = QueryArgs, - ?LOG_DEBUG("ici ~p", [ListName]), - case ListName of - nil -> couch_httpd_view:design_doc_view(Req, Db, DName, ViewName, nil); - _ -> - couch_httpd_show:handle_view_list(Req, DName, ListName, DName, ViewName, Db, nil) - end; - -handle_db_view_req(#httpd{method='POST', - path_parts=[_Db, _View, DName, ViewName]}=Req, Db) -> - QueryArgs = couch_httpd_view:parse_view_params(Req, nil, nil), - #view_query_args{ - list = ListName - } = QueryArgs, - case ListName of - nil -> - {Fields} = couch_httpd:json_body_obj(Req), - case proplists:get_value(<<"keys">>, Fields, nil) of - nil -> - Fmt = "POST to view ~p/~p in database ~p with no keys member.", - ?LOG_DEBUG(Fmt, [DName, ViewName, Db]), - couch_httpd_view:design_doc_view(Req, Db, DName, ViewName, nil); - Keys when is_list(Keys) -> - couch_httpd_view:design_doc_view(Req, Db, DName, ViewName, Keys); - _ -> - throw({bad_request, "`keys` member must be a array."}) - end; - _ -> - ReqBody = couch_httpd:body(Req), - {Props2} = ?JSON_DECODE(ReqBody), - Keys = proplists:get_value(<<"keys">>, Props2, nil), - couch_httpd_show:handle_view_list(Req#httpd{req_body=ReqBody}, - DName, ListName, DName, ViewName, Db, Keys) - end; - -handle_db_view_req(Req, _Db) -> +handle_view_req(Req, _Db, _DDoc) -> send_method_not_allowed(Req, "GET,POST,HEAD"). handle_temp_view_req(#httpd{method='POST'}=Req, Db) -> @@ -236,6 +193,35 @@ get_stale_type(Req) -> get_reduce_type(Req) -> list_to_atom(couch_httpd:qs_value(Req, "reduce", "true")). +load_view(Req, Db, {ViewDesignId, ViewName}, Keys) -> + Stale = couch_httpd_view:get_stale_type(Req), + Reduce = couch_httpd_view:get_reduce_type(Req), + case couch_view:get_map_view(Db, ViewDesignId, ViewName, Stale) of + {ok, View, Group} -> + QueryArgs = couch_httpd_view:parse_view_params(Req, Keys, map), + {map, View, Group, QueryArgs}; + {not_found, _Reason} -> + case couch_view:get_reduce_view(Db, ViewDesignId, ViewName, Stale) of + {ok, ReduceView, Group} -> + case Reduce of + false -> + QueryArgs = couch_httpd_view:parse_view_params(Req, Keys, map_red), + MapView = couch_view:extract_map_view(ReduceView), + {map, MapView, Group, QueryArgs}; + _ -> + QueryArgs = couch_httpd_view:parse_view_params(Req, Keys, reduce), + {reduce, ReduceView, Group, QueryArgs} + end; + {not_found, Reason} -> + throw({not_found, Reason}) + end + end. + +% query_parse_error could be removed +% we wouldn't need to pass the view type, it'd just parse params. +% I'm not sure what to do about the error handling, but +% it might simplify things to have a parse_view_params function +% that doesn't throw(). parse_view_params(Req, Keys, ViewType) -> QueryList = couch_httpd:qs(Req), QueryParams = @@ -258,6 +244,7 @@ parse_view_params(Req, Keys, ViewType) -> {reduce, _, false} -> QueryArgs; {reduce, _, _} -> + % we can simplify code if we just drop this error message. Msg = <<"Multi-key fetchs for reduce " "view must include `group=true`">>, throw({query_parse_error, Msg}); diff --git a/src/couchdb/couch_native_process.erl b/src/couchdb/couch_native_process.erl index 2b74073c..65e4e131 100644 --- a/src/couchdb/couch_native_process.erl +++ b/src/couchdb/couch_native_process.erl @@ -38,63 +38,102 @@ % extensions will evolve which offer useful layers on top of this view server % to help simplify your view code. -module(couch_native_process). +-behaviour(gen_server). --export([start_link/0]). --export([set_timeout/2, prompt/2, stop/1]). +-export([start_link/0,init/1,terminate/2,handle_call/3,handle_cast/2]). +-export([set_timeout/2, prompt/2]). -define(STATE, native_proc_state). --record(evstate, {funs=[], query_config=[], list_pid=nil, timeout=5000}). +-record(evstate, {ddocs, funs=[], query_config=[], list_pid=nil, timeout=5000}). -include("couch_db.hrl"). start_link() -> - {ok, self()}. + gen_server:start_link(?MODULE, [], []). -stop(_Pid) -> - ok. +% this is a bit messy, see also couch_query_servers handle_info +% stop(_Pid) -> +% ok. -set_timeout(_Pid, TimeOut) -> - NewState = case get(?STATE) of - undefined -> - #evstate{timeout=TimeOut}; - State -> - State#evstate{timeout=TimeOut} - end, - put(?STATE, NewState), - ok. +set_timeout(Pid, TimeOut) -> + gen_server:call(Pid, {set_timeout, TimeOut}). -prompt(Pid, Data) when is_pid(Pid), is_list(Data) -> - case get(?STATE) of - undefined -> - State = #evstate{}, - put(?STATE, State); - State -> - State - end, - case is_pid(State#evstate.list_pid) of - true -> - case hd(Data) of - <<"list_row">> -> ok; - <<"list_end">> -> ok; - _ -> throw({error, query_server_error}) - end; - _ -> - ok % Not listing - end, - {NewState, Resp} = run(State, to_binary(Data)), - put(?STATE, NewState), +prompt(Pid, Data) when is_list(Data) -> + gen_server:call(Pid, {prompt, Data}). + +% gen_server callbacks +init([]) -> + {ok, #evstate{ddocs=dict:new()}}. + +handle_call({set_timeout, TimeOut}, _From, State) -> + {reply, ok, State#evstate{timeout=TimeOut}}; + +handle_call({prompt, Data}, _From, State) -> + ?LOG_DEBUG("Prompt native qs: ~s",[?JSON_ENCODE(Data)]), + {NewState, Resp} = try run(State, to_binary(Data)) of + {S, R} -> {S, R} + catch + throw:{error, Why} -> + {State, [<<"error">>, Why, Why]} + end, + case Resp of {error, Reason} -> Msg = io_lib:format("couch native server error: ~p", [Reason]), - {[{<<"error">>, list_to_binary(Msg)}]}; - _ -> - Resp + {reply, [<<"error">>, <<"native_query_server">>, list_to_binary(Msg)], NewState}; + [<<"error">> | Rest] -> + Msg = io_lib:format("couch native server error: ~p", [Rest]), + {reply, [<<"error">> | Rest], NewState}; + [<<"fatal">> | Rest] -> + Msg = io_lib:format("couch native server error: ~p", [Rest]), + {stop, fatal, [<<"error">> | Rest], NewState}; + Resp -> + {reply, Resp, NewState} end. -run(_, [<<"reset">>]) -> - {#evstate{}, true}; -run(_, [<<"reset">>, QueryConfig]) -> - {#evstate{query_config=QueryConfig}, true}; +handle_cast(_Msg, State) -> {noreply, State}. +handle_info(_Msg, State) -> {noreply, State}. +terminate(_Reason, _State) -> ok. +code_change(_OldVersion, State, _Extra) -> {ok, State}. + +run(#evstate{list_pid=Pid}=State, [<<"list_row">>, Row]) when is_pid(Pid) -> + Pid ! {self(), list_row, Row}, + receive + {Pid, chunks, Data} -> + {State, [<<"chunks">>, Data]}; + {Pid, list_end, Data} -> + receive + {'EXIT', Pid, normal} -> ok + after State#evstate.timeout -> + throw({timeout, list_cleanup}) + end, + process_flag(trap_exit, erlang:get(do_trap)), + {State#evstate{list_pid=nil}, [<<"end">>, Data]} + after State#evstate.timeout -> + throw({timeout, list_row}) + end; +run(#evstate{list_pid=Pid}=State, [<<"list_end">>]) when is_pid(Pid) -> + Pid ! {self(), list_end}, + Resp = + receive + {Pid, list_end, Data} -> + receive + {'EXIT', Pid, normal} -> ok + after State#evstate.timeout -> + throw({timeout, list_cleanup}) + end, + [<<"end">>, Data] + after State#evstate.timeout -> + throw({timeout, list_end}) + end, + process_flag(trap_exit, erlang:get(do_trap)), + {State#evstate{list_pid=nil}, Resp}; +run(#evstate{list_pid=Pid}=State, _Command) when is_pid(Pid) -> + {State, [<<"error">>, list_error, list_error]}; +run(#evstate{ddocs=DDocs}, [<<"reset">>]) -> + {#evstate{ddocs=DDocs}, true}; +run(#evstate{ddocs=DDocs}, [<<"reset">>, QueryConfig]) -> + {#evstate{ddocs=DDocs, query_config=QueryConfig}, true}; run(#evstate{funs=Funs}=State, [<<"add_fun">> , BinFunc]) -> FunInfo = makefun(State, BinFunc), {State#evstate{funs=Funs ++ [FunInfo]}, true}; @@ -115,41 +154,55 @@ run(State, [<<"reduce">>, Funs, KVs]) -> {State, catch reduce(State, Funs, Keys2, Vals2, false)}; run(State, [<<"rereduce">>, Funs, Vals]) -> {State, catch reduce(State, Funs, null, Vals, true)}; -run(State, [<<"validate">>, BFun, NDoc, ODoc, Ctx]) -> - {_Sig, Fun} = makefun(State, BFun), - {State, catch Fun(NDoc, ODoc, Ctx)}; -run(State, [<<"filter">>, Docs, Req]) -> - {_Sig, Fun} = hd(State#evstate.funs), +run(#evstate{ddocs=DDocs}=State, [<<"ddoc">>, <<"new">>, DDocId, DDoc]) -> + DDocs2 = store_ddoc(DDocs, DDocId, DDoc), + {State#evstate{ddocs=DDocs2}, true}; +run(#evstate{ddocs=DDocs}=State, [<<"ddoc">>, DDocId | Rest]) -> + DDoc = load_ddoc(DDocs, DDocId), + ddoc(State, DDoc, Rest); +run(_, Unknown) -> + ?LOG_ERROR("Native Process: Unknown command: ~p~n", [Unknown]), + throw({error, unknown_command}). + +ddoc(State, {DDoc}, [FunPath, Args]) -> + % load fun from the FunPath + BFun = lists:foldl(fun + (Key, {Props}) when is_list(Props) -> + proplists:get_value(Key, Props, nil); + (Key, Fun) when is_binary(Fun) -> + Fun; + (Key, nil) -> + throw({error, not_found}); + (Key, Fun) -> + throw({error, malformed_ddoc}) + end, {DDoc}, FunPath), + ddoc(State, makefun(State, BFun, {DDoc}), FunPath, Args). + +ddoc(State, {_, Fun}, [<<"validate_doc_update">>], Args) -> + {State, (catch apply(Fun, Args))}; +ddoc(State, {_, Fun}, [<<"filters">>|_], [Docs, Req]) -> Resp = lists:map(fun(Doc) -> (catch Fun(Doc, Req)) =:= true end, Docs), {State, [true, Resp]}; -run(State, [<<"show">>, BFun, Doc, Req]) -> - {_Sig, Fun} = makefun(State, BFun), - Resp = case (catch Fun(Doc, Req)) of +ddoc(State, {_, Fun}, [<<"shows">>|_], Args) -> + Resp = case (catch apply(Fun, Args)) of FunResp when is_list(FunResp) -> FunResp; - FunResp when tuple_size(FunResp) =:= 1 -> - [<<"resp">>, FunResp]; + {FunResp} -> + [<<"resp">>, {FunResp}]; FunResp -> FunResp end, {State, Resp}; -run(State, [<<"update">>, BFun, Doc, Req]) -> - {_Sig, Fun} = makefun(State, BFun), - Resp = case (catch Fun(Doc, Req)) of +ddoc(State, {_, Fun}, [<<"updates">>|_], Args) -> + Resp = case (catch apply(Fun, Args)) of [JsonDoc, JsonResp] -> [<<"up">>, JsonDoc, JsonResp] end, {State, Resp}; -run(State, [<<"list">>, Head, Req]) -> - {Sig, Fun} = hd(State#evstate.funs), - % This is kinda dirty - case is_function(Fun, 2) of - false -> throw({error, render_error}); - true -> ok - end, +ddoc(State, {Sig, Fun}, [<<"lists">>|_], Args) -> Self = self(), SpawnFun = fun() -> - LastChunk = (catch Fun(Head, Req)), + LastChunk = (catch apply(Fun, Args)), case start_list_resp(Self, Sig) of started -> receive @@ -177,44 +230,20 @@ run(State, [<<"list">>, Head, Req]) -> after State#evstate.timeout -> throw({timeout, list_start}) end, - {State#evstate{list_pid=Pid}, Resp}; -run(#evstate{list_pid=Pid}=State, [<<"list_row">>, Row]) when is_pid(Pid) -> - Pid ! {self(), list_row, Row}, - receive - {Pid, chunks, Data} -> - {State, [<<"chunks">>, Data]}; - {Pid, list_end, Data} -> - receive - {'EXIT', Pid, normal} -> ok - after State#evstate.timeout -> - throw({timeout, list_cleanup}) - end, - process_flag(trap_exit, erlang:get(do_trap)), - {State#evstate{list_pid=nil}, [<<"end">>, Data]} - after State#evstate.timeout -> - throw({timeout, list_row}) - end; -run(#evstate{list_pid=Pid}=State, [<<"list_end">>]) when is_pid(Pid) -> - Pid ! {self(), list_end}, - Resp = - receive - {Pid, list_end, Data} -> - receive - {'EXIT', Pid, normal} -> ok - after State#evstate.timeout -> - throw({timeout, list_cleanup}) - end, - [<<"end">>, Data] - after State#evstate.timeout -> - throw({timeout, list_end}) - end, - process_flag(trap_exit, erlang:get(do_trap)), - {State#evstate{list_pid=nil}, Resp}; -run(_, Unknown) -> - ?LOG_ERROR("Native Process: Unknown command: ~p~n", [Unknown]), - throw({error, query_server_error}). + {State#evstate{list_pid=Pid}, Resp}. + +store_ddoc(DDocs, DDocId, DDoc) -> + dict:store(DDocId, DDoc, DDocs). +load_ddoc(DDocs, DDocId) -> + try dict:fetch(DDocId, DDocs) of + {DDoc} -> {DDoc} + catch + _:Else -> throw({error, ?l2b(io_lib:format("Native Query Server missing DDoc with Id: ~s",[DDocId]))}) + end. bindings(State, Sig) -> + bindings(State, Sig, nil). +bindings(State, Sig, DDoc) -> Self = self(), Log = fun(Msg) -> @@ -262,14 +291,19 @@ bindings(State, Sig) -> FoldRows = fun(Fun, Acc) -> foldrows(GetRow, Fun, Acc) end, - [ + Bindings = [ {'Log', Log}, {'Emit', Emit}, {'Start', Start}, {'Send', Send}, {'GetRow', GetRow}, {'FoldRows', FoldRows} - ]. + ], + case DDoc of + {Props} -> + Bindings ++ [{'DDoc', DDoc}]; + _Else -> Bindings + end. % thanks to erlview, via: % http://erlang.org/pipermail/erlang-questions/2003-November/010544.html @@ -277,8 +311,11 @@ makefun(State, Source) -> Sig = erlang:md5(Source), BindFuns = bindings(State, Sig), {Sig, makefun(State, Source, BindFuns)}. - -makefun(_State, Source, BindFuns) -> +makefun(State, Source, {DDoc}) -> + Sig = erlang:md5(lists:flatten([Source, term_to_binary(DDoc)])), + BindFuns = bindings(State, Sig, {DDoc}), + {Sig, makefun(State, Source, BindFuns)}; +makefun(_State, Source, BindFuns) when is_list(BindFuns) -> FunStr = binary_to_list(Source), {ok, Tokens, _} = erl_scan:string(FunStr), Form = case (catch erl_parse:parse_exprs(Tokens)) of diff --git a/src/couchdb/couch_os_process.erl b/src/couchdb/couch_os_process.erl index 72b715c3..5ac13715 100644 --- a/src/couchdb/couch_os_process.erl +++ b/src/couchdb/couch_os_process.erl @@ -53,7 +53,7 @@ prompt(Pid, Data) -> {ok, Result} -> Result; Error -> - ?LOG_ERROR("OS Process Error :: ~p",[Error]), + ?LOG_ERROR("OS Process Error ~p :: ~p",[Pid,Error]), throw(Error) end. @@ -80,22 +80,24 @@ readline(#os_proc{port = Port} = OsProc, Acc) -> % Standard JSON functions writejson(OsProc, Data) when is_record(OsProc, os_proc) -> - % ?LOG_DEBUG("OS Process Input :: ~p", [Data]), - true = writeline(OsProc, ?JSON_ENCODE(Data)). + JsonData = ?JSON_ENCODE(Data), + ?LOG_DEBUG("OS Process ~p Input :: ~s", [OsProc#os_proc.port, JsonData]), + true = writeline(OsProc, JsonData). readjson(OsProc) when is_record(OsProc, os_proc) -> Line = readline(OsProc), + ?LOG_DEBUG("OS Process ~p Output :: ~s", [OsProc#os_proc.port, Line]), case ?JSON_DECODE(Line) of [<<"log">>, Msg] when is_binary(Msg) -> % we got a message to log. Log it and continue - ?LOG_INFO("OS Process :: ~s", [Msg]), + ?LOG_INFO("OS Process ~p Log :: ~s", [OsProc#os_proc.port, Msg]), readjson(OsProc); - {[{<<"error">>, Id}, {<<"reason">>, Reason}]} -> + [<<"error">>, Id, Reason] -> throw({list_to_atom(binary_to_list(Id)),Reason}); - {[{<<"reason">>, Reason}, {<<"error">>, Id}]} -> + [<<"fatal">>, Id, Reason] -> + ?LOG_INFO("OS Process ~p Fatal Error :: ~s ~p",[OsProc#os_proc.port, Id, Reason]), throw({list_to_atom(binary_to_list(Id)),Reason}); Result -> - % ?LOG_DEBUG("OS Process Output :: ~p", [Result]), Result end. @@ -112,6 +114,7 @@ init([Command, Options, PortOptions]) -> }, KillCmd = readline(BaseProc), Pid = self(), + ?LOG_DEBUG("OS Process Start :: ~p", [BaseProc#os_proc.port]), spawn(fun() -> % this ensure the real os process is killed when this process dies. erlang:monitor(process, Pid), @@ -143,8 +146,12 @@ handle_call({prompt, Data}, _From, OsProc) -> Writer(OsProc, Data), {reply, {ok, Reader(OsProc)}, OsProc} catch - throw:OsError -> - {stop, normal, OsError, OsProc} + throw:{error, OsError} -> + {reply, OsError, OsProc}; + throw:{fatal, OsError} -> + {stop, normal, OsError, OsProc}; + throw:OtherError -> + {stop, normal, OtherError, OsProc} end. handle_cast({send, Data}, #os_proc{writer=Writer}=OsProc) -> diff --git a/src/couchdb/couch_query_servers.erl b/src/couchdb/couch_query_servers.erl index 4ac56727..30f4c4c7 100644 --- a/src/couchdb/couch_query_servers.erl +++ b/src/couchdb/couch_query_servers.erl @@ -17,10 +17,11 @@ -export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2,code_change/3,stop/0]). -export([start_doc_map/2, map_docs/2, stop_doc_map/1]). --export([reduce/3, rereduce/3,validate_doc_update/5]). --export([render_doc_show/6, render_doc_update/6, start_view_list/2, - render_list_head/4, render_list_row/4, render_list_tail/1]). +-export([reduce/3, rereduce/3,validate_doc_update/4]). -export([filter_docs/5]). + +-export([with_ddoc_proc/2, proc_prompt/2, ddoc_prompt/3, ddoc_proc_prompt/3, json_doc/1]). + % -export([test/0]). -include("couch_db.hrl"). @@ -28,6 +29,7 @@ -record(proc, { pid, lang, + ddoc_keys = [], prompt_fun, set_timeout_fun, stop_fun @@ -37,7 +39,7 @@ start_link() -> gen_server:start_link({local, couch_query_servers}, couch_query_servers, [], []). stop() -> - exit(whereis(couch_query_servers), close). + exit(whereis(couch_query_servers), normal). start_doc_map(Lang, Functions) -> Proc = get_os_process(Lang), @@ -91,21 +93,15 @@ group_reductions_results(List) -> rereduce(_Lang, [], _ReducedValues) -> {ok, []}; rereduce(Lang, RedSrcs, ReducedValues) -> - Proc = get_os_process(Lang), - Grouped = group_reductions_results(ReducedValues), - Results = try lists:zipwith( + Grouped = group_reductions_results(ReducedValues), + Results = lists:zipwith( fun (<<"_", _/binary>> = FunSrc, Values) -> {ok, [Result]} = builtin_reduce(rereduce, [FunSrc], [[[], V] || V <- Values], []), Result; (FunSrc, Values) -> - [true, [Result]] = - proc_prompt(Proc, [<<"rereduce">>, [FunSrc], Values]), - Result - end, RedSrcs, Grouped) - after - ok = ret_os_process(Proc) - end, + os_rereduce(Lang, [FunSrc], Values) + end, RedSrcs, Grouped), {ok, Results}. reduce(_Lang, [], _KVs) -> @@ -137,6 +133,17 @@ os_reduce(Lang, OsRedSrcs, KVs) -> end, {ok, OsResults}. +os_rereduce(_Lang, [], _KVs) -> + {ok, []}; +os_rereduce(Lang, OsRedSrcs, KVs) -> + Proc = get_os_process(Lang), + try proc_prompt(Proc, [<<"rereduce">>, OsRedSrcs, KVs]) of + [true, [Reduction]] -> Reduction + after + ok = ret_os_process(Proc) + end. + + builtin_reduce(_Re, [], _KVs, Acc) -> {ok, lists:reverse(Acc)}; builtin_reduce(Re, [<<"_sum">>|BuiltinReds], KVs, Acc) -> @@ -157,92 +164,49 @@ builtin_sum_rows(KVs) -> throw({invalid_value, <<"builtin _sum function requires map values to be numbers">>}) end, 0, KVs). -validate_doc_update(Lang, FunSrc, EditDoc, DiskDoc, Ctx) -> - Proc = get_os_process(Lang), - JsonEditDoc = couch_doc:to_json_obj(EditDoc, [revs]), - JsonDiskDoc = - if DiskDoc == nil -> - null; - true -> - couch_doc:to_json_obj(DiskDoc, [revs]) - end, - try proc_prompt(Proc, - [<<"validate">>, FunSrc, JsonEditDoc, JsonDiskDoc, Ctx]) of - 1 -> - ok; - {[{<<"forbidden">>, Message}]} -> - throw({forbidden, Message}); - {[{<<"unauthorized">>, Message}]} -> - throw({unauthorized, Message}) - after - ok = ret_os_process(Proc) - end. -% todo use json_apply_field -append_docid(DocId, JsonReqIn) -> - [{<<"docId">>, DocId} | JsonReqIn]. -render_doc_show(Lang, ShowSrc, DocId, Doc, Req, Db) -> - Proc = get_os_process(Lang), - {JsonReqIn} = couch_httpd_external:json_req_obj(Req, Db), - - {JsonReq, JsonDoc} = case {DocId, Doc} of - {nil, nil} -> {{JsonReqIn}, null}; - {DocId, nil} -> {{append_docid(DocId, JsonReqIn)}, null}; - _ -> {{append_docid(DocId, JsonReqIn)}, couch_doc:to_json_obj(Doc, [revs])} - end, - try proc_prompt(Proc, [<<"show">>, ShowSrc, JsonDoc, JsonReq]) - after - ok = ret_os_process(Proc) - end. - -render_doc_update(Lang, UpdateSrc, DocId, Doc, Req, Db) -> - Proc = get_os_process(Lang), - {JsonReqIn} = couch_httpd_external:json_req_obj(Req, Db), - - {JsonReq, JsonDoc} = case {DocId, Doc} of - {nil, nil} -> {{JsonReqIn}, null}; - {DocId, nil} -> {{append_docid(DocId, JsonReqIn)}, null}; - _ -> {{append_docid(DocId, JsonReqIn)}, couch_doc:to_json_obj(Doc, [revs])} - end, - try proc_prompt(Proc, [<<"update">>, UpdateSrc, JsonDoc, JsonReq]) - after - ok = ret_os_process(Proc) +% use the function stored in ddoc.validate_doc_update to test an update. +validate_doc_update(DDoc, EditDoc, DiskDoc, Ctx) -> + JsonEditDoc = couch_doc:to_json_obj(EditDoc, [revs]), + JsonDiskDoc = json_doc(DiskDoc), + case ddoc_prompt(DDoc, [<<"validate_doc_update">>], [JsonEditDoc, JsonDiskDoc, Ctx]) of + 1 -> + ok; + {[{<<"forbidden">>, Message}]} -> + throw({forbidden, Message}); + {[{<<"unauthorized">>, Message}]} -> + throw({unauthorized, Message}) end. -start_view_list(Lang, ListSrc) -> - Proc = get_os_process(Lang), - proc_prompt(Proc, [<<"add_fun">>, ListSrc]), - {ok, Proc}. - -render_list_head(Proc, Req, Db, Head) -> - JsonReq = couch_httpd_external:json_req_obj(Req, Db), - proc_prompt(Proc, [<<"list">>, Head, JsonReq]). - -render_list_row(Proc, Db, {{Key, DocId}, Value}, IncludeDoc) -> - JsonRow = couch_httpd_view:view_row_obj(Db, {{Key, DocId}, Value}, IncludeDoc), - proc_prompt(Proc, [<<"list_row">>, JsonRow]); - -render_list_row(Proc, _, {Key, Value}, _IncludeDoc) -> - JsonRow = {[{key, Key}, {value, Value}]}, - proc_prompt(Proc, [<<"list_row">>, JsonRow]). - -render_list_tail(Proc) -> - JsonResp = proc_prompt(Proc, [<<"list_end">>]), - ok = ret_os_process(Proc), - JsonResp. +json_doc(nil) -> null; +json_doc(Doc) -> + couch_doc:to_json_obj(Doc, [revs]). -filter_docs(Lang, Src, Docs, Req, Db) -> +filter_docs(Req, Db, DDoc, FName, Docs) -> JsonReq = couch_httpd_external:json_req_obj(Req, Db), JsonDocs = [couch_doc:to_json_obj(Doc, [revs]) || Doc <- Docs], JsonCtx = couch_util:json_user_ctx(Db), - Proc = get_os_process(Lang), - [true, Passes] = proc_prompt(Proc, - [<<"filter">>, Src, JsonDocs, JsonReq, JsonCtx]), - ret_os_process(Proc), - {ok, Passes}. + [true, Passes] = ddoc_prompt(DDoc, [<<"filters">>, FName], [JsonDocs, JsonReq, JsonCtx]), + {ok, Passes}. + +ddoc_proc_prompt({Proc, DDocId}, FunPath, Args) -> + proc_prompt(Proc, [<<"ddoc">>, DDocId, FunPath, Args]). + +ddoc_prompt(DDoc, FunPath, Args) -> + with_ddoc_proc(DDoc, fun({Proc, DDocId}) -> + proc_prompt(Proc, [<<"ddoc">>, DDocId, FunPath, Args]) + end). + +with_ddoc_proc(#doc{id=DDocId,revs={Start, [DiskRev|_]}}=DDoc, Fun) -> + Rev = couch_doc:rev_to_str({Start, DiskRev}), + DDocKey = {DDocId, Rev}, + Proc = get_ddoc_process(DDoc, DDocKey), + try Fun({Proc, DDocId}) + after + ok = ret_os_process(Proc) + end. init([]) -> - % read config and register for configuration changes % just stop if one of the config settings change. couch_server_sup @@ -282,7 +246,39 @@ init([]) -> terminate(_Reason, _Server) -> ok. - +handle_call({get_proc, #doc{body={Props}}=DDoc, DDocKey}, _From, {Langs, PidProcs, LangProcs, InUse}=Server) -> + % Note to future self. Add max process limit. + Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), + case ets:lookup(LangProcs, Lang) of + [{Lang, [P|Rest]}] -> + % find a proc in the set that has the DDoc + case proc_with_ddoc(DDoc, DDocKey, [P|Rest]) of + {ok, Proc} -> + % looks like the proc isn't getting dropped from the list. + % we need to change this to take a fun for equality checking + % so we can do a comparison on portnum + rem_from_list(LangProcs, Lang, Proc), + add_to_list(InUse, Lang, Proc), + {reply, {ok, Proc, get_query_server_config()}, Server}; + Error -> + {reply, Error, Server} + end; + _ -> + case (catch new_process(Langs, Lang)) of + {ok, Proc} -> + add_value(PidProcs, Proc#proc.pid, Proc), + case proc_with_ddoc(DDoc, DDocKey, [Proc]) of + {ok, Proc2} -> + rem_from_list(LangProcs, Lang, Proc), + add_to_list(InUse, Lang, Proc2), + {reply, {ok, Proc2, get_query_server_config()}, Server}; + Error -> + {reply, Error, Server} + end; + Error -> + {reply, Error, Server} + end + end; handle_call({get_proc, Lang}, _From, {Langs, PidProcs, LangProcs, InUse}=Server) -> % Note to future self. Add max process limit. case ets:lookup(LangProcs, Lang) of @@ -290,12 +286,13 @@ handle_call({get_proc, Lang}, _From, {Langs, PidProcs, LangProcs, InUse}=Server) add_value(PidProcs, Proc#proc.pid, Proc), rem_from_list(LangProcs, Lang, Proc), add_to_list(InUse, Lang, Proc), - {reply, {recycled, Proc, get_query_server_config()}, Server}; + {reply, {ok, Proc, get_query_server_config()}, Server}; _ -> case (catch new_process(Langs, Lang)) of {ok, Proc} -> + add_value(PidProcs, Proc#proc.pid, Proc), add_to_list(InUse, Lang, Proc), - {reply, {new, Proc}, Server}; + {reply, {ok, Proc, get_query_server_config()}, Server}; Error -> {reply, Error, Server} end @@ -350,6 +347,23 @@ new_process(Langs, Lang) -> {unknown_query_language, Lang} end. +proc_with_ddoc(DDoc, DDocKey, LangProcs) -> + DDocProcs = lists:filter(fun(#proc{ddoc_keys=Keys}) -> + lists:any(fun(Key) -> + Key == DDocKey + end, Keys) + end, LangProcs), + case DDocProcs of + [DDocProc|_] -> + ?LOG_DEBUG("DDocProc found for DDocKey: ~p",[DDocKey]), + {ok, DDocProc}; + [] -> + [TeachProc|_] = LangProcs, + ?LOG_DEBUG("Teach ddoc to new proc ~p with DDocKey: ~p",[TeachProc, DDocKey]), + {ok, SmartProc} = teach_ddoc(DDoc, DDocKey, TeachProc), + {ok, SmartProc} + end. + proc_prompt(Proc, Args) -> {Mod, Func} = Proc#proc.prompt_fun, apply(Mod, Func, [Proc#proc.pid, Args]). @@ -362,14 +376,44 @@ proc_set_timeout(Proc, Timeout) -> {Mod, Func} = Proc#proc.set_timeout_fun, apply(Mod, Func, [Proc#proc.pid, Timeout]). +teach_ddoc(DDoc, {DDocId, _Rev}=DDocKey, #proc{ddoc_keys=Keys}=Proc) -> + % send ddoc over the wire + % we only share the rev with the client we know to update code + % but it only keeps the latest copy, per each ddoc, around. + true = proc_prompt(Proc, [<<"ddoc">>, <<"new">>, DDocId, couch_doc:to_json_obj(DDoc, [])]), + % we should remove any other ddocs keys for this docid + % because the query server overwrites without the rev + Keys2 = [{D,R} || {D,R} <- Keys, D /= DDocId], + % add ddoc to the proc + {ok, Proc#proc{ddoc_keys=[DDocKey|Keys2]}}. + +get_ddoc_process(#doc{} = DDoc, DDocKey) -> + % remove this case statement + case gen_server:call(couch_query_servers, {get_proc, DDoc, DDocKey}) of + {ok, Proc, QueryConfig} -> + % process knows the ddoc + case (catch proc_prompt(Proc, [<<"reset">>, QueryConfig])) of + true -> + proc_set_timeout(Proc, list_to_integer(couch_config:get( + "couchdb", "os_process_timeout", "5000"))), + link(Proc#proc.pid), + Proc; + _ -> + catch proc_stop(Proc), + get_ddoc_process(DDoc, DDocKey) + end; + Error -> + throw(Error) + end. + +ret_ddoc_process(Proc) -> + true = gen_server:call(couch_query_servers, {ret_proc, Proc}), + catch unlink(Proc#proc.pid), + ok. + get_os_process(Lang) -> case gen_server:call(couch_query_servers, {get_proc, Lang}) of - {new, Proc} -> - proc_set_timeout(Proc, list_to_integer(couch_config:get( - "couchdb", "os_process_timeout", "5000"))), - link(Proc#proc.pid), - Proc; - {recycled, Proc, QueryConfig} -> + {ok, Proc, QueryConfig} -> case (catch proc_prompt(Proc, [<<"reset">>, QueryConfig])) of true -> proc_set_timeout(Proc, list_to_integer(couch_config:get( @@ -403,9 +447,20 @@ add_to_list(Tid, Key, Value) -> true = ets:insert(Tid, {Key, [Value]}) end. +rem_from_list(Tid, Key, Value) when is_record(Value, proc)-> + Pid = Value#proc.pid, + case ets:lookup(Tid, Key) of + [{Key, Vals}] -> + % make a new values list that doesn't include the Value arg + NewValues = [Val || #proc{pid=P}=Val <- Vals, P /= Pid], + ets:insert(Tid, {Key, NewValues}); + [] -> ok + end; rem_from_list(Tid, Key, Value) -> case ets:lookup(Tid, Key) of [{Key, Vals}] -> - ets:insert(Tid, {Key, [Val || Val <- Vals, Val /= Value]}); + % make a new values list that doesn't include the Value arg + NewValues = [Val || Val <- Vals, Val /= Value], + ets:insert(Tid, {Key, NewValues}); [] -> ok end. diff --git a/test/view_server/query_server_spec.rb b/test/view_server/query_server_spec.rb index c7b5902d..3d933fdc 100644 --- a/test/view_server/query_server_spec.rb +++ b/test/view_server/query_server_spec.rb @@ -12,6 +12,13 @@ # to run (requires ruby and rspec): # spec test/view_server/query_server_spec.rb -f specdoc --color +# +# environment options: +# QS_TRACE=true +# shows full output from the query server +# QS_LANG=lang +# run tests on the query server (for now, one of: js, erlang) +# COUCH_ROOT = "#{File.dirname(__FILE__)}/../.." unless defined?(COUCH_ROOT) LANGUAGE = ENV["QS_LANG"] || "js" @@ -48,6 +55,17 @@ class OSProcessRunner def add_fun(fun) run(["add_fun", fun]) end + def teach_ddoc(ddoc) + run(["ddoc", "new", ddoc_id(ddoc), ddoc]) + end + def ddoc_run(ddoc, fun_path, args) + run(["ddoc", ddoc_id(ddoc), fun_path, args]) + end + def ddoc_id(ddoc) + d_id = ddoc["_id"] + raise 'ddoc must have _id' unless d_id + d_id + end def get_chunks resp = jsgets raise "not a chunk" unless resp.first == "chunks" @@ -99,7 +117,7 @@ class QueryServerRunner < OSProcessRunner COMMANDS = { "js" => "#{COUCH_ROOT}/bin/couchjs_dev #{COUCH_ROOT}/share/server/main.js", - "erlang" => "#{COUCH_ROOT}/test/run_native_process.es" + "erlang" => "#{COUCH_ROOT}/test/view_server/run_native_process.es" } def self.run_command @@ -113,6 +131,8 @@ class ExternalRunner < OSProcessRunner end end +# we could organize this into a design document per language. +# that would make testing future languages really easy. functions = { "emit-twice" => { @@ -126,7 +146,11 @@ functions = { ERLANG }, "emit-once" => { - "js" => %{function(doc){emit("baz",doc.a)}}, + "js" => <<-JS, + function(doc){ + emit("baz",doc.a) + } + JS "erlang" => <<-ERLANG fun({Doc}) -> A = proplists:get_value(<<"a">>, Doc, null), @@ -370,6 +394,8 @@ functions = { "list-raw" => { "js" => <<-JS, function(head, req) { + // log(this.toSource()); + // log(typeof send); send("first chunk"); send(req.q); var row; @@ -420,9 +446,47 @@ functions = { [{Doc2}, {[{<<"body">>, <<"hello doc">>}]}] end. ERLANG + }, + "error" => { + "js" => <<-JS, + function() { + throw(["error","error_key","testing"]); + } + JS + "erlang" => <<-ERLANG + fun(A, B) -> + throw([<<"error">>,<<"error_key">>,<<"testing">>]) + end. + ERLANG + }, + "fatal" => { + "js" => <<-JS, + function() { + throw(["fatal","error_key","testing"]); + } + JS + "erlang" => <<-ERLANG + fun(A, B) -> + throw([<<"fatal">>,<<"error_key">>,<<"testing">>]) + end. + ERLANG } } +def make_ddoc(fun_path, fun_str) + doc = {"_id"=>"foo"} + d = doc + while p = fun_path.shift + l = p + if !fun_path.empty? + d[p] = {} + d = d[p] + end + end + d[l] = fun_str + doc +end + describe "query server normal case" do before(:all) do `cd #{COUCH_ROOT} && make` @@ -434,6 +498,17 @@ describe "query server normal case" do it "should reset" do @qs.run(["reset"]).should == true end + it "should not erase ddocs on reset" do + @fun = functions["show-simple"][LANGUAGE] + @ddoc = make_ddoc(["shows","simple"], @fun) + @qs.teach_ddoc(@ddoc) + @qs.run(["reset"]).should == true + @qs.ddoc_run(@ddoc, + ["shows","simple"], + [{:title => "Best ever", :body => "Doc body"}, {}]).should == + ["resp", {"body" => "Best ever - Doc body"}] + end + it "should run map funs" do @qs.reset! @qs.run(["add_fun", functions["emit-twice"][LANGUAGE]]).should == true @@ -464,173 +539,222 @@ describe "query server normal case" do end end + describe "design docs" do + before(:all) do + @ddoc = { + "_id" => "foo" + } + @qs.reset! + end + it "should learn design docs" do + @qs.teach_ddoc(@ddoc).should == true + end + end + # it "should validate" describe "validation" do before(:all) do @fun = functions["validate-forbidden"][LANGUAGE] - @qs.reset! + @ddoc = make_ddoc(["validate_doc_update"], @fun) + @qs.teach_ddoc(@ddoc) end it "should allow good updates" do - @qs.run(["validate", @fun, {"good" => true}, {}, {}]).should == 1 + @qs.ddoc_run(@ddoc, + ["validate_doc_update"], + [{"good" => true}, {}, {}]).should == 1 end it "should reject invalid updates" do - @qs.run(["validate", @fun, {"bad" => true}, {}, {}]).should == {"forbidden"=>"bad doc"} + @qs.ddoc_run(@ddoc, + ["validate_doc_update"], + [{"bad" => true}, {}, {}]).should == {"forbidden"=>"bad doc"} end end describe "show" do before(:all) do @fun = functions["show-simple"][LANGUAGE] - @qs.reset! + @ddoc = make_ddoc(["shows","simple"], @fun) + @qs.teach_ddoc(@ddoc) end it "should show" do - @qs.rrun(["show", @fun, - {:title => "Best ever", :body => "Doc body"}, {}]) - @qs.jsgets.should == ["resp", {"body" => "Best ever - Doc body"}] + @qs.ddoc_run(@ddoc, + ["shows","simple"], + [{:title => "Best ever", :body => "Doc body"}, {}]).should == + ["resp", {"body" => "Best ever - Doc body"}] end end describe "show with headers" do before(:all) do + # TODO we can make real ddocs up there. @fun = functions["show-headers"][LANGUAGE] - @qs.reset! + @ddoc = make_ddoc(["shows","headers"], @fun) + @qs.teach_ddoc(@ddoc) end it "should show headers" do - @qs.rrun(["show", @fun, - {:title => "Best ever", :body => "Doc body"}, {}]) - @qs.jsgets.should == ["resp", {"code"=>200,"headers" => {"X-Plankton"=>"Rusty"}, "body" => "Best ever - Doc body"}] + @qs.ddoc_run( + @ddoc, + ["shows","headers"], + [{:title => "Best ever", :body => "Doc body"}, {}] + ). + should == ["resp", {"code"=>200,"headers" => {"X-Plankton"=>"Rusty"}, "body" => "Best ever - Doc body"}] end end - -# end -# LIST TESTS -# __END__ - - describe "raw list with headers" do - before(:each) do - @fun = functions["show-sends"][LANGUAGE] - @qs.reset! - @qs.add_fun(@fun).should == true - end - it "should do headers proper" do - @qs.rrun(["list", {"total_rows"=>1000}, {"q" => "ok"}]) - @qs.jsgets.should == ["start", ["first chunk", 'second "chunk"'], {"headers"=>{"Content-Type"=>"text/plain"}}] - @qs.rrun(["list_end"]) - @qs.jsgets.should == ["end", ["tail"]] - end - end - - describe "list with rows" do - before(:each) do - @fun = functions["show-while-get-rows"][LANGUAGE] - @qs.run(["reset"]).should == true - @qs.add_fun(@fun).should == true - end - it "should list em" do - @qs.rrun(["list", {"foo"=>"bar"}, {"q" => "ok"}]) - @qs.jsgets.should == ["start", ["first chunk", "ok"], {"headers"=>{}}] - @qs.rrun(["list_row", {"key"=>"baz"}]) - @qs.get_chunks.should == ["baz"] - @qs.rrun(["list_row", {"key"=>"bam"}]) - @qs.get_chunks.should == ["bam"] - @qs.rrun(["list_end"]) - @qs.jsgets.should == ["end", ["tail"]] - end - it "should work with zero rows" do - @qs.rrun(["list", {"foo"=>"bar"}, {"q" => "ok"}]) - @qs.jsgets.should == ["start", ["first chunk", "ok"], {"headers"=>{}}] - @qs.rrun(["list_end"]) - @qs.jsgets.should == ["end", ["tail"]] - end - end - - describe "should buffer multiple chunks sent for a single row." do - before(:all) do - @fun = functions["show-while-get-rows-multi-send"][LANGUAGE] - @qs.reset! - @qs.add_fun(@fun).should == true - end - it "should should buffer em" do - @qs.rrun(["list", {"foo"=>"bar"}, {"q" => "ok"}]) - @qs.jsgets.should == ["start", ["bacon"], {"headers"=>{}}] - @qs.rrun(["list_row", {"key"=>"baz"}]) - @qs.get_chunks.should == ["baz", "eggs"] - @qs.rrun(["list_row", {"key"=>"bam"}]) - @qs.get_chunks.should == ["bam", "eggs"] - @qs.rrun(["list_end"]) - @qs.jsgets.should == ["end", ["tail"]] - end - end - - describe "example list" do - before(:all) do - @fun = functions["list-simple"][LANGUAGE] - @qs.reset! - @qs.add_fun(@fun).should == true - end - it "should run normal" do - @qs.run(["list", {"foo"=>"bar"}, {"q" => "ok"}]).should == ["start", ["first chunk", "ok"], {"headers"=>{}}] - @qs.run(["list_row", {"key"=>"baz"}]).should == ["chunks", ["baz"]] - @qs.run(["list_row", {"key"=>"bam"}]).should == ["chunks", ["bam"]] - @qs.run(["list_row", {"key"=>"foom"}]).should == ["chunks", ["foom"]] - @qs.run(["list_row", {"key"=>"fooz"}]).should == ["chunks", ["fooz"]] - @qs.run(["list_row", {"key"=>"foox"}]).should == ["chunks", ["foox"]] - @qs.run(["list_end"]).should == ["end" , ["early"]] - end - end - - describe "only goes to 2 list" do + + describe "recoverable error" do before(:all) do - @fun = functions["list-chunky"][LANGUAGE] - @qs.reset! - @qs.add_fun(@fun).should == true - end - it "should end early" do - @qs.run(["list", {"foo"=>"bar"}, {"q" => "ok"}]). - should == ["start", ["first chunk", "ok"], {"headers"=>{}}] - @qs.run(["list_row", {"key"=>"baz"}]). - should == ["chunks", ["baz"]] - - @qs.run(["list_row", {"key"=>"bam"}]). - should == ["chunks", ["bam"]] - - @qs.run(["list_row", {"key"=>"foom"}]). - should == ["end", ["foom", "early tail"]] - # here's where js has to discard quit properly - @qs.run(["reset"]). - should == true + @fun = functions["error"][LANGUAGE] + @ddoc = make_ddoc(["shows","error"], @fun) + @qs.teach_ddoc(@ddoc) + end + it "should not exit" do + @qs.ddoc_run(@ddoc, ["shows","error"], + [{"foo"=>"bar"}, {"q" => "ok"}]). + should == ["error", "error_key", "testing"] + # still running + @qs.run(["reset"]).should == true end end describe "changes filter" do before(:all) do @fun = functions["filter-basic"][LANGUAGE] - @qs.reset! + @ddoc = make_ddoc(["filters","basic"], @fun) + @qs.teach_ddoc(@ddoc) end it "should only return true for good docs" do - @qs.run(["filter", @fun, [{"key"=>"bam", "good" => true}, {"foo" => "bar"}, {"good" => true}], {"req" => "foo"}]). - should == [true, [true, false, true]] + @qs.ddoc_run(@ddoc, + ["filters","basic"], + [[{"key"=>"bam", "good" => true}, {"foo" => "bar"}, {"good" => true}], {"req" => "foo"}] + ). + should == [true, [true, false, true]] end end describe "update" do before(:all) do + # in another patch we can remove this duplication + # by setting up the design doc for each language ahead of time. @fun = functions["update-basic"][LANGUAGE] - @qs.reset! + @ddoc = make_ddoc(["updates","basic"], @fun) + @qs.teach_ddoc(@ddoc) end it "should return a doc and a resp body" do - up, doc, resp = @qs.run(["update", @fun, {"foo" => "gnarly"}, {"verb" => "POST"}]) + up, doc, resp = @qs.ddoc_run(@ddoc, + ["updates","basic"], + [{"foo" => "gnarly"}, {"verb" => "POST"}] + ) up.should == "up" doc.should == {"foo" => "gnarly", "world" => "hello"} resp["body"].should == "hello doc" end end -end + +# end +# LIST TESTS +# __END__ + + describe "ddoc list" do + before(:all) do + @ddoc = { + "_id" => "foo", + "lists" => { + "simple" => functions["list-simple"][LANGUAGE], + "headers" => functions["show-sends"][LANGUAGE], + "rows" => functions["show-while-get-rows"][LANGUAGE], + "buffer-chunks" => functions["show-while-get-rows-multi-send"][LANGUAGE], + "chunky" => functions["list-chunky"][LANGUAGE] + } + } + @qs.teach_ddoc(@ddoc) + end + + describe "example list" do + it "should run normal" do + @qs.ddoc_run(@ddoc, + ["lists","simple"], + [{"foo"=>"bar"}, {"q" => "ok"}] + ).should == ["start", ["first chunk", "ok"], {"headers"=>{}}] + @qs.run(["list_row", {"key"=>"baz"}]).should == ["chunks", ["baz"]] + @qs.run(["list_row", {"key"=>"bam"}]).should == ["chunks", ["bam"]] + @qs.run(["list_row", {"key"=>"foom"}]).should == ["chunks", ["foom"]] + @qs.run(["list_row", {"key"=>"fooz"}]).should == ["chunks", ["fooz"]] + @qs.run(["list_row", {"key"=>"foox"}]).should == ["chunks", ["foox"]] + @qs.run(["list_end"]).should == ["end" , ["early"]] + end + end + + describe "headers" do + it "should do headers proper" do + @qs.ddoc_run(@ddoc, ["lists","headers"], + [{"total_rows"=>1000}, {"q" => "ok"}] + ).should == ["start", ["first chunk", 'second "chunk"'], + {"headers"=>{"Content-Type"=>"text/plain"}}] + @qs.rrun(["list_end"]) + @qs.jsgets.should == ["end", ["tail"]] + end + end + + describe "with rows" do + it "should list em" do + @qs.ddoc_run(@ddoc, ["lists","rows"], + [{"foo"=>"bar"}, {"q" => "ok"}]). + should == ["start", ["first chunk", "ok"], {"headers"=>{}}] + @qs.rrun(["list_row", {"key"=>"baz"}]) + @qs.get_chunks.should == ["baz"] + @qs.rrun(["list_row", {"key"=>"bam"}]) + @qs.get_chunks.should == ["bam"] + @qs.rrun(["list_end"]) + @qs.jsgets.should == ["end", ["tail"]] + end + it "should work with zero rows" do + @qs.ddoc_run(@ddoc, ["lists","rows"], + [{"foo"=>"bar"}, {"q" => "ok"}]). + should == ["start", ["first chunk", "ok"], {"headers"=>{}}] + @qs.rrun(["list_end"]) + @qs.jsgets.should == ["end", ["tail"]] + end + end + + describe "should buffer multiple chunks sent for a single row." do + it "should should buffer em" do + @qs.ddoc_run(@ddoc, ["lists","buffer-chunks"], + [{"foo"=>"bar"}, {"q" => "ok"}]). + should == ["start", ["bacon"], {"headers"=>{}}] + @qs.rrun(["list_row", {"key"=>"baz"}]) + @qs.get_chunks.should == ["baz", "eggs"] + @qs.rrun(["list_row", {"key"=>"bam"}]) + @qs.get_chunks.should == ["bam", "eggs"] + @qs.rrun(["list_end"]) + @qs.jsgets.should == ["end", ["tail"]] + end + end + it "should end after 2" do + @qs.ddoc_run(@ddoc, ["lists","chunky"], + [{"foo"=>"bar"}, {"q" => "ok"}]). + should == ["start", ["first chunk", "ok"], {"headers"=>{}}] + + @qs.run(["list_row", {"key"=>"baz"}]). + should == ["chunks", ["baz"]] + + @qs.run(["list_row", {"key"=>"bam"}]). + should == ["chunks", ["bam"]] + + @qs.run(["list_row", {"key"=>"foom"}]). + should == ["end", ["foom", "early tail"]] + # here's where js has to discard quit properly + @qs.run(["reset"]). + should == true + end + end + end + + def should_have_exited qs begin qs.run(["reset"]) - "raise before this".should == true + "raise before this (except Erlang)".should == true rescue RuntimeError => e e.message.should == "no response" rescue Errno::EPIPE @@ -641,54 +765,60 @@ end describe "query server that exits" do before(:each) do @qs = QueryServerRunner.run + @ddoc = { + "_id" => "foo", + "lists" => { + "capped" => functions["list-capped"][LANGUAGE], + "raw" => functions["list-raw"][LANGUAGE] + }, + "shows" => { + "fatal" => functions["fatal"][LANGUAGE] + } + } + @qs.teach_ddoc(@ddoc) end after(:each) do @qs.close end - if LANGUAGE == "js" - describe "old style list" do - before(:each) do - @fun = functions["list-old-style"][LANGUAGE] - @qs.reset! - @qs.add_fun(@fun).should == true - end - it "should get a warning" do - resp = @qs.run(["list", {"foo"=>"bar"}, {"q" => "ok"}]) - resp["error"].should == "render_error" - resp["reason"].should include("the list API has changed") - end - end - end - describe "only goes to 2 list" do - before(:each) do - @fun = functions["list-capped"][LANGUAGE] - @qs.reset! - @qs.add_fun(@fun).should == true - end it "should exit if erlang sends too many rows" do - @qs.run(["list", {"foo"=>"bar"}, {"q" => "ok"}]).should == ["start", ["bacon"], {"headers"=>{}}] + @qs.ddoc_run(@ddoc, ["lists","capped"], + [{"foo"=>"bar"}, {"q" => "ok"}]). + should == ["start", ["bacon"], {"headers"=>{}}] @qs.run(["list_row", {"key"=>"baz"}]).should == ["chunks", ["baz"]] @qs.run(["list_row", {"key"=>"foom"}]).should == ["chunks", ["foom"]] @qs.run(["list_row", {"key"=>"fooz"}]).should == ["end", ["fooz", "early"]] - @qs.rrun(["list_row", {"key"=>"foox"}]) - @qs.jsgets["error"].should == "query_server_error" + e = @qs.run(["list_row", {"key"=>"foox"}]) + e[0].should == "error" + e[1].should == "unknown_command" should_have_exited @qs end end describe "raw list" do - before(:each) do - @fun = functions["list-raw"][LANGUAGE] - @qs.run(["reset"]).should == true - @qs.add_fun(@fun).should == true - end it "should exit if it gets a non-row in the middle" do - @qs.rrun(["list", {"foo"=>"bar"}, {"q" => "ok"}]) - @qs.jsgets.should == ["start", ["first chunk", "ok"], {"headers"=>{}}] - @qs.run(["reset"])["error"].should == "query_server_error" + @qs.ddoc_run(@ddoc, ["lists","raw"], + [{"foo"=>"bar"}, {"q" => "ok"}]). + should == ["start", ["first chunk", "ok"], {"headers"=>{}}] + e = @qs.run(["reset"]) + e[0].should == "error" + e[1].should == "list_error" + should_have_exited @qs + end + end + + describe "fatal error" do + it "should exit" do + @qs.ddoc_run(@ddoc, ["shows","fatal"], + [{"foo"=>"bar"}, {"q" => "ok"}]). + should == ["error", "error_key", "testing"] should_have_exited @qs end end end + +describe "thank you for using the tests" do + it "for more info run with QS_TRACE=true or see query_server_spec.rb file header" do + end +end
\ No newline at end of file diff --git a/test/view_server/run_native_process.es b/test/view_server/run_native_process.es index dfdc423e..fcf16d75 100755 --- a/test/view_server/run_native_process.es +++ b/test/view_server/run_native_process.es @@ -15,7 +15,7 @@ read() -> case io:get_line('') of eof -> stop; - Data -> mochijson2:decode(Data) + Data -> couch_util:json_decode(Data) end. send(Data) when is_binary(Data) -> @@ -24,15 +24,19 @@ send(Data) when is_list(Data) -> io:format(Data ++ "\n", []). write(Data) -> - case (catch mochijson2:encode(Data)) of + % log("~p", [Data]), + case (catch couch_util:json_encode(Data)) of + % when testing, this is what prints your errors {json_encode, Error} -> write({[{<<"error">>, Error}]}); Json -> send(Json) end. -%log(Mesg) -> +% log(Mesg) -> % log(Mesg, []). -%log(Mesg, Params) -> +% log(Mesg, Params) -> % io:format(standard_error, Mesg, Params). +% jlog(Mesg) -> +% write([<<"log">>, list_to_binary(io_lib:format("~p",[Mesg]))]). loop(Pid) -> case read() of @@ -40,7 +44,7 @@ loop(Pid) -> Json -> case (catch couch_native_process:prompt(Pid, Json)) of {error, Reason} -> - ok = write({[{error, Reason}]}); + ok = write([error, Reason, Reason]); Resp -> ok = write(Resp), loop(Pid) |