From 9007e2d21dea8b0185c0096b30364a8ee40a3867 Mon Sep 17 00:00:00 2001 From: John Christopher Anderson Date: Fri, 13 Mar 2009 22:15:34 +0000 Subject: Commit Damien's rep_security branch to trunk. Changes bulk_docs conflict checking. Breaks file format, see mailing list for data upgrade procedure, or http://wiki.apache.org/couchdb/Breaking_changes git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@753448 13f79535-47bb-0310-9956-ffa450edef68 --- share/Makefile.am | 1 + share/www/script/couch.js | 44 ++- share/www/script/couch_tests.js | 1 + share/www/script/test/attachment_names.js | 2 +- share/www/script/test/attachments.js | 4 +- share/www/script/test/basics.js | 12 +- share/www/script/test/bulk_docs.js | 71 ++-- share/www/script/test/compact.js | 8 +- share/www/script/test/design_docs.js | 2 +- share/www/script/test/etags_views.js | 3 +- share/www/script/test/invalid_docids.js | 14 +- share/www/script/test/list_views.js | 6 +- share/www/script/test/lots_of_docs.js | 2 +- share/www/script/test/purge.js | 2 +- share/www/script/test/reduce.js | 6 +- share/www/script/test/reduce_false.js | 2 +- share/www/script/test/replication.js | 41 ++- share/www/script/test/rev_stemming.js | 93 ++++++ share/www/script/test/security_validation.js | 165 ++++++++-- share/www/script/test/view_include_docs.js | 2 +- share/www/script/test/view_multi_key_all_docs.js | 2 +- share/www/script/test/view_multi_key_design.js | 2 +- share/www/script/test/view_multi_key_temp.js | 2 +- share/www/script/test/view_pagination.js | 2 +- src/couchdb/couch_db.erl | 395 ++++++++++++++++------- src/couchdb/couch_db.hrl | 24 +- src/couchdb/couch_db_updater.erl | 147 ++++----- src/couchdb/couch_doc.erl | 185 +++++++---- src/couchdb/couch_httpd.erl | 120 +++---- src/couchdb/couch_httpd_db.erl | 285 ++++++++-------- src/couchdb/couch_httpd_misc_handlers.erl | 55 ++-- src/couchdb/couch_httpd_show.erl | 15 +- src/couchdb/couch_httpd_view.erl | 12 +- src/couchdb/couch_key_tree.erl | 375 ++++++++++++++++----- src/couchdb/couch_rep.erl | 365 +++++++++++---------- src/couchdb/couch_util.erl | 15 +- 36 files changed, 1619 insertions(+), 863 deletions(-) create mode 100644 share/www/script/test/rev_stemming.js diff --git a/share/Makefile.am b/share/Makefile.am index 0f8daaab..3cabe166 100644 --- a/share/Makefile.am +++ b/share/Makefile.am @@ -104,6 +104,7 @@ nobase_dist_localdata_DATA = \ www/script/test/view_sandboxing.js \ www/script/test/view_xml.js \ www/script/test/replication.js \ + www/script/test/rev_stemming.js \ www/script/test/etags_head.js \ www/script/test/etags_views.js \ www/script/test/show_documents.js \ diff --git a/share/www/script/couch.js b/share/www/script/couch.js index 821fb694..4586cca8 100644 --- a/share/www/script/couch.js +++ b/share/www/script/couch.js @@ -98,15 +98,26 @@ function CouchDB(name, httpHeaders) { if (docs[i]._id == undefined) docs[i]._id = newUuids.pop(); } - this.last_req = this.request("POST", this.uri + "_bulk_docs" + encodeOptions(options), { - body: JSON.stringify({"docs": docs}) + var json = {"docs": docs}; + // put any options in the json + for (var option in options) { + json[option] = options[option]; + } + this.last_req = this.request("POST", this.uri + "_bulk_docs", { + body: JSON.stringify(json) }); - CouchDB.maybeThrowError(this.last_req); - var result = JSON.parse(this.last_req.responseText); - for (var i = 0; i < docs.length; i++) { - docs[i]._rev = result.new_revs[i].rev; + if (this.last_req.status == 417) { + return {errors: JSON.parse(this.last_req.responseText)}; + } + else { + CouchDB.maybeThrowError(this.last_req); + var results = JSON.parse(this.last_req.responseText); + for (var i = 0; i < docs.length; i++) { + if(results[i].rev) + docs[i]._rev = results[i].rev; + } + return results; } - return result; } this.ensureFullCommit = function() { @@ -203,6 +214,20 @@ function CouchDB(name, httpHeaders) { return JSON.parse(this.last_req.responseText); } + this.setDbProperty = function(propId, propValue) { + this.last_req = this.request("PUT", this.uri + propId,{ + body:JSON.stringify(propValue) + }); + CouchDB.maybeThrowError(this.last_req); + return JSON.parse(this.last_req.responseText); + } + + this.getDbProperty = function(propId) { + this.last_req = this.request("GET", this.uri + propId); + CouchDB.maybeThrowError(this.last_req); + return JSON.parse(this.last_req.responseText); + } + this.setAdmins = function(adminsArray) { this.last_req = this.request("PUT", this.uri + "_admins",{ body:JSON.stringify(adminsArray) @@ -283,8 +308,11 @@ CouchDB.getVersion = function() { return JSON.parse(CouchDB.last_req.responseText).version; } -CouchDB.replicate = function(source, target) { +CouchDB.replicate = function(source, target, rep_options) { + rep_options = rep_options || {}; + var headers = rep_options.headers || {}; CouchDB.last_req = CouchDB.request("POST", "/_replicate", { + headers: headers, body: JSON.stringify({source: source, target: target}) }); CouchDB.maybeThrowError(CouchDB.last_req); diff --git a/share/www/script/couch_tests.js b/share/www/script/couch_tests.js index 64c366b6..8ba1f76d 100644 --- a/share/www/script/couch_tests.js +++ b/share/www/script/couch_tests.js @@ -68,6 +68,7 @@ loadTest("purge.js"); loadTest("config.js"); loadTest("security_validation.js"); loadTest("stats.js"); +loadTest("rev_stemming.js"); function makeDocs(start, end, templateDoc) { var templateDocSrc = templateDoc ? JSON.stringify(templateDoc) : "{}" diff --git a/share/www/script/test/attachment_names.js b/share/www/script/test/attachment_names.js index 802abc08..3c694dd0 100644 --- a/share/www/script/test/attachment_names.js +++ b/share/www/script/test/attachment_names.js @@ -10,7 +10,7 @@ // License for the specific language governing permissions and limitations under // the License. -couchTests.attatchment_names = function(debug) { +couchTests.attachment_names = function(debug) { var db = new CouchDB("test_suite_db"); db.deleteDb(); db.createDb(); diff --git a/share/www/script/test/attachments.js b/share/www/script/test/attachments.js index 409426c7..4b43e31a 100644 --- a/share/www/script/test/attachments.js +++ b/share/www/script/test/attachments.js @@ -32,8 +32,8 @@ couchTests.attachments= function(debug) { var xhr = CouchDB.request("GET", "/test_suite_db/bin_doc/foo.txt"); T(xhr.responseText == "This is a base64 encoded text"); T(xhr.getResponseHeader("Content-Type") == "text/plain"); - T(xhr.getResponseHeader("Etag") == save_response.rev); - + T(xhr.getResponseHeader("Etag") == '"' + save_response.rev + '"'); + // empty attachment var binAttDoc2 = { _id: "bin_doc2", diff --git a/share/www/script/test/basics.js b/share/www/script/test/basics.js index 89e99a64..2837b77a 100644 --- a/share/www/script/test/basics.js +++ b/share/www/script/test/basics.js @@ -14,14 +14,14 @@ couchTests.basics = function(debug) { var result = JSON.parse(CouchDB.request("GET", "/").responseText); T(result.couchdb == "Welcome"); - + var db = new CouchDB("test_suite_db"); db.deleteDb(); // bug COUCHDB-100: DELETE on non-existent DB returns 500 instead of 404 db.deleteDb(); - -db.createDb(); + + db.createDb(); // PUT on existing DB should return 412 instead of 500 xhr = CouchDB.request("PUT", "/test_suite_db/"); @@ -122,14 +122,14 @@ db.createDb(); // make sure we can still open the old rev of the deleted doc T(db.open(existingDoc._id, {rev: existingDoc._rev}) != null); - + // make sure restart works T(db.ensureFullCommit().ok); restartServer(); - + // make sure we can still open T(db.open(existingDoc._id, {rev: existingDoc._rev}) != null); - + // test that the POST response has a Location header var xhr = CouchDB.request("POST", "/test_suite_db", { body: JSON.stringify({"foo":"bar"}) diff --git a/share/www/script/test/bulk_docs.js b/share/www/script/test/bulk_docs.js index d781b2c6..40a36331 100644 --- a/share/www/script/test/bulk_docs.js +++ b/share/www/script/test/bulk_docs.js @@ -19,38 +19,73 @@ couchTests.bulk_docs = function(debug) { var docs = makeDocs(5); // Create the docs - var result = db.bulkSave(docs); - T(result.ok); - T(result.new_revs.length == 5); + var results = db.bulkSave(docs); + + T(results.length == 5); for (var i = 0; i < 5; i++) { - T(result.new_revs[i].id == docs[i]._id); - T(result.new_revs[i].rev); + T(results[i].id == docs[i]._id); + T(results[i].rev); + // Update the doc docs[i].string = docs[i].string + ".00"; } - // Update the docs - result = db.bulkSave(docs); - T(result.ok); - T(result.new_revs.length == 5); + // Save the docs + results = db.bulkSave(docs); + T(results.length == 5); for (i = 0; i < 5; i++) { - T(result.new_revs[i].id == i.toString()); + T(results[i].id == i.toString()); + + // set the delete flag to delete the docs in the next step docs[i]._deleted = true; } + + // now test a bulk update with a conflict + // open and save + var doc = db.open("0"); + db.save(doc); - // Delete the docs - result = db.bulkSave(docs); - T(result.ok); - T(result.new_revs.length == 5); - for (i = 0; i < 5; i++) { + // Now bulk delete the docs + results = db.bulkSave(docs); + + // doc "0" should be a conflict + T(results.length == 5); + T(results[0].id == "0"); + T(results[0].error == "conflict"); + T(results[0].rev === undefined); // no rev member when a conflict + + // but the rest are not + for (i = 1; i < 5; i++) { + T(results[i].id == i.toString()); + T(results[i].rev) T(db.open(docs[i]._id) == null); } + + // now force a conflict to to save + + // save doc 0, this will cause a conflict when we save docs[0] + var doc = db.open("0"); + docs[0] = db.open("0") + db.save(doc); + + docs[0].shooby = "dooby"; + + // Now save the bulk docs, When we use all_or_nothing, we don't get conflict + // checking, all docs are saved regardless of conflict status, or none are + // saved. + results = db.bulkSave(docs,{all_or_nothing:true}); + T(results.error === undefined); + + var doc = db.open("0", {conflicts:true}); + var docConflict = db.open("0", {rev:doc._conflicts[0]}); + + T(doc.shooby == "dooby" || docConflict.shooby == "dooby"); // verify creating a document with no id returns a new id var req = CouchDB.request("POST", "/test_suite_db/_bulk_docs", { body: JSON.stringify({"docs": [{"foo":"bar"}]}) }); - result = JSON.parse(req.responseText); + results = JSON.parse(req.responseText); - T(result.new_revs[0].id != ""); - T(result.new_revs[0].rev != ""); + T(results[0].id != ""); + T(results[0].rev != ""); }; diff --git a/share/www/script/test/compact.js b/share/www/script/test/compact.js index 4612621e..51a57051 100644 --- a/share/www/script/test/compact.js +++ b/share/www/script/test/compact.js @@ -16,8 +16,7 @@ couchTests.compact = function(debug) { db.createDb(); if (debug) debugger; var docs = makeDocs(0, 10); - var saveResult = db.bulkSave(docs); - T(saveResult.ok); + db.bulkSave(docs); var binAttDoc = { _id: "bin_doc", @@ -36,12 +35,11 @@ couchTests.compact = function(debug) { for(var i in docs) { db.deleteDoc(docs[i]); } - db.setAdmins(["Foo bar"]); var deletesize = db.info().disk_size; T(deletesize > originalsize); - var xhr = CouchDB.request("POST", "/test_suite_db/_compact"); - T(xhr.status == 202); + T(db.compact().ok); + T(db.last_req.status == 202); // compaction isn't instantaneous, loop until done while (db.info().compact_running) {}; diff --git a/share/www/script/test/design_docs.js b/share/www/script/test/design_docs.js index aedf741a..ebf0e74d 100644 --- a/share/www/script/test/design_docs.js +++ b/share/www/script/test/design_docs.js @@ -43,7 +43,7 @@ couchTests.design_docs = function(debug) { } T(db.save(designDoc).ok); - T(db.bulkSave(makeDocs(1, numDocs + 1)).ok); + db.bulkSave(makeDocs(1, numDocs + 1)); // test that the _all_docs view returns correctly with keys var results = db.allDocs({startkey:"_design", endkey:"_design0"}); diff --git a/share/www/script/test/etags_views.js b/share/www/script/test/etags_views.js index cb38ccf6..018bdc25 100644 --- a/share/www/script/test/etags_views.js +++ b/share/www/script/test/etags_views.js @@ -42,8 +42,7 @@ couchTests.etags_views = function(debug) { T(db.save(designDoc).ok); var xhr; var docs = makeDocs(0, 10); - var saveResult = db.bulkSave(docs); - T(saveResult.ok); + db.bulkSave(docs); // verify get w/Etag on map view xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView"); diff --git a/share/www/script/test/invalid_docids.js b/share/www/script/test/invalid_docids.js index 6ed01f0b..814d1a21 100644 --- a/share/www/script/test/invalid_docids.js +++ b/share/www/script/test/invalid_docids.js @@ -23,32 +23,32 @@ couchTests.invalid_docids = function(debug) { //Test non-string try { db.save({"_id": 1}); - T(1 == 0); + T(1 == 0, "doc id must be string"); } catch(e) { T(db.last_req.status == 400); - T(e.error == "invalid_doc"); + T(e.error == "bad_request"); } // Test invalid _prefix try { db.save({"_id": "_invalid"}); - T(1 == 0); + T(1 == 0, "doc id may not start with underscore"); } catch(e) { T(db.last_req.status == 400); - T(e.error == "invalid_doc"); + T(e.error == "bad_request"); } // Test _bulk_docs explicitly. var docs = [{"_id": "_design/foo"}, {"_id": "_local/bar"}]; - T(db.bulkSave(docs).ok); + db.bulkSave(docs); docs.forEach(function(d) {T(db.open(d._id)._id == d._id);}); docs = [{"_id": "_invalid"}]; try { db.bulkSave(docs); - T(1 == 0); + T(1 == 0, "doc id may not start with underscore, even in bulk docs"); } catch(e) { T(db.last_req.status == 400); - T(e.error == "invalid_doc"); + T(e.error == "bad_request"); } }; diff --git a/share/www/script/test/list_views.js b/share/www/script/test/list_views.js index 055ef51c..e71ebc4e 100644 --- a/share/www/script/test/list_views.js +++ b/share/www/script/test/list_views.js @@ -137,8 +137,7 @@ couchTests.list_views = function(debug) { T(db.save(designDoc).ok); var docs = makeDocs(0, 10); - var saveResult = db.bulkSave(docs); - T(saveResult.ok); + db.bulkSave(docs); var view = db.view('lists/basicView'); T(view.total_rows == 10); @@ -207,8 +206,7 @@ couchTests.list_views = function(debug) { // verify the etags expire correctly var docs = makeDocs(11, 12); - var saveResult = db.bulkSave(docs); - T(saveResult.ok); + db.bulkSave(docs); xhr = CouchDB.request("GET", "/test_suite_db/_design/lists/_list/simpleForm/withReduce?group=true", { headers: {"if-none-match": etag} diff --git a/share/www/script/test/lots_of_docs.js b/share/www/script/test/lots_of_docs.js index 702317e5..4101e6a7 100644 --- a/share/www/script/test/lots_of_docs.js +++ b/share/www/script/test/lots_of_docs.js @@ -25,7 +25,7 @@ couchTests.lots_of_docs = function(debug) { for(var i=0; i < numDocsToCreate; i += 100) { var createNow = Math.min(numDocsToCreate - i, 100); var docs = makeDocs(i, i + createNow); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); } // query all documents, and return the doc.integer member as a key. diff --git a/share/www/script/test/purge.js b/share/www/script/test/purge.js index e05ac024..0b47c0d0 100644 --- a/share/www/script/test/purge.js +++ b/share/www/script/test/purge.js @@ -34,7 +34,7 @@ couchTests.purge = function(debug) { T(db.save(designDoc).ok); - T(db.bulkSave(makeDocs(1, numDocs + 1)).ok); + db.bulkSave(makeDocs(1, numDocs + 1)); // go ahead and validate the views before purging var rows = db.view("test/all_docs_twice").rows; diff --git a/share/www/script/test/reduce.js b/share/www/script/test/reduce.js index d3245f98..c8bdcd92 100644 --- a/share/www/script/test/reduce.js +++ b/share/www/script/test/reduce.js @@ -17,7 +17,7 @@ couchTests.reduce = function(debug) { if (debug) debugger; var numDocs = 500 var docs = makeDocs(1,numDocs + 1); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); var summate = function(N) {return (N+1)*N/2;}; var map = function (doc) { @@ -65,7 +65,7 @@ couchTests.reduce = function(debug) { docs.push({keys:["d", "a"]}); docs.push({keys:["d", "b"]}); docs.push({keys:["d", "c"]}); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); T(db.info().doc_count == ((i - 1) * 10 * 11) + ((j + 1) * 11)); } @@ -157,7 +157,7 @@ couchTests.reduce = function(debug) { docs.push({val:80}); docs.push({val:90}); docs.push({val:100}); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); } var results = db.query(map, reduceCombine); diff --git a/share/www/script/test/reduce_false.js b/share/www/script/test/reduce_false.js index 41f0c669..22ef2e8b 100644 --- a/share/www/script/test/reduce_false.js +++ b/share/www/script/test/reduce_false.js @@ -18,7 +18,7 @@ couchTests.reduce_false = function(debug) { var numDocs = 5; var docs = makeDocs(1,numDocs + 1); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); var summate = function(N) {return (N+1)*N/2;}; var designDoc = { diff --git a/share/www/script/test/replication.js b/share/www/script/test/replication.js index 59dc4700..944c8e47 100644 --- a/share/www/script/test/replication.js +++ b/share/www/script/test/replication.js @@ -59,7 +59,7 @@ couchTests.replication = function(debug) { simple_test: new function () { this.init = function(dbA, dbB) { var docs = makeDocs(0, numDocs); - T(dbA.bulkSave(docs).ok); + dbA.bulkSave(docs); }; this.afterAB1 = function(dbA, dbB) { @@ -160,7 +160,7 @@ couchTests.replication = function(debug) { dbA.deleteDoc({_id:"foo", _rev:docA._conflicts[0]}); }; - this.afterBA2 = function(dbA, dbB) { + this.afterBA2 = function(dbA, dbB) { // open documents and include the conflict meta data var docA = dbA.open("foo", {conflicts: true}); var docB = dbB.open("foo", {conflicts: true}); @@ -179,29 +179,56 @@ couchTests.replication = function(debug) { } } - T(CouchDB.replicate(A, B).ok); + var result = CouchDB.replicate(A, B); + + var seqA = result.source_last_seq; + T(0 == result.history[0].start_last_seq); + T(result.history[1] === undefined) for(test in repTests) { if(repTests[test].afterAB1) repTests[test].afterAB1(dbA, dbB); } - T(CouchDB.replicate(B, A).ok); + result = CouchDB.replicate(B, A); + + var seqB = result.source_last_seq; + T(0 == result.history[0].start_last_seq); + T(result.history[1] === undefined) for(test in repTests) { if(repTests[test].afterBA1) repTests[test].afterBA1(dbA, dbB); } - T(CouchDB.replicate(A, B).ok); + var result2 = CouchDB.replicate(A, B); + + // each successful replication produces a new session id + T(result2.session_id != result.session_id); + + T(seqA < result2.source_last_seq); + T(seqA == result2.history[0].start_last_seq); + T(result2.history[1].end_last_seq == seqA) + + seqA = result2.source_last_seq; for(test in repTests) { if(repTests[test].afterAB2) repTests[test].afterAB2(dbA, dbB); } - T(CouchDB.replicate(B, A).ok); + result = CouchDB.replicate(B, A) + + T(seqB < result.source_last_seq); + T(seqB == result.history[0].start_last_seq); + T(result.history[1].end_last_seq == seqB) + + seqB = result.source_last_seq; for(test in repTests) { if(repTests[test].afterBA2) repTests[test].afterBA2(dbA, dbB); } - + + // do an replication where nothing has changed + result2 = CouchDB.replicate(B, A); + T(result2.no_changes == true); + T(result2.session_id == result.session_id); } }; diff --git a/share/www/script/test/rev_stemming.js b/share/www/script/test/rev_stemming.js new file mode 100644 index 00000000..f629c299 --- /dev/null +++ b/share/www/script/test/rev_stemming.js @@ -0,0 +1,93 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +couchTests.rev_stemming = function(debug) { + var db = new CouchDB("test_suite_db_a"); + db.deleteDb(); + db.createDb(); + var dbB = new CouchDB("test_suite_db_b"); + dbB.deleteDb(); + dbB.createDb(); + if (debug) debugger; + + var newLimit = 5; + + T(db.getDbProperty("_revs_limit") == 1000); + + var doc = {_id:"foo",foo:0} + for( var i=0; i < newLimit + 1; i++) { + doc.foo++; + T(db.save(doc).ok); + } + var doc0 = db.open("foo", {revs:true}); + T(doc0._revisions.ids.length == newLimit + 1); + + var docBar = {_id:"bar",foo:0} + for( var i=0; i < newLimit + 1; i++) { + docBar.foo++; + T(db.save(docBar).ok); + } + T(db.open("bar", {revs:true})._revisions.ids.length == newLimit + 1); + + T(db.setDbProperty("_revs_limit", newLimit).ok); + + for( var i=0; i < newLimit + 1; i++) { + doc.foo++; + T(db.save(doc).ok); + } + doc0 = db.open("foo", {revs:true}); + T(doc0._revisions.ids.length == newLimit); + + + // If you replicate after you make more edits than the limit, you'll + // cause a spurious edit conflict. + CouchDB.replicate("test_suite_db_a", "test_suite_db_b"); + var docB1 = dbB.open("foo",{conflicts:true}) + T(docB1._conflicts == null); + + for( var i=0; i < newLimit - 1; i++) { + doc.foo++; + T(db.save(doc).ok); + } + + // one less edit than limit, no conflict + CouchDB.replicate("test_suite_db_a", "test_suite_db_b"); + var docB1 = dbB.open("foo",{conflicts:true}) + T(docB1._conflicts == null); + + //now we hit the limit + for( var i=0; i < newLimit; i++) { + doc.foo++; + T(db.save(doc).ok); + } + + CouchDB.replicate("test_suite_db_a", "test_suite_db_b"); + + var docB2 = dbB.open("foo",{conflicts:true}); + + // we have a conflict, but the previous replicated rev is always the losing + // conflict + T(docB2._conflicts[0] == docB1._rev) + + // We having already updated bar before setting the limit, so it's still got + // a long rev history. compact to stem the revs. + + T(db.open("bar", {revs:true})._revisions.ids.length == newLimit + 1); + + T(db.compact().ok); + + // compaction isn't instantaneous, loop until done + while (db.info().compact_running) {}; + + T(db.open("bar", {revs:true})._revisions.ids.length == newLimit); + +}; diff --git a/share/www/script/test/security_validation.js b/share/www/script/test/security_validation.js index 61f56b39..a41d8d70 100644 --- a/share/www/script/test/security_validation.js +++ b/share/www/script/test/security_validation.js @@ -31,28 +31,27 @@ couchTests.security_validation = function(debug) { // the "username:password" string, it's sent completely plain text. // Firefox and Safari both deal with this correctly (which is to say // they correctly do nothing special). - - + + var db = new CouchDB("test_suite_db"); db.deleteDb(); db.createDb(); if (debug) debugger; - + run_on_modified_server( [{section: "httpd", key: "authentication_handler", value: "{couch_httpd, special_test_authentication_handler}"}, - {section:"httpd", + {section:"httpd", key: "WWW-Authenticate", value: "X-Couch-Test-Auth"}], - - function () { + function () { // try saving document usin the wrong credentials var wrongPasswordDb = new CouchDB("test_suite_db", {"WWW-Authenticate": "X-Couch-Test-Auth Damien Katz:foo"} ); - + try { wrongPasswordDb.save({foo:1,author:"Damien Katz"}); T(false && "Can't get here. Should have thrown an error 1"); @@ -60,8 +59,8 @@ couchTests.security_validation = function(debug) { T(e.error == "unauthorized"); T(wrongPasswordDb.last_req.status == 401); } - - + + // Create the design doc that will run custom validation code var designDoc = { _id:"_design/test", @@ -83,9 +82,9 @@ couchTests.security_validation = function(debug) { var userDb = new CouchDB("test_suite_db", {"WWW-Authenticate": "X-Couch-Test-Auth Damien Katz:pecan pie"} ); - + T(userDb.save({_id:"testdoc", foo:1, author:"Damien Katz"}).ok); - + // Attempt to save the design as a non-admin try { userDb.save(designDoc); @@ -94,17 +93,17 @@ couchTests.security_validation = function(debug) { T(e.error == "unauthorized"); T(userDb.last_req.status == 401); } - - // add user as admin - db.setAdmins(["Damien Katz"]); - + + // set user as the admin + T(db.setDbProperty("_admins", ["Damien Katz"]).ok); + T(userDb.save(designDoc).ok); - + // update the document var doc = userDb.open("testdoc"); doc.foo=2; T(userDb.save(doc).ok); - + // Save a document that's missing an author field. try { userDb.save({foo:1}); @@ -113,12 +112,12 @@ couchTests.security_validation = function(debug) { T(e.error == "forbidden"); T(userDb.last_req.status == 403); } - + // Now attempt to update the document as a different user, Jan var user2Db = new CouchDB("test_suite_db", {"WWW-Authenticate": "X-Couch-Test-Auth Jan Lehnardt:apple"} ); - + var doc = user2Db.open("testdoc"); doc.foo=3; try { @@ -128,17 +127,17 @@ couchTests.security_validation = function(debug) { T(e.error == "unauthorized"); T(user2Db.last_req.status == 401); } - + // Now have Damien change the author to Jan doc = userDb.open("testdoc"); doc.author="Jan Lehnardt"; T(userDb.save(doc).ok); - + // Now update the document as Jan doc = user2Db.open("testdoc"); doc.foo = 3; T(user2Db.save(doc).ok); - + // Damien can't delete it try { userDb.deleteDoc(doc); @@ -147,8 +146,126 @@ couchTests.security_validation = function(debug) { T(e.error == "unauthorized"); T(userDb.last_req.status == 401); } - + // Now delete document - T(user2Db.deleteDoc(doc).ok); + T(user2Db.deleteDoc(doc).ok); + + // now test bulk docs + var docs = [{_id:"bahbah",author:"Damien Katz",foo:"bar"},{_id:"fahfah",foo:"baz"}]; + + // Create the docs + var results = db.bulkSave(docs); + + T(results[0].rev) + T(results[0].error == undefined) + T(results[1].rev === undefined) + T(results[1].error == "forbidden") + + T(db.open("bahbah")); + T(db.open("fahfah") == null); + + + // now all or nothing with a failure + var docs = [{_id:"booboo",author:"Damien Katz",foo:"bar"},{_id:"foofoo",foo:"baz"}]; + + // Create the docs + var results = db.bulkSave(docs, {all_or_nothing:true}); + + T(results.errors.length == 1); + T(results.errors[0].error == "forbidden"); + T(db.open("booboo") == null); + T(db.open("foofoo") == null); + + + // Now test replication + var AuthHeaders = {"WWW-Authenticate": "X-Couch-Test-Auth Christopher Lenz:dog food"}; + var host = CouchDB.host; + var dbPairs = [ + {source:"test_suite_db_a", + target:"test_suite_db_b"}, + + {source:"test_suite_db_a", + target:{url: "http://" + host + "/test_suite_db_b", + headers: AuthHeaders}}, + + {source:{url:"http://" + host + "/test_suite_db_a", + headers: AuthHeaders}, + target:"test_suite_db_b"}, + + {source:{url:"http://" + host + "/test_suite_db_a", + headers: AuthHeaders}, + target:{url:"http://" + host + "/test_suite_db_b", + headers: AuthHeaders}}, + ] + var adminDbA = new CouchDB("test_suite_db_a"); + var adminDbB = new CouchDB("test_suite_db_b"); + var dbA = new CouchDB("test_suite_db_a", + {"WWW-Authenticate": "X-Couch-Test-Auth Christopher Lenz:dog food"}); + var dbB = new CouchDB("test_suite_db_b", + {"WWW-Authenticate": "X-Couch-Test-Auth Christopher Lenz:dog food"}); + var xhr; + for (var testPair = 0; testPair < dbPairs.length; testPair++) { + var A = dbPairs[testPair].source + var B = dbPairs[testPair].target + + adminDbA.deleteDb(); + adminDbA.createDb(); + adminDbB.deleteDb(); + adminDbB.createDb(); + + // save and replicate a documents that will and will not pass our design + // doc validation function. + dbA.save({_id:"foo1",value:"a",author:"Noah Slater"}); + dbA.save({_id:"foo2",value:"a",author:"Christopher Lenz"}); + dbA.save({_id:"bad1",value:"a"}); + + T(CouchDB.replicate(A, B, {headers:AuthHeaders}).ok); + T(CouchDB.replicate(B, A, {headers:AuthHeaders}).ok); + + T(dbA.open("foo1")); + T(dbB.open("foo1")); + T(dbA.open("foo2")); + T(dbB.open("foo2")); + + // save the design doc to dbA + delete designDoc._rev; // clear rev from previous saves + adminDbA.save(designDoc); + + // no affect on already saved docs + T(dbA.open("bad1")); + + // Update some docs on dbB. Since the design hasn't replicated, anything + // is allowed. + + // this edit will fail validation on replication to dbA (no author) + T(dbB.save({_id:"bad2",value:"a"}).ok); + + // this edit will fail security on replication to dbA (wrong author + // replicating the change) + var foo1 = dbB.open("foo1"); + foo1.value = "b"; + dbB.save(foo1); + + // this is a legal edit + var foo2 = dbB.open("foo2"); + foo2.value = "b"; + dbB.save(foo2); + + var results = CouchDB.replicate(B, A, {headers:AuthHeaders}); + + T(results.ok); + + T(results.history[0].docs_written == 1); + T(results.history[0].doc_write_failures == 2); + + // bad2 should not be on dbA + T(dbA.open("bad2") == null); + + // The edit to foo1 should not have replicated. + T(dbA.open("foo1").value == "a"); + + // The edit to foo2 should have replicated. + T(dbA.open("foo2").value == "b"); + } }); }; diff --git a/share/www/script/test/view_include_docs.js b/share/www/script/test/view_include_docs.js index dd8ebb44..2c47b696 100644 --- a/share/www/script/test/view_include_docs.js +++ b/share/www/script/test/view_include_docs.js @@ -17,7 +17,7 @@ couchTests.view_include_docs = function(debug) { if (debug) debugger; var docs = makeDocs(0, 100); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); var designDoc = { _id:"_design/test", diff --git a/share/www/script/test/view_multi_key_all_docs.js b/share/www/script/test/view_multi_key_all_docs.js index f036db11..8e2464b4 100644 --- a/share/www/script/test/view_multi_key_all_docs.js +++ b/share/www/script/test/view_multi_key_all_docs.js @@ -17,7 +17,7 @@ couchTests.view_multi_key_all_docs = function(debug) { if (debug) debugger; var docs = makeDocs(0, 100); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); var keys = ["10","15","30","37","50"]; var rows = db.allDocs({},keys).rows; diff --git a/share/www/script/test/view_multi_key_design.js b/share/www/script/test/view_multi_key_design.js index 2464ff1c..c2833910 100644 --- a/share/www/script/test/view_multi_key_design.js +++ b/share/www/script/test/view_multi_key_design.js @@ -17,7 +17,7 @@ couchTests.view_multi_key_design = function(debug) { if (debug) debugger; var docs = makeDocs(0, 100); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); var designDoc = { _id:"_design/test", diff --git a/share/www/script/test/view_multi_key_temp.js b/share/www/script/test/view_multi_key_temp.js index 28cd42b4..545e2520 100644 --- a/share/www/script/test/view_multi_key_temp.js +++ b/share/www/script/test/view_multi_key_temp.js @@ -17,7 +17,7 @@ couchTests.view_multi_key_temp = function(debug) { if (debug) debugger; var docs = makeDocs(0, 100); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); var queryFun = function(doc) { emit(doc.integer, doc.integer) }; var reduceFun = function (keys, values) { return sum(values); }; diff --git a/share/www/script/test/view_pagination.js b/share/www/script/test/view_pagination.js index 7d466f19..21eab888 100644 --- a/share/www/script/test/view_pagination.js +++ b/share/www/script/test/view_pagination.js @@ -17,7 +17,7 @@ couchTests.view_pagination = function(debug) { if (debug) debugger; var docs = makeDocs(0, 100); - T(db.bulkSave(docs).ok); + db.bulkSave(docs); var queryFun = function(doc) { emit(doc.integer, null) }; var i; diff --git a/src/couchdb/couch_db.erl b/src/couchdb/couch_db.erl index 92026ff1..46de8745 100644 --- a/src/couchdb/couch_db.erl +++ b/src/couchdb/couch_db.erl @@ -17,6 +17,7 @@ -export([open_ref_counted/2,is_idle/1,monitor/1,count_changes_since/2]). -export([update_doc/3,update_docs/4,update_docs/2,update_docs/3,delete_doc/3]). -export([get_doc_info/2,open_doc/2,open_doc/3,open_doc_revs/4]). +-export([set_revs_limit/2,get_revs_limit/1]). -export([get_missing_revs/2,name/1,doc_to_tree/1,get_update_seq/1,get_committed_update_seq/1]). -export([enum_docs/4,enum_docs/5,enum_docs_since/4,enum_docs_since/5]). -export([enum_docs_since_reduce_to_count/1,enum_docs_reduce_to_count/1]). @@ -184,22 +185,42 @@ get_db_info(Db) -> ], {ok, InfoList}. +check_is_admin(#db{admins=Admins, user_ctx=#user_ctx{name=Name,roles=Roles}}) -> + DbAdmins = [<<"_admin">> | Admins], + case DbAdmins -- [Name | Roles] of + DbAdmins -> % same list, not an admin + throw({unauthorized, <<"You are not a db or server admin.">>}); + _ -> + ok + end. + get_admins(#db{admins=Admins}) -> Admins. -set_admins(#db{update_pid=UpdatePid,user_ctx=Ctx}, - Admins) when is_list(Admins) -> - case gen_server:call(UpdatePid, {set_admins, Admins, Ctx}, infinity) of - ok -> ok; - Error -> throw(Error) - end. +set_admins(#db{update_pid=Pid}=Db, Admins) when is_list(Admins) -> + check_is_admin(Db), + gen_server:call(Pid, {set_admins, Admins}, infinity). + + +get_revs_limit(#db{revs_limit=Limit}) -> + Limit. + +set_revs_limit(#db{update_pid=Pid}=Db, Limit) when Limit > 0 -> + check_is_admin(Db), + gen_server:call(Pid, {set_revs_limit, Limit}, infinity); +set_revs_limit(_Db, _Limit) -> + throw(invalid_revs_limit). name(#db{name=Name}) -> Name. update_doc(Db, Doc, Options) -> - {ok, [NewRev]} = update_docs(Db, [Doc], Options), - {ok, NewRev}. + case update_docs(Db, [Doc], Options) of + {ok, [{ok, NewRev}]} -> + {ok, NewRev}; + {ok, [Error]} -> + throw(Error) + end. update_docs(Db, Docs) -> update_docs(Db, Docs, []). @@ -227,146 +248,283 @@ group_alike_docs([Doc|Rest], [Bucket|RestBuckets]) -> validate_doc_update(#db{user_ctx=UserCtx, admins=Admins}, - #doc{id= <<"_design/",_/binary>>}=Doc, _GetDiskDocFun) -> + #doc{id= <<"_design/",_/binary>>}, _GetDiskDocFun) -> UserNames = [UserCtx#user_ctx.name | UserCtx#user_ctx.roles], % if the user is a server admin or db admin, allow the save case length(UserNames -- [<<"_admin">> | Admins]) == length(UserNames) of true -> % not an admin - throw({unauthorized, <<"You are not a server or database admin.">>}); + {unauthorized, <<"You are not a server or database admin.">>}; false -> - Doc + ok end; -validate_doc_update(#db{validate_doc_funs=[]}, Doc, _GetDiskDocFun) -> - Doc; -validate_doc_update(_Db, #doc{id= <<"_local/",_/binary>>}=Doc, _GetDiskDocFun) -> - Doc; +validate_doc_update(#db{validate_doc_funs=[]}, _Doc, _GetDiskDocFun) -> + ok; +validate_doc_update(_Db, #doc{id= <<"_local/",_/binary>>}, _GetDiskDocFun) -> + ok; validate_doc_update(#db{name=DbName,user_ctx=Ctx}=Db, Doc, GetDiskDocFun) -> DiskDoc = GetDiskDocFun(), JsonCtx = {[{<<"db">>, DbName}, {<<"name">>,Ctx#user_ctx.name}, {<<"roles">>,Ctx#user_ctx.roles}]}, - [case Fun(Doc, DiskDoc, JsonCtx) of - ok -> ok; - Error -> throw(Error) - end || Fun <- Db#db.validate_doc_funs], - Doc. + try [case Fun(Doc, DiskDoc, JsonCtx) of + ok -> ok; + Error -> throw(Error) + end || Fun <- Db#db.validate_doc_funs], + ok + catch + throw:Error -> + Error + end. -prep_and_validate_new_edit(Db, #doc{id=Id,revs=[NewRev|PrevRevs]}=Doc, +prep_and_validate_update(Db, #doc{id=Id,revs={RevStart, [_NewRev|PrevRevs]}}=Doc, OldFullDocInfo, LeafRevsDict) -> case PrevRevs of [PrevRev|_] -> - case dict:find(PrevRev, LeafRevsDict) of - {ok, {Deleted, Sp, DiskRevs}} -> - Doc2 = Doc#doc{revs=[NewRev|DiskRevs]}, - case couch_doc:has_stubs(Doc2) of + case dict:find({RevStart-1, PrevRev}, LeafRevsDict) of + {ok, {Deleted, DiskSp, DiskRevs}} -> + case couch_doc:has_stubs(Doc) of true -> - DiskDoc = make_doc(Db, Id, Deleted, Sp, DiskRevs), - Doc3 = couch_doc:merge_stubs(Doc2, DiskDoc), - validate_doc_update(Db, Doc3, fun() -> DiskDoc end); + DiskDoc = make_doc(Db, Id, Deleted, DiskSp, DiskRevs), + Doc2 = couch_doc:merge_stubs(Doc, DiskDoc), + {validate_doc_update(Db, Doc2, fun() -> DiskDoc end), Doc2}; false -> - LoadDiskDoc = fun() -> make_doc(Db,Id,Deleted,Sp,DiskRevs) end, - validate_doc_update(Db, Doc2, LoadDiskDoc) + LoadDiskDoc = fun() -> make_doc(Db,Id,Deleted,DiskSp,DiskRevs) end, + {validate_doc_update(Db, Doc, LoadDiskDoc), Doc} end; error -> - throw(conflict) + {conflict, Doc} end; [] -> % new doc, and we have existing revs. if OldFullDocInfo#full_doc_info.deleted -> % existing docs are deletions - validate_doc_update(Db, Doc, nil); + {validate_doc_update(Db, Doc, fun() -> nil end), Doc}; true -> - throw(conflict) + {conflict, Doc} end end. + + +prep_and_validate_updates(_Db, [], [], AccPrepped, AccFatalErrors) -> + {AccPrepped, AccFatalErrors}; +prep_and_validate_updates(Db, [DocBucket|RestBuckets], [not_found|RestLookups], AccPrepped, AccErrors) -> + [#doc{id=Id}|_]=DocBucket, + % no existing revs are known, + {PreppedBucket, AccErrors3} = lists:foldl( + fun(#doc{revs=Revs}=Doc, {AccBucket, AccErrors2}) -> + case Revs of + {Pos, [NewRev,_OldRev|_]} -> + % old revs specified but none exist, a conflict + {AccBucket, [{{Id, {Pos, NewRev}}, conflict} | AccErrors2]}; + {Pos, [NewRev]} -> + case validate_doc_update(Db, Doc, fun() -> nil end) of + ok -> + {[Doc | AccBucket], AccErrors2}; + Error -> + {AccBucket, [{{Id, {Pos, NewRev}}, Error} | AccErrors2]} + end + end + end, + {[], AccErrors}, DocBucket), + + prep_and_validate_updates(Db, RestBuckets, RestLookups, + [PreppedBucket | AccPrepped], AccErrors3); +prep_and_validate_updates(Db, [DocBucket|RestBuckets], + [{ok, #full_doc_info{rev_tree=OldRevTree}=OldFullDocInfo}|RestLookups], + AccPrepped, AccErrors) -> + Leafs = couch_key_tree:get_all_leafs(OldRevTree), + LeafRevsDict = dict:from_list([{{Start, RevId}, {Deleted, Sp, Revs}} || + {{Deleted, Sp}, {Start, [RevId|_]}=Revs} <- Leafs]), + {PreppedBucket, AccErrors3} = lists:foldl( + fun(Doc, {Docs2Acc, AccErrors2}) -> + case prep_and_validate_update(Db, Doc, OldFullDocInfo, + LeafRevsDict) of + {ok, Doc} -> + {[Doc | Docs2Acc], AccErrors2}; + {Error, #doc{id=Id,revs={Pos, [NewRev|_]}}} -> + % Record the error + {Docs2Acc, [{{Id, {Pos, NewRev}}, Error} |AccErrors2]} + end + end, + {[], AccErrors}, DocBucket), + prep_and_validate_updates(Db, RestBuckets, RestLookups, [PreppedBucket | AccPrepped], AccErrors3). + + update_docs(#db{update_pid=UpdatePid}=Db, Docs, Options) -> - update_docs(#db{update_pid=UpdatePid}=Db, Docs, Options, true). + update_docs(#db{update_pid=UpdatePid}=Db, Docs, Options, interactive_edit). -update_docs(Db, Docs, Options, false) -> + +prep_and_validate_replicated_updates(_Db, [], [], AccPrepped, AccErrors) -> + Errors2 = [{{Id, {Pos, Rev}}, Error} || + {#doc{id=Id,revs={Pos,[Rev|_]}}, Error} <- AccErrors], + {lists:reverse(AccPrepped), lists:reverse(Errors2)}; +prep_and_validate_replicated_updates(Db, [Bucket|RestBuckets], [OldInfo|RestOldInfo], AccPrepped, AccErrors) -> + case OldInfo of + not_found -> + {ValidatedBucket, AccErrors3} = lists:foldl( + fun(Doc, {AccPrepped2, AccErrors2}) -> + case validate_doc_update(Db, Doc, fun() -> nil end) of + ok -> + {[Doc | AccPrepped2], AccErrors2}; + Error -> + {AccPrepped2, [{Doc, Error} | AccErrors2]} + end + end, + {[], AccErrors}, Bucket), + prep_and_validate_replicated_updates(Db, RestBuckets, RestOldInfo, [ValidatedBucket | AccPrepped], AccErrors3); + {ok, #full_doc_info{rev_tree=OldTree}} -> + NewRevTree = lists:foldl( + fun(NewDoc, AccTree) -> + {NewTree, _} = couch_key_tree:merge(AccTree, [couch_db:doc_to_tree(NewDoc)]), + NewTree + end, + OldTree, Bucket), + Leafs = couch_key_tree:get_all_leafs_full(NewRevTree), + LeafRevsFullDict = dict:from_list( [{{Start, RevId}, FullPath} || {Start, [{RevId, _}|_]}=FullPath <- Leafs]), + {ValidatedBucket, AccErrors3} = + lists:foldl( + fun(#doc{id=Id,revs={Pos, [RevId|_]}}=Doc, {AccValidated, AccErrors2}) -> + case dict:find({Pos, RevId}, LeafRevsFullDict) of + {ok, {Start, Path}} -> + % our unflushed doc is a leaf node. Go back on the path + % to find the previous rev that's on disk. + PrevRevResult = + case couch_doc:has_stubs(Doc) of + true -> + [_PrevRevFull | [PrevRevFull | _]=PrevPath] = Path, + case PrevRevFull of + {_RevId, ?REV_MISSING} -> + conflict; + {RevId, {IsDel, DiskSp}} -> + DiskDoc = make_doc(Db, Id, IsDel, DiskSp, PrevPath), + Doc2 = couch_doc:merge_stubs(Doc, DiskDoc), + {ok, Doc2, fun() -> DiskDoc end} + end; + false -> + {ok, Doc, + fun() -> + make_first_doc_on_disk(Db,Id,Start-1, tl(Path)) + end} + end, + case PrevRevResult of + {ok, NewDoc, LoadPrevRevFun} -> + case validate_doc_update(Db, NewDoc, LoadPrevRevFun) of + ok -> + {[NewDoc | AccValidated], AccErrors2}; + Error -> + {AccValidated, [{NewDoc, Error} | AccErrors2]} + end; + Error -> + {AccValidated, [{Doc, Error} | AccErrors2]} + end; + _ -> + % this doc isn't a leaf or already exists in the tree. + % ignore but consider it a success. + {AccValidated, AccErrors2} + end + end, + {[], AccErrors}, Bucket), + prep_and_validate_replicated_updates(Db, RestBuckets, RestOldInfo, [ValidatedBucket | AccPrepped], AccErrors3) + end. + +update_docs(Db, Docs, Options, replicated_changes) -> couch_stats_collector:increment({couchdb, database_writes}), DocBuckets = group_alike_docs(Docs), - Ids = [Id || [#doc{id=Id}|_] <- DocBuckets], - ExistingDocs = get_full_doc_infos(Db, Ids), + case (Db#db.validate_doc_funs /= []) orelse + lists:any( + fun(#doc{id= <>}) -> true; + (_) -> false + end, Docs) of + true -> + Ids = [Id || [#doc{id=Id}|_] <- DocBuckets], + ExistingDocs = get_full_doc_infos(Db, Ids), - DocBuckets2 = lists:zipwith( - fun(Bucket, not_found) -> - [validate_doc_update(Db, Doc, fun()-> nil end) || Doc <- Bucket]; - (Bucket, {ok, #full_doc_info{rev_tree=OldRevTree}}) -> - NewTree = lists:foldl( - fun(Doc, RevTreeAcc) -> - couch_key_tree:merge(RevTreeAcc, doc_to_tree(Doc)) - end, - OldRevTree, Bucket), - Leafs = couch_key_tree:get_all_leafs_full(NewTree), - LeafRevsFullDict = dict:from_list( [{Rev, FullPath} || [{Rev, _}|_]=FullPath <- Leafs]), - lists:flatmap( - fun(#doc{revs=[Rev|_]}=Doc) -> - case dict:find(Rev, LeafRevsFullDict) of - {ok, [{Rev, #doc{id=Id}}|_]=Path} -> - % our unflushed doc is a leaf node. Go back on the path - % to find the previous rev that's on disk. - LoadPrevRev = fun() -> - make_first_doc_on_disk(Db, Id, Path) - end, - [validate_doc_update(Db, Doc, LoadPrevRev)]; - _ -> - % this doc isn't a leaf or is already exists in the tree. ignore - [] - end - end, Bucket) - end, - DocBuckets, ExistingDocs), - write_and_commit(Db, DocBuckets2, Options); + {DocBuckets2, DocErrors} = + prep_and_validate_replicated_updates(Db, DocBuckets, ExistingDocs, [], []), + DocBuckets3 = [Bucket || [_|_]=Bucket <- DocBuckets2]; % remove empty buckets + false -> + DocErrors = [], + DocBuckets3 = DocBuckets + end, + {ok, []} = write_and_commit(Db, DocBuckets3, [merge_conflicts | Options]), + {ok, DocErrors}; -update_docs(Db, Docs, Options, true) -> +update_docs(Db, Docs, Options, interactive_edit) -> couch_stats_collector:increment({couchdb, database_writes}), - + AllOrNothing = lists:member(all_or_nothing, Options), % go ahead and generate the new revision ids for the documents. Docs2 = lists:map( - fun(#doc{id=Id,revs=Revs}=Doc) -> + fun(#doc{id=Id,revs={Start, RevIds}}=Doc) -> case Id of <> -> - Rev = case Revs of [] -> 0; [Rev0|_] -> list_to_integer(binary_to_list(Rev0)) end, - Doc#doc{revs=[list_to_binary(integer_to_list(Rev + 1))]}; + Rev = case RevIds of [] -> 0; [Rev0|_] -> list_to_integer(?b2l(Rev0)) end, + Doc#doc{revs={Start, [?l2b(integer_to_list(Rev + 1))]}}; _ -> - Doc#doc{revs=[list_to_binary(integer_to_list(couch_util:rand32())) | Revs]} + Doc#doc{revs={Start+1, [?l2b(integer_to_list(couch_util:rand32())) | RevIds]}} end end, Docs), DocBuckets = group_alike_docs(Docs2), - Ids = [Id || [#doc{id=Id}|_] <- DocBuckets], - - % lookup the doc by id and get the most recent - - ExistingDocs = get_full_doc_infos(Db, Ids), - DocBuckets2 = lists:zipwith( - fun(Bucket, not_found) -> - % no existing revs on disk, make sure no old revs specified. - [throw(conflict) || #doc{revs=[_NewRev, _OldRev | _]} <- Bucket], - [validate_doc_update(Db, Doc, fun()-> nil end) || Doc <- Bucket]; - (Bucket, {ok, #full_doc_info{rev_tree=OldRevTree}=OldFullDocInfo}) -> - Leafs = couch_key_tree:get_all_leafs(OldRevTree), - LeafRevsDict = dict:from_list([{Rev, {Deleted, Sp, Revs}} || {Rev, {Deleted, Sp}, Revs} <- Leafs]), - [prep_and_validate_new_edit(Db, Doc, OldFullDocInfo, LeafRevsDict) || Doc <- Bucket] + case (Db#db.validate_doc_funs /= []) orelse + lists:any( + fun(#doc{id= <>}) -> + true; + (#doc{attachments=Atts}) -> + Atts /= [] + end, Docs) of + true -> + % lookup the doc by id and get the most recent + Ids = [Id || [#doc{id=Id}|_] <- DocBuckets], + ExistingDocInfos = get_full_doc_infos(Db, Ids), + + {DocBucketsPrepped, Failures} = + case AllOrNothing of + true -> + prep_and_validate_replicated_updates(Db, DocBuckets, + ExistingDocInfos, [], []); + false -> + prep_and_validate_updates(Db, DocBuckets, ExistingDocInfos, [], []) end, - DocBuckets, ExistingDocs), - ok = write_and_commit(Db, DocBuckets2, [new_edits | Options]), - {ok, [NewRev ||#doc{revs=[NewRev|_]} <- Docs2]}. + + % strip out any empty buckets + DocBuckets2 = [Bucket || [_|_] = Bucket <- DocBucketsPrepped]; + false -> + Failures = [], + DocBuckets2 = DocBuckets + end, + if (AllOrNothing) and (Failures /= []) -> + {aborted, Failures}; + true -> + Options2 = if AllOrNothing -> [merge_conflicts]; + true -> [] end ++ Options, + {ok, CommitFailures} = write_and_commit(Db, DocBuckets2, Options2), + FailDict = dict:from_list(CommitFailures ++ Failures), + % the output for each is either {ok, NewRev} or Error + {ok, lists:map( + fun(#doc{id=Id,revs={Pos, [NewRevId|_]}}) -> + case dict:find({Id, {Pos, NewRevId}}, FailDict) of + {ok, Error} -> + Error; + error -> + {ok, {Pos, NewRevId}} + end + end, Docs2)} + end. % Returns the first available document on disk. Input list is a full rev path % for the doc. -make_first_doc_on_disk(_Db, _Id, []) -> +make_first_doc_on_disk(_Db, _Id, _Pos, []) -> nil; -make_first_doc_on_disk(Db, Id, [{_Rev, ?REV_MISSING}|RestPath]) -> - make_first_doc_on_disk(Db, Id, RestPath); -make_first_doc_on_disk(Db, Id, [{_Rev, {IsDel, Sp}} |_]=DocPath) -> +make_first_doc_on_disk(Db, Id, Pos, [{_Rev, ?REV_MISSING}|RestPath]) -> + make_first_doc_on_disk(Db, Id, Pos - 1, RestPath); +make_first_doc_on_disk(Db, Id, Pos, [{_Rev, {IsDel, Sp}} |_]=DocPath) -> Revs = [Rev || {Rev, _} <- DocPath], - make_doc(Db, Id, IsDel, Sp, Revs). + make_doc(Db, Id, IsDel, Sp, {Pos, Revs}). write_and_commit(#db{update_pid=UpdatePid, user_ctx=Ctx}=Db, DocBuckets, @@ -374,20 +532,18 @@ write_and_commit(#db{update_pid=UpdatePid, user_ctx=Ctx}=Db, DocBuckets, % flush unwritten binaries to disk. DocBuckets2 = [[doc_flush_binaries(Doc, Db#db.fd) || Doc <- Bucket] || Bucket <- DocBuckets], case gen_server:call(UpdatePid, {update_docs, DocBuckets2, Options}, infinity) of - ok -> ok; + {ok, Conflicts} -> {ok, Conflicts}; retry -> % This can happen if the db file we wrote to was swapped out by - % compaction. Retry writing to the current file + % compaction. Retry by reopening the db and writing to the current file {ok, Db2} = open_ref_counted(Db#db.main_pid, Ctx), DocBuckets3 = [[doc_flush_binaries(Doc, Db2#db.fd) || Doc <- Bucket] || Bucket <- DocBuckets], % We only retry once close(Db2), case gen_server:call(UpdatePid, {update_docs, DocBuckets3, Options}, infinity) of - ok -> ok; - Else -> throw(Else) - end; - Else-> - throw(Else) + {ok, Conflicts} -> {ok, Conflicts}; + retry -> throw({update_error, compaction_retry}) + end end. @@ -506,7 +662,7 @@ enum_docs_reduce_to_count(Reds) -> {Count, _DelCount} = couch_btree:final_reduce( fun couch_db_updater:btree_by_id_reduce/2, Reds), Count. - + count_changes_since(Db, SinceSeq) -> {ok, Changes} = couch_btree:fold_reduce(Db#db.docinfo_by_seq_btree, @@ -518,7 +674,7 @@ count_changes_since(Db, SinceSeq) -> end, ok), Changes. - + enum_docs_since(Db, SinceSeq, Direction, InFun, Ctx) -> couch_btree:fold(Db#db.docinfo_by_seq_btree, SinceSeq + 1, Direction, InFun, Ctx). @@ -594,11 +750,11 @@ open_doc_revs_int(Db, IdRevs, Options) -> end end, FoundResults = - lists:map(fun({Rev, Value, FoundRevPath}) -> + lists:map(fun({Value, {Pos, [Rev|_]}=FoundRevPath}) -> case Value of ?REV_MISSING -> % we have the rev in our list but know nothing about it - {{not_found, missing}, Rev}; + {{not_found, missing}, {Pos, Rev}}; {IsDeleted, SummaryPtr} -> {ok, make_doc(Db, Id, IsDeleted, SummaryPtr, FoundRevPath)} end @@ -616,18 +772,18 @@ open_doc_revs_int(Db, IdRevs, Options) -> open_doc_int(Db, <> = Id, _Options) -> case couch_btree:lookup(Db#db.local_docs_btree, [Id]) of [{ok, {_, {Rev, BodyData}}}] -> - {ok, #doc{id=Id, revs=[list_to_binary(integer_to_list(Rev))], body=BodyData}}; + {ok, #doc{id=Id, revs={0, [list_to_binary(integer_to_list(Rev))]}, body=BodyData}}; [not_found] -> {not_found, missing} end; -open_doc_int(Db, #doc_info{id=Id,rev=Rev,deleted=IsDeleted,summary_pointer=Sp}=DocInfo, Options) -> - Doc = make_doc(Db, Id, IsDeleted, Sp, [Rev]), +open_doc_int(Db, #doc_info{id=Id,rev={Pos,RevId},deleted=IsDeleted,summary_pointer=Sp}=DocInfo, Options) -> + Doc = make_doc(Db, Id, IsDeleted, Sp, {Pos,[RevId]}), {ok, Doc#doc{meta=doc_meta_info(DocInfo, [], Options)}}; open_doc_int(Db, #full_doc_info{id=Id,rev_tree=RevTree}=FullDocInfo, Options) -> #doc_info{deleted=IsDeleted,rev=Rev,summary_pointer=Sp} = DocInfo = couch_doc:to_doc_info(FullDocInfo), - {[{_Rev,_Value, Revs}], []} = couch_key_tree:get(RevTree, [Rev]), - Doc = make_doc(Db, Id, IsDeleted, Sp, Revs), + {[{_, RevPath}], []} = couch_key_tree:get(RevTree, [Rev]), + Doc = make_doc(Db, Id, IsDeleted, Sp, RevPath), {ok, Doc#doc{meta=doc_meta_info(DocInfo, RevTree, Options)}}; open_doc_int(Db, Id, Options) -> case get_full_doc_info(Db, Id) of @@ -641,9 +797,10 @@ doc_meta_info(DocInfo, RevTree, Options) -> case lists:member(revs_info, Options) of false -> []; true -> - {[RevPath],[]} = + {[{Pos, RevPath}],[]} = couch_key_tree:get_full_key_paths(RevTree, [DocInfo#doc_info.rev]), - [{revs_info, lists:map( + + [{revs_info, Pos, lists:map( fun({Rev, {true, _Sp}}) -> {Rev, deleted}; ({Rev, {false, _Sp}}) -> @@ -670,13 +827,15 @@ doc_meta_info(DocInfo, RevTree, Options) -> end. -doc_to_tree(Doc) -> - doc_to_tree(Doc, lists:reverse(Doc#doc.revs)). +doc_to_tree(#doc{revs={Start, RevIds}}=Doc) -> + [Tree] = doc_to_tree_simple(Doc, lists:reverse(RevIds)), + {Start - length(RevIds) + 1, Tree}. + -doc_to_tree(Doc, [RevId]) -> +doc_to_tree_simple(Doc, [RevId]) -> [{RevId, Doc, []}]; -doc_to_tree(Doc, [RevId | Rest]) -> - [{RevId, ?REV_MISSING, doc_to_tree(Doc, Rest)}]. +doc_to_tree_simple(Doc, [RevId | Rest]) -> + [{RevId, ?REV_MISSING, doc_to_tree_simple(Doc, Rest)}]. make_doc(Db, FullDocInfo) -> {#doc_info{id=Id,deleted=Deleted,summary_pointer=Sp}, RevPath} @@ -703,4 +862,4 @@ make_doc(#db{fd=Fd}=Db, Id, Deleted, BodySp, RevisionPath) -> }. - \ No newline at end of file + diff --git a/src/couchdb/couch_db.hrl b/src/couchdb/couch_db.hrl index 026afe14..f460f450 100644 --- a/src/couchdb/couch_db.hrl +++ b/src/couchdb/couch_db.hrl @@ -69,7 +69,7 @@ -record(doc, { id = <<"">>, - revs = [], + revs = {0, []}, % the json body object. body = {[]}, @@ -104,7 +104,7 @@ % if the disk revision is incremented, then new upgrade logic will need to be % added to couch_db_updater:init_db. --define(LATEST_DISK_VERSION, 0). +-define(LATEST_DISK_VERSION, 1). -record(db_header, {disk_version = ?LATEST_DISK_VERSION, @@ -115,13 +115,14 @@ local_docs_btree_state = nil, purge_seq = 0, purged_docs = nil, - admins_ptr = nil + admins_ptr = nil, + revs_limit = 1000 }). -record(db, - {main_pid=nil, - update_pid=nil, - compactor_pid=nil, + {main_pid = nil, + update_pid = nil, + compactor_pid = nil, instance_start_time, % number of microsecs since jan 1 1970 as a binary string fd, fd_ref_counter, @@ -133,11 +134,12 @@ update_seq, name, filepath, - validate_doc_funs=[], - admins=[], - admins_ptr=nil, - user_ctx=#user_ctx{}, - waiting_delayed_commit=nil + validate_doc_funs = [], + admins = [], + admins_ptr = nil, + user_ctx = #user_ctx{}, + waiting_delayed_commit = nil, + revs_limit = 1000 }). diff --git a/src/couchdb/couch_db_updater.erl b/src/couchdb/couch_db_updater.erl index 7790d7a4..7752a577 100644 --- a/src/couchdb/couch_db_updater.erl +++ b/src/couchdb/couch_db_updater.erl @@ -44,15 +44,13 @@ handle_call(get_db, _From, Db) -> {reply, {ok, Db}, Db}; handle_call({update_docs, DocActions, Options}, _From, Db) -> try update_docs_int(Db, DocActions, Options) of - {ok, Db2} -> + {ok, Conflicts, Db2} -> ok = gen_server:call(Db#db.main_pid, {db_updated, Db2}), couch_db_update_notifier:notify({updated, Db2#db.name}), - {reply, ok, Db2} + {reply, {ok, Conflicts}, Db2} catch throw: retry -> - {reply, retry, Db}; - throw: conflict -> - {reply, conflict, Db} + {reply, retry, Db} end; handle_call(full_commit, _From, #db{waiting_delayed_commit=nil}=Db) -> {reply, ok, Db}; % no data waiting, return ok immediately @@ -64,18 +62,18 @@ handle_call(increment_update_seq, _From, Db) -> couch_db_update_notifier:notify({updated, Db#db.name}), {reply, {ok, Db2#db.update_seq}, Db2}; -handle_call({set_admins, NewAdmins, #user_ctx{roles=Roles}}, _From, Db) -> - DbAdmins = [<<"_admin">> | Db#db.admins], - case length(DbAdmins -- Roles) == length(DbAdmins) of - true -> - {reply, {unauthorized, <<"You are not a db or server admin.">>}, Db}; - false -> - {ok, Ptr} = couch_file:append_term(Db#db.fd, NewAdmins), - Db2 = commit_data(Db#db{admins=NewAdmins, admins_ptr=Ptr, - update_seq=Db#db.update_seq+1}), - ok = gen_server:call(Db2#db.main_pid, {db_updated, Db2}), - {reply, ok, Db2} - end; +handle_call({set_admins, NewAdmins}, _From, Db) -> + {ok, Ptr} = couch_file:append_term(Db#db.fd, NewAdmins), + Db2 = commit_data(Db#db{admins=NewAdmins, admins_ptr=Ptr, + update_seq=Db#db.update_seq+1}), + ok = gen_server:call(Db2#db.main_pid, {db_updated, Db2}), + {reply, ok, Db2}; + +handle_call({set_revs_limit, Limit}, _From, Db) -> + Db2 = commit_data(Db#db{revs_limit=Limit, + update_seq=Db#db.update_seq+1}), + ok = gen_server:call(Db2#db.main_pid, {db_updated, Db2}), + {reply, ok, Db2}; handle_call({purge_docs, _IdRevs}, _From, #db{compactor_pid=Pid}=Db) when Pid /= nil -> @@ -298,7 +296,8 @@ init_db(DbName, Filepath, Fd, Header0) -> filepath = Filepath, admins = Admins, admins_ptr = AdminsPtr, - instance_start_time = StartTime + instance_start_time = StartTime, + revs_limit = Header#db_header.revs_limit }. @@ -358,40 +357,31 @@ flush_trees(#db{fd=Fd}=Db, [InfoUnflushed | RestUnflushed], AccFlushed) -> end, Unflushed), flush_trees(Db, RestUnflushed, [InfoUnflushed#full_doc_info{rev_tree=Flushed} | AccFlushed]). -merge_rev_trees(_NoConflicts, [], [], AccNewInfos, AccSeq) -> - {ok, lists:reverse(AccNewInfos), AccSeq}; -merge_rev_trees(NoConflicts, [NewDocs|RestDocsList], - [OldDocInfo|RestOldInfo], AccNewInfos, AccSeq) -> - #full_doc_info{id=Id,rev_tree=OldTree}=OldDocInfo, - UpdatesRevTree = lists:foldl( - fun(NewDoc, AccTree) -> - couch_key_tree:merge(AccTree, couch_db:doc_to_tree(NewDoc)) +merge_rev_trees(_MergeConflicts, [], [], AccNewInfos, AccConflicts, AccSeq) -> + {ok, lists:reverse(AccNewInfos), AccConflicts, AccSeq}; +merge_rev_trees(MergeConflicts, [NewDocs|RestDocsList], + [OldDocInfo|RestOldInfo], AccNewInfos, AccConflicts, AccSeq) -> + #full_doc_info{id=Id,rev_tree=OldTree,deleted=OldDeleted}=OldDocInfo, + {NewRevTree, NewConflicts} = lists:foldl( + fun(#doc{revs={Pos,[Rev|_]}}=NewDoc, {AccTree, AccConflicts2}) -> + case couch_key_tree:merge(AccTree, [couch_db:doc_to_tree(NewDoc)]) of + {_NewTree, conflicts} + when (not OldDeleted) and (not MergeConflicts) -> + {AccTree, [{{Id, {Pos,Rev}}, conflict} | AccConflicts2]}; + {NewTree, _} -> + {NewTree, AccConflicts2} + end end, - [], NewDocs), - NewRevTree = couch_key_tree:merge(OldTree, UpdatesRevTree), + {OldTree, AccConflicts}, NewDocs), if NewRevTree == OldTree -> % nothing changed - merge_rev_trees(NoConflicts, RestDocsList, RestOldInfo, AccNewInfos, AccSeq); + merge_rev_trees(MergeConflicts, RestDocsList, RestOldInfo, AccNewInfos, + NewConflicts, AccSeq); true -> - if NoConflicts andalso OldTree /= [] -> - OldConflicts = couch_key_tree:count_leafs(OldTree), - NewConflicts = couch_key_tree:count_leafs(NewRevTree), - if NewConflicts > OldConflicts -> - % if all the old docs are deletions, allow this new conflict - case [1 || {_Rev,{IsDel,_Sp},_Path} <- - couch_key_tree:get_all_leafs(OldTree), IsDel==false] of - [] -> - ok; - _ -> - throw(conflict) - end; - true -> ok - end; - true -> ok - end, + % we have updated the document, give it a new seq # NewInfo = #full_doc_info{id=Id,update_seq=AccSeq+1,rev_tree=NewRevTree}, - merge_rev_trees(NoConflicts, RestDocsList,RestOldInfo, - [NewInfo|AccNewInfos],AccSeq+1) + merge_rev_trees(MergeConflicts, RestDocsList,RestOldInfo, + [NewInfo|AccNewInfos], NewConflicts, AccSeq+1) end. new_index_entries([], AccById, AccBySeq) -> @@ -402,19 +392,23 @@ new_index_entries([FullDocInfo|RestInfos], AccById, AccBySeq) -> [FullDocInfo#full_doc_info{deleted=Deleted}|AccById], [DocInfo|AccBySeq]). + +stem_full_doc_infos(#db{revs_limit=Limit}, DocInfos) -> + [Info#full_doc_info{rev_tree=couch_key_tree:stem(Tree, Limit)} || + #full_doc_info{rev_tree=Tree}=Info <- DocInfos]. + + update_docs_int(Db, DocsList, Options) -> #db{ fulldocinfo_by_id_btree = DocInfoByIdBTree, docinfo_by_seq_btree = DocInfoBySeqBTree, update_seq = LastSeq } = Db, - % separate out the NonRep documents from the rest of the documents {DocsList2, NonRepDocs} = lists:foldl( - fun([#doc{id=Id}=Doc | Rest]=Docs, {DocsListAcc, NonRepDocsAcc}) -> + fun([#doc{id=Id}=Doc | _]=Docs, {DocsListAcc, NonRepDocsAcc}) -> case Id of - <> when Rest==[] -> - % when saving NR (non rep) documents, you can only save a single rev + <> -> {DocsListAcc, [Doc | NonRepDocsAcc]}; Id-> {[Docs | DocsListAcc], NonRepDocsAcc} @@ -434,23 +428,26 @@ update_docs_int(Db, DocsList, Options) -> Ids, OldDocLookups), % Merge the new docs into the revision trees. - NoConflicts = lists:member(new_edits, Options), - {ok, NewDocInfos, NewSeq} = merge_rev_trees(NoConflicts, DocsList2, OldDocInfos, [], LastSeq), + {ok, NewDocInfos0, Conflicts, NewSeq} = merge_rev_trees( + lists:member(merge_conflicts, Options), + DocsList2, OldDocInfos, [], [], LastSeq), + + NewDocInfos = stem_full_doc_infos(Db, NewDocInfos0), RemoveSeqs = - [ OldSeq || {ok, #full_doc_info{update_seq=OldSeq}} <- OldDocLookups], + [OldSeq || {ok, #full_doc_info{update_seq=OldSeq}} <- OldDocLookups], - % All regular documents are now ready to write. + % All documents are now ready to write. - % Try to write the local documents first, a conflict might be generated - {ok, Db2} = update_local_docs(Db, NonRepDocs), + {ok, LocalConflicts, Db2} = update_local_docs(Db, NonRepDocs), - % Write out the document summaries (they are stored in the nodes of the rev trees) + % Write out the document summaries (the bodies are stored in the nodes of + % the trees, the attachments are already written to disk) {ok, FlushedDocInfos} = flush_trees(Db2, NewDocInfos, []), {ok, InfoById, InfoBySeq} = new_index_entries(FlushedDocInfos, [], []), - % and the indexes to the documents + % and the indexes {ok, DocInfoBySeqBTree2} = couch_btree:add_remove(DocInfoBySeqBTree, InfoBySeq, RemoveSeqs), {ok, DocInfoByIdBTree2} = couch_btree:add_remove(DocInfoByIdBTree, InfoById, []), @@ -459,6 +456,8 @@ update_docs_int(Db, DocsList, Options) -> docinfo_by_seq_btree = DocInfoBySeqBTree2, update_seq = NewSeq}, + % Check if we just updated any design documents, and update the validation + % funs if we did. case [1 || <<"_design/",_/binary>> <- Ids] of [] -> Db4 = Db3; @@ -466,18 +465,15 @@ update_docs_int(Db, DocsList, Options) -> Db4 = refresh_validate_doc_funs(Db3) end, - {ok, commit_data(Db4, not lists:member(full_commit, Options))}. - + {ok, LocalConflicts ++ Conflicts, + commit_data(Db4, not lists:member(full_commit, Options))}. + update_local_docs(#db{local_docs_btree=Btree}=Db, Docs) -> Ids = [Id || #doc{id=Id} <- Docs], OldDocLookups = couch_btree:lookup(Btree, Ids), BtreeEntries = lists:zipwith( - fun(#doc{id=Id,deleted=Delete,revs=Revs,body=Body}, OldDocLookup) -> - NewRev = - case Revs of - [] -> 0; - [RevStr|_] -> list_to_integer(binary_to_list(RevStr)) - end, + fun(#doc{id=Id,deleted=Delete,revs={0,[RevStr]},body=Body}, OldDocLookup) -> + NewRev = list_to_integer(?b2l(RevStr)), OldRev = case OldDocLookup of {ok, {_, {OldRev0, _}}} -> OldRev0; @@ -490,18 +486,19 @@ update_local_docs(#db{local_docs_btree=Btree}=Db, Docs) -> true -> {remove, Id} end; false -> - throw(conflict) + {conflict, {Id, {0, RevStr}}} end end, Docs, OldDocLookups), BtreeIdsRemove = [Id || {remove, Id} <- BtreeEntries], BtreeIdsUpdate = [ByIdDocInfo || {update, ByIdDocInfo} <- BtreeEntries], - + Conflicts = [{conflict, IdRev} || {conflict, IdRev} <- BtreeEntries], + {ok, Btree2} = couch_btree:add_remove(Btree, BtreeIdsUpdate, BtreeIdsRemove), - {ok, Db#db{local_docs_btree = Btree2}}. + {ok, Conflicts, Db#db{local_docs_btree = Btree2}}. commit_data(Db) -> @@ -515,7 +512,8 @@ commit_data(#db{fd=Fd, header=Header} = Db, Delay) -> docinfo_by_seq_btree_state = couch_btree:get_state(Db#db.docinfo_by_seq_btree), fulldocinfo_by_id_btree_state = couch_btree:get_state(Db#db.fulldocinfo_by_id_btree), local_docs_btree_state = couch_btree:get_state(Db#db.local_docs_btree), - admins_ptr = Db#db.admins_ptr + admins_ptr = Db#db.admins_ptr, + revs_limit = Db#db.revs_limit }, if Header == Header2 -> Db; @@ -549,6 +547,10 @@ copy_raw_doc(SrcFd, SrcSp, DestFd, DestStream) -> copy_rev_tree(_SrcFd, _DestFd, _DestStream, []) -> []; +copy_rev_tree(SrcFd, DestFd, DestStream, [{Start, Tree} | RestTree]) -> + % root nner node, only copy info/data from leaf nodes + [Tree2] = copy_rev_tree(SrcFd, DestFd, DestStream, [Tree]), + [{Start, Tree2} | copy_rev_tree(SrcFd, DestFd, DestStream, RestTree)]; copy_rev_tree(SrcFd, DestFd, DestStream, [{RevId, {IsDel, Sp}, []} | RestTree]) -> % This is a leaf node, copy it over NewSp = copy_raw_doc(SrcFd, Sp, DestFd, DestStream), @@ -560,10 +562,11 @@ copy_rev_tree(SrcFd, DestFd, DestStream, [{RevId, _, SubTree} | RestTree]) -> copy_docs(#db{fd=SrcFd}=Db, #db{fd=DestFd,summary_stream=DestStream}=NewDb, InfoBySeq, Retry) -> Ids = [Id || #doc_info{id=Id} <- InfoBySeq], LookupResults = couch_btree:lookup(Db#db.fulldocinfo_by_id_btree, Ids), - NewFullDocInfos = lists:map( + NewFullDocInfos0 = lists:map( fun({ok, #full_doc_info{rev_tree=RevTree}=Info}) -> Info#full_doc_info{rev_tree=copy_rev_tree(SrcFd, DestFd, DestStream, RevTree)} end, LookupResults), + NewFullDocInfos = stem_full_doc_infos(Db, NewFullDocInfos0), NewDocInfos = [couch_doc:to_doc_info(Info) || Info <- NewFullDocInfos], RemoveSeqs = case Retry of diff --git a/src/couchdb/couch_doc.erl b/src/couchdb/couch_doc.erl index 9860ac0c..fc817d56 100644 --- a/src/couchdb/couch_doc.erl +++ b/src/couchdb/couch_doc.erl @@ -12,41 +12,53 @@ -module(couch_doc). --export([to_doc_info/1,to_doc_info_path/1]). +-export([to_doc_info/1,to_doc_info_path/1,parse_rev/1,parse_revs/1,rev_to_str/1,rev_to_strs/1]). -export([bin_foldl/3,bin_size/1,bin_to_binary/1,get_validate_doc_fun/1]). -export([from_json_obj/1,to_json_obj/2,has_stubs/1, merge_stubs/2]). -include("couch_db.hrl"). % helpers used by to_json_obj -to_json_rev([]) -> +to_json_rev(0, []) -> []; -to_json_rev(Revs) -> - [{<<"_rev">>, lists:nth(1, Revs)}]. +to_json_rev(Start, [FirstRevId|_]) -> + [{<<"_rev">>, ?l2b([integer_to_list(Start),"-",FirstRevId])}]. to_json_body(true, _Body) -> [{<<"_deleted">>, true}]; to_json_body(false, {Body}) -> Body. -to_json_revs(Options, Revs) -> +to_json_revisions(Options, Start, RevIds) -> case lists:member(revs, Options) of false -> []; true -> - [{<<"_revs">>, Revs}] + [{<<"_revisions">>, {[{<<"start">>, Start}, + {<<"ids">>, RevIds}]}}] end. -to_json_revs_info(Meta) -> +rev_to_str({Pos, RevId}) -> + ?l2b([integer_to_list(Pos),"-",RevId]). + +rev_to_strs([]) -> + []; +rev_to_strs([{Pos, RevId}| Rest]) -> + [rev_to_str({Pos, RevId}) | rev_to_strs(Rest)]. + +to_json_meta(Meta) -> lists:map( - fun({revs_info, RevsInfo}) -> - JsonRevsInfo = - [{[{rev, Rev}, {status, list_to_binary(atom_to_list(Status))}]} || - {Rev, Status} <- RevsInfo], + fun({revs_info, Start, RevsInfo}) -> + {JsonRevsInfo, _Pos} = lists:mapfoldl( + fun({RevId, Status}, PosAcc) -> + JsonObj = {[{<<"rev">>, rev_to_str({PosAcc, RevId})}, + {<<"status">>, ?l2b(atom_to_list(Status))}]}, + {JsonObj, PosAcc - 1} + end, Start, RevsInfo), {<<"_revs_info">>, JsonRevsInfo}; ({conflicts, Conflicts}) -> - {<<"_conflicts">>, Conflicts}; - ({deleted_conflicts, Conflicts}) -> - {<<"_deleted_conflicts">>, Conflicts} + {<<"_conflicts">>, rev_to_strs(Conflicts)}; + ({deleted_conflicts, DConflicts}) -> + {<<"_deleted_conflicts">>, rev_to_strs(DConflicts)} end, Meta). to_json_attachment_stubs(Attachments) -> @@ -98,17 +110,62 @@ to_json_attachments(Attachments, Options) -> to_json_attachment_stubs(Attachments) end. -to_json_obj(#doc{id=Id,deleted=Del,body=Body,revs=Revs,meta=Meta}=Doc,Options)-> +to_json_obj(#doc{id=Id,deleted=Del,body=Body,revs={Start, RevIds}, + meta=Meta}=Doc,Options)-> {[{<<"_id">>, Id}] - ++ to_json_rev(Revs) + ++ to_json_rev(Start, RevIds) ++ to_json_body(Del, Body) - ++ to_json_revs(Options, Revs) - ++ to_json_revs_info(Meta) + ++ to_json_revisions(Options, Start, RevIds) + ++ to_json_meta(Meta) ++ to_json_attachments(Doc#doc.attachments, Options) }. from_json_obj({Props}) -> - {JsonBins} = proplists:get_value(<<"_attachments">>, Props, {[]}), + transfer_fields(Props, #doc{body=[]}); + +from_json_obj(_Other) -> + throw({bad_request, "Document must be a JSON object"}). + +parse_rev(Rev) when is_binary(Rev) -> + parse_rev(?b2l(Rev)); +parse_rev(Rev) -> + {Pos, [$- | RevId]} = lists:splitwith(fun($-) -> false; (_) -> true end, Rev), + {list_to_integer(Pos), ?l2b(RevId)}. + +parse_revs([]) -> + []; +parse_revs([Rev | Rest]) -> + [parse_rev(Rev) | parse_revs(Rest)]. + + +transfer_fields([], #doc{body=Fields}=Doc) -> + % convert fields back to json object + Doc#doc{body={lists:reverse(Fields)}}; + +transfer_fields([{<<"_id">>, Id} | Rest], Doc) when is_binary(Id) -> + case Id of + <<"_design/", _/binary>> -> ok; + <<"_local/", _/binary>> -> ok; + <<"_", _/binary>> -> + throw({bad_request, <<"Only reserved document ids may start with underscore.">>}); + _Else -> ok + end, + transfer_fields(Rest, Doc#doc{id=Id}); + +transfer_fields([{<<"_id">>, Id} | _Rest], _Doc) -> + ?LOG_DEBUG("Document id is not a string: ~p", [Id]), + throw({bad_request, <<"Document id must be a string">>}); + +transfer_fields([{<<"_rev">>, Rev} | Rest], #doc{revs={0, []}}=Doc) -> + {Pos, RevId} = parse_rev(Rev), + transfer_fields(Rest, + Doc#doc{revs={Pos, [RevId]}}); + +transfer_fields([{<<"_rev">>, _Rev} | Rest], Doc) -> + % we already got the rev from the _revisions + transfer_fields(Rest,Doc); + +transfer_fields([{<<"_attachments">>, {JsonBins}} | Rest], Doc) -> Bins = lists:flatmap(fun({Name, {BinProps}}) -> case proplists:get_value(<<"stub">>, BinProps) of true -> @@ -122,51 +179,40 @@ from_json_obj({Props}) -> [{Name, {Type, couch_util:decodeBase64(Value)}}] end end, JsonBins), - AllowedSpecialMembers = [<<"id">>, <<"revs">>, <<"rev">>, <<"attachments">>, <<"revs_info">>, - <<"conflicts">>, <<"deleted_conflicts">>, <<"deleted">>], - % collect all the doc-members that start with "_" - % if any aren't in the AllowedSpecialMembers list - % then throw a invalid_doc error - [case lists:member(Name, AllowedSpecialMembers) of - true -> - ok; - false -> - throw({invalid_doc, io_lib:format("Bad special document member: _~s", [Name])}) - end - || {<<$_,Name/binary>>, _Value} <- Props], - Revs = - case proplists:get_value(<<"_revs">>, Props, []) of - [] -> - case proplists:get_value(<<"_rev">>, Props) of - undefined -> []; - Rev -> [Rev] - end; - Revs0 -> - Revs0 - end, - case proplists:get_value(<<"_id">>, Props, <<>>) of - <<"_design/", _/binary>> = Id -> ok; - <<"_local/", _/binary>> = Id -> ok; - <<"_", _/binary>> = Id -> - throw({invalid_doc, "Document Ids must not start with underscore."}); - Id when is_binary(Id) -> ok; - Id -> - ?LOG_DEBUG("Document id is not a string: ~p", [Id]), - throw({invalid_doc, "Document id is not a string"}) + transfer_fields(Rest, Doc#doc{attachments=Bins}); + +transfer_fields([{<<"_revisions">>, {Props}} | Rest], Doc) -> + RevIds = proplists:get_value(<<"ids">>, Props), + Start = proplists:get_value(<<"start">>, Props), + if not is_integer(Start) -> + throw({doc_validation, "_revisions.start isn't an integer."}); + not is_list(RevIds) -> + throw({doc_validation, "_revisions.ids isn't a array."}); + true -> + ok end, + [throw({doc_validation, "RevId isn't a string"}) || + RevId <- RevIds, not is_binary(RevId)], + transfer_fields(Rest, Doc#doc{revs={Start, RevIds}}); - % strip out the all props beginning with _ - NewBody = {[{K, V} || {<>=K, V} <- Props, First /= $_]}, - #doc{ - id = Id, - revs = Revs, - deleted = proplists:get_value(<<"_deleted">>, Props, false), - body = NewBody, - attachments = Bins - }; +transfer_fields([{<<"_deleted">>, B} | Rest], Doc) when (B==true) or (B==false) -> + transfer_fields(Rest, Doc#doc{deleted=B}); -from_json_obj(_Other) -> - throw({invalid_doc, "Document must be a JSON object"}). +% ignored fields +transfer_fields([{<<"_revs_info">>, _} | Rest], Doc) -> + transfer_fields(Rest, Doc); +transfer_fields([{<<"_conflicts">>, _} | Rest], Doc) -> + transfer_fields(Rest, Doc); +transfer_fields([{<<"_deleted_conflicts">>, _} | Rest], Doc) -> + transfer_fields(Rest, Doc); + +% unknown special field +transfer_fields([{<<"_",Name/binary>>, Start} | _], _) when is_integer(Start) -> + throw({doc_validation, + ?l2b(io_lib:format("Bad special document member: _~s", [Name]))}); + +transfer_fields([Field | Rest], #doc{body=Fields}=Doc) -> + transfer_fields(Rest, Doc#doc{body=[Field|Fields]}). to_doc_info(FullDocInfo) -> {DocInfo, _Path} = to_doc_info_path(FullDocInfo), @@ -175,27 +221,26 @@ to_doc_info(FullDocInfo) -> to_doc_info_path(#full_doc_info{id=Id,update_seq=Seq,rev_tree=Tree}) -> LeafRevs = couch_key_tree:get_all_leafs(Tree), SortedLeafRevs = - lists:sort(fun({RevIdA, {IsDeletedA, _}, PathA}, {RevIdB, {IsDeletedB, _}, PathB}) -> + lists:sort(fun({{IsDeletedA, _}, {StartA, [RevIdA|_]}}, {{IsDeletedB, _}, {StartB, [RevIdB|_]}}) -> % sort descending by {not deleted, then Depth, then RevisionId} - A = {not IsDeletedA, length(PathA), RevIdA}, - B = {not IsDeletedB, length(PathB), RevIdB}, + A = {not IsDeletedA, StartA, RevIdA}, + B = {not IsDeletedB, StartB, RevIdB}, A > B end, LeafRevs), - [{RevId, {IsDeleted, SummaryPointer}, Path} | Rest] = SortedLeafRevs, - + [{{IsDeleted, SummaryPointer}, {Start, [RevId|_]}=Path} | Rest] = SortedLeafRevs, {ConflictRevTuples, DeletedConflictRevTuples} = - lists:splitwith(fun({_ConflictRevId, {IsDeleted1, _Sp}, _}) -> + lists:splitwith(fun({{IsDeleted1, _Sp}, _}) -> not IsDeleted1 end, Rest), - ConflictRevs = [RevId1 || {RevId1, _, _} <- ConflictRevTuples], - DeletedConflictRevs = [RevId2 || {RevId2, _, _} <- DeletedConflictRevTuples], + ConflictRevs = [{Start1, RevId1} || {_, {Start1, [RevId1|_]}} <- ConflictRevTuples], + DeletedConflictRevs = [{Start1, RevId1} || {_, {Start1, [RevId1|_]}} <- DeletedConflictRevTuples], DocInfo = #doc_info{ id=Id, update_seq=Seq, - rev = RevId, + rev = {Start, RevId}, summary_pointer = SummaryPointer, conflict_revs = ConflictRevs, deleted_conflict_revs = DeletedConflictRevs, diff --git a/src/couchdb/couch_httpd.erl b/src/couchdb/couch_httpd.erl index 48ff403b..2ba684a8 100644 --- a/src/couchdb/couch_httpd.erl +++ b/src/couchdb/couch_httpd.erl @@ -16,7 +16,7 @@ -export([start_link/0, stop/0, handle_request/4]). -export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,path/1,absolute_uri/2]). --export([verify_is_server_admin/1,unquote/1,quote/1,recv/2,recv_chunked/4]). +-export([verify_is_server_admin/1,unquote/1,quote/1,recv/2,recv_chunked/4,error_info/1]). -export([parse_form/1,json_body/1,body/1,doc_etag/1, make_etag/1, etag_respond/3]). -export([primary_header_value/2,partition/1,serve_file/3]). -export([start_chunked_response/3,send_chunk/2]). @@ -166,7 +166,7 @@ handle_request(MochiReq, UrlHandlers, DbUrlHandlers, DesignUrlHandlers) -> catch throw:Error -> send_error(HttpReq, Error); - Tag:Error -> + Tag:Error when Error ==foo -> ?LOG_ERROR("Uncaught error in HTTP request: ~p",[{Tag, Error}]), ?LOG_DEBUG("Stacktrace: ~p",[erlang:get_stacktrace()]), send_error(HttpReq, Error) @@ -295,8 +295,8 @@ body(#httpd{mochi_req=MochiReq}) -> json_body(Httpd) -> ?JSON_DECODE(body(Httpd)). -doc_etag(#doc{revs=[DiskRev|_]}) -> - "\"" ++ binary_to_list(DiskRev) ++ "\"". +doc_etag(#doc{revs={Start, [DiskRev|_]}}) -> + "\"" ++ ?b2l(couch_doc:rev_to_str({Start, DiskRev})) ++ "\"". make_etag(Term) -> <> = erlang:md5(term_to_binary(Term)), @@ -392,75 +392,55 @@ end_json_response(Resp) -> send_chunk(Resp, []). -send_error(Req, bad_request) -> - send_error(Req, 400, <<"bad_request">>, <<>>); -send_error(Req, {query_parse_error, Reason}) -> - send_error(Req, 400, <<"query_parse_error">>, Reason); -send_error(Req, {bad_request, Reason}) -> - send_error(Req, 400, <<"bad_request">>, Reason); -send_error(Req, not_found) -> - send_error(Req, 404, <<"not_found">>, <<"Missing">>); -send_error(Req, {not_found, Reason}) -> - send_error(Req, 404, <<"not_found">>, Reason); -send_error(Req, conflict) -> - send_error(Req, 409, <<"conflict">>, <<"Document update conflict.">>); -send_error(Req, {invalid_doc, Reason}) -> - send_error(Req, 400, <<"invalid_doc">>, Reason); -send_error(Req, {forbidden, Msg}) -> - send_json(Req, 403, - {[{<<"error">>, <<"forbidden">>}, - {<<"reason">>, Msg}]}); -send_error(Req, {unauthorized, Msg}) -> - case couch_config:get("httpd", "WWW-Authenticate", nil) of - nil -> - Headers = []; - Type -> - Headers = [{"WWW-Authenticate", Type}] - end, - send_json(Req, 401, Headers, - {[{<<"error">>, <<"unauthorized">>}, - {<<"reason">>, Msg}]}); -send_error(Req, {http_error, Code, Headers, Error, Reason}) -> - send_json(Req, Code, Headers, - {[{<<"error">>, Error}, {<<"reason">>, Reason}]}); -send_error(Req, {user_error, {Props}}) -> - {Headers} = proplists:get_value(<<"headers">>, Props, {[]}), - send_json(Req, - proplists:get_value(<<"http_status">>, Props, 500), - Headers, - {[{<<"error">>, proplists:get_value(<<"error">>, Props)}, - {<<"reason">>, proplists:get_value(<<"reason">>, Props)}]}); -send_error(Req, file_exists) -> - send_error(Req, 412, <<"file_exists">>, <<"The database could not be " - "created, the file already exists.">>); -send_error(Req, {Error, Reason}) -> - send_error(Req, 500, Error, Reason); -send_error(Req, Error) -> - send_error(Req, 500, <<"error">>, Error). - +error_info(bad_request) -> + {400, <<"bad_request">>, <<>>}; +error_info({bad_request, Reason}) -> + {400, <<"bad_request">>, Reason}; +error_info({query_parse_error, Reason}) -> + {400, <<"query_parse_error">>, Reason}; +error_info(not_found) -> + {404, <<"not_found">>, <<"Missing">>}; +error_info({not_found, Reason}) -> + {404, <<"not_found">>, Reason}; +error_info(conflict) -> + {409, <<"conflict">>, <<"Document update conflict.">>}; +error_info({forbidden, Msg}) -> + {403, <<"forbidden">>, Msg}; +error_info({unauthorized, Msg}) -> + {401, <<"unauthorized">>, Msg}; +error_info(file_exists) -> + {412, <<"file_exists">>, <<"The database could not be " + "created, the file already exists.">>}; +error_info({Error, Reason}) -> + {500, couch_util:to_binary(Error), couch_util:to_binary(Reason)}; +error_info(Error) -> + {500, <<"unknown_error">>, couch_util:to_binary(Error)}. +send_error(Req, Error) -> + {Code, ErrorStr, ReasonStr} = error_info(Error), + if Code == 401 -> + case couch_config:get("httpd", "WWW-Authenticate", nil) of + nil -> + Headers = []; + Type -> + Headers = [{"WWW-Authenticate", Type}] + end; + true -> + Headers = [] + end, + send_error(Req, Code, Headers, ErrorStr, ReasonStr). -send_error(Req, Code, Error, Msg) when is_atom(Error) -> - send_error(Req, Code, list_to_binary(atom_to_list(Error)), Msg); -send_error(Req, Code, Error, Msg) when is_list(Msg) -> - case (catch list_to_binary(Msg)) of - Bin when is_binary(Bin) -> - send_error(Req, Code, Error, Bin); - _ -> - send_error(Req, Code, Error, io_lib:format("~p", [Msg])) - end; -send_error(Req, Code, Error, Msg) when not is_binary(Error) -> - send_error(Req, Code, list_to_binary(io_lib:format("~p", [Error])), Msg); -send_error(Req, Code, Error, Msg) when not is_binary(Msg) -> - send_error(Req, Code, Error, list_to_binary(io_lib:format("~p", [Msg]))); -send_error(Req, Code, Error, <<>>) -> - send_json(Req, Code, {[{<<"error">>, Error}]}); -send_error(Req, Code, Error, Msg) -> - send_json(Req, Code, {[{<<"error">>, Error}, {<<"reason">>, Msg}]}). +send_error(Req, Code, ErrorStr, ReasonStr) -> + send_error(Req, Code, [], ErrorStr, ReasonStr). -send_redirect(Req, Path) -> - Headers = [{"Location", couch_httpd:absolute_uri(Req, Path)}], - send_response(Req, 301, Headers, <<>>). +send_error(Req, Code, Headers, ErrorStr, ReasonStr) -> + send_json(Req, Code, Headers, + {[{<<"error">>, ErrorStr}, + {<<"reason">>, ReasonStr}]}). + + send_redirect(Req, Path) -> + Headers = [{"Location", couch_httpd:absolute_uri(Req, Path)}], + send_response(Req, 301, Headers, <<>>). negotiate_content_type(#httpd{mochi_req=MochiReq}) -> %% Determine the appropriate Content-Type header for a JSON response diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl index 75022cd3..3680d73b 100644 --- a/src/couchdb/couch_httpd_db.erl +++ b/src/couchdb/couch_httpd_db.erl @@ -22,8 +22,8 @@ -record(doc_query_args, { options = [], - rev = "", - open_revs = "" + rev = nil, + open_revs = [] }). % Database request handlers @@ -89,13 +89,13 @@ db_req(#httpd{method='GET',path_parts=[_DbName]}=Req, Db) -> db_req(#httpd{method='POST',path_parts=[DbName]}=Req, Db) -> Doc = couch_doc:from_json_obj(couch_httpd:json_body(Req)), DocId = couch_util:new_uuid(), - {ok, NewRev} = couch_db:update_doc(Db, Doc#doc{id=DocId, revs=[]}, []), + {ok, NewRev} = couch_db:update_doc(Db, Doc#doc{id=DocId}, []), DocUrl = absolute_uri(Req, binary_to_list(<<"/",DbName/binary,"/",DocId/binary>>)), send_json(Req, 201, [{"Location", DocUrl}], {[ {ok, true}, {id, DocId}, - {rev, NewRev} + {rev, couch_doc:rev_to_str(NewRev)} ]}); db_req(#httpd{path_parts=[_DbName]}=Req, _Db) -> @@ -131,32 +131,59 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>]}=Req, Db) -> <<>> -> couch_util:new_uuid(); Id0 -> Id0 end, - Revs = case proplists:get_value(<<"_rev">>, ObjProps) of - undefined -> []; - Rev -> [Rev] + case proplists:get_value(<<"_rev">>, ObjProps) of + undefined -> + Revs = {0, []}; + Rev -> + {Pos, RevId} = couch_doc:parse_rev(Rev), + Revs = {Pos, [RevId]} end, Doc#doc{id=Id,revs=Revs} end, DocsArray), - {ok, ResultRevs} = couch_db:update_docs(Db, Docs, Options), - - % output the results - DocResults = lists:zipwith( - fun(Doc, NewRev) -> - {[{<<"id">>, Doc#doc.id}, {<<"rev">>, NewRev}]} - end, - Docs, ResultRevs), - send_json(Req, 201, {[ - {ok, true}, - {new_revs, DocResults} - ]}); - + Options2 = + case proplists:get_value(<<"all_or_nothing">>, JsonProps) of + true -> [all_or_nothing|Options]; + _ -> Options + end, + case couch_db:update_docs(Db, Docs, Options2) of + {ok, Results} -> + % output the results + DocResults = lists:zipwith( + fun(Doc, {ok, NewRev}) -> + {[{<<"id">>, Doc#doc.id}, {<<"rev">>, couch_doc:rev_to_str(NewRev)}]}; + (Doc, Error) -> + {_Code, Err, Msg} = couch_httpd:error_info(Error), + % maybe we should add the http error code to the json? + {[{<<"id">>, Doc#doc.id}, {<<"error">>, Err}, {"reason", Msg}]} + end, + Docs, Results), + send_json(Req, 201, DocResults); + {aborted, Errors} -> + ErrorsJson = + lists:map( + fun({{Id, Rev}, Error}) -> + {_Code, Err, Msg} = couch_httpd:error_info(Error), + {[{<<"id">>, Id}, + {<<"rev">>, couch_doc:rev_to_str(Rev)}, + {<<"error">>, Err}, + {"reason", Msg}]} + end, Errors), + send_json(Req, 417, ErrorsJson) + end; false -> Docs = [couch_doc:from_json_obj(JsonObj) || JsonObj <- DocsArray], - ok = couch_db:update_docs(Db, Docs, Options, false), - send_json(Req, 201, {[ - {ok, true} - ]}) + {ok, Errors} = couch_db:update_docs(Db, Docs, Options, replicated_changes), + ErrorsJson = + lists:map( + fun({{Id, Rev}, Error}) -> + {_Code, Err, Msg} = couch_httpd:error_info(Error), + {[{<<"id">>, Id}, + {<<"rev">>, couch_doc:rev_to_str(Rev)}, + {<<"error">>, Err}, + {"reason", Msg}]} + end, Errors), + send_json(Req, 201, ErrorsJson) end; db_req(#httpd{path_parts=[_,<<"_bulk_docs">>]}=Req, _Db) -> send_method_not_allowed(Req, "POST"); @@ -170,12 +197,12 @@ db_req(#httpd{path_parts=[_,<<"_compact">>]}=Req, _Db) -> db_req(#httpd{method='POST',path_parts=[_,<<"_purge">>]}=Req, Db) -> {IdsRevs} = couch_httpd:json_body(Req), - % validate the json input - [{_Id, [_|_]=_Revs} = IdRevs || IdRevs <- IdsRevs], + IdsRevs2 = [{Id, couch_doc:parse_revs(Revs)} || {Id, Revs} <- IdsRevs], - case couch_db:purge_docs(Db, IdsRevs) of + case couch_db:purge_docs(Db, IdsRevs2) of {ok, PurgeSeq, PurgedIdsRevs} -> - send_json(Req, 200, {[{<<"purge_seq">>, PurgeSeq}, {<<"purged">>, {PurgedIdsRevs}}]}); + PurgedIdsRevs2 = [{Id, couch_doc:rev_to_strs(Revs)} || {Id, Revs} <- PurgedIdsRevs], + send_json(Req, 200, {[{<<"purge_seq">>, PurgeSeq}, {<<"purged">>, {PurgedIdsRevs2}}]}); Error -> throw(Error) end; @@ -204,7 +231,7 @@ db_req(#httpd{method='GET',path_parts=[_,<<"_all_docs_by_seq">>]}=Req, Db) -> {ok, Info} = couch_db:get_db_info(Db), CurrentEtag = couch_httpd:make_etag(proplists:get_value(update_seq, Info)), - couch_httpd:etag_respond(Req, CurrentEtag, fun() -> + couch_httpd:etag_respond(Req, CurrentEtag, fun() -> TotalRowCount = proplists:get_value(doc_count, Info), FoldlFun = couch_httpd_view:make_view_fold_fun(Req, QueryArgs, CurrentEtag, Db, TotalRowCount, #view_fold_helper_funs{ @@ -227,14 +254,14 @@ db_req(#httpd{method='GET',path_parts=[_,<<"_all_docs_by_seq">>]}=Req, Db) -> deleted_conflict_revs=DelConflictRevs } = DocInfo, Json = { - [{<<"rev">>, Rev}] ++ + [{<<"rev">>, couch_doc:rev_to_str(Rev)}] ++ case ConflictRevs of [] -> []; - _ -> [{<<"conflicts">>, ConflictRevs}] + _ -> [{<<"conflicts">>, couch_doc:rev_to_strs(ConflictRevs)}] end ++ case DelConflictRevs of [] -> []; - _ -> [{<<"deleted_conflicts">>, DelConflictRevs}] + _ -> [{<<"deleted_conflicts">>, couch_doc:rev_to_strs(DelConflictRevs)}] end ++ case Deleted of true -> [{<<"deleted">>, true}]; @@ -251,9 +278,11 @@ db_req(#httpd{path_parts=[_,<<"_all_docs_by_seq">>]}=Req, _Db) -> db_req(#httpd{method='POST',path_parts=[_,<<"_missing_revs">>]}=Req, Db) -> {JsonDocIdRevs} = couch_httpd:json_body(Req), - {ok, Results} = couch_db:get_missing_revs(Db, JsonDocIdRevs), + JsonDocIdRevs2 = [{Id, [couch_doc:parse_rev(RevStr) || RevStr <- RevStrs]} || {Id, RevStrs} <- JsonDocIdRevs], + {ok, Results} = couch_db:get_missing_revs(Db, JsonDocIdRevs2), + Results2 = [{Id, [couch_doc:rev_to_str(Rev) || Rev <- Revs]} || {Id, Revs} <- Results], send_json(Req, {[ - {missing_revs, {Results}} + {missing_revs, {Results2}} ]}); db_req(#httpd{path_parts=[_,<<"_missing_revs">>]}=Req, _Db) -> @@ -271,6 +300,18 @@ db_req(#httpd{method='GET',path_parts=[_,<<"_admins">>]}=Req, Db) -> db_req(#httpd{path_parts=[_,<<"_admins">>]}=Req, _Db) -> send_method_not_allowed(Req, "PUT,GET"); +db_req(#httpd{method='PUT',path_parts=[_,<<"_revs_limit">>]}=Req, + Db) -> + Limit = couch_httpd:json_body(Req), + ok = couch_db:set_revs_limit(Db, Limit), + send_json(Req, {[{<<"ok">>, true}]}); + +db_req(#httpd{method='GET',path_parts=[_,<<"_revs_limit">>]}=Req, Db) -> + send_json(Req, couch_db:get_revs_limit(Db)); + +db_req(#httpd{path_parts=[_,<<"_revs_limit">>]}=Req, _Db) -> + send_method_not_allowed(Req, "PUT,GET"); + % Special case to enable using an unencoded slash in the URL of design docs, % as slashes in document IDs must otherwise be URL encoded. db_req(#httpd{method='GET',mochi_req=MochiReq, path_parts=[DbName,<<"_design/",_/binary>>|_]}=Req, _Db) -> @@ -334,7 +375,7 @@ all_docs_view(Req, Db, Keys) -> AdapterFun = fun(#full_doc_info{id=Id}=FullDocInfo, Offset, Acc) -> case couch_doc:to_doc_info(FullDocInfo) of #doc_info{deleted=false, rev=Rev} -> - FoldlFun({{Id, Id}, {[{rev, Rev}]}}, Offset, Acc); + FoldlFun({{Id, Id}, {[{rev, couch_doc:rev_to_str(Rev)}]}}, Offset, Acc); #doc_info{deleted=true} -> {ok, Acc} end @@ -358,9 +399,9 @@ all_docs_view(Req, Db, Keys) -> DocInfo = (catch couch_db:get_doc_info(Db, Key)), Doc = case DocInfo of {ok, #doc_info{id=Id, rev=Rev, deleted=false}} = DocInfo -> - {{Id, Id}, {[{rev, Rev}]}}; + {{Id, Id}, {[{rev, couch_doc:rev_to_str(Rev)}]}}; {ok, #doc_info{id=Id, rev=Rev, deleted=true}} = DocInfo -> - {{Id, Id}, {[{rev, Rev}, {deleted, true}]}}; + {{Id, Id}, {[{rev, couch_doc:rev_to_str(Rev)}, {deleted, true}]}}; not_found -> {{Key, error}, not_found}; _ -> @@ -381,20 +422,12 @@ all_docs_view(Req, Db, Keys) -> - - db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) -> - case extract_header_rev(Req, couch_httpd:qs_value(Req, "rev")) of - missing_rev -> - couch_httpd:send_error(Req, 409, <<"missing_rev">>, - <<"Document rev/etag must be specified to delete">>); - RevToDelete -> - {ok, NewRev} = couch_db:delete_doc(Db, DocId, [RevToDelete]), - send_json(Req, 200, {[ - {ok, true}, - {id, DocId}, - {rev, NewRev} - ]}) + case couch_httpd:qs_value(Req, "rev") of + undefined -> + update_doc(Req, Db, DocId, {[{<<"_deleted">>,true}]}); + Rev -> + update_doc(Req, Db, DocId, {[{<<"_rev">>, ?l2b(Rev)},{<<"_deleted">>,true}]}) end; db_doc_req(#httpd{method='GET'}=Req, Db, DocId) -> @@ -438,82 +471,31 @@ db_doc_req(#httpd{method='GET'}=Req, Db, DocId) -> end_json_response(Resp) end; -db_doc_req(#httpd{method='POST'}=Req, Db, DocId) -> - Form = couch_httpd:parse_form(Req), - Rev = list_to_binary(proplists:get_value("_rev", Form)), - Doc = case couch_db:open_doc_revs(Db, DocId, [Rev], []) of - {ok, [{ok, Doc0}]} -> Doc0#doc{revs=[Rev]}; - {ok, [Error]} -> throw(Error) - end, - - NewAttachments = [ - {validate_attachment_name(Name), {list_to_binary(ContentType), Content}} || - {Name, {ContentType, _}, Content} <- - proplists:get_all_values("_attachments", Form) - ], - #doc{attachments=Attachments} = Doc, - NewDoc = Doc#doc{ - attachments = Attachments ++ NewAttachments - }, - {ok, NewRev} = couch_db:update_doc(Db, NewDoc, []), - - send_json(Req, 201, [{"Etag", "\"" ++ NewRev ++ "\""}], {obj, [ - {ok, true}, - {id, DocId}, - {rev, NewRev} - ]}); - db_doc_req(#httpd{method='PUT'}=Req, Db, DocId) -> - Json = couch_httpd:json_body(Req), - Doc = couch_doc:from_json_obj(Json), - ExplicitRev = - case Doc#doc.revs of - [Rev0|_] -> Rev0; - [] -> undefined - end, - validate_attachment_names(Doc), - case couch_httpd:header_value(Req, "X-Couch-Full-Commit", "false") of - "true" -> - Options = [full_commit]; - _ -> - Options = [] - end, - case extract_header_rev(Req, ExplicitRev) of - missing_rev -> - Revs = []; - Rev -> - Revs = [Rev] - end, - {ok, NewRev} = couch_db:update_doc(Db, Doc#doc{id=DocId, revs=Revs}, Options), - send_json(Req, 201, [{"Etag", <<"\"", NewRev/binary, "\"">>}], {[ - {ok, true}, - {id, DocId}, - {rev, NewRev} - ]}); + update_doc(Req, Db, DocId, couch_httpd:json_body(Req)); db_doc_req(#httpd{method='COPY'}=Req, Db, SourceDocId) -> SourceRev = case extract_header_rev(Req, couch_httpd:qs_value(Req, "rev")) of - missing_rev -> []; + missing_rev -> nil; Rev -> Rev end, - {TargetDocId, TargetRev} = parse_copy_destination_header(Req), + {TargetDocId, TargetRevs} = parse_copy_destination_header(Req), % open revision Rev or Current Doc = couch_doc_open(Db, SourceDocId, SourceRev, []), - % save new doc - {ok, NewTargetRev} = couch_db:update_doc(Db, Doc#doc{id=TargetDocId, revs=TargetRev}, []), - - send_json(Req, 201, [{"Etag", "\"" ++ binary_to_list(NewTargetRev) ++ "\""}], {[ - {ok, true}, - {id, TargetDocId}, - {rev, NewTargetRev} - ]}); + case couch_db:update_doc(Db, Doc#doc{id=TargetDocId, revs=TargetRevs}, []) of + {ok, NewTargetRev} -> + send_json(Req, 201, [{"Etag", "\"" ++ ?b2l(couch_doc:rev_to_str(NewTargetRev)) ++ "\""}], + update_result_to_json({ok, NewTargetRev})); + Error -> + throw(Error) + end; db_doc_req(#httpd{method='MOVE'}=Req, Db, SourceDocId) -> - SourceRev = + SourceRev = {SourceRevPos, SourceRevId} = case extract_header_rev(Req, couch_httpd:qs_value(Req, "rev")) of missing_rev -> throw({bad_request, "MOVE requires a specified rev parameter" @@ -521,37 +503,68 @@ db_doc_req(#httpd{method='MOVE'}=Req, Db, SourceDocId) -> Rev -> Rev end, - {TargetDocId, TargetRev} = parse_copy_destination_header(Req), + {TargetDocId, TargetRevs} = parse_copy_destination_header(Req), % open revision Rev or Current Doc = couch_doc_open(Db, SourceDocId, SourceRev, []), % save new doc & delete old doc in one operation Docs = [ - Doc#doc{id=TargetDocId, revs=TargetRev}, - #doc{id=SourceDocId, revs=[SourceRev], deleted=true} + #doc{id=SourceDocId, revs={SourceRevPos, [SourceRevId]}, deleted=true}, + Doc#doc{id=TargetDocId, revs=TargetRevs} ], - {ok, ResultRevs} = couch_db:update_docs(Db, Docs, []), + {ok, [SourceResult, TargetResult]} = couch_db:update_docs(Db, Docs, []), - DocResults = lists:zipwith( - fun(FDoc, NewRev) -> - {[{id, FDoc#doc.id}, {rev, NewRev}]} - end, - Docs, ResultRevs), send_json(Req, 201, {[ - {ok, true}, - {new_revs, DocResults} + {SourceDocId, update_result_to_json(SourceResult)}, + {TargetDocId, update_result_to_json(TargetResult)} ]}); db_doc_req(Req, _Db, _DocId) -> send_method_not_allowed(Req, "DELETE,GET,HEAD,POST,PUT,COPY,MOVE"). +update_result_to_json({ok, NewRev}) -> + {[{rev, couch_doc:rev_to_str(NewRev)}]}; +update_result_to_json(Error) -> + {_Code, ErrorStr, Reason} = couch_httpd:error_info(Error), + {[{error, ErrorStr}, {reason, Reason}]}. + + +update_doc(Req, Db, DocId, Json) -> + #doc{deleted=Deleted} = Doc = couch_doc:from_json_obj(Json), + validate_attachment_names(Doc), + ExplicitDocRev = + case Doc#doc.revs of + {Start,[RevId|_]} -> {Start, RevId}; + _ -> undefined + end, + case extract_header_rev(Req, ExplicitDocRev) of + missing_rev -> + Revs = {0, []}; + {Pos, Rev} -> + Revs = {Pos, [Rev]} + end, + + case couch_httpd:header_value(Req, "X-Couch-Full-Commit", "false") of + "true" -> + Options = [full_commit]; + _ -> + Options = [] + end, + {ok, NewRev} = couch_db:update_doc(Db, Doc#doc{id=DocId, revs=Revs}, Options), + NewRevStr = couch_doc:rev_to_str(NewRev), + send_json(Req, if Deleted -> 200; true -> 201 end, + [{"Etag", <<"\"", NewRevStr/binary, "\"">>}], {[ + {ok, true}, + {id, DocId}, + {rev, NewRevStr}]}). + % Useful for debugging % couch_doc_open(Db, DocId) -> % couch_doc_open(Db, DocId, [], []). couch_doc_open(Db, DocId, Rev, Options) -> case Rev of - "" -> % open most recent rev + nil -> % open most recent rev case couch_db:open_doc(Db, DocId, Options) of {ok, Doc} -> Doc; @@ -572,13 +585,13 @@ couch_doc_open(Db, DocId, Rev, Options) -> db_attachment_req(#httpd{method='GET'}=Req, Db, DocId, FileNameParts) -> FileName = list_to_binary(mochiweb_util:join(lists:map(fun binary_to_list/1, FileNameParts),"/")), case couch_db:open_doc(Db, DocId, []) of - {ok, #doc{attachments=Attachments, revs=[LastRev|_OldRevs]}} -> + {ok, #doc{attachments=Attachments}=Doc} -> case proplists:get_value(FileName, Attachments) of undefined -> throw({not_found, "Document is missing attachment"}); {Type, Bin} -> {ok, Resp} = start_chunked_response(Req, 200, [ - {"ETag", binary_to_list(LastRev)}, + {"ETag", couch_httpd:doc_etag(Doc)}, {"Cache-Control", "must-revalidate"}, {"Content-Type", binary_to_list(Type)}%, % My understanding of http://www.faqs.org/rfcs/rfc2616.html @@ -640,7 +653,7 @@ db_attachment_req(#httpd{method=Method}=Req, Db, DocId, FileNameParts) #doc{id=DocId}; Rev -> case couch_db:open_doc_revs(Db, DocId, [Rev], []) of - {ok, [{ok, Doc0}]} -> Doc0#doc{revs=[Rev]}; + {ok, [{ok, Doc0}]} -> Doc0; {ok, [Error]} -> throw(Error) end end, @@ -653,7 +666,7 @@ db_attachment_req(#httpd{method=Method}=Req, Db, DocId, FileNameParts) send_json(Req, case Method of 'DELETE' -> 200; _ -> 201 end, {[ {ok, true}, {id, DocId}, - {rev, UpdatedRev} + {rev, couch_doc:rev_to_str(UpdatedRev)} ]}); db_attachment_req(Req, _Db, _DocId, _FileNameParts) -> @@ -682,25 +695,24 @@ parse_doc_query(Req) -> Options = [deleted_conflicts | Args#doc_query_args.options], Args#doc_query_args{options=Options}; {"rev", Rev} -> - Args#doc_query_args{rev=list_to_binary(Rev)}; + Args#doc_query_args{rev=couch_doc:parse_rev(Rev)}; {"open_revs", "all"} -> Args#doc_query_args{open_revs=all}; {"open_revs", RevsJsonStr} -> JsonArray = ?JSON_DECODE(RevsJsonStr), - Args#doc_query_args{open_revs=JsonArray}; + Args#doc_query_args{open_revs=[couch_doc:parse_rev(Rev) || Rev <- JsonArray]}; _Else -> % unknown key value pair, ignore. Args end end, #doc_query_args{}, couch_httpd:qs(Req)). - -extract_header_rev(Req, ExplicitRev) when is_list(ExplicitRev)-> - extract_header_rev(Req, list_to_binary(ExplicitRev)); +extract_header_rev(Req, ExplicitRev) when is_binary(ExplicitRev) or is_list(ExplicitRev)-> + extract_header_rev(Req, couch_doc:parse_rev(ExplicitRev)); extract_header_rev(Req, ExplicitRev) -> Etag = case couch_httpd:header_value(Req, "If-Match") of undefined -> undefined; - Value -> list_to_binary(string:strip(Value, both, $")) + Value -> couch_doc:parse_rev(string:strip(Value, both, $")) end, case {ExplicitRev, Etag} of {undefined, undefined} -> missing_rev; @@ -716,11 +728,12 @@ parse_copy_destination_header(Req) -> Destination = couch_httpd:header_value(Req, "Destination"), case regexp:match(Destination, "\\?") of nomatch -> - {list_to_binary(Destination), []}; + {list_to_binary(Destination), {0, []}}; {match, _, _} -> {ok, [DocId, RevQueryOptions]} = regexp:split(Destination, "\\?"), {ok, [_RevQueryKey, Rev]} = regexp:split(RevQueryOptions, "="), - {list_to_binary(DocId), [list_to_binary(Rev)]} + {Pos, RevId} = couch_doc:parse_rev(Rev), + {list_to_binary(DocId), {Pos, [RevId]}} end. validate_attachment_names(Doc) -> diff --git a/src/couchdb/couch_httpd_misc_handlers.erl b/src/couchdb/couch_httpd_misc_handlers.erl index 583a87c2..46fed095 100644 --- a/src/couchdb/couch_httpd_misc_handlers.erl +++ b/src/couchdb/couch_httpd_misc_handlers.erl @@ -70,35 +70,36 @@ handle_task_status_req(#httpd{method='GET'}=Req) -> handle_task_status_req(Req) -> send_method_not_allowed(Req, "GET,HEAD"). +% add trailing slash if missing +fix_db_url(UrlBin) -> + ?l2b(case lists:last(Url = ?b2l(UrlBin)) of + $/ -> Url; + _ -> Url ++ "/" + end). + -handle_replicate_req(#httpd{user_ctx=UserCtx,method='POST'}=Req) -> +get_rep_endpoint(_Req, {Props}) -> + Url = proplists:get_value(<<"url">>, Props), + {BinHeaders} = proplists:get_value(<<"headers">>, Props, {[]}), + {remote, fix_db_url(Url), [{?b2l(K),?b2l(V)} || {K,V} <- BinHeaders]}; +get_rep_endpoint(_Req, <<"http://",_/binary>>=Url) -> + {remote, fix_db_url(Url), []}; +get_rep_endpoint(_Req, <<"https://",_/binary>>=Url) -> + {remote, fix_db_url(Url), []}; +get_rep_endpoint(#httpd{user_ctx=UserCtx}, <>) -> + {local, DbName, UserCtx}. + +handle_replicate_req(#httpd{method='POST'}=Req) -> {Props} = couch_httpd:json_body(Req), - Source = proplists:get_value(<<"source">>, Props), - Target = proplists:get_value(<<"target">>, Props), - - {SrcOpts} = proplists:get_value(<<"source_options">>, Props, {[]}), - {SrcHeadersBinary} = proplists:get_value(<<"headers">>, SrcOpts, {[]}), - SrcHeaders = [{?b2l(K),(V)} || {K,V} <- SrcHeadersBinary], - - {TgtOpts} = proplists:get_value(<<"target_options">>, Props, {[]}), - {TgtHeadersBinary} = proplists:get_value(<<"headers">>, TgtOpts, {[]}), - TgtHeaders = [{?b2l(K),(V)} || {K,V} <- TgtHeadersBinary], - - {Options} = proplists:get_value(<<"options">>, Props, {[]}), - Options2 = [{source_options, - [{headers, SrcHeaders}, - {user_ctx, UserCtx}]}, - {target_options, - [{headers, TgtHeaders}, - {user_ctx, UserCtx}]} - | Options], - case couch_rep:replicate(Source, Target, Options2) of - {ok, {JsonResults}} -> - send_json(Req, {[{ok, true} | JsonResults]}); - {error, {Type, Details}} -> - send_json(Req, 500, {[{error, Type}, {reason, Details}]}); - {error, Reason} -> - send_json(Req, 500, {[{error, Reason}]}) + Source = get_rep_endpoint(Req, proplists:get_value(<<"source">>, Props)), + Target = get_rep_endpoint(Req, proplists:get_value(<<"target">>, Props)), + case couch_rep:replicate(Source, Target) of + {ok, {JsonResults}} -> + send_json(Req, {[{ok, true} | JsonResults]}); + {error, {Type, Details}} -> + send_json(Req, 500, {[{error, Type}, {reason, Details}]}); + {error, Reason} -> + send_json(Req, 500, {[{error, Reason}]}) end; handle_replicate_req(Req) -> send_method_not_allowed(Req, "POST"). diff --git a/src/couchdb/couch_httpd_show.erl b/src/couchdb/couch_httpd_show.erl index 7b6f2832..dd337ced 100644 --- a/src/couchdb/couch_httpd_show.erl +++ b/src/couchdb/couch_httpd_show.erl @@ -27,10 +27,10 @@ handle_doc_show_req(#httpd{ path_parts=[_DbName, _Design, DesignName, _Show, ShowName, DocId] }=Req, Db) -> DesignId = <<"_design/", DesignName/binary>>, - #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, [], []), + #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), ShowSrc = get_nested_json_value({Props}, [<<"shows">>, ShowName]), - Doc = try couch_httpd_db:couch_doc_open(Db, DocId, [], []) of + Doc = try couch_httpd_db:couch_doc_open(Db, DocId, nil, []) of FoundDoc -> FoundDoc catch _ -> nil @@ -42,7 +42,7 @@ handle_doc_show_req(#httpd{ path_parts=[_DbName, _Design, DesignName, _Show, ShowName] }=Req, Db) -> DesignId = <<"_design/", DesignName/binary>>, - #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, [], []), + #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), ShowSrc = get_nested_json_value({Props}, [<<"shows">>, ShowName]), send_doc_show_response(Lang, ShowSrc, nil, nil, Req, Db); @@ -56,7 +56,7 @@ handle_doc_show_req(Req, _Db) -> handle_view_list_req(#httpd{method='GET', path_parts=[_DbName, _Design, DesignName, _List, ListName, ViewName]}=Req, Db) -> DesignId = <<"_design/", DesignName/binary>>, - #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, [], []), + #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), ListSrc = get_nested_json_value({Props}, [<<"lists">>, ListName]), send_view_list_response(Lang, ListSrc, ViewName, DesignId, Req, Db, nil); @@ -67,7 +67,7 @@ handle_view_list_req(#httpd{method='GET'}=Req, _Db) -> handle_view_list_req(#httpd{method='POST', path_parts=[_DbName, _Design, DesignName, _List, ListName, ViewName]}=Req, Db) -> DesignId = <<"_design/", DesignName/binary>>, - #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, [], []), + #doc{body={Props}} = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []), Lang = proplists:get_value(<<"language">>, Props, <<"javascript">>), ListSrc = get_nested_json_value({Props}, [<<"lists">>, ListName]), ReqBody = couch_httpd:body(Req), @@ -370,13 +370,12 @@ send_doc_show_response(Lang, ShowSrc, DocId, nil, #httpd{mochi_req=MReq}=Req, Db couch_httpd_external:send_external_response(Req, JsonResp) end); -send_doc_show_response(Lang, ShowSrc, DocId, #doc{revs=[DocRev|_]}=Doc, - #httpd{mochi_req=MReq}=Req, Db) -> +send_doc_show_response(Lang, ShowSrc, DocId, #doc{revs=Revs}=Doc, #httpd{mochi_req=MReq}=Req, Db) -> % calculate the etag Headers = MReq:get(headers), Hlist = mochiweb_headers:to_list(Headers), Accept = proplists:get_value('Accept', Hlist), - CurrentEtag = couch_httpd:make_etag({Lang, ShowSrc, DocRev, Accept}), + CurrentEtag = couch_httpd:make_etag({Lang, ShowSrc, Revs, Accept}), % We know our etag now couch_httpd:etag_respond(Req, CurrentEtag, fun() -> ExternalResp = couch_query_servers:render_doc_show(Lang, ShowSrc, diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl index 322cf945..7f338655 100644 --- a/src/couchdb/couch_httpd_view.erl +++ b/src/couchdb/couch_httpd_view.erl @@ -565,14 +565,14 @@ view_row_obj(Db, {{Key, DocId}, Value}, IncludeDocs) -> true -> Rev = case Value of {Props} -> - case is_list(Props) of - true -> - proplists:get_value(<<"_rev">>, Props, []); - _ -> - [] + case proplists:get_value(<<"_rev">>, Props) of + undefined -> + nil; + Rev0 -> + couch_doc:parse_rev(Rev0) end; _ -> - [] + nil end, ?LOG_DEBUG("Include Doc: ~p ~p", [DocId, Rev]), case (catch couch_httpd_db:couch_doc_open(Db, DocId, Rev, [])) of diff --git a/src/couchdb/couch_key_tree.erl b/src/couchdb/couch_key_tree.erl index 82c42265..df24d4e1 100644 --- a/src/couchdb/couch_key_tree.erl +++ b/src/couchdb/couch_key_tree.erl @@ -13,7 +13,8 @@ -module(couch_key_tree). -export([merge/2, find_missing/2, get_key_leafs/2, get_full_key_paths/2, get/2]). --export([map/2, get_all_leafs/1, get_leaf_keys/1, count_leafs/1, remove_leafs/2,get_all_leafs_full/1]). +-export([map/2, get_all_leafs/1, count_leafs/1, remove_leafs/2, + get_all_leafs_full/1,stem/2,test/0]). % a key tree looks like this: % Tree -> [] or [{Key, Value, ChildTree} | SiblingTree] @@ -22,70 +23,150 @@ % And each Key < SiblingKey +% partial trees arranged by how much they are cut off. -% key tree functions +merge(A, B) -> + {Merged, HasConflicts} = + lists:foldl( + fun(InsertTree, {AccTrees, AccConflicts}) -> + case merge_one(AccTrees, InsertTree, [], false) of + {ok, Merged, Conflicts} -> + {Merged, Conflicts or AccConflicts}; + no -> + {[InsertTree | AccTrees], true} + end + end, + {A, false}, B), + if HasConflicts or + ((length(Merged) /= length(A)) and (length(Merged) /= length(B))) -> + Conflicts = conflicts; + true -> + Conflicts = no_conflicts + end, + {lists:sort(Merged), Conflicts}. + +merge_one([], Insert, OutAcc, ConflictsAcc) -> + {ok, [Insert | OutAcc], ConflictsAcc}; +merge_one([{Start, Tree}|Rest], {StartInsert, TreeInsert}, OutAcc, ConflictsAcc) -> + if Start =< StartInsert -> + StartA = Start, + StartB = StartInsert, + TreeA = Tree, + TreeB = TreeInsert; + true -> + StartB = Start, + StartA = StartInsert, + TreeB = Tree, + TreeA = TreeInsert + end, + case merge_at([TreeA], StartB - StartA, TreeB) of + {ok, [CombinedTrees], Conflicts} -> + merge_one(Rest, {StartA, CombinedTrees}, OutAcc, Conflicts or ConflictsAcc); + no -> + merge_one(Rest, {StartB, TreeB}, [{StartA, TreeA} | OutAcc], ConflictsAcc) + end. + +merge_at([], _Place, _Insert) -> + no; +merge_at([{Key, Value, SubTree}|Sibs], 0, {InsertKey, InsertValue, InsertSubTree}) -> + if Key == InsertKey -> + {Merge, Conflicts} = merge_simple(SubTree, InsertSubTree), + {ok, [{Key, Value, Merge} | Sibs], Conflicts}; + true -> + case merge_at(Sibs, 0, {InsertKey, InsertValue, InsertSubTree}) of + {ok, Merged, Conflicts} -> + {ok, [{Key, Value, SubTree} | Merged], Conflicts}; + no -> + no + end + end; +merge_at([{Key, Value, SubTree}|Sibs], Place, Insert) -> + case merge_at(SubTree, Place - 1,Insert) of + {ok, Merged, Conflicts} -> + {ok, [{Key, Value, Merged} | Sibs], Conflicts}; + no -> + case merge_at(Sibs, Place, Insert) of + {ok, Merged} -> + [{Key, Value, SubTree} | Merged]; + no -> + no + end + end. -% When the same key is found in the trees, the value in tree B is discarded. -merge([], B) -> - B; -merge(A, []) -> - A; -merge([ATree | ANextTree], [BTree | BNextTree]) -> +% key tree functions +merge_simple([], B) -> + {B, false}; +merge_simple(A, []) -> + {A, false}; +merge_simple([ATree | ANextTree], [BTree | BNextTree]) -> {AKey, AValue, ASubTree} = ATree, {BKey, _BValue, BSubTree} = BTree, if AKey == BKey -> %same key - MergedSubTree = merge(ASubTree, BSubTree), - MergedNextTree = merge(ANextTree, BNextTree), - [{AKey, AValue, MergedSubTree} | MergedNextTree]; + {MergedSubTree, Conflict1} = merge_simple(ASubTree, BSubTree), + {MergedNextTree, Conflict2} = merge_simple(ANextTree, BNextTree), + {[{AKey, AValue, MergedSubTree} | MergedNextTree], Conflict1 or Conflict2}; AKey < BKey -> - [ATree | merge(ANextTree, [BTree | BNextTree])]; + {MTree, _} = merge_simple(ANextTree, [BTree | BNextTree]), + {[ATree | MTree], true}; true -> - [BTree | merge([ATree | ANextTree], BNextTree)] + {MTree, _} = merge_simple([ATree | ANextTree], BNextTree), + {[BTree | MTree], true} end. find_missing(_Tree, []) -> []; -find_missing([], Keys) -> - Keys; -find_missing([{Key, _, SubTree} | RestTree], Keys) -> - SrcKeys2 = Keys -- [Key], - SrcKeys3 = find_missing(SubTree, SrcKeys2), - find_missing(RestTree, SrcKeys3). - - -get_all_key_paths_rev([], KeyPathAcc) -> - KeyPathAcc; -get_all_key_paths_rev([{Key, Value, SubTree} | RestTree], KeyPathAcc) -> - get_all_key_paths_rev(SubTree, [{Key, Value} | KeyPathAcc]) ++ - get_all_key_paths_rev(RestTree, KeyPathAcc). - +find_missing([], SeachKeys) -> + SeachKeys; +find_missing([{Start, {Key, Value, SubTree}} | RestTree], SeachKeys) -> + PossibleKeys = [{KeyPos, KeyValue} || {KeyPos, KeyValue} <- SeachKeys, KeyPos >= Start], + ImpossibleKeys = [{KeyPos, KeyValue} || {KeyPos, KeyValue} <- SeachKeys, KeyPos < Start], + Missing = find_missing_simple(Start, [{Key, Value, SubTree}], PossibleKeys), + find_missing(RestTree, ImpossibleKeys ++ Missing). + +find_missing_simple(_Pos, _Tree, []) -> + []; +find_missing_simple(_Pos, [], SeachKeys) -> + SeachKeys; +find_missing_simple(Pos, [{Key, _, SubTree} | RestTree], SeachKeys) -> + PossibleKeys = [{KeyPos, KeyValue} || {KeyPos, KeyValue} <- SeachKeys, KeyPos >= Pos], + ImpossibleKeys = [{KeyPos, KeyValue} || {KeyPos, KeyValue} <- SeachKeys, KeyPos < Pos], + + SrcKeys2 = PossibleKeys -- [{Pos, Key}], + SrcKeys3 = find_missing_simple(Pos + 1, SubTree, SrcKeys2), + ImpossibleKeys ++ find_missing_simple(Pos, RestTree, SrcKeys3). + + +filter_leafs([], _Keys, FilteredAcc, RemovedKeysAcc) -> + {FilteredAcc, RemovedKeysAcc}; +filter_leafs([{Pos, [{LeafKey, _}|_]} = Path |Rest], Keys, FilteredAcc, RemovedKeysAcc) -> + FilteredKeys = lists:delete({Pos, LeafKey}, Keys), + if FilteredKeys == Keys -> + % this leaf is not a key we are looking to remove + filter_leafs(Rest, Keys, [Path | FilteredAcc], RemovedKeysAcc); + true -> + % this did match a key, remove both the node and the input key + filter_leafs(Rest, FilteredKeys, FilteredAcc, [{Pos, LeafKey} | RemovedKeysAcc]) + end. % Removes any branches from the tree whose leaf node(s) are in the Keys -remove_leafs(Tree, Keys) -> +remove_leafs(Trees, Keys) -> % flatten each branch in a tree into a tree path - Paths = get_all_key_paths_rev(Tree, []), + Paths = get_all_leafs_full(Trees), % filter out any that are in the keys list. - {FoundKeys, FilteredPaths} = lists:mapfoldl( - fun(Key, PathsAcc) -> - case [Path || [{LeafKey,_}|_]=Path <- PathsAcc, LeafKey /= Key] of - PathsAcc -> - {nil, PathsAcc}; - PathsAcc2 -> - {Key, PathsAcc2} - end - end, Paths, Keys), - + {FilteredPaths, RemovedKeys} = filter_leafs(Paths, Keys, [], []), + % convert paths back to trees NewTree = lists:foldl( - fun(Path,TreeAcc) -> - SingleTree = lists:foldl( + fun({PathPos, Path},TreeAcc) -> + [SingleTree] = lists:foldl( fun({K,V},NewTreeAcc) -> [{K,V,NewTreeAcc}] end, [], Path), - merge(TreeAcc, SingleTree) + {NewTrees, _} = merge(TreeAcc, [{PathPos + 1 - length(Path), SingleTree}]), + NewTrees end, [], FilteredPaths), - {NewTree, FoundKeys}. + {NewTree, RemovedKeys}. % get the leafs in the tree matching the keys. The matching key nodes can be @@ -94,87 +175,211 @@ remove_leafs(Tree, Keys) -> get_key_leafs(Tree, Keys) -> get_key_leafs(Tree, Keys, []). -get_key_leafs(_Tree, [], _KeyPathAcc) -> +get_key_leafs(_, [], Acc) -> + {Acc, []}; +get_key_leafs([], Keys, Acc) -> + {Acc, Keys}; +get_key_leafs([{Pos, Tree}|Rest], Keys, Acc) -> + {Gotten, RemainingKeys} = get_key_leafs_simple(Pos, [Tree], Keys, []), + get_key_leafs(Rest, RemainingKeys, Gotten ++ Acc). + +get_key_leafs_simple(_Pos, _Tree, [], _KeyPathAcc) -> {[], []}; -get_key_leafs([], KeysToGet, _KeyPathAcc) -> +get_key_leafs_simple(_Pos, [], KeysToGet, _KeyPathAcc) -> {[], KeysToGet}; -get_key_leafs([{Key, _Value, SubTree}=Tree | RestTree], KeysToGet, KeyPathAcc) -> - case KeysToGet -- [Key] of +get_key_leafs_simple(Pos, [{Key, _Value, SubTree}=Tree | RestTree], KeysToGet, KeyPathAcc) -> + case lists:delete({Pos, Key}, KeysToGet) of KeysToGet -> % same list, key not found - {LeafsFound, KeysToGet2} = get_key_leafs(SubTree, KeysToGet, [Key | KeyPathAcc]), - {RestLeafsFound, KeysRemaining} = get_key_leafs(RestTree, KeysToGet2, KeyPathAcc), + {LeafsFound, KeysToGet2} = get_key_leafs_simple(Pos + 1, SubTree, KeysToGet, [Key | KeyPathAcc]), + {RestLeafsFound, KeysRemaining} = get_key_leafs_simple(Pos, RestTree, KeysToGet2, KeyPathAcc), {LeafsFound ++ RestLeafsFound, KeysRemaining}; KeysToGet2 -> - LeafsFound = get_all_leafs([Tree], KeyPathAcc), + LeafsFound = get_all_leafs_simple(Pos, [Tree], KeyPathAcc), LeafKeysFound = [LeafKeyFound || {LeafKeyFound, _, _} <- LeafsFound], KeysToGet2 = KeysToGet2 -- LeafKeysFound, - {RestLeafsFound, KeysRemaining} = get_key_leafs(RestTree, KeysToGet2, KeyPathAcc), + {RestLeafsFound, KeysRemaining} = get_key_leafs_simple(Pos, RestTree, KeysToGet2, KeyPathAcc), {LeafsFound ++ RestLeafsFound, KeysRemaining} end. get(Tree, KeysToGet) -> {KeyPaths, KeysNotFound} = get_full_key_paths(Tree, KeysToGet), - FixedResults = [ {Key, Value, [Key0 || {Key0, _} <- Path]} || [{Key, Value}|_] = Path <- KeyPaths], + FixedResults = [ {Value, {Pos, [Key0 || {Key0, _} <- Path]}} || {Pos, [{_Key, Value}|_]=Path} <- KeyPaths], {FixedResults, KeysNotFound}. get_full_key_paths(Tree, Keys) -> get_full_key_paths(Tree, Keys, []). -get_full_key_paths(_Tree, [], _KeyPathAcc) -> +get_full_key_paths(_, [], Acc) -> + {Acc, []}; +get_full_key_paths([], Keys, Acc) -> + {Acc, Keys}; +get_full_key_paths([{Pos, Tree}|Rest], Keys, Acc) -> + {Gotten, RemainingKeys} = get_full_key_paths(Pos, [Tree], Keys, []), + get_full_key_paths(Rest, RemainingKeys, Gotten ++ Acc). + + +get_full_key_paths(_Pos, _Tree, [], _KeyPathAcc) -> {[], []}; -get_full_key_paths([], KeysToGet, _KeyPathAcc) -> +get_full_key_paths(_Pos, [], KeysToGet, _KeyPathAcc) -> {[], KeysToGet}; -get_full_key_paths([{KeyId, Value, SubTree} | RestTree], KeysToGet, KeyPathAcc) -> - KeysToGet2 = KeysToGet -- [KeyId], +get_full_key_paths(Pos, [{KeyId, Value, SubTree} | RestTree], KeysToGet, KeyPathAcc) -> + KeysToGet2 = KeysToGet -- [{Pos, KeyId}], CurrentNodeResult = case length(KeysToGet2) == length(KeysToGet) of true -> % not in the key list. []; false -> % this node is the key list. return it - [[{KeyId, Value} | KeyPathAcc]] + [{Pos, [{KeyId, Value} | KeyPathAcc]}] end, - {KeysGotten, KeysRemaining} = get_full_key_paths(SubTree, KeysToGet2, [{KeyId, Value} | KeyPathAcc]), - {KeysGotten2, KeysRemaining2} = get_full_key_paths(RestTree, KeysRemaining, KeyPathAcc), + {KeysGotten, KeysRemaining} = get_full_key_paths(Pos + 1, SubTree, KeysToGet2, [{KeyId, Value} | KeyPathAcc]), + {KeysGotten2, KeysRemaining2} = get_full_key_paths(Pos, RestTree, KeysRemaining, KeyPathAcc), {CurrentNodeResult ++ KeysGotten ++ KeysGotten2, KeysRemaining2}. get_all_leafs_full(Tree) -> get_all_leafs_full(Tree, []). -get_all_leafs_full([], _KeyPathAcc) -> +get_all_leafs_full([], Acc) -> + Acc; +get_all_leafs_full([{Pos, Tree} | Rest], Acc) -> + get_all_leafs_full(Rest, get_all_leafs_full_simple(Pos, [Tree], []) ++ Acc). + +get_all_leafs_full_simple(_Pos, [], _KeyPathAcc) -> []; -get_all_leafs_full([{KeyId, Value, []} | RestTree], KeyPathAcc) -> - [[{KeyId, Value} | KeyPathAcc] | get_all_leafs_full(RestTree, KeyPathAcc)]; -get_all_leafs_full([{KeyId, Value, SubTree} | RestTree], KeyPathAcc) -> - get_all_leafs_full(SubTree, [{KeyId, Value} | KeyPathAcc]) ++ get_all_leafs_full(RestTree, KeyPathAcc). +get_all_leafs_full_simple(Pos, [{KeyId, Value, []} | RestTree], KeyPathAcc) -> + [{Pos, [{KeyId, Value} | KeyPathAcc]} | get_all_leafs_full_simple(Pos, RestTree, KeyPathAcc)]; +get_all_leafs_full_simple(Pos, [{KeyId, Value, SubTree} | RestTree], KeyPathAcc) -> + get_all_leafs_full_simple(Pos + 1, SubTree, [{KeyId, Value} | KeyPathAcc]) ++ get_all_leafs_full_simple(Pos, RestTree, KeyPathAcc). -get_all_leafs(Tree) -> - get_all_leafs(Tree, []). +get_all_leafs(Trees) -> + get_all_leafs(Trees, []). + +get_all_leafs([], Acc) -> + Acc; +get_all_leafs([{Pos, Tree}|Rest], Acc) -> + get_all_leafs(Rest, get_all_leafs_simple(Pos, [Tree], []) ++ Acc). -get_all_leafs([], _KeyPathAcc) -> +get_all_leafs_simple(_Pos, [], _KeyPathAcc) -> []; -get_all_leafs([{KeyId, Value, []} | RestTree], KeyPathAcc) -> - [{KeyId, Value, [KeyId | KeyPathAcc]} | get_all_leafs(RestTree, KeyPathAcc)]; -get_all_leafs([{KeyId, _Value, SubTree} | RestTree], KeyPathAcc) -> - get_all_leafs(SubTree, [KeyId | KeyPathAcc]) ++ get_all_leafs(RestTree, KeyPathAcc). +get_all_leafs_simple(Pos, [{KeyId, Value, []} | RestTree], KeyPathAcc) -> + [{Value, {Pos, [KeyId | KeyPathAcc]}} | get_all_leafs_simple(Pos, RestTree, KeyPathAcc)]; +get_all_leafs_simple(Pos, [{KeyId, _Value, SubTree} | RestTree], KeyPathAcc) -> + get_all_leafs_simple(Pos + 1, SubTree, [KeyId | KeyPathAcc]) ++ get_all_leafs_simple(Pos, RestTree, KeyPathAcc). + -get_leaf_keys([]) -> - []; -get_leaf_keys([{Key, _Value, []} | RestTree]) -> - [Key | get_leaf_keys(RestTree)]; -get_leaf_keys([{_Key, _Value, SubTree} | RestTree]) -> - get_leaf_keys(SubTree) ++ get_leaf_keys(RestTree). - count_leafs([]) -> 0; -count_leafs([{_Key, _Value, []} | RestTree]) -> - 1 + count_leafs(RestTree); -count_leafs([{_Key, _Value, SubTree} | RestTree]) -> - count_leafs(SubTree) + count_leafs(RestTree). +count_leafs([{_Pos,Tree}|Rest]) -> + count_leafs_simple([Tree]) + count_leafs(Rest). +count_leafs_simple([]) -> + 0; +count_leafs_simple([{_Key, _Value, []} | RestTree]) -> + 1 + count_leafs_simple(RestTree); +count_leafs_simple([{_Key, _Value, SubTree} | RestTree]) -> + count_leafs_simple(SubTree) + count_leafs_simple(RestTree). + map(_Fun, []) -> []; -map(Fun, [{Key, Value, SubTree} | RestTree]) -> - Value2 = Fun(Key, Value), - [{Key, Value2, map(Fun, SubTree)} | map(Fun, RestTree)]. +map(Fun, [{Pos, Tree}|Rest]) -> + [NewTree] = map_simple(Fun, Pos, [Tree]), + [{Pos, NewTree} | map(Fun, Rest)]. + +map_simple(_Fun, _Pos, []) -> + []; +map_simple(Fun, Pos, [{Key, Value, SubTree} | RestTree]) -> + Value2 = Fun({Pos, Key}, Value), + [{Key, Value2, map_simple(Fun, Pos + 1, SubTree)} | map_simple(Fun, Pos, RestTree)]. + + +stem(Trees, Limit) -> + % flatten each branch in a tree into a tree path + Paths = get_all_leafs_full(Trees), + + Paths2 = [{Pos, lists:sublist(Path, Limit)} || {Pos, Path} <- Paths], + + % convert paths back to trees + lists:foldl( + fun({PathPos, Path},TreeAcc) -> + [SingleTree] = lists:foldl( + fun({K,V},NewTreeAcc) -> [{K,V,NewTreeAcc}] end, [], Path), + {NewTrees, _} = merge(TreeAcc, [{PathPos + 1 - length(Path), SingleTree}]), + NewTrees + end, [], Paths2). +test() -> + EmptyTree = [], + One = [{0, {"1","foo",[]}}], + TwoSibs = [{0, {"1","foo",[]}}, + {0, {"2","foo",[]}}], + OneChild = [{0, {"1","foo",[{"1a", "bar", []}]}}], + TwoChild = [{0, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}], + TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, + {"1b", "bar", []}]}}], + Stemmed1a = [{1, {"1a", "bar", [{"1aa", "bar", []}]}}], + Stemmed1aa = [{2, {"1aa", "bar", []}}], + + {EmptyTree, no_conflicts} = merge(EmptyTree, EmptyTree), + {One, no_conflicts} = merge(EmptyTree, One), + {One, no_conflicts} = merge(One, EmptyTree), + {TwoSibs, no_conflicts} = merge(One, TwoSibs), + {One, no_conflicts} = merge(One, One), + {TwoChild, no_conflicts} = merge(TwoChild, TwoChild), + {TwoChildSibs, no_conflicts} = merge(TwoChildSibs, TwoChildSibs), + {TwoChild, no_conflicts} = merge(TwoChild, Stemmed1aa), + {TwoChild, no_conflicts} = merge(TwoChild, Stemmed1a), + {Stemmed1a, no_conflicts} = merge(Stemmed1a, Stemmed1aa), + Expect1 = OneChild ++ Stemmed1aa, + {Expect1, conflicts} = merge(OneChild, Stemmed1aa), + {TwoChild, no_conflicts} = merge(Expect1, TwoChild), + + []=find_missing(TwoChildSibs, [{0,"1"}, {1,"1a"}]), + [{0, "10"}, {100, "x"}]=find_missing(TwoChildSibs, [{0,"1"}, {0, "10"}, {1,"1a"}, {100, "x"}]), + [{0, "1"}, {100, "x"}]=find_missing(Stemmed1a, [{0,"1"}, {1,"1a"}, {100, "x"}]), + [{0, "1"}, {1,"1a"}, {100, "x"}]=find_missing(Stemmed1aa, [{0,"1"}, {1,"1a"}, {100, "x"}]), + + {TwoChildSibs, []} = remove_leafs(TwoChildSibs, []), + {TwoChildSibs, []} = remove_leafs(TwoChildSibs, [{0, "1"}]), + {OneChild, [{1, "1b"}]} = remove_leafs(TwoChildSibs, [{1, "1b"}]), + {[], [{1, "1b"},{1, "1a"}]} = remove_leafs(TwoChildSibs, [{1, "1a"}, {1, "1b"}]), + {Stemmed1a, []} = remove_leafs(Stemmed1a, [{1, "1a"}]), + {[], [{2, "1aa"}]} = remove_leafs(Stemmed1a, [{2, "1aa"}]), + {TwoChildSibs, []} = remove_leafs(TwoChildSibs, []), + + {[],[{0,"x"}]} = get_key_leafs(TwoChildSibs, [{0, "x"}]), + + {[{"bar", {1, ["1a","1"]}}],[]} = get_key_leafs(TwoChildSibs, [{1, "1a"}]), + {[{"bar", {1, ["1a","1"]}},{"bar",{1, ["1b","1"]}}],[]} = get_key_leafs(TwoChildSibs, [{0, "1"}]), + + {[{"foo", {0, ["1"]}}],[]} = get(TwoChildSibs, [{0, "1"}]), + {[{"bar", {1, ["1a", "1"]}}],[]} = get(TwoChildSibs, [{1, "1a"}]), + + {[{0,[{"1", "foo"}]}],[]} = get_full_key_paths(TwoChildSibs, [{0, "1"}]), + {[{1,[{"1a", "bar"},{"1", "foo"}]}],[]} = get_full_key_paths(TwoChildSibs, [{1, "1a"}]), + + [{2, [{"1aa", "bar"},{"1a", "bar"}]}] = get_all_leafs_full(Stemmed1a), + [{1, [{"1a", "bar"},{"1", "foo"}]}, {1, [{"1b", "bar"},{"1", "foo"}]}] = get_all_leafs_full(TwoChildSibs), + + [{"bar", {2, ["1aa","1a"]}}] = get_all_leafs(Stemmed1a), + [{"bar", {1, ["1a", "1"]}}, {"bar", {1, ["1b","1"]}}] = get_all_leafs(TwoChildSibs), + + 0 = count_leafs(EmptyTree), + 1 = count_leafs(One), + 2 = count_leafs(TwoChildSibs), + 1 = count_leafs(Stemmed1a), + + TwoChild = stem(TwoChild, 3), + Stemmed1a = stem(TwoChild, 2), + Stemmed1aa = stem(TwoChild, 1), + ok. + + + + + + + + + + + \ No newline at end of file diff --git a/src/couchdb/couch_rep.erl b/src/couchdb/couch_rep.erl index 89d40be3..3647f6db 100644 --- a/src/couchdb/couch_rep.erl +++ b/src/couchdb/couch_rep.erl @@ -15,11 +15,11 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --export([replicate/3]). +-export([replicate/2]). -include_lib("couch_db.hrl"). -%% @spec replicate(Source::binary(), Target::binary(), Options::proplist()) -> +%% @spec replicate(Source::binary(), Target::binary()) -> %% {ok, Stats} | {error, Reason} %% @doc Triggers a replication. Stats is a JSON Object with the following %% keys: session_id (UUID), source_last_seq (integer), and history (array). @@ -30,26 +30,29 @@ %% The supervisor will try to restart the replication in case of any error %% other than shutdown. Just call this function again to listen for the %% result of the retry. -replicate(Source, Target, Options) -> - Id = <>, - Args = [?MODULE, [Source,Target,Options], []], +replicate(Source, Target) -> - Replicator = {Id, + {ok, HostName} = inet:gethostname(), + RepId = couch_util:to_hex( + erlang:md5(term_to_binary([HostName, Source, Target]))), + Args = [?MODULE, [RepId, Source,Target], []], + + Replicator = {RepId, {gen_server, start_link, Args}, transient, - 10000, + 1, worker, [?MODULE] }, Server = case supervisor:start_child(couch_rep_sup, Replicator) of {ok, Pid} -> - ?LOG_INFO("starting new replication ~p at ~p", [Id, Pid]), + ?LOG_INFO("starting new replication ~p at ~p", [RepId, Pid]), Pid; {error, already_present} -> - case supervisor:restart_child(couch_rep_sup, Id) of + case supervisor:restart_child(couch_rep_sup, RepId) of {ok, Pid} -> - ?LOG_INFO("starting replication ~p at ~p", [Id, Pid]), + ?LOG_INFO("starting replication ~p at ~p", [RepId, Pid]), Pid; {error, running} -> %% this error occurs if multiple replicators are racing @@ -57,16 +60,16 @@ replicate(Source, Target, Options) -> %% the Pid by calling start_child again. {error, {already_started, Pid}} = supervisor:start_child(couch_rep_sup, Replicator), - ?LOG_INFO("replication ~p already running at ~p", [Id, Pid]), + ?LOG_INFO("replication ~p already running at ~p", [RepId, Pid]), Pid end; {error, {already_started, Pid}} -> - ?LOG_INFO("replication ~p already running at ~p", [Id, Pid]), + ?LOG_INFO("replication ~p already running at ~p", [RepId, Pid]), Pid end, case gen_server:call(Server, get_result, infinity) of - retry -> replicate(Source, Target, Options); + retry -> replicate(Source, Target); Else -> Else end. @@ -79,6 +82,7 @@ replicate(Source, Target, Options) -> headers }). + -record(state, { context, current_seq, @@ -90,18 +94,14 @@ replicate(Source, Target, Options) -> listeners = [] }). -init([Source, Target, Options]) -> + +init([RepId, Source, Target]) -> process_flag(trap_exit, true), - {ok, DbSrc} = - open_db(Source, proplists:get_value(source_options, Options, [])), - {ok, DbTgt} = - open_db(Target, proplists:get_value(target_options, Options, [])), + {ok, DbSrc, SrcName} = open_db(Source), + {ok, DbTgt, TgtName} = open_db(Target), - {ok, Host} = inet:gethostname(), - HostBin = list_to_binary(Host), - DocKey = <>, + DocKey = ?l2b(?LOCAL_DOC_PREFIX ++ RepId), {ok, InfoSrc} = get_db_info(DbSrc), {ok, InfoTgt} = get_db_info(DbTgt), @@ -110,49 +110,49 @@ init([Source, Target, Options]) -> SrcInstanceStartTime = proplists:get_value(instance_start_time, InfoSrc), TgtInstanceStartTime = proplists:get_value(instance_start_time, InfoTgt), - case proplists:get_value(full, Options, false) - orelse proplists:get_value("full", Options, false) of + RepRecDocSrc = + case open_doc(DbSrc, DocKey, []) of + {ok, SrcDoc} -> + ?LOG_DEBUG("Found existing replication record on source", []), + SrcDoc; + _ -> #doc{id=DocKey} + end, + + RepRecDocTgt = + case open_doc(DbTgt, DocKey, []) of + {ok, TgtDoc} -> + ?LOG_DEBUG("Found existing replication record on target", []), + TgtDoc; + _ -> #doc{id=DocKey} + end, + + #doc{body={RepRecProps}} = RepRecDocSrc, + #doc{body={RepRecPropsTgt}} = RepRecDocTgt, + + case proplists:get_value(<<"session_id">>, RepRecProps) == + proplists:get_value(<<"session_id">>, RepRecPropsTgt) of true -> - RepRecSrc = RepRecTgt = #doc{id=DocKey}; + % if the records have the same session id, + % then we have a valid replication history + OldSeqNum = proplists:get_value(<<"source_last_seq">>, RepRecProps, 0), + OldHistory = proplists:get_value(<<"history">>, RepRecProps, []); false -> - RepRecSrc = case open_doc(DbSrc, DocKey, []) of - {ok, SrcDoc} -> - ?LOG_DEBUG("Found existing replication record on source", []), - SrcDoc; - _ -> #doc{id=DocKey} - end, - - RepRecTgt = case open_doc(DbTgt, DocKey, []) of - {ok, TgtDoc} -> - ?LOG_DEBUG("Found existing replication record on target", []), - TgtDoc; - _ -> #doc{id=DocKey} - end - end, - - #doc{body={OldRepHistoryProps}} = RepRecSrc, - #doc{body={OldRepHistoryPropsTrg}} = RepRecTgt, - - SeqNum = case OldRepHistoryProps == OldRepHistoryPropsTrg of - true -> - % if the records are identical, then we have a valid replication history - proplists:get_value(<<"source_last_seq">>, OldRepHistoryProps, 0); - false -> - ?LOG_INFO("Replication records differ. " + ?LOG_INFO("Replication records differ. " "Performing full replication instead of incremental.", []), - ?LOG_DEBUG("Record on source:~p~nRecord on target:~p~n", - [OldRepHistoryProps, OldRepHistoryPropsTrg]), - 0 + ?LOG_DEBUG("Record on source:~p~nRecord on target:~p~n", + [RepRecProps, RepRecPropsTgt]), + OldSeqNum = 0, + OldHistory = [] end, Context = [ - {start_seq, SeqNum}, - {history, OldRepHistoryProps}, + {start_seq, OldSeqNum}, + {history, OldHistory}, {rep_starttime, ReplicationStartTime}, {src_starttime, SrcInstanceStartTime}, {tgt_starttime, TgtInstanceStartTime}, - {src_record, RepRecSrc}, - {tgt_record, RepRecTgt} + {src_record, RepRecDocSrc}, + {tgt_record, RepRecDocTgt} ], Stats = ets:new(replication_stats, [set, private]), @@ -160,16 +160,17 @@ init([Source, Target, Options]) -> ets:insert(Stats, {missing_revs, 0}), ets:insert(Stats, {docs_read, 0}), ets:insert(Stats, {docs_written, 0}), + ets:insert(Stats, {doc_write_failures, 0}), - couch_task_status:add_task("Replication", < ", - Target/binary>>, "Starting"), + couch_task_status:add_task("Replication", < ", + TgtName/binary>>, "Starting"), Parent = self(), - Pid = spawn_link(fun() -> enum_docs_since(Parent,DbSrc,DbTgt,{SeqNum,0}) end), + Pid = spawn_link(fun() -> enum_docs_since(Parent,DbSrc,DbTgt,{OldSeqNum,0}) end), State = #state{ context = Context, - current_seq = SeqNum, + current_seq = OldSeqNum, enum_pid = Pid, source = DbSrc, target = DbTgt, @@ -178,7 +179,6 @@ init([Source, Target, Options]) -> {ok, State}. - handle_call(get_result, From, #state{listeners=L} = State) -> {noreply, State#state{listeners=[From|L]}}; @@ -191,7 +191,7 @@ handle_call({replicate_doc, {Id, Revs}}, {Pid,_}, #state{enum_pid=Pid} = State) target = Target, stats = Stats } = State, - + ets:update_counter(Stats, missing_revs, length(Revs)), %% get document(s) @@ -203,8 +203,11 @@ handle_call({replicate_doc, {Id, Revs}}, {Pid,_}, #state{enum_pid=Pid} = State) {NewBuffer, NewContext} = case couch_util:should_flush() of true -> Docs2 = lists:flatten([Docs|Buffer]), - ok = update_docs(Target, Docs2, [], false), - ets:update_counter(Stats, docs_written, length(Docs2)), + {ok, Errors} = update_docs(Target, Docs2, [], replicated_changes), + dump_update_errors(Errors), + ets:update_counter(Stats, doc_write_failures, length(Errors)), + ets:update_counter(Stats, docs_written, length(Docs2) - + length(Errors)), {ok, _, Ctxt} = do_checkpoint(Source, Target, Context, Seq, Stats), {[], Ctxt}; false -> @@ -255,8 +258,11 @@ terminate(normal, State) -> stats = Stats } = State, - ok = update_docs(Target, lists:flatten(Buffer), [], false), - ets:update_counter(Stats, docs_written, lists:flatlength(Buffer)), + {ok, Errors} = update_docs(Target, lists:flatten(Buffer), [], replicated_changes), + dump_update_errors(Errors), + ets:update_counter(Stats, doc_write_failures, length(Errors)), + ets:update_counter(Stats, docs_written, lists:flatlength(Buffer) - + length(Errors)), couch_task_status:update("Finishing"), @@ -264,9 +270,12 @@ terminate(normal, State) -> ets:delete(Stats), close_db(Target), - %% reply to original requester - [Original|Rest] = Listeners, - gen_server:reply(Original, {ok, NewRepHistory}), + case Listeners of + [Original|Rest] -> + %% reply to original requester + gen_server:reply(Original, {ok, NewRepHistory}); + Rest -> ok + end, %% maybe trigger another replication. If this replicator uses a local %% source Db, changes to that Db since we started will not be included in @@ -304,6 +313,16 @@ code_change(_OldVsn, State, _Extra) -> %% internal functions %%============================================================================= + +% we should probably write these to a special replication log +% or have a callback where the caller decides what to do with replication +% errors. +dump_update_errors([]) -> ok; +dump_update_errors([{{Id, Rev}, Error}|Rest]) -> + ?LOG_INFO("error replicating document \"~s\" rev \"~s\":~p", + [Id, couch_doc:rev_to_str(Rev), Error]), + dump_update_errors(Rest). + attachment_loop(ReqId) -> couch_util:should_flush(), receive @@ -354,6 +373,16 @@ attachment_stub_converter(DbS, Id, {Name, {stub, Type, Length}}) -> end, {Name, {Type, {RcvFun, Length}}}. + +open_db({remote, Url, Headers})-> + {ok, #http_db{uri=?b2l(Url), headers=Headers}, Url}; +open_db({local, DbName, UserCtx})-> + case couch_db:open(DbName, [{user_ctx, UserCtx}]) of + {ok, Db} -> {ok, Db, DbName}; + Error -> Error + end. + + close_db(#http_db{})-> ok; close_db(Db)-> @@ -362,27 +391,38 @@ close_db(Db)-> do_checkpoint(Source, Target, Context, NewSeqNum, Stats) -> ?LOG_INFO("recording a checkpoint at source update_seq ~p", [NewSeqNum]), [ - {start_seq, SeqNum}, - {history, OldRepHistoryProps}, + {start_seq, StartSeqNum}, + {history, OldHistory}, {rep_starttime, ReplicationStartTime}, {src_starttime, SrcInstanceStartTime}, {tgt_starttime, TgtInstanceStartTime}, - {src_record, RepRecSrc}, - {tgt_record, RepRecTgt} + {src_record, #doc{body={LastRepRecord}}=RepRecDocSrc}, + {tgt_record, RepRecDocTgt} ] = Context, - NewHistory = case NewSeqNum == SeqNum andalso OldRepHistoryProps /= [] of + case NewSeqNum == StartSeqNum andalso OldHistory /= [] of true -> % nothing changed, don't record results - {OldRepHistoryProps}; + {ok, {[{<<"no_changes">>, true} | LastRepRecord]}, Context}; false -> + % something changed, record results for incremental replication, + % commit changes to both src and tgt. The src because if changes - % we replicated are lost, we'll record the a seq number of ahead - % of what was committed and therefore lose future changes with the - % same seq nums. - {ok, SrcInstanceStartTime2} = ensure_full_commit(Source), + % we replicated are lost, we'll record the a seq number ahead + % of what was committed. If those changes are lost and the seq number + % reverts to a previous committed value, we will skip future changes + % when new doc updates are given our already replicated seq nums. + + % commit the src async + ParentPid = self(), + SrcCommitPid = spawn_link(fun() -> + ParentPid ! {self(), ensure_full_commit(Source)} end), + + % commit tgt sync {ok, TgtInstanceStartTime2} = ensure_full_commit(Target), + receive {SrcCommitPid, {ok, SrcInstanceStartTime2}} -> ok end, + RecordSeqNum = if SrcInstanceStartTime2 == SrcInstanceStartTime andalso TgtInstanceStartTime2 == TgtInstanceStartTime -> @@ -391,60 +431,57 @@ do_checkpoint(Source, Target, Context, NewSeqNum, Stats) -> ?LOG_INFO("A server has restarted sinced replication start. " "Not recording the new sequence number to ensure the " "replication is redone and documents reexamined.", []), - SeqNum + StartSeqNum end, - %% format replication history - JsonStats = [ + NewHistoryEntry = { + [{<<"start_time">>, list_to_binary(ReplicationStartTime)}, + {<<"end_time">>, list_to_binary(httpd_util:rfc1123_date())}, + {<<"start_last_seq">>, StartSeqNum}, + {<<"end_last_seq">>, NewSeqNum}, {<<"missing_checked">>, ets:lookup_element(Stats, total_revs, 2)}, {<<"missing_found">>, ets:lookup_element(Stats, missing_revs, 2)}, {<<"docs_read">>, ets:lookup_element(Stats, docs_read, 2)}, - {<<"docs_written">>, ets:lookup_element(Stats, docs_written, 2)} + {<<"docs_written">>, ets:lookup_element(Stats, docs_written, 2)}, + {<<"doc_write_failures">>, ets:lookup_element(Stats, doc_write_failures, 2)} + ]}, + % limit history to 50 entries + HistEntries =lists:sublist([NewHistoryEntry | OldHistory], 50), + + NewRepHistory = + {[{<<"session_id">>, couch_util:new_uuid()}, + {<<"source_last_seq">>, RecordSeqNum}, + {<<"history">>, HistEntries}]}, + + {ok, {SrcRevPos,SrcRevId}} = update_doc(Source, + RepRecDocSrc#doc{body=NewRepHistory}, []), + {ok, {TgtRevPos,TgtRevId}} = update_doc(Target, + RepRecDocTgt#doc{body=NewRepHistory}, []), + + NewContext = [ + {start_seq, StartSeqNum}, + {history, OldHistory}, + {rep_starttime, ReplicationStartTime}, + {src_starttime, SrcInstanceStartTime}, + {tgt_starttime, TgtInstanceStartTime}, + {src_record, RepRecDocSrc#doc{revs={SrcRevPos,[SrcRevId]}}}, + {tgt_record, RepRecDocTgt#doc{revs={TgtRevPos,[TgtRevId]}}} ], - - HistEntries =[ - { - [{<<"start_time">>, list_to_binary(ReplicationStartTime)}, - {<<"end_time">>, list_to_binary(httpd_util:rfc1123_date())}, - {<<"start_last_seq">>, SeqNum}, - {<<"end_last_seq">>, NewSeqNum} | JsonStats]} - | proplists:get_value(<<"history">>, OldRepHistoryProps, [])], - % something changed, record results - {[ - {<<"session_id">>, couch_util:new_uuid()}, - {<<"source_last_seq">>, RecordSeqNum}, - {<<"history">>, lists:sublist(HistEntries, 50)} - ]} - end, - %% update local documents - RepRecSrc = proplists:get_value(src_record, Context), - RepRecTgt = proplists:get_value(tgt_record, Context), - {ok, TgtRev} = update_local_doc(Target, RepRecTgt#doc{body=NewHistory}, []), - {ok, SrcRev} = update_local_doc(Source, RepRecSrc#doc{body=NewHistory}, []), + {ok, NewRepHistory, NewContext} - NewContext = [ - {start_seq, SeqNum}, - {history, OldRepHistoryProps}, - {rep_starttime, ReplicationStartTime}, - {src_starttime, SrcInstanceStartTime}, - {tgt_starttime, TgtInstanceStartTime}, - {src_record, RepRecSrc#doc{revs=[SrcRev]}}, - {tgt_record, RepRecTgt#doc{revs=[TgtRev]}} - ], - - {ok, NewHistory, NewContext}. + end. do_http_request(Url, Action, Headers) -> do_http_request(Url, Action, Headers, []). do_http_request(Url, Action, Headers, JsonBody) -> - do_http_request(?b2l(?l2b(Url)), Action, Headers, JsonBody, 10). + do_http_request(Url, Action, Headers, JsonBody, 10). do_http_request(Url, Action, _Headers, _JsonBody, 0) -> ?LOG_ERROR("couch_rep HTTP ~p request failed after 10 retries: ~s", [Action, Url]), - exit({http_request_failed, ?l2b(Url)}); + exit({http_request_failed, Url}); do_http_request(Url, Action, Headers, JsonBody, Retries) -> ?LOG_DEBUG("couch_rep HTTP ~p request: ~s", [Action, Url]), Body = @@ -498,7 +535,6 @@ enum_docs_since(Pid, DbSource, DbTarget, {StartSeq, RevsCount}) -> [] -> gen_server:call(Pid, {fin, {StartSeq, RevsCount}}, infinity); DocInfoList -> - % UpdateSeqs = [D#doc_info.update_seq || D <- DocInfoList], SrcRevsList = lists:map(fun(SrcDocInfo) -> #doc_info{id=Id, rev=Rev, @@ -521,13 +557,8 @@ enum_docs_since(Pid, DbSource, DbTarget, {StartSeq, RevsCount}) -> enum_docs_since(Pid, DbSource, DbTarget, {LastSeq, RevsCount2}) end. -fix_url(UrlBin) -> - Url = binary_to_list(UrlBin), - case lists:last(Url) of - $/ -> Url; - _ -> Url ++ "/" - end. + get_db_info(#http_db{uri=DbUrl, headers=Headers}) -> {DbProps} = do_http_request(DbUrl, get, Headers), {ok, [{list_to_existing_atom(?b2l(K)), V} || {K,V} <- DbProps]}; @@ -542,12 +573,12 @@ get_doc_info_list(#http_db{uri=DbUrl, headers=Headers}, StartSeq) -> {RowValueProps} = proplists:get_value(<<"value">>, RowInfoList), #doc_info{ id=proplists:get_value(<<"id">>, RowInfoList), - rev=proplists:get_value(<<"rev">>, RowValueProps), + rev=couch_doc:parse_rev(proplists:get_value(<<"rev">>, RowValueProps)), update_seq = proplists:get_value(<<"key">>, RowInfoList), conflict_revs = - proplists:get_value(<<"conflicts">>, RowValueProps, []), + couch_doc:parse_revs(proplists:get_value(<<"conflicts">>, RowValueProps, [])), deleted_conflict_revs = - proplists:get_value(<<"deleted_conflicts">>, RowValueProps, []), + couch_doc:parse_revs(proplists:get_value(<<"deleted_conflicts">>, RowValueProps, [])), deleted = proplists:get_value(<<"deleted">>, RowValueProps, false) } end, proplists:get_value(<<"rows">>, Results)); @@ -561,25 +592,18 @@ get_doc_info_list(DbSource, StartSeq) -> lists:reverse(DocInfoList). get_missing_revs(#http_db{uri=DbUrl, headers=Headers}, DocIdRevsList) -> + DocIdRevsList2 = [{Id, couch_doc:rev_to_strs(Revs)} || {Id, Revs} <- DocIdRevsList], {ResponseMembers} = do_http_request(DbUrl ++ "_missing_revs", post, Headers, - {DocIdRevsList}), - {MissingRevs} = proplists:get_value(<<"missing_revs">>, ResponseMembers), - {ok, MissingRevs}; + {DocIdRevsList2}), + {DocMissingRevsList} = proplists:get_value(<<"missing_revs">>, ResponseMembers), + DocMissingRevsList2 = [{Id, couch_doc:parse_revs(MissingRevStrs)} || {Id, MissingRevStrs} <- DocMissingRevsList], + {ok, DocMissingRevsList2}; get_missing_revs(Db, DocId) -> couch_db:get_missing_revs(Db, DocId). -open_http_db(UrlBin, Options) -> - Headers = proplists:get_value(headers, Options, {[]}), - {ok, #http_db{uri=fix_url(UrlBin), headers=Headers}}. - -open_db(<<"http://", _/binary>>=Url, Options)-> - open_http_db(Url, Options); -open_db(<<"https://", _/binary>>=Url, Options)-> - open_http_db(Url, Options); -open_db(DbName, Options)-> - couch_db:open(DbName, Options). - -open_doc(#http_db{uri=DbUrl, headers=Headers}, DocId, []) -> + +open_doc(#http_db{uri=DbUrl, headers=Headers}, DocId, Options) -> + [] = Options, case do_http_request(DbUrl ++ url_encode(DocId), get, Headers) of {[{<<"error">>, ErrId}, {<<"reason">>, Reason}]} -> {couch_util:to_existing_atom(ErrId), Reason}; @@ -589,7 +613,9 @@ open_doc(#http_db{uri=DbUrl, headers=Headers}, DocId, []) -> open_doc(Db, DocId, Options) -> couch_db:open_doc(Db, DocId, Options). -open_doc_revs(#http_db{uri=DbUrl, headers=Headers} = DbS, DocId, Revs, _Opts) -> +open_doc_revs(#http_db{uri=DbUrl, headers=Headers} = DbS, DocId, Revs0, + [latest]) -> + Revs = couch_doc:rev_to_strs(Revs0), BaseUrl = DbUrl ++ url_encode(DocId) ++ "?revs=true&latest=true", %% MochiWeb expects URLs < 8KB long, so maybe split into multiple requests @@ -612,39 +638,52 @@ open_doc_revs(#http_db{uri=DbUrl, headers=Headers} = DbS, DocId, Revs, _Opts) -> lists:flatten(?JSON_ENCODE(lists:reverse(Rest))), get, Headers) end, - Results = - lists:map(fun({[{<<"missing">>, Rev}]}) -> - {{not_found, missing}, Rev}; - ({[{<<"ok">>, JsonDoc}]}) -> + Results = + lists:map( + fun({[{<<"missing">>, Rev}]}) -> + {{not_found, missing}, couch_doc:parse_rev(Rev)}; + ({[{<<"ok">>, JsonDoc}]}) -> #doc{id=Id, attachments=Attach} = Doc = couch_doc:from_json_obj(JsonDoc), Attach2 = [attachment_stub_converter(DbS,Id,A) || A <- Attach], {ok, Doc#doc{attachments=Attach2}} - end, JsonResults), + end, JsonResults), {ok, Results}; open_doc_revs(Db, DocId, Revs, Options) -> couch_db:open_doc_revs(Db, DocId, Revs, Options). -update_docs(_, [], _, _) -> - ok; -update_docs(#http_db{uri=DbUrl, headers=Headers}, Docs, [], NewEdits) -> - JsonDocs = [couch_doc:to_json_obj(Doc, [revs,attachments]) || Doc <- Docs], - {Returned} = - do_http_request(DbUrl ++ "_bulk_docs", post, Headers, - {[{new_edits, NewEdits}, {docs, JsonDocs}]}), - true = proplists:get_value(<<"ok">>, Returned), - ok; -update_docs(Db, Docs, Options, NewEdits) -> - couch_db:update_docs(Db, Docs, Options, NewEdits). -update_local_doc(#http_db{uri=DbUrl, headers=Headers}, #doc{id=DocId}=Doc, []) -> +update_doc(#http_db{uri=DbUrl, headers=Headers}, #doc{id=DocId}=Doc, Options) -> + [] = Options, Url = DbUrl ++ url_encode(DocId), {ResponseMembers} = do_http_request(Url, put, Headers, - couch_doc:to_json_obj(Doc, [revs,attachments])), - RevId = proplists:get_value(<<"rev">>, ResponseMembers), - {ok, RevId}; -update_local_doc(Db, Doc, Options) -> + couch_doc:to_json_obj(Doc, [attachments])), + Rev = proplists:get_value(<<"rev">>, ResponseMembers), + {ok, couch_doc:parse_rev(Rev)}; +update_doc(Db, Doc, Options) -> couch_db:update_doc(Db, Doc, Options). +update_docs(_, [], _, _) -> + {ok, []}; +update_docs(#http_db{uri=DbUrl, headers=Headers}, Docs, [], replicated_changes) -> + JsonDocs = [couch_doc:to_json_obj(Doc, [revs,attachments]) || Doc <- Docs], + ErrorsJson = + do_http_request(DbUrl ++ "_bulk_docs", post, Headers, + {[{new_edits, false}, {docs, JsonDocs}]}), + ErrorsList = + lists:map( + fun({Props}) -> + Id = proplists:get_value(<<"id">>, Props), + Rev = couch_doc:parse_rev(proplists:get_value(<<"rev">>, Props)), + ErrId = couch_util:to_existing_atom( + proplists:get_value(<<"error">>, Props)), + Reason = proplists:get_value(<<"reason">>, Props), + Error = {ErrId, Reason}, + {{Id, Rev}, Error} + end, ErrorsJson), + {ok, ErrorsList}; +update_docs(Db, Docs, Options, UpdateType) -> + couch_db:update_docs(Db, Docs, Options, UpdateType). + up_to_date(#http_db{}, _Seq) -> true; up_to_date(Source, Seq) -> diff --git a/src/couchdb/couch_util.erl b/src/couchdb/couch_util.erl index 21d6eb4c..bd377d80 100644 --- a/src/couchdb/couch_util.erl +++ b/src/couchdb/couch_util.erl @@ -13,7 +13,7 @@ -module(couch_util). -export([start_driver/1]). --export([should_flush/0, should_flush/1, to_existing_atom/1]). +-export([should_flush/0, should_flush/1, to_existing_atom/1, to_binary/1]). -export([new_uuid/0, rand32/0, implode/2, collate/2, collate/3]). -export([abs_pathname/1,abs_pathname/2, trim/1, ascii_lower/1]). -export([encodeBase64/1, decodeBase64/1, to_hex/1,parse_term/1,dict_find/3]). @@ -57,6 +57,19 @@ to_digit(N) when N < 10 -> $0 + N; to_digit(N) -> $a + N-10. +to_binary(V) when is_binary(V) -> + V; +to_binary(V) when is_list(V) -> + try list_to_binary(V) + catch + _ -> list_to_binary(io_lib:format("~p", [V])) + end; +to_binary(V) when is_atom(V) -> + list_to_binary(atom_to_list(V)); +to_binary(V) -> + list_to_binary(io_lib:format("~p", [V])). + + parse_term(Bin) when is_binary(Bin)-> parse_term(binary_to_list(Bin)); parse_term(List) -> -- cgit v1.2.3