diff options
| -rw-r--r-- | etc/couchdb/default.ini.tpl.in | 1 | ||||
| -rw-r--r-- | share/www/script/test/delayed_commits.js | 106 | ||||
| -rw-r--r-- | src/couchdb/couch_db.erl | 20 | ||||
| -rw-r--r-- | src/couchdb/couch_httpd_db.erl | 8 |
4 files changed, 84 insertions, 51 deletions
diff --git a/etc/couchdb/default.ini.tpl.in b/etc/couchdb/default.ini.tpl.in index 28c4e294..3260449b 100644 --- a/etc/couchdb/default.ini.tpl.in +++ b/etc/couchdb/default.ini.tpl.in @@ -10,6 +10,7 @@ max_document_size = 4294967296 ; 4 GB max_attachment_chunk_size = 4294967296 ; 4GB os_process_timeout = 5000 ; 5 seconds. for view and external servers. max_dbs_open = 100 +delayed_commits = false batch_save_size = 1000 ; number of docs at which to save a batch batch_save_interval = 1000 ; milliseconds after which to save batches diff --git a/share/www/script/test/delayed_commits.js b/share/www/script/test/delayed_commits.js index 27ef24c8..80d345b1 100644 --- a/share/www/script/test/delayed_commits.js +++ b/share/www/script/test/delayed_commits.js @@ -16,79 +16,86 @@ couchTests.delayed_commits = function(debug) { db.createDb(); if (debug) debugger; - // By default, couchdb doesn't fully commit documents to disk right away, - // it waits about a second to batch the full commit flush along with any - // other updates. If it crashes or is restarted you may lose the most - // recent commits. + run_on_modified_server( + [{section: "couchdb", + key: "delayed_commits", + value: "true"}], - T(db.save({_id:"1",a:2,b:4}).ok); - T(db.open("1") != null); + function () { + // By default, couchdb doesn't fully commit documents to disk right away, + // it waits about a second to batch the full commit flush along with any + // other updates. If it crashes or is restarted you may lose the most + // recent commits. - restartServer(); + T(db.save({_id:"1",a:2,b:4}).ok); + T(db.open("1") != null); - T(db.open("1") == null); // lost the update. - // note if we waited > 1 sec before the restart, the doc would likely - // commit. + restartServer(); + T(db.open("1") == null); // lost the update. + // note if we waited > 1 sec before the restart, the doc would likely + // commit. - // Retry the same thing but with full commits on. - var db2 = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"true"}); + // Retry the same thing but with full commits on. - T(db2.save({_id:"1",a:2,b:4}).ok); - T(db2.open("1") != null); + var db2 = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"true"}); - restartServer(); + T(db2.save({_id:"1",a:2,b:4}).ok); + T(db2.open("1") != null); - T(db2.open("1") != null); + restartServer(); - // You can update but without committing immediately, and then ensure - // everything is commited in the last step. + T(db2.open("1") != null); - T(db.save({_id:"2",a:2,b:4}).ok); - T(db.open("2") != null); - T(db.ensureFullCommit().ok); - restartServer(); + // You can update but without committing immediately, and then ensure + // everything is commited in the last step. - T(db.open("2") != null); + T(db.save({_id:"2",a:2,b:4}).ok); + T(db.open("2") != null); + T(db.ensureFullCommit().ok); + restartServer(); - // However, it's possible even when flushed, that the server crashed between - // the update and the commit, and you don't want to check to make sure - // every doc you updated actually made it to disk. So record the instance - // start time of the database before the updates and then check it again - // after the flush (the instance start time is returned by the flush - // operation). if they are the same, we know everything was updated - // safely. + T(db.open("2") != null); - // First try it with a crash. + // However, it's possible even when flushed, that the server crashed between + // the update and the commit, and you don't want to check to make sure + // every doc you updated actually made it to disk. So record the instance + // start time of the database before the updates and then check it again + // after the flush (the instance start time is returned by the flush + // operation). if they are the same, we know everything was updated + // safely. - var instanceStartTime = db.info().instance_start_time; + // First try it with a crash. - T(db.save({_id:"3",a:2,b:4}).ok); - T(db.open("3") != null); + var instanceStartTime = db.info().instance_start_time; - restartServer(); + T(db.save({_id:"3",a:2,b:4}).ok); + T(db.open("3") != null); - var commitResult = db.ensureFullCommit(); - T(commitResult.ok && commitResult.instance_start_time != instanceStartTime); - // start times don't match, meaning the server lost our change + restartServer(); - T(db.open("3") == null); // yup lost it + var commitResult = db.ensureFullCommit(); + T(commitResult.ok && commitResult.instance_start_time != instanceStartTime); + // start times don't match, meaning the server lost our change - // retry with no server restart + T(db.open("3") == null); // yup lost it - var instanceStartTime = db.info().instance_start_time; + // retry with no server restart - T(db.save({_id:"4",a:2,b:4}).ok); - T(db.open("4") != null); + var instanceStartTime = db.info().instance_start_time; - var commitResult = db.ensureFullCommit(); - T(commitResult.ok && commitResult.instance_start_time == instanceStartTime); - // Successful commit, start times match! + T(db.save({_id:"4",a:2,b:4}).ok); + T(db.open("4") != null); - restartServer(); + var commitResult = db.ensureFullCommit(); + T(commitResult.ok && commitResult.instance_start_time == instanceStartTime); + // Successful commit, start times match! - T(db.open("4") != null); + restartServer(); + + T(db.open("4") != null); + }); // Now test that when we exceed the max_dbs_open, pending commits are safely // written. @@ -96,6 +103,9 @@ couchTests.delayed_commits = function(debug) { var max = 2; run_on_modified_server( [{section: "couchdb", + key: "delayed_commits", + value: "true"}, + {section: "couchdb", key: "max_dbs_open", value: max.toString()}], diff --git a/src/couchdb/couch_db.erl b/src/couchdb/couch_db.erl index aff5b743..7118145d 100644 --- a/src/couchdb/couch_db.erl +++ b/src/couchdb/couch_db.erl @@ -567,9 +567,27 @@ make_first_doc_on_disk(Db, Id, Pos, [{_Rev, {IsDel, Sp, _Seq}} |_]=DocPath) -> Revs = [Rev || {Rev, _} <- DocPath], make_doc(Db, Id, IsDel, Sp, {Pos, Revs}). +set_commit_option(Options) -> + CommitSettings = { + [true || O <- Options, O==full_commit orelse O==delay_commit], + couch_config:get("couchdb", "delayed_commits", "false") + }, + case CommitSettings of + {[true], _} -> + Options; % user requested explicit commit setting, do not change it + {_, "true"} -> + Options; % delayed commits are enabled, do nothing + {_, "false"} -> + [full_commit|Options]; + {_, Else} -> + ?LOG_ERROR("[couchdb] delayed_commits setting must be true/false, not ~p", + [Else]), + [full_commit|Options] + end. write_and_commit(#db{update_pid=UpdatePid, user_ctx=Ctx}=Db, DocBuckets, - NonRepDocs, Options) -> + NonRepDocs, Options0) -> + Options = set_commit_option(Options0), case gen_server:call(UpdatePid, {update_docs, DocBuckets, NonRepDocs, Options}, infinity) of {ok, Results} -> {ok, Results}; diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl index 55429cef..915a5eae 100644 --- a/src/couchdb/couch_httpd_db.erl +++ b/src/couchdb/couch_httpd_db.erl @@ -313,9 +313,11 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>]}=Req, Db) -> couch_stats_collector:increment({httpd, bulk_requests}), {JsonProps} = couch_httpd:json_body_obj(Req), DocsArray = proplists:get_value(<<"docs">>, JsonProps), - case couch_httpd:header_value(Req, "X-Couch-Full-Commit", "false") of + case couch_httpd:header_value(Req, "X-Couch-Full-Commit") of "true" -> Options = [full_commit]; + "false" -> + Options = [delay_commit]; _ -> Options = [] end, @@ -770,9 +772,11 @@ update_doc(Req, Db, DocId, Json) -> update_doc(Req, Db, DocId, Json, Headers) -> #doc{deleted=Deleted} = Doc = couch_doc_from_req(Req, DocId, Json), - case couch_httpd:header_value(Req, "X-Couch-Full-Commit", "false") of + case couch_httpd:header_value(Req, "X-Couch-Full-Commit") of "true" -> Options = [full_commit]; + "false" -> + Options = [delay_commit]; _ -> Options = [] end, |