From 51b27cd943b38c3b6e0e9c25915ee3aa4f092c9a Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Fri, 3 Jun 2011 11:30:05 +0000 Subject: set HttpOnly on auth cookies on SSL. git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1130996 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_httpd_auth.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/couchdb/couch_httpd_auth.erl b/src/couchdb/couch_httpd_auth.erl index 155865e5..9f6ed18a 100644 --- a/src/couchdb/couch_httpd_auth.erl +++ b/src/couchdb/couch_httpd_auth.erl @@ -231,7 +231,7 @@ cookie_auth_cookie(Req, User, Secret, TimeStamp) -> Hash = crypto:sha_mac(Secret, SessionData), mochiweb_cookies:cookie("AuthSession", couch_util:encodeBase64Url(SessionData ++ ":" ++ ?b2l(Hash)), - [{path, "/"}, cookie_scheme(Req)]). + [{path, "/"}] ++ cookie_scheme(Req)). hash_password(Password, Salt) -> ?l2b(couch_util:to_hex(crypto:sha(<>))). @@ -292,7 +292,7 @@ handle_session_req(#httpd{method='POST', mochi_req=MochiReq}=Req) -> ]}); _Else -> % clear the session - Cookie = mochiweb_cookies:cookie("AuthSession", "", [{path, "/"}, cookie_scheme(Req)]), + Cookie = mochiweb_cookies:cookie("AuthSession", "", [{path, "/"}] ++ cookie_scheme(Req)), send_json(Req, 401, [Cookie], {[{error, <<"unauthorized">>},{reason, <<"Name or password is incorrect.">>}]}) end; % get user info @@ -322,7 +322,7 @@ handle_session_req(#httpd{method='GET', user_ctx=UserCtx}=Req) -> end; % logout by deleting the session handle_session_req(#httpd{method='DELETE'}=Req) -> - Cookie = mochiweb_cookies:cookie("AuthSession", "", [{path, "/"}, cookie_scheme(Req)]), + Cookie = mochiweb_cookies:cookie("AuthSession", "", [{path, "/"}] ++ cookie_scheme(Req)), {Code, Headers} = case couch_httpd:qs_value(Req, "next", nil) of nil -> {200, [Cookie]}; @@ -353,7 +353,8 @@ make_cookie_time() -> NowMS * 1000000 + NowS. cookie_scheme(#httpd{mochi_req=MochiReq}) -> + [{http_only, true}] ++ case MochiReq:get(scheme) of - http -> {http_only, true}; - https -> {secure, true} + http -> []; + https -> [{secure, true}] end. -- cgit v1.2.3 From b7e11f2ee481a351a37354cc278cb5f98a959802 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Fri, 3 Jun 2011 11:30:24 +0000 Subject: unfreeze 1.1.x - prep for 1.1.1 git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1130997 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 5 +++++ NEWS | 5 +++++ acinclude.m4.in | 4 ++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 7c522b96..b4ace919 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Apache CouchDB CHANGES ====================== +Version 1.1.1 +------------- + +This version has not been released yet. + Version 1.1.0 ------------- diff --git a/NEWS b/NEWS index b709ef25..a49d3c19 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,11 @@ For details about backwards incompatible changes, see: Each release section notes when backwards incompatible changes have been made. +Version 1.1.1 +------------- + +This version has not been released yet. + Version 1.1.0 ------------- diff --git a/acinclude.m4.in b/acinclude.m4.in index e1efe10c..e74f6336 100644 --- a/acinclude.m4.in +++ b/acinclude.m4.in @@ -18,8 +18,8 @@ m4_define([LOCAL_PACKAGE_NAME], [Apache CouchDB]) m4_define([LOCAL_BUG_URI], [https://issues.apache.org/jira/browse/COUCHDB]) m4_define([LOCAL_VERSION_MAJOR], [1]) m4_define([LOCAL_VERSION_MINOR], [1]) -m4_define([LOCAL_VERSION_REVISION], [0]) -m4_define([LOCAL_VERSION_STAGE], []) +m4_define([LOCAL_VERSION_REVISION], [1]) +m4_define([LOCAL_VERSION_STAGE], [a]) m4_define([LOCAL_VERSION_RELEASE], [%release%]) m4_define([LOCAL_VERSION_PRIMARY], [LOCAL_VERSION_MAJOR.LOCAL_VERSION_MINOR.LOCAL_VERSION_REVISION]) -- cgit v1.2.3 From cee8283b7e0e5d6c330398f9b6d9ffe5ebc4fe9a Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Fri, 3 Jun 2011 14:10:54 +0000 Subject: import version 1.0.2 info to NEWS/CHANGES. git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1131049 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ NEWS | 22 ++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/CHANGES b/CHANGES index b4ace919..54a2e03c 100644 --- a/CHANGES +++ b/CHANGES @@ -61,6 +61,65 @@ URL Rewriter & Vhosts: * Fix for variable substituion +Version 1.0.2 +------------- + +Futon: + + * Make test suite work with Safari and Chrome. + * Fixed animated progress spinner. + * Fix raw view document link due to overzealous URI encoding. + * Spell javascript correctly in loadScript(uri). + +Storage System: + + * Fix leaking file handles after compacting databases and views. + * Fix databases forgetting their validation function after compaction. + * Fix occasional timeout errors after successfully compacting large databases. + * Fix ocassional error when writing to a database that has just been compacted. + * Fix occasional timeout errors on systems with slow or heavily loaded IO. + * Fix for OOME when compactions include documents with many conflicts. + * Fix for missing attachment compression when MIME types included parameters. + * Preserve purge metadata during compaction to avoid spurious view rebuilds. + * Fix spurious conflicts introduced when uploading an attachment after + a doc has been in a conflict. See COUCHDB-902 for details. + * Fix for frequently edited documents in multi-master deployments being + duplicated in _changes and _all_docs. See COUCHDDB-968 for details on how + to repair. + * Significantly higher read and write throughput against database and + view index files. + +Log System: + + * Reduce lengthy stack traces. + * Allow logging of native types. + +HTTP Interface: + + * Allow reduce=false parameter in map-only views. + * Fix parsing of Accept headers. + * Fix for multipart GET APIs when an attachment was created during a + local-local replication. See COUCHDB-1022 for details. + +Replicator: + + * Updated ibrowse library to 2.1.2 fixing numerous replication issues. + * Make sure that the replicator respects HTTP settings defined in the config. + * Fix error when the ibrowse connection closes unexpectedly. + * Fix authenticated replication (with HTTP basic auth) of design documents + with attachments. + * Various fixes to make replication more resilient for edge-cases. + +View Server: + + * Don't trigger view updates when requesting `_design/doc/_info`. + * Fix for circular references in CommonJS requires. + * Made isArray() function available to functions executed in the query server. + * Documents are now sealed before being passed to map functions. + * Force view compaction failure when duplicated document data exists. When + this error is seen in the logs users should rebuild their views from + scratch to fix the issue. See COUCHDB-999 for details. + Version 1.0.1 ------------- diff --git a/NEWS b/NEWS index a49d3c19..97eb58e7 100644 --- a/NEWS +++ b/NEWS @@ -37,6 +37,28 @@ This release contains backwards incompatible changes. to lack of permissions. * Added a "change password"-feature to Futon. +Version 1.0.2 +------------- + + * Make test suite work with Safari and Chrome. + * Fix leaking file handles after compacting databases and views. + * Fix databases forgetting their validation function after compaction. + * Fix occasional timeout errors. + * Reduce lengthy stack traces. + * Allow logging of native types. + * Updated ibrowse library to 2.1.2 fixing numerous replication issues. + * Fix authenticated replication of design documents with attachments. + * Fix multipart GET APIs by always sending attachments in compressed + form when the source attachment is compressed on disk. Fixes a possible + edge case when an attachment underwent local-local replication. + * Various fixes to make replicated more resilient for edge-cases. + * Don't trigger a view update when requesting `_design/doc/_info`. + * Fix for circular references in CommonJS requires. + * Fix for frequently edited documents in multi-master deployments being + duplicated in _changes and _all_docs. + * Fix spurious conflict generation during attachment uploads. + * Fix for various bugs in Futon. + Version 1.0.1 ------------- -- cgit v1.2.3 From ea57780780730eb5f2d98f30697e6a8c2b3cf7f7 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Wed, 8 Jun 2011 07:53:55 +0000 Subject: bump minimum erlang to R13B02 (COUCHDB-1191) git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1133272 13f79535-47bb-0310-9956-ffa450edef68 --- INSTALL.Unix | 2 +- configure.ac | 8 ++++---- src/couchdb/couch_app.erl | 3 --- src/erlang-oauth/Makefile.am | 4 +--- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/INSTALL.Unix b/INSTALL.Unix index 768e3846..720134d4 100644 --- a/INSTALL.Unix +++ b/INSTALL.Unix @@ -8,7 +8,7 @@ Dependencies You will need the following installed: - * Erlang OTP (>=R12B5) (http://erlang.org/) + * Erlang OTP (>=R13B2) (http://erlang.org/) * ICU (http://icu.sourceforge.net/) * OpenSSL (http://www.openssl.org/) * Mozilla SpiderMonkey (1.8) (http://www.mozilla.org/js/spidermonkey/) diff --git a/configure.ac b/configure.ac index b776a258..0cc277a4 100644 --- a/configure.ac +++ b/configure.ac @@ -241,7 +241,7 @@ if test x${ERL} = x; then AC_MSG_ERROR([Could not find the `erl' executable. Is Erlang installed?]) fi -erlang_version_error="The installed Erlang version is less than 5.6.5 (R12B05)." +erlang_version_error="The installed Erlang version is less than erts-5.7.3 (R13B02)." version="`${ERL} -version 2>&1 | ${SED} "s/[[^0-9]]/ /g"`" @@ -249,12 +249,12 @@ if test `echo $version | ${AWK} "{print \\$1}"` -lt 5; then AC_MSG_ERROR([$erlang_version_error]) fi -if test `echo $version | ${AWK} "{print \\$2}"` -lt 6; then +if test `echo $version | ${AWK} "{print \\$2}"` -lt 7; then AC_MSG_ERROR([$erlang_version_error]) fi -if test `echo $version | ${AWK} "{print \\$2}"` -eq 6; then - if test `echo $version | ${AWK} "{print \\$3}"` -lt 5; then +if test `echo $version | ${AWK} "{print \\$2}"` -eq 7; then + if test `echo $version | ${AWK} "{print \\$3}"` -lt 3; then AC_MSG_ERROR([$erlang_version_error]) fi fi diff --git a/src/couchdb/couch_app.erl b/src/couchdb/couch_app.erl index 232953d9..c16412a3 100644 --- a/src/couchdb/couch_app.erl +++ b/src/couchdb/couch_app.erl @@ -48,9 +48,6 @@ start_apps([App|Rest]) -> start_apps(Rest); {error, {already_started, App}} -> start_apps(Rest); - {error, _Reason} when App =:= public_key -> - % ignore on R12B5 - start_apps(Rest); {error, _Reason} -> {error, {app_would_not_start, App}} end. diff --git a/src/erlang-oauth/Makefile.am b/src/erlang-oauth/Makefile.am index 48b76482..76239484 100644 --- a/src/erlang-oauth/Makefile.am +++ b/src/erlang-oauth/Makefile.am @@ -22,15 +22,13 @@ oauth_file_collection = \ oauth_unix.erl \ oauth_uri.erl -# Removed oauth_rsa_sha1.beam until we require R12B5 or -# we add a ./configure option to enable it. - oauthebin_make_generated_file_list = \ oauth.app \ oauth.beam \ oauth_hmac_sha1.beam \ oauth_http.beam \ oauth_plaintext.beam \ + oauth_rsa_sha1.erl \ oauth_unix.beam \ oauth_uri.beam -- cgit v1.2.3 From 218f61ca2844ef4326157c6db958574c29c1ea40 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Wed, 8 Jun 2011 18:57:43 +0000 Subject: backport r1133312 from trunk _view_cleanup with no _design docs - COUCHDB-1136 git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1133510 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_view.erl | 17 +++--- test/etap/072-cleanup.t | 130 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 6 deletions(-) create mode 100755 test/etap/072-cleanup.t diff --git a/src/couchdb/couch_view.erl b/src/couchdb/couch_view.erl index 911f1aa6..c9a9b2cc 100644 --- a/src/couchdb/couch_view.erl +++ b/src/couchdb/couch_view.erl @@ -92,13 +92,18 @@ cleanup_index_files(Db) -> FileList = list_index_files(Db), - % regex that matches all ddocs - RegExp = "("++ string:join(Sigs, "|") ++")", + DeleteFiles = + if length(Sigs) =:= 0 -> + FileList; + true -> + % regex that matches all ddocs + RegExp = "("++ string:join(Sigs, "|") ++")", + + % filter out the ones in use + [FilePath || FilePath <- FileList, + re:run(FilePath, RegExp, [{capture, none}]) =:= nomatch] + end, - % filter out the ones in use - DeleteFiles = [FilePath - || FilePath <- FileList, - re:run(FilePath, RegExp, [{capture, none}]) =:= nomatch], % delete unused files ?LOG_DEBUG("deleting unused view index files: ~p",[DeleteFiles]), RootDir = couch_config:get("couchdb", "view_index_dir"), diff --git a/test/etap/072-cleanup.t b/test/etap/072-cleanup.t new file mode 100755 index 00000000..61790bc6 --- /dev/null +++ b/test/etap/072-cleanup.t @@ -0,0 +1,130 @@ +#!/usr/bin/env escript +%% -*- erlang -*- + +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-define(TEST_DB, <<"etap-test-db">>). + +-record(user_ctx, { + name = null, + roles = [], + handler +}). + +-define(ADMIN_USER, #user_ctx{roles=[<<"_admin">>]}). + +main(_) -> + test_util:init_code_path(), + + etap:plan(7), + try test() of + ok -> + etap:end_tests() + catch + Other -> + etap:diag(io_lib:format("Test died abnormally: ~p", [Other])), + timer:sleep(1000), + etap:bail(Other) + end, + ok. + +test() -> + + {ok, _} = couch_server_sup:start_link(test_util:config_files()), + ok = application:start(inets), + couch_server:delete(?TEST_DB, []), + timer:sleep(1000), + + couch_db:create(?TEST_DB, []), + + {ok, AllDbs} = couch_server:all_databases(), + etap:ok(lists:member(?TEST_DB, AllDbs), "Database was created."), + + FooRev = create_design_doc(<<"_design/foo">>, <<"bar">>), + query_view("foo", "bar"), + + BoozRev = create_design_doc(<<"_design/booz">>, <<"baz">>), + query_view("booz", "baz"), + + {ok, Db} = couch_db:open(?TEST_DB, [{user_ctx, ?ADMIN_USER}]), + view_cleanup(), + etap:is(count_index_files(), 2, + "Two index files before any deletions."), + + delete_design_doc(<<"_design/foo">>, FooRev), + view_cleanup(), + etap:is(count_index_files(), 1, + "One index file after first deletion and cleanup."), + + delete_design_doc(<<"_design/booz">>, BoozRev), + view_cleanup(), + etap:is(count_index_files(), 0, + "No index files after second deletion and cleanup."), + + couch_server:delete(?TEST_DB, []), + {ok, AllDbs2} = couch_server:all_databases(), + etap:ok(not lists:member(?TEST_DB, AllDbs2), + "Database was deleted."), + ok. + +create_design_doc(DDName, ViewName) -> + {ok, Db} = couch_db:open(?TEST_DB, [{user_ctx, ?ADMIN_USER}]), + DDoc = couch_doc:from_json_obj({[ + {<<"_id">>, DDName}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, {[ + {ViewName, {[ + {<<"map">>, <<"function(doc) { emit(doc.value, 1); }">>} + ]}} + ]}} + ]}), + {ok, Rev} = couch_db:update_doc(Db, DDoc, []), + couch_db:ensure_full_commit(Db), + couch_db:close(Db), + Rev. + +delete_design_doc(DDName, Rev) -> + {ok, Db} = couch_db:open(?TEST_DB, [{user_ctx, ?ADMIN_USER}]), + DDoc = couch_doc:from_json_obj({[ + {<<"_id">>, DDName}, + {<<"_rev">>, couch_doc:rev_to_str(Rev)}, + {<<"_deleted">>, true} + ]}), + {ok, _} = couch_db:update_doc(Db, DDoc, [Rev]), + couch_db:close(Db). + +db_url() -> + Addr = couch_config:get("httpd", "bind_address", "127.0.0.1"), + Port = integer_to_list(mochiweb_socket_server:get(couch_httpd, port)), + "http://" ++ Addr ++ ":" ++ Port ++ "/" ++ + binary_to_list(?TEST_DB). + +query_view(DDoc, View) -> + {ok, {{_, Code, _}, _Headers, _Body}} = http:request( + get, + {db_url() ++ "/_design/" ++ DDoc ++ "/_view/" ++ View, []}, + [], + [{sync, true}]), + etap:is(Code, 200, "Built view index for " ++ DDoc ++ "."), + ok. + +view_cleanup() -> + {ok, Db} = couch_db:open(?TEST_DB, [{user_ctx, ?ADMIN_USER}]), + couch_view:cleanup_index_files(Db), + couch_db:close(Db). + +count_index_files() -> + % call server to fetch the index files + RootDir = couch_config:get("couchdb", "view_index_dir"), + length(filelib:wildcard(RootDir ++ "/." ++ + binary_to_list(?TEST_DB) ++ "_design"++"/*")). -- cgit v1.2.3 From a217146276375f6247b027305723b0734ab280f5 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Sat, 11 Jun 2011 19:14:16 +0000 Subject: Backport r113686 from trunk add 072-cleanup.t to etap Makefile.am git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1134729 13f79535-47bb-0310-9956-ffa450edef68 --- test/etap/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/test/etap/Makefile.am b/test/etap/Makefile.am index ce52d430..ecbc3a93 100644 --- a/test/etap/Makefile.am +++ b/test/etap/Makefile.am @@ -53,6 +53,7 @@ EXTRA_DIST = \ 064-kt-counting.t \ 065-kt-stemming.t \ 070-couch-db.t \ + 072-cleanup.t \ 080-config-get-set.t \ 081-config-override.1.ini \ 081-config-override.2.ini \ -- cgit v1.2.3 From e192fe836b55d7e58273554a3a739218bd78798e Mon Sep 17 00:00:00 2001 From: Filipe David Borba Manana Date: Sun, 12 Jun 2011 16:14:33 +0000 Subject: Fix erlang-oauth Makefile.am (wrong file extension) git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1134941 13f79535-47bb-0310-9956-ffa450edef68 --- src/erlang-oauth/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlang-oauth/Makefile.am b/src/erlang-oauth/Makefile.am index 76239484..50782e75 100644 --- a/src/erlang-oauth/Makefile.am +++ b/src/erlang-oauth/Makefile.am @@ -28,7 +28,7 @@ oauthebin_make_generated_file_list = \ oauth_hmac_sha1.beam \ oauth_http.beam \ oauth_plaintext.beam \ - oauth_rsa_sha1.erl \ + oauth_rsa_sha1.beam \ oauth_unix.beam \ oauth_uri.beam -- cgit v1.2.3 From d5da180b06b8711c72bf38e35c380a25804e018f Mon Sep 17 00:00:00 2001 From: Filipe David Borba Manana Date: Sun, 12 Jun 2011 16:16:29 +0000 Subject: Backport revision 1129897 from trunk Fixes to the doc PUT multipart API Don't hold the connection forever if the document is rejected by a validate_doc_update function. The solution is to discard all the attachments' data if the document was rejected. git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1134942 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_doc.erl | 19 ++++++++++++++++++- src/couchdb/couch_httpd_db.erl | 14 ++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/couchdb/couch_doc.erl b/src/couchdb/couch_doc.erl index 5cd6ac80..531eb6bb 100644 --- a/src/couchdb/couch_doc.erl +++ b/src/couchdb/couch_doc.erl @@ -18,6 +18,7 @@ -export([validate_docid/1]). -export([doc_from_multi_part_stream/2]). -export([doc_to_multi_part_stream/5, len_doc_to_multi_part_stream/4]). +-export([abort_multi_part_stream/1]). -include("couch_db.hrl"). @@ -501,7 +502,7 @@ doc_from_multi_part_stream(ContentType, DataFun) -> receive {Parser, finished} -> ok end, erlang:put(mochiweb_request_recv, true) end, - {ok, Doc#doc{atts=Atts2}, WaitFun} + {ok, Doc#doc{atts=Atts2}, WaitFun, Parser} end. mp_parse_doc({headers, H}, []) -> @@ -542,3 +543,19 @@ mp_parse_atts(body_end) -> end. +abort_multi_part_stream(Parser) -> + abort_multi_part_stream(Parser, erlang:monitor(process, Parser)). + +abort_multi_part_stream(Parser, MonRef) -> + case is_process_alive(Parser) of + true -> + Parser ! {get_bytes, self()}, + receive + {bytes, _Bytes} -> + abort_multi_part_stream(Parser, MonRef); + {'DOWN', MonRef, _, _, _} -> + ok + end; + false -> + erlang:demonitor(MonRef, [flush]) + end. diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl index e3638b25..71204598 100644 --- a/src/couchdb/couch_httpd_db.erl +++ b/src/couchdb/couch_httpd_db.erl @@ -692,12 +692,18 @@ db_doc_req(#httpd{method='PUT'}=Req, Db, DocId) -> RespHeaders = [{"Location", Loc}], case couch_util:to_list(couch_httpd:header_value(Req, "Content-Type")) of ("multipart/related;" ++ _) = ContentType -> - {ok, Doc0, WaitFun} = couch_doc:doc_from_multi_part_stream( + {ok, Doc0, WaitFun, Parser} = couch_doc:doc_from_multi_part_stream( ContentType, fun() -> receive_request_data(Req) end), Doc = couch_doc_from_req(Req, DocId, Doc0), - Result = update_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType), - WaitFun(), - Result; + try + Result = update_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType), + WaitFun(), + Result + catch throw:Err -> + % Document rejected by a validate_doc_update function. + couch_doc:abort_multi_part_stream(Parser), + throw(Err) + end; _Else -> case couch_httpd:qs_value(Req, "batch") of "ok" -> -- cgit v1.2.3 From f0aaa2c3c7f6bbb858b40a55267781f1bcde0f67 Mon Sep 17 00:00:00 2001 From: Filipe David Borba Manana Date: Thu, 16 Jun 2011 20:06:15 +0000 Subject: Merge revision 1136639 from trunk Human readable message on view compaction error When a view has duplicated document IDs in the main btree, the view compactor exists. Unfortunatelly its exit reason is not human readable because it's an IOList. This patch improves the error message and logs it with an 'error' level. Issue reported by Mike Leddy in ticket COUCHDB-999. git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1136640 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_view_compactor.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/couchdb/couch_view_compactor.erl b/src/couchdb/couch_view_compactor.erl index 9a47f5f8..8eda43e9 100644 --- a/src/couchdb/couch_view_compactor.erl +++ b/src/couchdb/couch_view_compactor.erl @@ -50,8 +50,10 @@ compact_group(Group, EmptyGroup) -> Fun = fun({DocId, _ViewIdKeys} = KV, {Bt, Acc, TotalCopied, LastId}) -> if DocId =:= LastId -> % COUCHDB-999 - Msg = "Duplicates of ~s detected in ~s ~s - rebuild required", - exit(io_lib:format(Msg, [DocId, DbName, GroupId])); + ?LOG_ERROR("Duplicates of document `~s` detected in view group `~s`" + ", database `~s` - view rebuild, from scratch, is required", + [DocId, GroupId, DbName]), + exit({view_duplicated_id, DocId}); true -> ok end, if TotalCopied rem 10000 =:= 0 -> couch_task_status:update("Copied ~p of ~p Ids (~p%)", -- cgit v1.2.3 From 0827cb483ef4cf74123313e8f2d0d2bd5ce8e46b Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Mon, 20 Jun 2011 21:22:02 +0000 Subject: Fix spurious declarations of new merge conflicts This patch also adds extra tests of the key tree merging logic as well as edoc-formatted documentation for the module and a few of the merge functions. Closes COUCHDB-902. Thanks Paul Davis, Bob Dionne, Klaus Trainer. backported from trunk@1065471 Conflicts: src/couchdb/couch_key_tree.erl git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1137789 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_key_tree.erl | 48 +++++++++++++++++++-- test/etap/060-kt-merging.t | 95 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 122 insertions(+), 21 deletions(-) diff --git a/src/couchdb/couch_key_tree.erl b/src/couchdb/couch_key_tree.erl index e5d3549f..48a76b1d 100644 --- a/src/couchdb/couch_key_tree.erl +++ b/src/couchdb/couch_key_tree.erl @@ -10,6 +10,41 @@ % License for the specific language governing permissions and limitations under % the License. +%% @doc Data structure used to represent document edit histories. + +%% A key tree is used to represent the edit history of a document. Each node of +%% the tree represents a particular version. Relations between nodes represent +%% the order that these edits were applied. For instance, a set of three edits +%% would produce a tree of versions A->B->C indicating that edit C was based on +%% version B which was in turn based on A. In a world without replication (and +%% no ability to disable MVCC checks), all histories would be forced to be +%% linear lists of edits due to constraints imposed by MVCC (ie, new edits must +%% be based on the current version). However, we have replication, so we must +%% deal with not so easy cases, which lead to trees. +%% +%% Consider a document in state A. This doc is replicated to a second node. We +%% then edit the document on each node leaving it in two different states, B +%% and C. We now have two key trees, A->B and A->C. When we go to replicate a +%% second time, the key tree must combine these two trees which gives us +%% A->(B|C). This is how conflicts are introduced. In terms of the key tree, we +%% say that we have two leaves (B and C) that are not deleted. The presense of +%% the multiple leaves indicate conflict. To remove a conflict, one of the +%% edits (B or C) can be deleted, which results in, A->(B|C->D) where D is an +%% edit that is specially marked with the a deleted=true flag. +%% +%% What makes this a bit more complicated is that there is a limit to the +%% number of revisions kept, specified in couch_db.hrl (default is 1000). When +%% this limit is exceeded only the last 1000 are kept. This comes in to play +%% when branches are merged. The comparison has to begin at the same place in +%% the branches. A revision id is of the form N-XXXXXXX where N is the current +%% revision. So each path will have a start number, calculated in +%% couch_doc:to_path using the formula N - length(RevIds) + 1 So, .eg. if a doc +%% was edit 1003 times this start number would be 4, indicating that 3 +%% revisions were truncated. +%% +%% This comes into play in @see merge_at/3 which recursively walks down one +%% tree or the other until they begin at the same revision. + -module(couch_key_tree). -export([merge/3, find_missing/2, get_key_leafs/2, get_full_key_paths/2, get/2]). @@ -31,6 +66,9 @@ merge(Paths, Path, Depth) -> {Merged, Conflicts} = merge(Paths, Path), {stem(Merged, Depth), Conflicts}. +%% @doc Merge a path with an existing list of paths, returning a new list of +%% paths. A return of conflicts indicates a new conflict was discovered in this +%% merge. Conflicts may already exist in the original list of paths. -spec merge([path()], path()) -> {[path()], conflicts | no_conflicts}. merge(Paths, Path) -> {ok, Merged, HasConflicts} = merge_one(Paths, Path, [], false), @@ -69,6 +107,7 @@ merge_at([{Key, Value, SubTree}|Sibs], Place, InsertTree) when Place > 0 -> {ok, Merged, Conflicts} -> {ok, [{Key, Value, Merged} | Sibs], Conflicts}; no -> + % first branch didn't merge, move to next branch case merge_at(Sibs, Place, InsertTree) of {ok, Merged, Conflicts} -> {ok, [{Key, Value, SubTree} | Merged], Conflicts}; @@ -111,11 +150,12 @@ merge_simple([{Key, V1, SubA} | NextA], [{Key, V2, SubB} | NextB]) -> Value = value_pref(V1, V2), {[{Key, Value, MergedSubTree} | MergedNextTree], Conflict1 or Conflict2}; merge_simple([{A, _, _} = Tree | Next], [{B, _, _} | _] = Insert) when A < B -> - {Merged, _} = merge_simple(Next, Insert), - {[Tree | Merged], true}; + {Merged, Conflict} = merge_simple(Next, Insert), + % if Merged has more branches than the input we added a new conflict + {[Tree | Merged], Conflict orelse (length(Merged) > length(Next))}; merge_simple(Ours, [Tree | Next]) -> - {Merged, _} = merge_simple(Ours, Next), - {[Tree | Merged], true}. + {Merged, Conflict} = merge_simple(Ours, Next), + {[Tree | Merged], Conflict orelse (length(Merged) > length(Next))}. find_missing(_Tree, []) -> []; diff --git a/test/etap/060-kt-merging.t b/test/etap/060-kt-merging.t index 0e481a52..efbdbf69 100755 --- a/test/etap/060-kt-merging.t +++ b/test/etap/060-kt-merging.t @@ -15,7 +15,7 @@ main(_) -> test_util:init_code_path(), - etap:plan(12), + etap:plan(16), case (catch test()) of ok -> etap:end_tests(); @@ -26,25 +26,21 @@ main(_) -> ok. test() -> - One = {0, {"1","foo",[]}}, - TwoSibs = [{0, {"1","foo",[]}}, - {0, {"2","foo",[]}}], - OneChild = {0, {"1","foo",[{"1a", "bar", []}]}}, - TwoChild = {0, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, - TwoChildSibs = {0, {"1","foo", [{"1a", "bar", []}, - {"1b", "bar", []}]}}, - TwoChildSibs2 = {0, {"1","foo", [{"1a", "bar", []}, - {"1b", "bar", [{"1bb", "boo", []}]}]}}, - Stemmed1b = {1, {"1a", "bar", []}}, - Stemmed1a = {1, {"1a", "bar", [{"1aa", "bar", []}]}}, - Stemmed1aa = {2, {"1aa", "bar", []}}, - Stemmed1bb = {2, {"1bb", "boo", []}}, + One = {1, {"1","foo",[]}}, etap:is( {[One], no_conflicts}, couch_key_tree:merge([], One, 10), "The empty tree is the identity for merge." ), + etap:is( + {[One], no_conflicts}, + couch_key_tree:merge([One], One, 10), + "Merging is reflexive." + ), + + TwoSibs = [{1, {"1","foo",[]}}, + {1, {"2","foo",[]}}], etap:is( {TwoSibs, no_conflicts}, @@ -52,41 +48,75 @@ test() -> "Merging a prefix of a tree with the tree yields the tree." ), + Three = {1, {"3","foo",[]}}, + ThreeSibs = [{1, {"1","foo",[]}}, + {1, {"2","foo",[]}}, + {1, {"3","foo",[]}}], + etap:is( - {[One], no_conflicts}, - couch_key_tree:merge([One], One, 10), - "Merging is reflexive." + {ThreeSibs, conflicts}, + couch_key_tree:merge(TwoSibs, Three, 10), + "Merging a third unrelated branch leads to a conflict." ), + + TwoChild = {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, + etap:is( {[TwoChild], no_conflicts}, couch_key_tree:merge([TwoChild], TwoChild, 10), "Merging two children is still reflexive." ), + TwoChildSibs = {1, {"1","foo", [{"1a", "bar", []}, + {"1b", "bar", []}]}}, etap:is( {[TwoChildSibs], no_conflicts}, couch_key_tree:merge([TwoChildSibs], TwoChildSibs, 10), "Merging a tree to itself is itself."), + TwoChildPlusSibs = + {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}, + {"1b", "bar", []}]}}, + + etap:is( + {[TwoChildPlusSibs], no_conflicts}, + couch_key_tree:merge([TwoChild], TwoChildSibs, 10), + "Merging tree of uneven length at node 2."), + + Stemmed1b = {2, {"1a", "bar", []}}, etap:is( {[TwoChildSibs], no_conflicts}, couch_key_tree:merge([TwoChildSibs], Stemmed1b, 10), "Merging a tree with a stem." ), + TwoChildSibs2 = {1, {"1","foo", [{"1a", "bar", []}, + {"1b", "bar", [{"1bb", "boo", []}]}]}}, + Stemmed1bb = {3, {"1bb", "boo", []}}, etap:is( {[TwoChildSibs2], no_conflicts}, couch_key_tree:merge([TwoChildSibs2], Stemmed1bb, 10), "Merging a stem at a deeper level." ), + StemmedTwoChildSibs2 = [{2,{"1a", "bar", []}}, + {2,{"1b", "bar", [{"1bb", "boo", []}]}}], + + etap:is( + {StemmedTwoChildSibs2, no_conflicts}, + couch_key_tree:merge(StemmedTwoChildSibs2, Stemmed1bb, 10), + "Merging a stem at a deeper level against paths at deeper levels." + ), + + Stemmed1aa = {3, {"1aa", "bar", []}}, etap:is( {[TwoChild], no_conflicts}, couch_key_tree:merge([TwoChild], Stemmed1aa, 10), "Merging a single tree with a deeper stem." ), + Stemmed1a = {2, {"1a", "bar", [{"1aa", "bar", []}]}}, etap:is( {[TwoChild], no_conflicts}, couch_key_tree:merge([TwoChild], Stemmed1a, 10), @@ -99,6 +129,7 @@ test() -> "More merging." ), + OneChild = {1, {"1","foo",[{"1a", "bar", []}]}}, Expect1 = [OneChild, Stemmed1aa], etap:is( {Expect1, conflicts}, @@ -112,4 +143,34 @@ test() -> "Merge should have no conflicts." ), + %% this test is based on couch-902-test-case2.py + %% foo has conflicts from replication at depth two + %% foo3 is the current value + Foo = {1, {"foo", + "val1", + [{"foo2","val2",[]}, + {"foo3", "val3", []} + ]}}, + %% foo now has an attachment added, which leads to foo4 and val4 + %% off foo3 + Bar = {1, {"foo", + [], + [{"foo3", + [], + [{"foo4","val4",[]} + ]}]}}, + %% this is what the merge returns + %% note that it ignore the conflicting branch as there's no match + FooBar = {1, {"foo", + "val1", + [{"foo2","val2",[]}, + {"foo3", "val3", [{"foo4","val4",[]}]} + ]}}, + + etap:is( + {[FooBar], no_conflicts}, + couch_key_tree:merge([Foo],Bar,10), + "Merging trees with conflicts ought to behave." + ), + ok. -- cgit v1.2.3 From d8dc16093a26c53407d8bf702698848104ba01d6 Mon Sep 17 00:00:00 2001 From: Filipe David Borba Manana Date: Tue, 21 Jun 2011 10:15:30 +0000 Subject: Backport revision 1137928 from trunk Fix server crash associated with the replicator database If there's an exception when calculating the replication ID for a replication document, it crashes the replication manager gen_server 10 times. 10 is the maximum number of restarts per hour specified for the couch_server_sup supervisor. An easy way to trigger such exception is to specify a non existent filter in a replication document. This closes COUCHDB-1199. git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.1.x@1137929 13f79535-47bb-0310-9956-ffa450edef68 --- share/www/script/test/replicator_db.js | 67 +++++++++++++++++++++++++++++++ src/couchdb/couch_rep.erl | 45 +++++++++++++++++---- src/couchdb/couch_replication_manager.erl | 41 +++++++++++++++++-- 3 files changed, 142 insertions(+), 11 deletions(-) diff --git a/share/www/script/test/replicator_db.js b/share/www/script/test/replicator_db.js index 4434124e..058b6a7a 100644 --- a/share/www/script/test/replicator_db.js +++ b/share/www/script/test/replicator_db.js @@ -1279,6 +1279,68 @@ couchTests.replicator_db = function(debug) { } + function test_invalid_filter() { + // COUCHDB-1199 - replication document with a filter field that was invalid + // crashed the CouchDB server. + var repDoc1 = { + _id: "rep1", + source: "couch_foo_test_db", + target: "couch_bar_test_db", + filter: "test/foofilter" + }; + + TEquals(true, repDb.save(repDoc1).ok); + + waitForRep(repDb, repDoc1, "error"); + repDoc1 = repDb.open(repDoc1._id); + TEquals("undefined", typeof repDoc1._replication_id); + TEquals("error", repDoc1._replication_state); + + populate_db(dbA, docs1); + populate_db(dbB, []); + + var repDoc2 = { + _id: "rep2", + source: dbA.name, + target: dbB.name, + filter: "test/foofilter" + }; + + TEquals(true, repDb.save(repDoc2).ok); + + waitForRep(repDb, repDoc2, "error"); + repDoc2 = repDb.open(repDoc2._id); + TEquals("undefined", typeof repDoc2._replication_id); + TEquals("error", repDoc2._replication_state); + + var ddoc = { + _id: "_design/mydesign", + language : "javascript", + filters : { + myfilter : (function(doc, req) { + return true; + }).toString() + } + }; + + TEquals(true, dbA.save(ddoc).ok); + + var repDoc3 = { + _id: "rep3", + source: dbA.name, + target: dbB.name, + filter: "mydesign/myfilter" + }; + + TEquals(true, repDb.save(repDoc3).ok); + + waitForRep(repDb, repDoc3, "completed"); + repDoc3 = repDb.open(repDoc3._id); + TEquals("string", typeof repDoc3._replication_id); + TEquals("completed", repDoc3._replication_state); + } + + // run all the tests var server_config = [ { @@ -1355,6 +1417,11 @@ couchTests.replicator_db = function(debug) { restartServer(); run_on_modified_server(server_config, rep_doc_field_validation); + + repDb.deleteDb(); + restartServer(); + run_on_modified_server(server_config, test_invalid_filter); + /* * Disabled, since error state would be set on the document only after * the exponential backoff retry done by the replicator database listener diff --git a/src/couchdb/couch_rep.erl b/src/couchdb/couch_rep.erl index 6e4295ea..9d86e7a5 100644 --- a/src/couchdb/couch_rep.erl +++ b/src/couchdb/couch_rep.erl @@ -486,7 +486,17 @@ make_replication_id(RepProps, UserCtx) -> % add a new clause and increase ?REP_ID_VERSION at the top make_replication_id({Props}, UserCtx, 2) -> {ok, HostName} = inet:gethostname(), - Port = mochiweb_socket_server:get(couch_httpd, port), + Port = case (catch mochiweb_socket_server:get(couch_httpd, port)) of + P when is_number(P) -> + P; + _ -> + % On restart we might be called before the couch_httpd process is + % started. + % TODO: we might be under an SSL socket server only, or both under + % SSL and a non-SSL socket. + % ... mochiweb_socket_server:get(https, port) + list_to_integer(couch_config:get("httpd", "port", "5984")) + end, Src = get_rep_endpoint(UserCtx, couch_util:get_value(<<"source">>, Props)), Tgt = get_rep_endpoint(UserCtx, couch_util:get_value(<<"target">>, Props)), maybe_append_filters({Props}, [HostName, Port, Src, Tgt], UserCtx); @@ -513,16 +523,37 @@ maybe_append_filters({Props}, Base, UserCtx) -> couch_util:to_hex(couch_util:md5(term_to_binary(Base2))). filter_code(Filter, Props, UserCtx) -> - {match, [DDocName, FilterName]} = - re:run(Filter, "(.*?)/(.*)", [{capture, [1, 2], binary}]), + {DDocName, FilterName} = + case re:run(Filter, "(.*?)/(.*)", [{capture, [1, 2], binary}]) of + {match, [DDocName0, FilterName0]} -> + {DDocName0, FilterName0}; + _ -> + throw({error, <<"Invalid filter. Must match `ddocname/filtername`.">>}) + end, ProxyParams = parse_proxy_params( couch_util:get_value(<<"proxy">>, Props, [])), - Source = open_db( - couch_util:get_value(<<"source">>, Props), UserCtx, ProxyParams), + DbName = couch_util:get_value(<<"source">>, Props), + Source = try + open_db(DbName, UserCtx, ProxyParams) + catch + _Tag:DbError -> + DbErrorMsg = io_lib:format("Could not open source database `~s`: ~s", + [couch_util:url_strip_password(DbName), couch_util:to_binary(DbError)]), + throw({error, iolist_to_binary(DbErrorMsg)}) + end, try - {ok, DDoc} = open_doc(Source, <<"_design/", DDocName/binary>>), + Body = case (catch open_doc(Source, <<"_design/", DDocName/binary>>)) of + {ok, #doc{body = Body0}} -> + Body0; + DocError -> + DocErrorMsg = io_lib:format( + "Couldn't open document `_design/~s` from source " + "database `~s`: ~s", + [dbname(Source), DDocName, couch_util:to_binary(DocError)]), + throw({error, iolist_to_binary(DocErrorMsg)}) + end, Code = couch_util:get_nested_json_value( - DDoc#doc.body, [<<"filters">>, FilterName]), + Body, [<<"filters">>, FilterName]), re:replace(Code, "^\s*(.*?)\s*$", "\\1", [{return, binary}]) after close_db(Source) diff --git a/src/couchdb/couch_replication_manager.erl b/src/couchdb/couch_replication_manager.erl index 6537c8b2..e3d97c37 100644 --- a/src/couchdb/couch_replication_manager.erl +++ b/src/couchdb/couch_replication_manager.erl @@ -32,7 +32,8 @@ -import(couch_util, [ get_value/2, - get_value/3 + get_value/3, + to_binary/1 ]). @@ -62,8 +63,16 @@ init(_) -> }}. -handle_call({rep_db_update, Change}, _From, State) -> - {reply, ok, process_update(State, Change)}; +handle_call({rep_db_update, {ChangeProps} = Change}, _From, State) -> + NewState = try + process_update(State, Change) + catch + _Tag:Error -> + JsonRepDoc = get_value(doc, ChangeProps), + rep_db_update_error(Error, JsonRepDoc), + State + end, + {reply, ok, NewState}; handle_call({triggered, {BaseId, _}}, _From, State) -> [{BaseId, {DocId, true}}] = ets:lookup(?REP_ID_TO_DOC_ID, BaseId), @@ -250,6 +259,19 @@ process_update(State, {Change}) -> end. +rep_db_update_error(Error, {Props} = JsonRepDoc) -> + case Error of + {bad_rep_doc, Reason} -> + ok; + _ -> + Reason = to_binary(Error) + end, + ?LOG_ERROR("Replication manager, error processing document `~s`: ~s", + [get_value(<<"_id">>, Props), Reason]), + couch_rep:update_rep_doc( + JsonRepDoc, [{<<"_replication_state">>, <<"error">>}]). + + rep_user_ctx({RepDoc}) -> case get_value(<<"user_ctx">>, RepDoc) of undefined -> @@ -265,7 +287,7 @@ rep_user_ctx({RepDoc}) -> maybe_start_replication(#state{max_retries = MaxRetries} = State, DocId, JsonRepDoc) -> UserCtx = rep_user_ctx(JsonRepDoc), - {BaseId, _} = RepId = couch_rep:make_replication_id(JsonRepDoc, UserCtx), + {BaseId, _} = RepId = make_rep_id(JsonRepDoc, UserCtx), case ets:lookup(?REP_ID_TO_DOC_ID, BaseId) of [] -> true = ets:insert(?REP_ID_TO_DOC_ID, {BaseId, {DocId, true}}), @@ -290,6 +312,17 @@ maybe_start_replication(#state{max_retries = MaxRetries} = State, end. +make_rep_id(RepDoc, UserCtx) -> + try + couch_rep:make_replication_id(RepDoc, UserCtx) + catch + throw:{error, Reason} -> + throw({bad_rep_doc, Reason}); + Tag:Err -> + throw({bad_rep_doc, to_binary({Tag, Err})}) + end. + + maybe_tag_rep_doc({Props} = JsonRepDoc, RepId) -> case get_value(<<"_replication_id">>, Props) of RepId -> -- cgit v1.2.3