From a6352493651f6f8e3430269555421b5599a77809 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 26 Oct 2011 14:04:54 -0400 Subject: Fix retrieval of headers larger than 4k Our headers start with a <<1>> and then four bytes indicating the length of the header and its checksum. When the header is larger than 4090 bytes it will be split across multiple blocks in the file and will need to be reassembled on read. The reassembly consists of stripping out <<0>> from the beginning of each subsequent block in the remove_block_prefixes/2 function. The bug here is that we tell remove_block_prefixes that we're starting 1 byte into the current block instead of 5, so it ends up removing one good byte from the header and injecting one or more random <<0>>s. Headers larger than 4k are very rare and generally require a view group with a huge number of indexes or indexes with fairly large reductions, which explains why this bug has gone undetected until now. Closes COUCHDB-1319. --- apps/couch/src/couch_file.erl | 2 +- apps/couch/test/etap/011-file-headers.t | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'apps') diff --git a/apps/couch/src/couch_file.erl b/apps/couch/src/couch_file.erl index dfc1f822..1985f5eb 100644 --- a/apps/couch/src/couch_file.erl +++ b/apps/couch/src/couch_file.erl @@ -528,7 +528,7 @@ load_header(Fd, Block) -> RawBin = <> end, <> = - iolist_to_binary(remove_block_prefixes(1, RawBin)), + iolist_to_binary(remove_block_prefixes(5, RawBin)), Md5Sig = couch_util:md5(HeaderBin), {ok, HeaderBin}. diff --git a/apps/couch/test/etap/011-file-headers.t b/apps/couch/test/etap/011-file-headers.t index 4705f629..764b10df 100755 --- a/apps/couch/test/etap/011-file-headers.t +++ b/apps/couch/test/etap/011-file-headers.t @@ -22,7 +22,7 @@ main(_) -> {S1, S2, S3} = now(), random:seed(S1, S2, S3), - etap:plan(17), + etap:plan(18), case (catch test()) of ok -> etap:end_tests(); @@ -68,6 +68,13 @@ test() -> etap:is({ok, Size2}, couch_file:bytes(Fd), "Rewriting the same second header returns the same second size."), + couch_file:write_header(Fd, erlang:make_tuple(5000, <<"CouchDB">>)), + etap:is( + couch_file:read_header(Fd), + {ok, erlang:make_tuple(5000, <<"CouchDB">>)}, + "Headers larger than the block size can be saved (COUCHDB-1319)" + ), + ok = couch_file:close(Fd), % Now for the fun stuff. Try corrupting the second header and see -- cgit v1.2.3 From 223646af60465522572cfea014f35bfa68496d0e Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Wed, 9 Nov 2011 13:08:49 +0000 Subject: Conditionally apply JSON encoding to update_seq values BigCouch 0.3 cannot parse requests of the form /db/_changes?since="123-foo" so the recent ?JSON_ENCODE addition to Since in two places causes 0.3 <-> 0.4 replication to fail with json_encode/badterm errors. This patch applies JSON encoding only when the Since value is not already a binary (i.e, when it's a [integer(), binary()]) and interop is restored. BugzID: 12833 --- apps/couch/src/couch_rep.erl | 3 ++- apps/couch/src/couch_rep_changes_feed.erl | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'apps') diff --git a/apps/couch/src/couch_rep.erl b/apps/couch/src/couch_rep.erl index 8f2508f5..82b69bc9 100644 --- a/apps/couch/src/couch_rep.erl +++ b/apps/couch/src/couch_rep.erl @@ -834,7 +834,8 @@ ensure_full_commit(#http_db{headers = Headers} = Source, RequiredSeq) -> Req = Source#http_db{ resource = "_ensure_full_commit", method = post, - qs = [{seq, iolist_to_binary(?JSON_ENCODE(RequiredSeq))}], + qs = [{seq, case RequiredSeq of Bin when is_binary(Bin) -> Bin; + Else -> iolist_to_binary(?JSON_ENCODE(Else)) end}], headers = Headers1 }, {ResultProps} = couch_rep_httpc:request(Req), diff --git a/apps/couch/src/couch_rep_changes_feed.erl b/apps/couch/src/couch_rep_changes_feed.erl index 70db52a0..651069fb 100644 --- a/apps/couch/src/couch_rep_changes_feed.erl +++ b/apps/couch/src/couch_rep_changes_feed.erl @@ -64,7 +64,8 @@ init([Parent, #http_db{headers = Headers0} = Source, Since, PostProps]) -> BaseQS = [ {"style", all_docs}, {"heartbeat", 10000}, - {"since", iolist_to_binary(?JSON_ENCODE(Since))}, + {"since", case Since of Bin when is_binary(Bin) -> Bin; + Else -> iolist_to_binary(?JSON_ENCODE(Else)) end}, {"feed", Feed} ], {QS, Method, Body, Headers} = case get_value(<<"doc_ids">>, PostProps) of -- cgit v1.2.3 From effae95921f30d4847f8a8082f7e6f619c5108d7 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Mon, 16 Jan 2012 19:03:03 +0000 Subject: Use spawn_monitor to await process completion BugzID: 13133 --- apps/couch/src/couch_doc.erl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'apps') diff --git a/apps/couch/src/couch_doc.erl b/apps/couch/src/couch_doc.erl index 63ac0892..2c841835 100644 --- a/apps/couch/src/couch_doc.erl +++ b/apps/couch/src/couch_doc.erl @@ -484,13 +484,10 @@ atts_to_mp([Att | RestAtts], Boundary, WriteFun, doc_from_multi_part_stream(ContentType, DataFun) -> - Parent = self(), - Parser = spawn_link(fun() -> + {Parser, ParserRef} = spawn_monitor(fun() -> {<<"--">>, _, _} = couch_httpd:parse_multipart_request( ContentType, DataFun, - fun(Next) -> mp_parse_doc(Next, []) end), - unlink(Parent), - Parent ! {self(), finished} + fun(Next) -> mp_parse_doc(Next, []) end) end), Parser ! {get_doc_bytes, self()}, receive @@ -505,7 +502,7 @@ doc_from_multi_part_stream(ContentType, DataFun) -> A end, Doc#doc.atts), WaitFun = fun() -> - receive {Parser, finished} -> ok end, + receive {'DOWN', ParserRef, _, _, _} -> ok end, erlang:put(mochiweb_request_recv, true) end, {ok, Doc#doc{atts=Atts2}, WaitFun, Parser} -- cgit v1.2.3 From 5a7a1a536f32794910e8730ad93891f23a4e8895 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Mon, 16 Jan 2012 22:11:20 +0000 Subject: Only set the process dictionary if exit was ok --- apps/couch/src/couch_doc.erl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'apps') diff --git a/apps/couch/src/couch_doc.erl b/apps/couch/src/couch_doc.erl index 2c841835..923fbb0e 100644 --- a/apps/couch/src/couch_doc.erl +++ b/apps/couch/src/couch_doc.erl @@ -487,7 +487,8 @@ doc_from_multi_part_stream(ContentType, DataFun) -> {Parser, ParserRef} = spawn_monitor(fun() -> {<<"--">>, _, _} = couch_httpd:parse_multipart_request( ContentType, DataFun, - fun(Next) -> mp_parse_doc(Next, []) end) + fun(Next) -> mp_parse_doc(Next, []) end), + exit(ok) end), Parser ! {get_doc_bytes, self()}, receive @@ -502,8 +503,14 @@ doc_from_multi_part_stream(ContentType, DataFun) -> A end, Doc#doc.atts), WaitFun = fun() -> - receive {'DOWN', ParserRef, _, _, _} -> ok end, - erlang:put(mochiweb_request_recv, true) + receive {'DOWN', ParserRef, _, _, Result} -> ok end, + case Result of + ok -> + erlang:put(mochiweb_request_recv, true); + _Else -> + ?LOG_ERROR("Unexpected msg while parsing multipart stream: ~p", + [Result]) + end end, {ok, Doc#doc{atts=Atts2}, WaitFun, Parser} end. -- cgit v1.2.3 From 855bb7dcda1cef59c449d62de35d9c4d74683731 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Wed, 18 Jan 2012 13:39:37 +0000 Subject: Revert use of spawn_monitor As Filipe correctly points out, we want the parent to die if the child dies. --- apps/couch/src/couch_doc.erl | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'apps') diff --git a/apps/couch/src/couch_doc.erl b/apps/couch/src/couch_doc.erl index 923fbb0e..63ac0892 100644 --- a/apps/couch/src/couch_doc.erl +++ b/apps/couch/src/couch_doc.erl @@ -484,11 +484,13 @@ atts_to_mp([Att | RestAtts], Boundary, WriteFun, doc_from_multi_part_stream(ContentType, DataFun) -> - {Parser, ParserRef} = spawn_monitor(fun() -> + Parent = self(), + Parser = spawn_link(fun() -> {<<"--">>, _, _} = couch_httpd:parse_multipart_request( ContentType, DataFun, fun(Next) -> mp_parse_doc(Next, []) end), - exit(ok) + unlink(Parent), + Parent ! {self(), finished} end), Parser ! {get_doc_bytes, self()}, receive @@ -503,14 +505,8 @@ doc_from_multi_part_stream(ContentType, DataFun) -> A end, Doc#doc.atts), WaitFun = fun() -> - receive {'DOWN', ParserRef, _, _, Result} -> ok end, - case Result of - ok -> - erlang:put(mochiweb_request_recv, true); - _Else -> - ?LOG_ERROR("Unexpected msg while parsing multipart stream: ~p", - [Result]) - end + receive {Parser, finished} -> ok end, + erlang:put(mochiweb_request_recv, true) end, {ok, Doc#doc{atts=Atts2}, WaitFun, Parser} end. -- cgit v1.2.3 From 6696d6fd42620f87f29f09f03daaf5fbe454e10d Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 24 Jan 2012 03:29:40 -0600 Subject: Minor bug getting the reduce_limit setting --- apps/couch/src/couch_proc_manager.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apps') diff --git a/apps/couch/src/couch_proc_manager.erl b/apps/couch/src/couch_proc_manager.erl index 438f7973..509da9ba 100644 --- a/apps/couch/src/couch_proc_manager.erl +++ b/apps/couch/src/couch_proc_manager.erl @@ -150,7 +150,7 @@ make_proc(Pid, Lang, Mod) -> {ok, Proc}. get_query_server_config() -> - Limit = couch_config:get("query_server_config", <<"reduce_limit">>, "true"), + Limit = couch_config:get("query_server_config", "reduce_limit", "true"), {[{<<"reduce_limit">>, list_to_atom(Limit)}]}. proc_with_ddoc(DDoc, DDocKey, Procs) -> -- cgit v1.2.3 From f199c336b304b1a9abc03d6898b10dafb7a17ad6 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 31 Jan 2012 14:34:59 -0500 Subject: Fix bug in replicator request piplining A replication with both an HTTP source and target on the same host and port could end up in a dead lock due to ibrowse replication pipelining when attachments are present on the source. The ibrowse http worker would end up forming a multipart/mime body using anonymous reader functions for attachment stubs. When the attachment stub functions are executed it is possible that they end up assigned to the same ibrowse worker. This is a bit of a long path but then end result is equivalent to calling gen_server:call(self(), Args, infinity) from a gen_server callback. A quick work around for users is to set up a DNA alias (possibly in /etc/hosts) or to use a combination of hostname and ip address so that ibrowse assigns the requests to different pools. --- apps/couch/src/couch_rep_writer.erl | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'apps') diff --git a/apps/couch/src/couch_rep_writer.erl b/apps/couch/src/couch_rep_writer.erl index 40323925..6f557107 100644 --- a/apps/couch/src/couch_rep_writer.erl +++ b/apps/couch/src/couch_rep_writer.erl @@ -126,13 +126,18 @@ write_multi_part_doc(#http_db{headers=Headers} = Db, #doc{atts=Atts} = Doc) -> {"Content-Length", Len} | Headers ] }, - Result = case couch_rep_httpc:request(Request) of - {[{<<"error">>, Error}, {<<"reason">>, Reason}]} -> - {Pos, [RevId | _]} = Doc#doc.revs, - ErrId = couch_util:to_existing_atom(Error), - [{Doc#doc.id, couch_doc:rev_to_str({Pos, RevId})}, {ErrId, Reason}]; - _ -> - [] + Conn = couch_rep_httpc:spawn_link_worker_process(Request), + Result = try + case couch_rep_httpc:request(Request) of + {[{<<"error">>, Error}, {<<"reason">>, Reason}]} -> + {Pos, [RevId | _]} = Doc#doc.revs, + ErrId = couch_util:to_existing_atom(Error), + [{Doc#doc.id, couch_doc:rev_to_str({Pos, RevId})}, {ErrId, Reason}]; + _ -> + [] + end + after + ibrowse:stop_worker_process(Conn) end, StreamerPid ! stop, Result. -- cgit v1.2.3 From db23528537fed7b9b0e85d3f3617f560cb52d630 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Tue, 31 Jan 2012 16:59:28 -0500 Subject: Fix previous commit --- apps/couch/src/couch_rep_writer.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apps') diff --git a/apps/couch/src/couch_rep_writer.erl b/apps/couch/src/couch_rep_writer.erl index 6f557107..cea4408e 100644 --- a/apps/couch/src/couch_rep_writer.erl +++ b/apps/couch/src/couch_rep_writer.erl @@ -128,7 +128,7 @@ write_multi_part_doc(#http_db{headers=Headers} = Db, #doc{atts=Atts} = Doc) -> }, Conn = couch_rep_httpc:spawn_link_worker_process(Request), Result = try - case couch_rep_httpc:request(Request) of + case couch_rep_httpc:request(Request#http_db{conn=Conn}) of {[{<<"error">>, Error}, {<<"reason">>, Reason}]} -> {Pos, [RevId | _]} = Doc#doc.revs, ErrId = couch_util:to_existing_atom(Error), -- cgit v1.2.3