From 2a3b0a8d47430053541a84a3eeacaaef505308b1 Mon Sep 17 00:00:00 2001 From: Filipe David Borba Manana Date: Tue, 11 Jan 2011 21:29:55 +0000 Subject: Merged revision 1057875 from trunk: Make the doc multipart GET APIs always send attachments compressed For attachments that are stored in compressed (gzip) form, make sure the document multipart/related and multipart/mixed APIs don't decompress the attachments before sending them through the socket. This is to avoid multipart parser issues when the attachment's identity length is unknown or lost due to a local to local replication triggered by CouchDB versions up to 1.0.1 Closes COUCHDB-1022. git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.0.x@1057878 13f79535-47bb-0310-9956-ffa450edef68 --- share/www/script/test/attachments_multipart.js | 6 +++--- src/couchdb/couch_httpd_db.erl | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/share/www/script/test/attachments_multipart.js b/share/www/script/test/attachments_multipart.js index fecf9d01..f173d2bb 100644 --- a/share/www/script/test/attachments_multipart.js +++ b/share/www/script/test/attachments_multipart.js @@ -29,17 +29,17 @@ couchTests.attachments_multipart= function(debug) { "_attachments":{ "foo.txt": { "follows":true, - "content_type":"text/plain", + "content_type":"application/test", "length":21 }, "bar.txt": { "follows":true, - "content_type":"text/plain", + "content_type":"application/test", "length":20 }, "baz.txt": { "follows":true, - "content_type":"text/plain", + "content_type":"application/test", "length":19 } } diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl index 7b09bf57..1767d9cc 100644 --- a/src/couchdb/couch_httpd_db.erl +++ b/src/couchdb/couch_httpd_db.erl @@ -736,34 +736,34 @@ send_doc_efficiently(Req, #doc{atts=Atts}=Doc, Headers, Options) -> JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, [attachments, follows|Options])), {ContentType, Len} = couch_doc:len_doc_to_multi_part_stream( - Boundary,JsonBytes, Atts,false), + Boundary,JsonBytes, Atts, true), CType = {<<"Content-Type">>, ContentType}, {ok, Resp} = start_response_length(Req, 200, [CType|Headers], Len), couch_doc:doc_to_multi_part_stream(Boundary,JsonBytes,Atts, - fun(Data) -> couch_httpd:send(Resp, Data) end, false) + fun(Data) -> couch_httpd:send(Resp, Data) end, true) end; false -> send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)) end. -send_docs_multipart(Req, Results, Options) -> +send_docs_multipart(Req, Results, Options1) -> OuterBoundary = couch_uuids:random(), InnerBoundary = couch_uuids:random(), + Options = [attachments, follows, att_encoding_info | Options1], CType = {"Content-Type", "multipart/mixed; boundary=\"" ++ ?b2l(OuterBoundary) ++ "\""}, {ok, Resp} = start_chunked_response(Req, 200, [CType]), couch_httpd:send_chunk(Resp, <<"--", OuterBoundary/binary>>), lists:foreach( fun({ok, #doc{atts=Atts}=Doc}) -> - JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, - [attachments,follows|Options])), + JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, Options)), {ContentType, _Len} = couch_doc:len_doc_to_multi_part_stream( - InnerBoundary, JsonBytes, Atts, false), + InnerBoundary, JsonBytes, Atts, true), couch_httpd:send_chunk(Resp, <<"\r\nContent-Type: ", ContentType/binary, "\r\n\r\n">>), couch_doc:doc_to_multi_part_stream(InnerBoundary, JsonBytes, Atts, fun(Data) -> couch_httpd:send_chunk(Resp, Data) - end, false), + end, true), couch_httpd:send_chunk(Resp, <<"\r\n--", OuterBoundary/binary>>); ({{not_found, missing}, RevId}) -> RevStr = couch_doc:rev_to_str(RevId), -- cgit v1.2.3 From 43da5dcc08fd83b6255f14d7a97ea177018f07e6 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 12 Jan 2011 04:55:16 +0000 Subject: Preserve purge metadata during compaction, thanks Mike Leddy Closes COUCHDB-1021 git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.0.x@1057988 13f79535-47bb-0310-9956-ffa450edef68 --- THANKS | 1 + share/www/script/test/purge.js | 8 ++++++++ src/couchdb/couch_db_updater.erl | 14 +++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/THANKS b/THANKS index 5f57b196..15072e2a 100644 --- a/THANKS +++ b/THANKS @@ -69,5 +69,6 @@ suggesting improvements or submitting changes. Some of these people are: * Juuso Väänänen * Benjamin Young * Gabriel Farrell + * Mike Leddy For a list of authors see the `AUTHORS` file. diff --git a/share/www/script/test/purge.js b/share/www/script/test/purge.js index af72ea4f..f8f45138 100644 --- a/share/www/script/test/purge.js +++ b/share/www/script/test/purge.js @@ -76,6 +76,14 @@ couchTests.purge = function(debug) { } T(db.view("test/single_doc").total_rows == 0); + // purge sequences are preserved after compaction (COUCHDB-1021) + T(db.compact().ok); + T(db.last_req.status == 202); + // compaction isn't instantaneous, loop until done + while (db.info().compact_running) {}; + var compactInfo = db.info(); + T(compactInfo.purge_seq == newInfo.purge_seq); + // purge documents twice in a row without loading views // (causes full view rebuilds) diff --git a/src/couchdb/couch_db_updater.erl b/src/couchdb/couch_db_updater.erl index eb1a3edc..e660800d 100644 --- a/src/couchdb/couch_db_updater.erl +++ b/src/couchdb/couch_db_updater.erl @@ -847,7 +847,7 @@ copy_compact(Db, NewDb0, Retry) -> commit_data(NewDb4#db{update_seq=Db#db.update_seq}). -start_copy_compact(#db{name=Name,filepath=Filepath}=Db) -> +start_copy_compact(#db{name=Name,filepath=Filepath,header=#db_header{purge_seq=PurgeSeq}}=Db) -> CompactFile = Filepath ++ ".compact", ?LOG_DEBUG("Compaction process spawned for db \"~s\"", [Name]), case couch_file:open(CompactFile) of @@ -867,8 +867,16 @@ start_copy_compact(#db{name=Name,filepath=Filepath}=Db) -> ok = couch_file:write_header(Fd, Header=#db_header{}) end, NewDb = init_db(Name, CompactFile, Fd, Header), + NewDb2 = if PurgeSeq > 0 -> + {ok, PurgedIdsRevs} = couch_db:get_last_purged(Db), + {ok, Pointer} = couch_file:append_term(Fd, PurgedIdsRevs), + NewDb#db{header=Header#db_header{purge_seq=PurgeSeq, purged_docs=Pointer}}; + true -> + NewDb + end, unlink(Fd), - NewDb2 = copy_compact(Db, NewDb, Retry), - close_db(NewDb2), + + NewDb3 = copy_compact(Db, NewDb2, Retry), + close_db(NewDb3), gen_server:cast(Db#db.update_pid, {compact_done, CompactFile}). -- cgit v1.2.3 From 3674f2a2ef778bdf211426d9e804192e22cd26ad Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Wed, 12 Jan 2011 10:45:14 +0000 Subject: ensure write_streamed_attachment bails on negative LenLeft values While Filipe has identified the fix for COUCHDB-1021, this patch will ensure that no other bug will cause negative values to be passed to this function, in turn leading to database inflation problems, etc. git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.0.x@1058058 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_db.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/couchdb/couch_db.erl b/src/couchdb/couch_db.erl index f005a2ea..b7055b5c 100644 --- a/src/couchdb/couch_db.erl +++ b/src/couchdb/couch_db.erl @@ -920,7 +920,7 @@ with_stream(Fd, #att{md5=InMd5,type=Type,encoding=Enc}=Att, Fun) -> write_streamed_attachment(_Stream, _F, 0) -> ok; -write_streamed_attachment(Stream, F, LenLeft) -> +write_streamed_attachment(Stream, F, LenLeft) when LenLeft > 0 -> Bin = F(), ok = couch_stream:write(Stream, Bin), write_streamed_attachment(Stream, F, LenLeft - size(Bin)). -- cgit v1.2.3 From 8432c0e8f31a683b13419dc591edca49933d1f81 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 18 Jan 2011 00:42:36 +0000 Subject: Avoid bug in rev tree logic during attachment uploads This is a workaround for a bug in couch_key_tree, described in COUCHDB-902, which would cause uploads to fail with spurious conflicts. A patch for the key tree itself will be landing on trunk. Thanks Bob Dionne, Klaus Trainer. git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.0.x@1060149 13f79535-47bb-0310-9956-ffa450edef68 --- share/Makefile.am | 1 + share/www/script/couch_tests.js | 1 + share/www/script/test/attachment_conflicts.js | 56 +++++++++++++++++++++++++++ src/couchdb/couch_httpd_db.erl | 4 +- 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 share/www/script/test/attachment_conflicts.js diff --git a/share/Makefile.am b/share/Makefile.am index 752fa9f9..f72db769 100644 --- a/share/Makefile.am +++ b/share/Makefile.am @@ -114,6 +114,7 @@ nobase_dist_localdata_DATA = \ www/script/test/all_docs.js \ www/script/test/attachments.js \ www/script/test/attachments_multipart.js \ + www/script/test/attachment_conflicts.js \ www/script/test/attachment_names.js \ www/script/test/attachment_paths.js \ www/script/test/attachment_views.js \ diff --git a/share/www/script/couch_tests.js b/share/www/script/couch_tests.js index c5257ea6..896b3538 100644 --- a/share/www/script/couch_tests.js +++ b/share/www/script/couch_tests.js @@ -32,6 +32,7 @@ loadTest("basics.js"); loadTest("all_docs.js"); loadTest("attachments.js"); loadTest("attachments_multipart.js"); +loadTest("attachment_conflicts.js"); loadTest("attachment_names.js"); loadTest("attachment_paths.js"); loadTest("attachment_views.js"); diff --git a/share/www/script/test/attachment_conflicts.js b/share/www/script/test/attachment_conflicts.js new file mode 100644 index 00000000..c400277e --- /dev/null +++ b/share/www/script/test/attachment_conflicts.js @@ -0,0 +1,56 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Do some edit conflict detection tests for attachments. +couchTests.attachment_conflicts = function(debug) { + + var dbA = new CouchDB("test_suite_db_a", {"X-Couch-Full-Commit":"false"}); + var dbB = new CouchDB("test_suite_db_b", {"X-Couch-Full-Commit":"false"}); + dbA.deleteDb(); + dbA.createDb(); + dbB.deleteDb(); + dbB.createDb(); + + if (debug) debugger; + + T(dbA.save({"_id":"doc", "foo":"bar"}).ok); + + // create conflict + T(CouchDB.replicate("test_suite_db_a", "test_suite_db_b").ok); + + var doc = dbA.open("doc"); + var rev11 = doc._rev; + T(dbA.save({"_id":"doc", "foo":"bar2","_rev":rev11}).ok); + + doc = dbB.open("doc"); + var rev12 = doc._rev; + T(dbB.save({"_id":"doc", "foo":"bar3","_rev":rev12}).ok); + + T(CouchDB.replicate("test_suite_db_a", "test_suite_db_b").ok); + + // the attachment + var bin_data = "JHAPDO*AU£PN ){(3u[d 93DQ9¡€])} ææøo'∂ƒæ≤çæππ•¥∫¶®#†π¶®¥π€ª®˙π8np"; + + doc = dbB.open("doc"); + var rev13 = doc._rev; + + // test that we can can attach to conflicting documents + var xhr = CouchDB.request("PUT", "/test_suite_db_b/doc/attachment.txt", { + headers: { + "Content-Type": "text/plain;charset=utf-8", + "If-Match": rev13 + }, + body: bin_data + }); + T(xhr.status == 201); + +}; diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl index 1767d9cc..217a2d03 100644 --- a/src/couchdb/couch_httpd_db.erl +++ b/src/couchdb/couch_httpd_db.erl @@ -1020,8 +1020,10 @@ db_attachment_req(#httpd{method=Method,mochi_req=MochiReq}=Req, Db, DocId, FileN end end, - #doc{atts=Atts} = Doc, + #doc{atts=Atts, revs = {Pos, Revs}} = Doc, DocEdited = Doc#doc{ + % prune revision list as a workaround for key tree bug (COUCHDB-902) + revs = {Pos, case Revs of [] -> []; [Hd|_] -> [Hd] end}, atts = NewAtt ++ [A || A <- Atts, A#att.name /= FileName] }, {ok, UpdatedRev} = couch_db:update_doc(Db, DocEdited, []), -- cgit v1.2.3 From c9c334db9fe384265df316459555fe2411fd8231 Mon Sep 17 00:00:00 2001 From: Paul Joseph Davis Date: Thu, 20 Jan 2011 00:50:18 +0000 Subject: Update CHANGES and NEWS before the 1.0.2 release. git-svn-id: https://svn.apache.org/repos/asf/couchdb/branches/1.0.x@1061083 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 7 +++++++ NEWS | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/CHANGES b/CHANGES index bfabc018..fdfd31d2 100644 --- a/CHANGES +++ b/CHANGES @@ -8,6 +8,8 @@ Futon: * Make test suite work with Safari and Chrome. * Fixed animated progress spinner. + * Fix raw view document link due to overzealous URI encoding. + * Spell javascript correctly in loadScript(uri). Storage System: @@ -18,6 +20,9 @@ Storage System: * Fix occasional timeout errors on systems with slow or heavily loaded IO. * Fix for OOME when compactions include documents with many conflicts. * Fix for missing attachment compression when MIME types included parameters. + * Preserve purge metadata during compaction to avoid spurious view rebuilds. + * Fix spurious conflicts introduced when uploading an attachment after + a doc has been in a conflict. See COUCHDB-902 for details. * Fix for frequently edited documents in multi-master deployments being duplicated in _changes and _all_docs. See COUCHDDB-968 for details on how to repair. @@ -33,6 +38,8 @@ HTTP Interface: * Allow reduce=false parameter in map-only views. * Fix parsing of Accept headers. + * Fix for multipart GET APIs when an attachment was created during a + local-local replication. See COUCHDB-1022 for details. Replicator: diff --git a/NEWS b/NEWS index 1a70a0f1..9550856e 100644 --- a/NEWS +++ b/NEWS @@ -18,11 +18,16 @@ Version 1.0.2 * Allow logging of native types. * Updated ibrowse library to 2.1.2 fixing numerous replication issues. * Fix authenticated replication of design documents with attachments. + * Fix multipart GET APIs by always sending attachments in compressed + form when the source attachment is compressed on disk. Fixes a possible + edge case when an attachment underwent local-local replication. * Various fixes to make replicated more resilient for edge-cases. * Don't trigger a view update when requesting `_design/doc/_info`. * Fix for circular references in CommonJS requires. * Fix for frequently edited documents in multi-master deployments being duplicated in _changes and _all_docs. + * Fix spurious conflict generation during attachment uploads. + * Fix for various bugs in Futon. Version 1.0.1 ------------- -- cgit v1.2.3