diff options
| -rw-r--r-- | THANKS | 1 | ||||
| -rw-r--r-- | share/www/script/test/design_docs.js | 2 | ||||
| -rw-r--r-- | share/www/script/test/etags_views.js | 102 | ||||
| -rw-r--r-- | src/couchdb/couch_db.hrl | 2 | ||||
| -rw-r--r-- | src/couchdb/couch_httpd_show.erl | 6 | ||||
| -rw-r--r-- | src/couchdb/couch_httpd_view.erl | 27 | ||||
| -rw-r--r-- | src/couchdb/couch_view_group.erl | 32 | ||||
| -rw-r--r-- | src/couchdb/couch_view_updater.erl | 19 |
8 files changed, 148 insertions, 43 deletions
@@ -68,5 +68,6 @@ suggesting improvements or submitting changes. Some of these people are: * David Rose <doppler@gmail.com> * Lim Yue Chuan <shasderias@gmail.com> * David Davis <xantus@xantus.org> + * Klaus Trainer <klaus.trainer@web.de> For a list of authors see the `AUTHORS` file. diff --git a/share/www/script/test/design_docs.js b/share/www/script/test/design_docs.js index 62606341..bfe8cde2 100644 --- a/share/www/script/test/design_docs.js +++ b/share/www/script/test/design_docs.js @@ -110,7 +110,7 @@ function() { var vinfo = dinfo.view_index; TEquals(51, vinfo.disk_size); TEquals(false, vinfo.compact_running); - TEquals("dc3264b45b74cc6d94666e3043e07154", vinfo.signature, 'ddoc sig'); + TEquals("86e9b34892b4df35cd2f7c27da30c94d", vinfo.signature, 'ddoc sig'); db.bulkSave(makeDocs(1, numDocs + 1)); diff --git a/share/www/script/test/etags_views.js b/share/www/script/test/etags_views.js index 7e1537bd..f556d6ac 100644 --- a/share/www/script/test/etags_views.js +++ b/share/www/script/test/etags_views.js @@ -11,23 +11,34 @@ // the License. couchTests.etags_views = function(debug) { - var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"false"}); + var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"true"}); db.deleteDb(); db.createDb(); if (debug) debugger; var designDoc = { - _id:"_design/etags", + _id: "_design/etags", language: "javascript", views : { + fooView: { + map: stringFun(function(doc) { + if (doc.foo) { + emit("bar", 1); + } + }), + }, basicView : { map : stringFun(function(doc) { - emit(doc.integer, doc.string); + if(doc.integer && doc.string) { + emit(doc.integer, doc.string); + } }) }, withReduce : { map : stringFun(function(doc) { - emit(doc.integer, doc.string); + if(doc.integer && doc.string) { + emit(doc.integer, doc.string); + } }), reduce : stringFun(function(keys, values, rereduce) { if (rereduce) { @@ -40,9 +51,9 @@ couchTests.etags_views = function(debug) { } }; T(db.save(designDoc).ok); + db.bulkSave(makeDocs(0, 10)); + var xhr; - var docs = makeDocs(0, 10); - db.bulkSave(docs); // verify get w/Etag on map view xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView"); @@ -52,17 +63,92 @@ couchTests.etags_views = function(debug) { headers: {"if-none-match": etag} }); T(xhr.status == 304); - // TODO GET with keys (when that is available) + + // verify ETag doesn't change when an update + // doesn't change the view group's index + T(db.save({"_id":"doc1", "foo":"bar"}).ok); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView"); + var etag1 = xhr.getResponseHeader("etag"); + T(etag1 == etag); + + // Verify that purges affect etags + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView"); + var foo_etag = xhr.getResponseHeader("etag"); + var doc1 = db.open("doc1"); + xhr = CouchDB.request("POST", "/test_suite_db/_purge", { + body: JSON.stringify({"doc1":[doc1._rev]}) + }); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView"); + var etag1 = xhr.getResponseHeader("etag"); + T(etag1 != foo_etag); + + // Test that _purge didn't affect the other view etags. + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView"); + var etag1 = xhr.getResponseHeader("etag"); + T(etag1 == etag); + + // verify different views in the same view group may have different ETags + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView"); + var etag1 = xhr.getResponseHeader("etag"); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView"); + var etag2 = xhr.getResponseHeader("etag"); + T(etag1 != etag2); + + // verify ETag changes when an update changes the view group's index. + db.bulkSave(makeDocs(10, 20)); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView"); + var etag1 = xhr.getResponseHeader("etag"); + T(etag1 != etag); + + // verify ETag is the same after a restart + restartServer(); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView"); + var etag2 = xhr.getResponseHeader("etag"); + T(etag1 == etag2); // reduce view xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce"); T(xhr.status == 200); var etag = xhr.getResponseHeader("etag"); - xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce", { + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce",{ headers: {"if-none-match": etag} }); T(xhr.status == 304); + // verify ETag doesn't change when an update + // doesn't change the view group's index + T(db.save({"_id":"doc3", "foo":"bar"}).ok); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce"); + var etag1 = xhr.getResponseHeader("etag"); + T(etag1 == etag); + // purge + var doc3 = db.open("doc3"); + xhr = CouchDB.request("POST", "/test_suite_db/_purge", { + body: JSON.stringify({"doc3":[doc3._rev]}) + }); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce"); + var etag1 = xhr.getResponseHeader("etag"); + T(etag1 == etag); + + // verify different views in the same view group may have different ETags + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView"); + var etag1 = xhr.getResponseHeader("etag"); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce"); + var etag2 = xhr.getResponseHeader("etag"); + T(etag1 != etag2); + + // verify ETag changes when an update changes the view group's index + db.bulkSave(makeDocs(20, 30)); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce"); + var etag1 = xhr.getResponseHeader("etag"); + T(etag1 != etag); + + // verify ETag is the same after a restart + restartServer(); + xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce"); + var etag2 = xhr.getResponseHeader("etag"); + T(etag1 == etag2); + // confirm ETag changes with different POST bodies xhr = CouchDB.request("POST", "/test_suite_db/_design/etags/_view/basicView", {body: JSON.stringify({keys:[1]})} diff --git a/src/couchdb/couch_db.hrl b/src/couchdb/couch_db.hrl index f861219e..74d2c630 100644 --- a/src/couchdb/couch_db.hrl +++ b/src/couchdb/couch_db.hrl @@ -238,6 +238,8 @@ -record(view, {id_num, + update_seq=0, + purge_seq=0, map_names=[], def, btree=nil, diff --git a/src/couchdb/couch_httpd_show.erl b/src/couchdb/couch_httpd_show.erl index d50ca83a..fa8cd972 100644 --- a/src/couchdb/couch_httpd_show.erl +++ b/src/couchdb/couch_httpd_show.erl @@ -188,14 +188,14 @@ handle_view_list_req(Req, _Db, _DDoc) -> handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) -> ViewDesignId = <<"_design/", ViewDesignName/binary>>, {ViewType, View, Group, QueryArgs} = couch_httpd_view:load_view(Req, Db, {ViewDesignId, ViewName}, Keys), - Etag = list_etag(Req, Db, Group, {couch_httpd:doc_etag(DDoc), Keys}), + Etag = list_etag(Req, Db, Group, View, {couch_httpd:doc_etag(DDoc), Keys}), couch_httpd:etag_respond(Req, Etag, fun() -> output_list(ViewType, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group) end). -list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, More) -> +list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, View, More) -> Accept = couch_httpd:header_value(Req, "Accept"), - couch_httpd_view:view_group_etag(Group, Db, {More, Accept, UserCtx#user_ctx.roles}). + couch_httpd_view:view_etag(Db, Group, View, {More, Accept, UserCtx#user_ctx.roles}). output_list(map, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group) -> output_map_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group); diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl index 65598a68..8829de02 100644 --- a/src/couchdb/couch_httpd_view.erl +++ b/src/couchdb/couch_httpd_view.erl @@ -17,7 +17,7 @@ -export([parse_view_params/3]). -export([make_view_fold_fun/7, finish_view_fold/4, finish_view_fold/5, view_row_obj/3]). --export([view_group_etag/2, view_group_etag/3, make_reduce_fold_funs/6]). +-export([view_etag/3, view_etag/4, make_reduce_fold_funs/6]). -export([design_doc_view/5, parse_bool_param/1, doc_member/2]). -export([make_key_options/1, load_view/4]). @@ -113,7 +113,7 @@ output_map_view(Req, View, Group, Db, QueryArgs, nil) -> limit = Limit, skip = SkipCount } = QueryArgs, - CurrentEtag = view_group_etag(Group, Db), + CurrentEtag = view_etag(Db, Group, View), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> {ok, RowCount} = couch_view:get_row_count(View), FoldlFun = make_view_fold_fun(Req, QueryArgs, CurrentEtag, Db, Group#group.current_seq, RowCount, #view_fold_helper_funs{reduce_count=fun couch_view:reduce_to_count/1}), @@ -129,7 +129,7 @@ output_map_view(Req, View, Group, Db, QueryArgs, Keys) -> limit = Limit, skip = SkipCount } = QueryArgs, - CurrentEtag = view_group_etag(Group, Db, Keys), + CurrentEtag = view_etag(Db, Group, View, Keys), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> {ok, RowCount} = couch_view:get_row_count(View), FoldAccInit = {Limit, SkipCount, undefined, []}, @@ -154,7 +154,7 @@ output_reduce_view(Req, Db, View, Group, QueryArgs, nil) -> skip = Skip, group_level = GroupLevel } = QueryArgs, - CurrentEtag = view_group_etag(Group, Db), + CurrentEtag = view_etag(Db, Group, View), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel, QueryArgs, CurrentEtag, Group#group.current_seq, @@ -172,7 +172,7 @@ output_reduce_view(Req, Db, View, Group, QueryArgs, Keys) -> skip = Skip, group_level = GroupLevel } = QueryArgs, - CurrentEtag = view_group_etag(Group, Db, Keys), + CurrentEtag = view_etag(Db, Group, View, Keys), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel, QueryArgs, CurrentEtag, Group#group.current_seq, @@ -607,16 +607,15 @@ send_json_reduce_row(Resp, {Key, Value}, RowFront) -> send_chunk(Resp, RowFront ++ ?JSON_ENCODE({[{key, Key}, {value, Value}]})), {ok, ",\r\n"}. -view_group_etag(Group, Db) -> - view_group_etag(Group, Db, nil). +view_etag(Db, Group, View) -> + view_etag(Db, Group, View, nil). -view_group_etag(#group{sig=Sig,current_seq=CurrentSeq}, _Db, Extra) -> - % ?LOG_ERROR("Group ~p",[Group]), - % This is not as granular as it could be. - % If there are updates to the db that do not effect the view index, - % they will change the Etag. For more granular Etags we'd need to keep - % track of the last Db seq that caused an index change. - couch_httpd:make_etag({Sig, CurrentSeq, Extra}). +view_etag(Db, Group, {reduce, _, _, View}, Extra) -> + view_etag(Db, Group, View, Extra); +view_etag(Db, Group, {temp_reduce, View}, Extra) -> + view_etag(Db, Group, View, Extra); +view_etag(_Db, #group{sig=Sig}, #view{update_seq=UpdateSeq, purge_seq=PurgeSeq}, Extra) -> + couch_httpd:make_etag({Sig, UpdateSeq, PurgeSeq, Extra}). % the view row has an error view_row_obj(_Db, {{Key, error}, Value}, _IncludeDocs) -> diff --git a/src/couchdb/couch_view_group.erl b/src/couchdb/couch_view_group.erl index 962f4aae..3a3ffe99 100644 --- a/src/couchdb/couch_view_group.erl +++ b/src/couchdb/couch_view_group.erl @@ -404,11 +404,15 @@ prepare_group({RootDir, DbName, #group{sig=Sig}=Group}, ForceReset)-> get_index_header_data(#group{current_seq=Seq, purge_seq=PurgeSeq, id_btree=IdBtree,views=Views}) -> - ViewStates = [couch_btree:get_state(Btree) || #view{btree=Btree} <- Views], - #index_header{seq=Seq, - purge_seq=PurgeSeq, - id_btree_state=couch_btree:get_state(IdBtree), - view_states=ViewStates}. + ViewStates = [ + {couch_btree:get_state(V#view.btree), V#view.update_seq, V#view.purge_seq} || V <- Views + ], + #index_header{ + seq=Seq, + purge_seq=PurgeSeq, + id_btree_state=couch_btree:get_state(IdBtree), + view_states=ViewStates + }. hex_sig(GroupSig) -> couch_util:to_hex(?b2l(GroupSig)). @@ -460,13 +464,15 @@ set_view_sig(#group{ lib={[]}, def_lang=Language, design_options=DesignOptions}=G) -> - G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions}))}; + ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views], + G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions}))}; set_view_sig(#group{ views=Views, lib=Lib, def_lang=Language, design_options=DesignOptions}=G) -> - G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions, sort_lib(Lib)}))}. + ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views], + G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions, sort_lib(Lib)}))}. sort_lib({Lib}) -> sort_lib(Lib, []). @@ -574,14 +580,14 @@ delete_index_file(RootDir, DbName, GroupSig) -> init_group(Db, Fd, #group{views=Views}=Group, nil) -> init_group(Db, Fd, Group, #index_header{seq=0, purge_seq=couch_db:get_purge_seq(Db), - id_btree_state=nil, view_states=[nil || _ <- Views]}); + id_btree_state=nil, view_states=[{nil, 0, 0} || _ <- Views]}); init_group(Db, Fd, #group{def_lang=Lang,views=Views}= Group, IndexHeader) -> #index_header{seq=Seq, purge_seq=PurgeSeq, id_btree_state=IdBtreeState, view_states=ViewStates} = IndexHeader, {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd), Views2 = lists:zipwith( - fun(BtreeState, #view{reduce_funs=RedFuns,options=Options}=View) -> + fun({BTState, USeq, PSeq}, #view{reduce_funs=RedFuns,options=Options}=View) -> FunSrcs = [FunSrc || {_Name, FunSrc} <- RedFuns], ReduceFun = fun(reduce, KVs) -> @@ -604,10 +610,10 @@ init_group(Db, Fd, #group{def_lang=Lang,views=Views}= <<"raw">> -> Less = fun(A,B) -> A < B end end, - {ok, Btree} = couch_btree:open(BtreeState, Fd, - [{less, Less}, - {reduce, ReduceFun}]), - View#view{btree=Btree} + {ok, Btree} = couch_btree:open(BTState, Fd, + [{less, Less}, {reduce, ReduceFun}] + ), + View#view{btree=Btree, update_seq=USeq, purge_seq=PSeq} end, ViewStates, Views), Group#group{db=Db, fd=Fd, current_seq=Seq, purge_seq=PurgeSeq, diff --git a/src/couchdb/couch_view_updater.erl b/src/couchdb/couch_view_updater.erl index bfef73ba..8e089fa9 100644 --- a/src/couchdb/couch_view_updater.erl +++ b/src/couchdb/couch_view_updater.erl @@ -96,19 +96,25 @@ purge_index(#group{db=Db, views=Views, id_btree=IdBtree}=Group) -> end, dict:new(), Lookups), % Now remove the values from the btrees + PurgeSeq = couch_db:get_purge_seq(Db), Views2 = lists:map( fun(#view{id_num=Num,btree=Btree}=View) -> case dict:find(Num, ViewKeysToRemoveDict) of {ok, RemoveKeys} -> - {ok, Btree2} = couch_btree:add_remove(Btree, [], RemoveKeys), - View#view{btree=Btree2}; + {ok, ViewBtree2} = couch_btree:add_remove(Btree, [], RemoveKeys), + case ViewBtree2 =/= Btree of + true -> + View#view{btree=ViewBtree2, purge_seq=PurgeSeq}; + _ -> + View#view{btree=ViewBtree2} + end; error -> % no keys to remove in this view View end end, Views), Group#group{id_btree=IdBtree2, views=Views2, - purge_seq=couch_db:get_purge_seq(Db)}. + purge_seq=PurgeSeq}. load_doc(Db, DocInfo, MapQueue, DocOpts, IncludeDesign) -> @@ -247,7 +253,12 @@ write_changes(Group, ViewKeyValuesToAdd, DocIdViewIdKeys, NewSeq, InitialBuild) Views2 = lists:zipwith(fun(View, {_View, AddKeyValues}) -> KeysToRemove = couch_util:dict_find(View#view.id_num, KeysToRemoveByView, []), {ok, ViewBtree2} = couch_btree:add_remove(View#view.btree, AddKeyValues, KeysToRemove), - View#view{btree = ViewBtree2} + case ViewBtree2 =/= View#view.btree of + true -> + View#view{btree=ViewBtree2, update_seq=NewSeq}; + _ -> + View#view{btree=ViewBtree2} + end end, Group#group.views, ViewKeyValuesToAdd), Group#group{views=Views2, current_seq=NewSeq, id_btree=IdBtree2}. |