From c67ba1f96daaec66b01c80b27ff92f33696b0900 Mon Sep 17 00:00:00 2001 From: Paul Joseph Davis Date: Sun, 10 Oct 2010 19:07:10 +0000 Subject: Fixes COUCHDB-799 - More granular ETags for views. ETags for views now only change when their underlying view index changes due to indexing or purges. ETags are also specific to each view. Thanks to Klaus Trainer for the patch. git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@1006339 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_db.hrl | 2 ++ src/couchdb/couch_httpd_show.erl | 6 +++--- src/couchdb/couch_httpd_view.erl | 27 +++++++++++++-------------- src/couchdb/couch_view_group.erl | 32 +++++++++++++++++++------------- src/couchdb/couch_view_updater.erl | 19 +++++++++++++++---- 5 files changed, 52 insertions(+), 34 deletions(-) (limited to 'src') diff --git a/src/couchdb/couch_db.hrl b/src/couchdb/couch_db.hrl index f861219e..74d2c630 100644 --- a/src/couchdb/couch_db.hrl +++ b/src/couchdb/couch_db.hrl @@ -238,6 +238,8 @@ -record(view, {id_num, + update_seq=0, + purge_seq=0, map_names=[], def, btree=nil, diff --git a/src/couchdb/couch_httpd_show.erl b/src/couchdb/couch_httpd_show.erl index d50ca83a..fa8cd972 100644 --- a/src/couchdb/couch_httpd_show.erl +++ b/src/couchdb/couch_httpd_show.erl @@ -188,14 +188,14 @@ handle_view_list_req(Req, _Db, _DDoc) -> handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) -> ViewDesignId = <<"_design/", ViewDesignName/binary>>, {ViewType, View, Group, QueryArgs} = couch_httpd_view:load_view(Req, Db, {ViewDesignId, ViewName}, Keys), - Etag = list_etag(Req, Db, Group, {couch_httpd:doc_etag(DDoc), Keys}), + Etag = list_etag(Req, Db, Group, View, {couch_httpd:doc_etag(DDoc), Keys}), couch_httpd:etag_respond(Req, Etag, fun() -> output_list(ViewType, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group) end). -list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, More) -> +list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, View, More) -> Accept = couch_httpd:header_value(Req, "Accept"), - couch_httpd_view:view_group_etag(Group, Db, {More, Accept, UserCtx#user_ctx.roles}). + couch_httpd_view:view_etag(Db, Group, View, {More, Accept, UserCtx#user_ctx.roles}). output_list(map, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group) -> output_map_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group); diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl index 65598a68..8829de02 100644 --- a/src/couchdb/couch_httpd_view.erl +++ b/src/couchdb/couch_httpd_view.erl @@ -17,7 +17,7 @@ -export([parse_view_params/3]). -export([make_view_fold_fun/7, finish_view_fold/4, finish_view_fold/5, view_row_obj/3]). --export([view_group_etag/2, view_group_etag/3, make_reduce_fold_funs/6]). +-export([view_etag/3, view_etag/4, make_reduce_fold_funs/6]). -export([design_doc_view/5, parse_bool_param/1, doc_member/2]). -export([make_key_options/1, load_view/4]). @@ -113,7 +113,7 @@ output_map_view(Req, View, Group, Db, QueryArgs, nil) -> limit = Limit, skip = SkipCount } = QueryArgs, - CurrentEtag = view_group_etag(Group, Db), + CurrentEtag = view_etag(Db, Group, View), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> {ok, RowCount} = couch_view:get_row_count(View), FoldlFun = make_view_fold_fun(Req, QueryArgs, CurrentEtag, Db, Group#group.current_seq, RowCount, #view_fold_helper_funs{reduce_count=fun couch_view:reduce_to_count/1}), @@ -129,7 +129,7 @@ output_map_view(Req, View, Group, Db, QueryArgs, Keys) -> limit = Limit, skip = SkipCount } = QueryArgs, - CurrentEtag = view_group_etag(Group, Db, Keys), + CurrentEtag = view_etag(Db, Group, View, Keys), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> {ok, RowCount} = couch_view:get_row_count(View), FoldAccInit = {Limit, SkipCount, undefined, []}, @@ -154,7 +154,7 @@ output_reduce_view(Req, Db, View, Group, QueryArgs, nil) -> skip = Skip, group_level = GroupLevel } = QueryArgs, - CurrentEtag = view_group_etag(Group, Db), + CurrentEtag = view_etag(Db, Group, View), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel, QueryArgs, CurrentEtag, Group#group.current_seq, @@ -172,7 +172,7 @@ output_reduce_view(Req, Db, View, Group, QueryArgs, Keys) -> skip = Skip, group_level = GroupLevel } = QueryArgs, - CurrentEtag = view_group_etag(Group, Db, Keys), + CurrentEtag = view_etag(Db, Group, View, Keys), couch_httpd:etag_respond(Req, CurrentEtag, fun() -> {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel, QueryArgs, CurrentEtag, Group#group.current_seq, @@ -607,16 +607,15 @@ send_json_reduce_row(Resp, {Key, Value}, RowFront) -> send_chunk(Resp, RowFront ++ ?JSON_ENCODE({[{key, Key}, {value, Value}]})), {ok, ",\r\n"}. -view_group_etag(Group, Db) -> - view_group_etag(Group, Db, nil). +view_etag(Db, Group, View) -> + view_etag(Db, Group, View, nil). -view_group_etag(#group{sig=Sig,current_seq=CurrentSeq}, _Db, Extra) -> - % ?LOG_ERROR("Group ~p",[Group]), - % This is not as granular as it could be. - % If there are updates to the db that do not effect the view index, - % they will change the Etag. For more granular Etags we'd need to keep - % track of the last Db seq that caused an index change. - couch_httpd:make_etag({Sig, CurrentSeq, Extra}). +view_etag(Db, Group, {reduce, _, _, View}, Extra) -> + view_etag(Db, Group, View, Extra); +view_etag(Db, Group, {temp_reduce, View}, Extra) -> + view_etag(Db, Group, View, Extra); +view_etag(_Db, #group{sig=Sig}, #view{update_seq=UpdateSeq, purge_seq=PurgeSeq}, Extra) -> + couch_httpd:make_etag({Sig, UpdateSeq, PurgeSeq, Extra}). % the view row has an error view_row_obj(_Db, {{Key, error}, Value}, _IncludeDocs) -> diff --git a/src/couchdb/couch_view_group.erl b/src/couchdb/couch_view_group.erl index 962f4aae..3a3ffe99 100644 --- a/src/couchdb/couch_view_group.erl +++ b/src/couchdb/couch_view_group.erl @@ -404,11 +404,15 @@ prepare_group({RootDir, DbName, #group{sig=Sig}=Group}, ForceReset)-> get_index_header_data(#group{current_seq=Seq, purge_seq=PurgeSeq, id_btree=IdBtree,views=Views}) -> - ViewStates = [couch_btree:get_state(Btree) || #view{btree=Btree} <- Views], - #index_header{seq=Seq, - purge_seq=PurgeSeq, - id_btree_state=couch_btree:get_state(IdBtree), - view_states=ViewStates}. + ViewStates = [ + {couch_btree:get_state(V#view.btree), V#view.update_seq, V#view.purge_seq} || V <- Views + ], + #index_header{ + seq=Seq, + purge_seq=PurgeSeq, + id_btree_state=couch_btree:get_state(IdBtree), + view_states=ViewStates + }. hex_sig(GroupSig) -> couch_util:to_hex(?b2l(GroupSig)). @@ -460,13 +464,15 @@ set_view_sig(#group{ lib={[]}, def_lang=Language, design_options=DesignOptions}=G) -> - G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions}))}; + ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views], + G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions}))}; set_view_sig(#group{ views=Views, lib=Lib, def_lang=Language, design_options=DesignOptions}=G) -> - G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions, sort_lib(Lib)}))}. + ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views], + G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions, sort_lib(Lib)}))}. sort_lib({Lib}) -> sort_lib(Lib, []). @@ -574,14 +580,14 @@ delete_index_file(RootDir, DbName, GroupSig) -> init_group(Db, Fd, #group{views=Views}=Group, nil) -> init_group(Db, Fd, Group, #index_header{seq=0, purge_seq=couch_db:get_purge_seq(Db), - id_btree_state=nil, view_states=[nil || _ <- Views]}); + id_btree_state=nil, view_states=[{nil, 0, 0} || _ <- Views]}); init_group(Db, Fd, #group{def_lang=Lang,views=Views}= Group, IndexHeader) -> #index_header{seq=Seq, purge_seq=PurgeSeq, id_btree_state=IdBtreeState, view_states=ViewStates} = IndexHeader, {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd), Views2 = lists:zipwith( - fun(BtreeState, #view{reduce_funs=RedFuns,options=Options}=View) -> + fun({BTState, USeq, PSeq}, #view{reduce_funs=RedFuns,options=Options}=View) -> FunSrcs = [FunSrc || {_Name, FunSrc} <- RedFuns], ReduceFun = fun(reduce, KVs) -> @@ -604,10 +610,10 @@ init_group(Db, Fd, #group{def_lang=Lang,views=Views}= <<"raw">> -> Less = fun(A,B) -> A < B end end, - {ok, Btree} = couch_btree:open(BtreeState, Fd, - [{less, Less}, - {reduce, ReduceFun}]), - View#view{btree=Btree} + {ok, Btree} = couch_btree:open(BTState, Fd, + [{less, Less}, {reduce, ReduceFun}] + ), + View#view{btree=Btree, update_seq=USeq, purge_seq=PSeq} end, ViewStates, Views), Group#group{db=Db, fd=Fd, current_seq=Seq, purge_seq=PurgeSeq, diff --git a/src/couchdb/couch_view_updater.erl b/src/couchdb/couch_view_updater.erl index bfef73ba..8e089fa9 100644 --- a/src/couchdb/couch_view_updater.erl +++ b/src/couchdb/couch_view_updater.erl @@ -96,19 +96,25 @@ purge_index(#group{db=Db, views=Views, id_btree=IdBtree}=Group) -> end, dict:new(), Lookups), % Now remove the values from the btrees + PurgeSeq = couch_db:get_purge_seq(Db), Views2 = lists:map( fun(#view{id_num=Num,btree=Btree}=View) -> case dict:find(Num, ViewKeysToRemoveDict) of {ok, RemoveKeys} -> - {ok, Btree2} = couch_btree:add_remove(Btree, [], RemoveKeys), - View#view{btree=Btree2}; + {ok, ViewBtree2} = couch_btree:add_remove(Btree, [], RemoveKeys), + case ViewBtree2 =/= Btree of + true -> + View#view{btree=ViewBtree2, purge_seq=PurgeSeq}; + _ -> + View#view{btree=ViewBtree2} + end; error -> % no keys to remove in this view View end end, Views), Group#group{id_btree=IdBtree2, views=Views2, - purge_seq=couch_db:get_purge_seq(Db)}. + purge_seq=PurgeSeq}. load_doc(Db, DocInfo, MapQueue, DocOpts, IncludeDesign) -> @@ -247,7 +253,12 @@ write_changes(Group, ViewKeyValuesToAdd, DocIdViewIdKeys, NewSeq, InitialBuild) Views2 = lists:zipwith(fun(View, {_View, AddKeyValues}) -> KeysToRemove = couch_util:dict_find(View#view.id_num, KeysToRemoveByView, []), {ok, ViewBtree2} = couch_btree:add_remove(View#view.btree, AddKeyValues, KeysToRemove), - View#view{btree = ViewBtree2} + case ViewBtree2 =/= View#view.btree of + true -> + View#view{btree=ViewBtree2, update_seq=NewSeq}; + _ -> + View#view{btree=ViewBtree2} + end end, Group#group.views, ViewKeyValuesToAdd), Group#group{views=Views2, current_seq=NewSeq, id_btree=IdBtree2}. -- cgit v1.2.3