Fixes COUCHDB-799 - More granular ETags for views.

ETags for views now only change when their underlying view index
changes due to indexing or purges. ETags are also specific to each
view.

Thanks to Klaus Trainer for the patch.



git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@1006339 13f79535-47bb-0310-9956-ffa450edef68

8 files changed, 148 insertions, 43 deletions

diff --git a/THANKS b/THANKS
index 6aef37e0..d6ad6a59 100644
--- a/THANKS
+++ b/THANKS
@@ -68,5 +68,6 @@ suggesting improvements or submitting changes. Some of these people are:
  * David Rose <doppler@gmail.com>
  * Lim Yue Chuan <shasderias@gmail.com>
  * David Davis <xantus@xantus.org>
+ * Klaus Trainer <klaus.trainer@web.de>
 
 For a list of authors see the `AUTHORS` file.
diff --git a/share/www/script/test/design_docs.js b/share/www/script/test/design_docs.js
index 62606341..bfe8cde2 100644
--- a/share/www/script/test/design_docs.js
+++ b/share/www/script/test/design_docs.js
@@ -110,7 +110,7 @@ function() {
   var vinfo = dinfo.view_index;
   TEquals(51, vinfo.disk_size);
   TEquals(false, vinfo.compact_running);
-  TEquals("dc3264b45b74cc6d94666e3043e07154", vinfo.signature, 'ddoc sig');
+  TEquals("86e9b34892b4df35cd2f7c27da30c94d", vinfo.signature, 'ddoc sig');
 
   db.bulkSave(makeDocs(1, numDocs + 1));
 
diff --git a/share/www/script/test/etags_views.js b/share/www/script/test/etags_views.js
index 7e1537bd..f556d6ac 100644
--- a/share/www/script/test/etags_views.js
+++ b/share/www/script/test/etags_views.js
@@ -11,23 +11,34 @@
 // the License.
 
 couchTests.etags_views = function(debug) {
-  var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"false"});
+  var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"true"});
   db.deleteDb();
   db.createDb();
   if (debug) debugger;
 
   var designDoc = {
-    _id:"_design/etags",
+    _id: "_design/etags",
     language: "javascript",
     views : {
+      fooView: {
+        map: stringFun(function(doc) {
+          if (doc.foo) {
+            emit("bar", 1);
+          }
+        }),
+      },
       basicView : {
         map : stringFun(function(doc) {
-          emit(doc.integer, doc.string);
+          if(doc.integer && doc.string) {
+            emit(doc.integer, doc.string);
+          }
         })
       },
       withReduce : {
         map : stringFun(function(doc) {
-          emit(doc.integer, doc.string);
+          if(doc.integer && doc.string) {
+            emit(doc.integer, doc.string);
+          }
         }),
         reduce : stringFun(function(keys, values, rereduce) {
           if (rereduce) {
@@ -40,9 +51,9 @@ couchTests.etags_views = function(debug) {
     }
   };
   T(db.save(designDoc).ok);
+  db.bulkSave(makeDocs(0, 10));
+
   var xhr;
-  var docs = makeDocs(0, 10);
-  db.bulkSave(docs);
 
   // verify get w/Etag on map view
   xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
@@ -52,17 +63,92 @@ couchTests.etags_views = function(debug) {
     headers: {"if-none-match": etag}
   });
   T(xhr.status == 304);
-  // TODO GET with keys (when that is available)
+
+  // verify ETag doesn't change when an update
+  // doesn't change the view group's index
+  T(db.save({"_id":"doc1", "foo":"bar"}).ok);
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 == etag);
+ 
+  // Verify that purges affect etags
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+  var foo_etag = xhr.getResponseHeader("etag");
+  var doc1 = db.open("doc1");
+  xhr = CouchDB.request("POST", "/test_suite_db/_purge", {
+    body: JSON.stringify({"doc1":[doc1._rev]})
+  });
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 != foo_etag);
+
+  // Test that _purge didn't affect the other view etags.
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 == etag);
+
+  // verify different views in the same view group may have different ETags
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+  var etag1 = xhr.getResponseHeader("etag");
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag2 = xhr.getResponseHeader("etag");
+  T(etag1 != etag2);
+
+  // verify ETag changes when an update changes the view group's index.
+  db.bulkSave(makeDocs(10, 20));
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 != etag);
+
+  // verify ETag is the same after a restart
+  restartServer();
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag2 = xhr.getResponseHeader("etag");
+  T(etag1 == etag2);
 
   // reduce view
   xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
   T(xhr.status == 200);
   var etag = xhr.getResponseHeader("etag");
-  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce", {
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce",{
     headers: {"if-none-match": etag}
   });
   T(xhr.status == 304);
 
+  // verify ETag doesn't change when an update
+  // doesn't change the view group's index
+  T(db.save({"_id":"doc3", "foo":"bar"}).ok);
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 == etag);
+  // purge
+  var doc3 = db.open("doc3");
+  xhr = CouchDB.request("POST", "/test_suite_db/_purge", {
+    body: JSON.stringify({"doc3":[doc3._rev]})
+  });
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 == etag);
+
+  // verify different views in the same view group may have different ETags
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+  var etag1 = xhr.getResponseHeader("etag");
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag2 = xhr.getResponseHeader("etag");
+  T(etag1 != etag2);
+
+  // verify ETag changes when an update changes the view group's index
+  db.bulkSave(makeDocs(20, 30));
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 != etag);
+
+  // verify ETag is the same after a restart
+  restartServer();
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag2 = xhr.getResponseHeader("etag");
+  T(etag1 == etag2);
+
   // confirm ETag changes with different POST bodies
   xhr = CouchDB.request("POST", "/test_suite_db/_design/etags/_view/basicView",
     {body: JSON.stringify({keys:[1]})}
diff --git a/src/couchdb/couch_db.hrl b/src/couchdb/couch_db.hrl
index f861219e..74d2c630 100644
--- a/src/couchdb/couch_db.hrl
+++ b/src/couchdb/couch_db.hrl
@@ -238,6 +238,8 @@
 
 -record(view,
     {id_num,
+    update_seq=0,
+    purge_seq=0,
     map_names=[],
     def,
     btree=nil,
diff --git a/src/couchdb/couch_httpd_show.erl b/src/couchdb/couch_httpd_show.erl
index d50ca83a..fa8cd972 100644
--- a/src/couchdb/couch_httpd_show.erl
+++ b/src/couchdb/couch_httpd_show.erl
@@ -188,14 +188,14 @@ handle_view_list_req(Req, _Db, _DDoc) ->
 handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) ->
     ViewDesignId = <<"_design/", ViewDesignName/binary>>,
     {ViewType, View, Group, QueryArgs} = couch_httpd_view:load_view(Req, Db, {ViewDesignId, ViewName}, Keys),
-    Etag = list_etag(Req, Db, Group, {couch_httpd:doc_etag(DDoc), Keys}),
+    Etag = list_etag(Req, Db, Group, View, {couch_httpd:doc_etag(DDoc), Keys}),
     couch_httpd:etag_respond(Req, Etag, fun() ->
             output_list(ViewType, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group)
         end).
 
-list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, More) ->
+list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, View, More) ->
     Accept = couch_httpd:header_value(Req, "Accept"),
-    couch_httpd_view:view_group_etag(Group, Db, {More, Accept, UserCtx#user_ctx.roles}).
+    couch_httpd_view:view_etag(Db, Group, View, {More, Accept, UserCtx#user_ctx.roles}).
 
 output_list(map, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group) ->
     output_map_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group);
diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl
index 65598a68..8829de02 100644
--- a/src/couchdb/couch_httpd_view.erl
+++ b/src/couchdb/couch_httpd_view.erl
@@ -17,7 +17,7 @@
 
 -export([parse_view_params/3]).
 -export([make_view_fold_fun/7, finish_view_fold/4, finish_view_fold/5, view_row_obj/3]).
--export([view_group_etag/2, view_group_etag/3, make_reduce_fold_funs/6]).
+-export([view_etag/3, view_etag/4, make_reduce_fold_funs/6]).
 -export([design_doc_view/5, parse_bool_param/1, doc_member/2]).
 -export([make_key_options/1, load_view/4]).
 
@@ -113,7 +113,7 @@ output_map_view(Req, View, Group, Db, QueryArgs, nil) ->
         limit = Limit,
         skip = SkipCount
     } = QueryArgs,
-    CurrentEtag = view_group_etag(Group, Db),
+    CurrentEtag = view_etag(Db, Group, View),
     couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
         {ok, RowCount} = couch_view:get_row_count(View),
         FoldlFun = make_view_fold_fun(Req, QueryArgs, CurrentEtag, Db, Group#group.current_seq, RowCount, #view_fold_helper_funs{reduce_count=fun couch_view:reduce_to_count/1}),
@@ -129,7 +129,7 @@ output_map_view(Req, View, Group, Db, QueryArgs, Keys) ->
         limit = Limit,
         skip = SkipCount
     } = QueryArgs,
-    CurrentEtag = view_group_etag(Group, Db, Keys),
+    CurrentEtag = view_etag(Db, Group, View, Keys),
     couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
         {ok, RowCount} = couch_view:get_row_count(View),
         FoldAccInit = {Limit, SkipCount, undefined, []},
@@ -154,7 +154,7 @@ output_reduce_view(Req, Db, View, Group, QueryArgs, nil) ->
         skip = Skip,
         group_level = GroupLevel
     } = QueryArgs,
-    CurrentEtag = view_group_etag(Group, Db),
+    CurrentEtag = view_etag(Db, Group, View),
     couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
         {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel,
                 QueryArgs, CurrentEtag, Group#group.current_seq,
@@ -172,7 +172,7 @@ output_reduce_view(Req, Db, View, Group, QueryArgs, Keys) ->
         skip = Skip,
         group_level = GroupLevel
     } = QueryArgs,
-    CurrentEtag = view_group_etag(Group, Db, Keys),
+    CurrentEtag = view_etag(Db, Group, View, Keys),
     couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
         {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel,
                 QueryArgs, CurrentEtag, Group#group.current_seq,
@@ -607,16 +607,15 @@ send_json_reduce_row(Resp, {Key, Value}, RowFront) ->
     send_chunk(Resp, RowFront ++ ?JSON_ENCODE({[{key, Key}, {value, Value}]})),
     {ok, ",\r\n"}.
 
-view_group_etag(Group, Db) ->
-    view_group_etag(Group, Db, nil).
+view_etag(Db, Group, View) ->
+    view_etag(Db, Group, View, nil).
 
-view_group_etag(#group{sig=Sig,current_seq=CurrentSeq}, _Db, Extra) ->
-    % ?LOG_ERROR("Group ~p",[Group]),
-    % This is not as granular as it could be.
-    % If there are updates to the db that do not effect the view index,
-    % they will change the Etag. For more granular Etags we'd need to keep
-    % track of the last Db seq that caused an index change.
-    couch_httpd:make_etag({Sig, CurrentSeq, Extra}).
+view_etag(Db, Group, {reduce, _, _, View}, Extra) ->
+    view_etag(Db, Group, View, Extra);
+view_etag(Db, Group, {temp_reduce, View}, Extra) ->
+    view_etag(Db, Group, View, Extra);
+view_etag(_Db, #group{sig=Sig}, #view{update_seq=UpdateSeq, purge_seq=PurgeSeq}, Extra) ->
+    couch_httpd:make_etag({Sig, UpdateSeq, PurgeSeq, Extra}).
 
 % the view row has an error
 view_row_obj(_Db, {{Key, error}, Value}, _IncludeDocs) ->
diff --git a/src/couchdb/couch_view_group.erl b/src/couchdb/couch_view_group.erl
index 962f4aae..3a3ffe99 100644
--- a/src/couchdb/couch_view_group.erl
+++ b/src/couchdb/couch_view_group.erl
@@ -404,11 +404,15 @@ prepare_group({RootDir, DbName, #group{sig=Sig}=Group}, ForceReset)->
 
 get_index_header_data(#group{current_seq=Seq, purge_seq=PurgeSeq,
             id_btree=IdBtree,views=Views}) ->
-    ViewStates = [couch_btree:get_state(Btree) || #view{btree=Btree} <- Views],
-    #index_header{seq=Seq,
-            purge_seq=PurgeSeq,
-            id_btree_state=couch_btree:get_state(IdBtree),
-            view_states=ViewStates}.
+    ViewStates = [
+        {couch_btree:get_state(V#view.btree), V#view.update_seq, V#view.purge_seq} || V <- Views
+    ],
+    #index_header{
+        seq=Seq,
+        purge_seq=PurgeSeq,
+        id_btree_state=couch_btree:get_state(IdBtree),
+        view_states=ViewStates
+    }.
 
 hex_sig(GroupSig) ->
     couch_util:to_hex(?b2l(GroupSig)).
@@ -460,13 +464,15 @@ set_view_sig(#group{
             lib={[]},
             def_lang=Language,
             design_options=DesignOptions}=G) ->
-    G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions}))};
+    ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views],
+    G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions}))};
 set_view_sig(#group{
             views=Views,
             lib=Lib,
             def_lang=Language,
             design_options=DesignOptions}=G) ->
-    G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions, sort_lib(Lib)}))}.
+    ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views],
+    G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions, sort_lib(Lib)}))}.
 
 sort_lib({Lib}) ->
     sort_lib(Lib, []).
@@ -574,14 +580,14 @@ delete_index_file(RootDir, DbName, GroupSig) ->
 init_group(Db, Fd, #group{views=Views}=Group, nil) ->
     init_group(Db, Fd, Group,
         #index_header{seq=0, purge_seq=couch_db:get_purge_seq(Db),
-            id_btree_state=nil, view_states=[nil || _ <- Views]});
+            id_btree_state=nil, view_states=[{nil, 0, 0} || _ <- Views]});
 init_group(Db, Fd, #group{def_lang=Lang,views=Views}=
             Group, IndexHeader) ->
      #index_header{seq=Seq, purge_seq=PurgeSeq,
             id_btree_state=IdBtreeState, view_states=ViewStates} = IndexHeader,
     {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd),
     Views2 = lists:zipwith(
-        fun(BtreeState, #view{reduce_funs=RedFuns,options=Options}=View) ->
+        fun({BTState, USeq, PSeq}, #view{reduce_funs=RedFuns,options=Options}=View) ->
             FunSrcs = [FunSrc || {_Name, FunSrc} <- RedFuns],
             ReduceFun =
                 fun(reduce, KVs) ->
@@ -604,10 +610,10 @@ init_group(Db, Fd, #group{def_lang=Lang,views=Views}=
             <<"raw">> ->
                 Less = fun(A,B) -> A < B end
             end,
-            {ok, Btree} = couch_btree:open(BtreeState, Fd,
-                        [{less, Less},
-                            {reduce, ReduceFun}]),
-            View#view{btree=Btree}
+            {ok, Btree} = couch_btree:open(BTState, Fd,
+                    [{less, Less}, {reduce, ReduceFun}]
+            ),
+            View#view{btree=Btree, update_seq=USeq, purge_seq=PSeq}
         end,
         ViewStates, Views),
     Group#group{db=Db, fd=Fd, current_seq=Seq, purge_seq=PurgeSeq,
diff --git a/src/couchdb/couch_view_updater.erl b/src/couchdb/couch_view_updater.erl
index bfef73ba..8e089fa9 100644
--- a/src/couchdb/couch_view_updater.erl
+++ b/src/couchdb/couch_view_updater.erl
@@ -96,19 +96,25 @@ purge_index(#group{db=Db, views=Views, id_btree=IdBtree}=Group) ->
         end, dict:new(), Lookups),
 
     % Now remove the values from the btrees
+    PurgeSeq = couch_db:get_purge_seq(Db),
     Views2 = lists:map(
         fun(#view{id_num=Num,btree=Btree}=View) ->
             case dict:find(Num, ViewKeysToRemoveDict) of
             {ok, RemoveKeys} ->
-                {ok, Btree2} = couch_btree:add_remove(Btree, [], RemoveKeys),
-                View#view{btree=Btree2};
+                {ok, ViewBtree2} = couch_btree:add_remove(Btree, [], RemoveKeys),
+                case ViewBtree2 =/= Btree of
+                    true ->
+                        View#view{btree=ViewBtree2, purge_seq=PurgeSeq};
+                    _ ->
+                        View#view{btree=ViewBtree2}
+                end;
             error -> % no keys to remove in this view
                 View
             end
         end, Views),
     Group#group{id_btree=IdBtree2,
             views=Views2,
-            purge_seq=couch_db:get_purge_seq(Db)}.
+            purge_seq=PurgeSeq}.
 
 
 load_doc(Db, DocInfo, MapQueue, DocOpts, IncludeDesign) ->
@@ -247,7 +253,12 @@ write_changes(Group, ViewKeyValuesToAdd, DocIdViewIdKeys, NewSeq, InitialBuild)
     Views2 = lists:zipwith(fun(View, {_View, AddKeyValues}) ->
             KeysToRemove = couch_util:dict_find(View#view.id_num, KeysToRemoveByView, []),
             {ok, ViewBtree2} = couch_btree:add_remove(View#view.btree, AddKeyValues, KeysToRemove),
-            View#view{btree = ViewBtree2}
+            case ViewBtree2 =/= View#view.btree of
+                true ->
+                    View#view{btree=ViewBtree2, update_seq=NewSeq};
+                _ ->
+                    View#view{btree=ViewBtree2}
+            end
         end,    Group#group.views, ViewKeyValuesToAdd),
     Group#group{views=Views2, current_seq=NewSeq, id_btree=IdBtree2}.