From d32d766aced9413a7500e17bb670220f38e15205 Mon Sep 17 00:00:00 2001 From: "Damien F. Katz" Date: Thu, 12 Jun 2008 21:32:20 +0000 Subject: fix for problem when view index header data exceeds 2k. git-svn-id: https://svn.apache.org/repos/asf/incubator/couchdb/trunk@667236 13f79535-47bb-0310-9956-ffa450edef68 --- src/couchdb/couch_view.erl | 83 ++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 43 deletions(-) (limited to 'src/couchdb/couch_view.erl') diff --git a/src/couchdb/couch_view.erl b/src/couchdb/couch_view.erl index ee493c7d..ecb8e06b 100644 --- a/src/couchdb/couch_view.erl +++ b/src/couchdb/couch_view.erl @@ -22,12 +22,12 @@ -include("couch_db.hrl"). -record(group, - {db, - fd, + {sig=nil, + db=nil, + fd=nil, name, def_lang, views, - reductions=[], % list of reduction names and id_num of view that contains it. id_btree=nil, current_seq=0, query_server=nil @@ -189,7 +189,8 @@ design_doc_to_view_group(#doc{id=Id,body={obj, Fields}}) -> {View#view{id_num=N},N+1} end, 0, dict:to_list(DictBySrc)), - reset_group(#group{name=Id, views=Views, def_lang=Language}). + Group = #group{name=Id, views=Views, def_lang=Language}, + Group#group{sig=erlang:md5(term_to_binary(Group))}. @@ -331,7 +332,7 @@ start_temp_update_loop(DbName, Fd, Lang, MapSrc, RedSrc) -> current_seq=0, def_lang=Lang, id_btree=nil}, - Group2 = disk_group_to_mem(Db, Fd, Group), + Group2 = init_group(Db, Fd, Group,nil), temp_update_loop(Group2, NotifyPids); Else -> exit(Else) @@ -355,7 +356,7 @@ start_update_loop(RootDir, DbName, GroupId) -> start_update_loop(RootDir, DbName, GroupId, get_notify_pids(1000)). start_update_loop(RootDir, DbName, GroupId, NotifyPids) -> - {Db, DbGroup} = + {Db, Group} = case (catch couch_server:open(DbName)) of {ok, Db0} -> case (catch couch_db:open_doc(Db0, GroupId)) of @@ -370,38 +371,37 @@ start_update_loop(RootDir, DbName, GroupId, NotifyPids) -> exit(Else) end, FileName = RootDir ++ "/." ++ DbName ++ GroupId ++".view", - Group = + Group2 = case couch_file:open(FileName) of {ok, Fd} -> + Sig = Group#group.sig, case (catch couch_file:read_header(Fd, <<$r, $c, $k, 0>>)) of - {ok, ExistingDiskGroup} -> - % validate all the view definitions in the index are correct. - case reset_group(ExistingDiskGroup) == reset_group(DbGroup) of - true -> disk_group_to_mem(Db, Fd, ExistingDiskGroup); - false -> reset_file(Db, Fd, DbName, DbGroup) - end; + {ok, {Sig, HeaderInfo}} -> + % sigs match! + init_group(Db, Fd, Group, HeaderInfo); _ -> - reset_file(Db, Fd, DbName, DbGroup) + reset_file(Db, Fd, DbName, Group) end; {error, enoent} -> case couch_file:open(FileName, [create]) of - {ok, Fd} -> reset_file(Db, Fd, DbName, DbGroup); + {ok, Fd} -> reset_file(Db, Fd, DbName, Group); Error -> throw(Error) end end, - update_loop(RootDir, DbName, GroupId, Group, NotifyPids). + update_loop(RootDir, DbName, GroupId, Group2, NotifyPids). -reset_file(Db, Fd, DbName, #group{name=Name} = DiskReadyGroup) -> +reset_file(Db, Fd, DbName, #group{sig=Sig,name=Name} = Group) -> ?LOG_DEBUG("Reseting group index \"~s\" in db ~s", [Name, DbName]), ok = couch_file:truncate(Fd, 0), - ok = couch_file:write_header(Fd, <<$r, $c, $k, 0>>, DiskReadyGroup), - disk_group_to_mem(Db, Fd, DiskReadyGroup). + ok = couch_file:write_header(Fd, <<$r, $c, $k, 0>>, {Sig, nil}), + init_group(Db, Fd, reset_group(Group), nil). -update_loop(RootDir, DbName, GroupId, #group{fd=Fd}=Group, NotifyPids) -> +update_loop(RootDir, DbName, GroupId, #group{sig=Sig,fd=Fd}=Group, NotifyPids) -> try update_group(Group) of {ok, Group2} -> - ok = couch_file:write_header(Fd, <<$r, $c, $k, 0>>, mem_group_to_disk(Group2)), + HeaderData = {Sig, get_index_header_data(Group2)}, + ok = couch_file:write_header(Fd, <<$r, $c, $k, 0>>, HeaderData), [Pid ! {self(), {ok, Group2}} || Pid <- NotifyPids], garbage_collect(), update_loop(RootDir, DbName, GroupId, Group2, get_notify_pids(100000)) @@ -475,11 +475,13 @@ nuke_dir(Dir) -> delete_index_file(RootDir, DbName, GroupId) -> file:delete(RootDir ++ "/." ++ DbName ++ GroupId ++ ".view"). -% Given a disk ready group structure, return an initialized, in-memory version. -disk_group_to_mem(Db, Fd, #group{id_btree=IdState,def_lang=Lang,views=Views}=Group) -> - {ok, IdBtree} = couch_btree:open(IdState, Fd), - Views2 = lists:map( - fun(#view{btree=BtreeState,reduce_funs=RedFuns}=View) -> +init_group(Db, Fd, #group{views=Views}=Group, nil = _IndexHeaderData) -> + init_group(Db, Fd, Group, {0, nil, [nil || _ <- Views]}); +init_group(Db, Fd, #group{def_lang=Lang,views=Views}=Group, + {Seq, IdBtreeState, ViewStates} = _IndexHeaderData) -> + {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd), + Views2 = lists:zipwith( + fun(BtreeState, #view{btree=BtreeState,reduce_funs=RedFuns}=View) -> FunSrcs = [FunSrc || {_Name, FunSrc} <- RedFuns], ReduceFun = fun(reduce, KVs) -> @@ -495,18 +497,13 @@ disk_group_to_mem(Db, Fd, #group{id_btree=IdState,def_lang=Lang,views=Views}=Gro [{less, fun less_json/2},{reduce, ReduceFun}]), View#view{btree=Btree} end, - Views), - Group#group{db=Db, fd=Fd, id_btree=IdBtree, views=Views2}. - -% Given an initialized, in-memory group structure, return a disk ready version. -mem_group_to_disk(#group{id_btree=IdBtree,views=Views}=Group) -> - Views2 = lists:map( - fun(#view{btree=Btree}=View) -> - State = couch_btree:get_state(Btree), - View#view{btree=State} - end, - Views), - Group#group{db=nil, fd=nil, id_btree=couch_btree:get_state(IdBtree), views=Views2}. + ViewStates, Views), + Group#group{db=Db, fd=Fd, current_seq=Seq, id_btree=IdBtree, views=Views2}. + + +get_index_header_data(#group{current_seq=Seq,id_btree=IdBtree,views=Views}) -> + ViewStates = [couch_btree:get_state(Btree) || #view{btree=Btree} <- Views], + {Seq, couch_btree:get_state(IdBtree), ViewStates}. @@ -582,7 +579,7 @@ less_list([A|RestA], [B|RestB]) -> end end. -process_doc(Db, DocInfo, {Docs, #group{name=GroupId}=Group, ViewKVs, DocIdViewIdKeys, _LastSeq}) -> +process_doc(Db, DocInfo, {Docs, #group{sig=Sig,name=GroupId}=Group, ViewKVs, DocIdViewIdKeys, _LastSeq}) -> % This fun computes once for each document #doc_info{id=DocId, update_seq=Seq, deleted=Deleted} = DocInfo, case DocId of @@ -591,11 +588,11 @@ process_doc(Db, DocInfo, {Docs, #group{name=GroupId}=Group, ViewKVs, DocIdViewId % anything in the definition changed. case couch_db:open_doc(Db, DocInfo) of {ok, Doc} -> - case design_doc_to_view_group(Doc) == reset_group(Group) of - true -> - % nothing changed, keeping on computing + case design_doc_to_view_group(Doc) of + #group{sig=Sig} -> + % The same md5 signature, keep on computing {ok, {Docs, Group, ViewKVs, DocIdViewIdKeys, Seq}}; - false -> + _ -> throw(restart) end; {not_found, deleted} -> -- cgit v1.2.3